feat(api-pull): 完善全悉设备接入并修复 NDR 边界重复采集
- 增加全悉设备 API 日志拉取及消费处理 - 完善全悉日志解析、字段映射和标准化逻辑 - NDR 采集窗口统一采用左闭右开规则 - 对设备秒级查询范围向外取整，并在本地精确过滤 - 区分接口原始返回数和实际接收数，保证分页偏移正确推进 - 补充时间窗口、边界过滤和分页推进测试 - 更新设备 API 拉取及对接说明文档
This commit is contained in:
王绍宏
+86
@@ -212,6 +212,92 @@ class ApiPullRunnerTest {
|
||||
any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void duplicateDocumentsOrChangedMetadataNeverCommit() {
|
||||
for (boolean duplicate : new boolean[]{true, false}) {
|
||||
RunnerFixture f = new RunnerFixture();
|
||||
f.config.setPageSize(1);
|
||||
when(f.adapter.requiresDistinctExternalIds()).thenReturn(true);
|
||||
PullPage first = new PullPage(page(2, 1, 0).getEvents(), 2, 200, 20, 0, 0, "2:2");
|
||||
PullPage second = new PullPage(page(2, 1, duplicate ? 0 : 1).getEvents(), 2, 200, 20, 0, 0,
|
||||
duplicate ? "2:2" : "3:2");
|
||||
when(f.adapter.pull(any())).thenReturn(first, second);
|
||||
f.runner.execute(f.config, f.checkpoint, f.run);
|
||||
assertEquals("FAILED", f.run.getStatus());
|
||||
assertEquals(duplicate ? "PAGINATION_DUPLICATE_DOCUMENT" : "PAGINATION_DATASET_CHANGED", f.run.getErrorCode());
|
||||
verify(f.mapper, never()).commitCheckpoint(anyInt(), anyLong(), anyString(), anyLong(),
|
||||
anyLong(), anyString(), any(), any(), any());
|
||||
verify(f.ingress, times(1)).tryAppend(any());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void secondPageFailureRestartsSameWindowAtOffsetZero() {
|
||||
RunnerFixture f = new RunnerFixture(); f.config.setPageSize(1);
|
||||
when(f.adapter.pull(any())).thenReturn(page(2, 1, 0))
|
||||
.thenThrow(new com.haobang.apipull.adapter.AdapterException("HTTP_TIMEOUT", "offline timeout"))
|
||||
.thenReturn(page(2, 1, 0), page(2, 1, 1));
|
||||
f.runner.execute(f.config, f.checkpoint, f.run);
|
||||
assertEquals("FAILED", f.run.getStatus());
|
||||
verify(f.mapper, never()).commitCheckpoint(anyInt(), anyLong(), anyString(), anyLong(),
|
||||
anyLong(), anyString(), any(), any(), any());
|
||||
f.runner.execute(f.config, f.checkpoint, run());
|
||||
org.mockito.ArgumentCaptor<PullRequest> requests = org.mockito.ArgumentCaptor.forClass(PullRequest.class);
|
||||
verify(f.adapter, times(4)).pull(requests.capture());
|
||||
assertEquals(0, requests.getAllValues().get(2).getOffset());
|
||||
assertEquals(requests.getAllValues().get(0).getQueryStart(), requests.getAllValues().get(2).getQueryStart());
|
||||
assertEquals(requests.getAllValues().get(0).getQueryEnd(), requests.getAllValues().get(2).getQueryEnd());
|
||||
verify(f.mapper).commitCheckpoint(anyInt(), anyLong(), anyString(), anyLong(), anyLong(), anyString(), any(), any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedDurabilityEmptyEarlyPageAndTotalDriftDoNotCommit() {
|
||||
for (int scenario = 0; scenario < 3; scenario++) {
|
||||
RunnerFixture f = new RunnerFixture(); f.config.setPageSize(1);
|
||||
if (scenario == 0) {
|
||||
CompletableFuture<DurableReceipt> failed = new CompletableFuture<>();
|
||||
failed.completeExceptionally(new java.io.IOException("offline force failure"));
|
||||
when(f.ingress.tryAppend(any())).thenReturn(AppendAttempt.accepted(failed));
|
||||
}
|
||||
when(f.adapter.pull(any())).thenReturn(page(2, 1, 0),
|
||||
scenario == 1 ? page(2, 0, 1) : page(3, 1, 1));
|
||||
f.runner.execute(f.config, f.checkpoint, f.run);
|
||||
assertEquals("FAILED", f.run.getStatus());
|
||||
verify(f.mapper, never()).commitCheckpoint(anyInt(), anyLong(), anyString(), anyLong(),
|
||||
anyLong(), anyString(), any(), any(), any());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void filteredBoundaryPageAdvancesVendorOffsetWithoutWritingWal() {
|
||||
RunnerFixture fixture = new RunnerFixture();
|
||||
fixture.config.setPageSize(1);
|
||||
PullPage filteredBoundary =
|
||||
new PullPage(
|
||||
java.util.Collections.emptyList(),
|
||||
2,
|
||||
200,
|
||||
20L,
|
||||
10L,
|
||||
1L,
|
||||
null,
|
||||
1);
|
||||
when(fixture.adapter.pull(any()))
|
||||
.thenReturn(filteredBoundary, page(2L, 1, 1L));
|
||||
|
||||
fixture.runner.execute(fixture.config, fixture.checkpoint, fixture.run);
|
||||
|
||||
assertEquals("SUCCESS", fixture.run.getStatus());
|
||||
assertEquals(Long.valueOf(1L), fixture.run.getRecordsFetched());
|
||||
assertEquals(Long.valueOf(1L), fixture.run.getRecordsDurable());
|
||||
verify(fixture.ingress, times(1)).tryAppend(any(IngressEvent.class));
|
||||
org.mockito.ArgumentCaptor<PullRequest> requests =
|
||||
org.mockito.ArgumentCaptor.forClass(PullRequest.class);
|
||||
verify(fixture.adapter, times(2)).pull(requests.capture());
|
||||
assertEquals(0L, requests.getAllValues().get(0).getOffset());
|
||||
assertEquals(1L, requests.getAllValues().get(1).getOffset());
|
||||
}
|
||||
|
||||
private static PullPage page(long total, int count, long offset) {
|
||||
List<PulledEvent> events = new ArrayList<>();
|
||||
for (int index = 0; index < count; index++) {
|
||||
|
||||
+99
@@ -227,6 +227,93 @@ class NdrServerApiAdapterTest {
|
||||
assertEquals(500, new ObjectMapper().readTree(requestBody).path("limit").asInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void adjacentLogicalWindowsEmitTheirSharedBoundaryOnlyOnce() throws Exception {
|
||||
CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
CloseableHttpResponse challenge =
|
||||
response(200, "{\"result\":1,\"data\":{\"challenge\":\"c\"}}");
|
||||
CloseableHttpResponse verified =
|
||||
response(200, "{\"result\":1,\"data\":{\"token\":\"t\"}}");
|
||||
String boundaryResponse =
|
||||
"{\"result\":1,\"data\":{\"alerts\":[{\"index\":\"i\",\"id\":\"boundary\","
|
||||
+ "\"timestamp\":\"2024-07-02 15:01:00\"}],\"total\":1}}";
|
||||
CloseableHttpResponse previousBusiness = response(200, boundaryResponse);
|
||||
CloseableHttpResponse nextBusiness = response(200, boundaryResponse);
|
||||
when(http.execute(any(HttpUriRequest.class)))
|
||||
.thenReturn(challenge, verified, previousBusiness, nextBusiness);
|
||||
NdrServerApiAdapter adapter = adapter(http);
|
||||
|
||||
PullPage previous =
|
||||
adapter.pull(
|
||||
new PullRequest(
|
||||
config(),
|
||||
Instant.parse("2024-07-02T07:00:00Z"),
|
||||
Instant.parse("2024-07-02T07:01:00Z"),
|
||||
100,
|
||||
0));
|
||||
PullPage next =
|
||||
adapter.pull(
|
||||
new PullRequest(
|
||||
config(),
|
||||
Instant.parse("2024-07-02T07:01:00Z"),
|
||||
Instant.parse("2024-07-02T07:02:00Z"),
|
||||
100,
|
||||
0));
|
||||
|
||||
assertEquals(1, previous.getSourceRecordCount());
|
||||
assertTrue(previous.getEvents().isEmpty(), "右端点不属于前一个半开窗口");
|
||||
assertEquals(1, next.getSourceRecordCount());
|
||||
assertEquals(1, next.getEvents().size(), "左端点属于后一个半开窗口");
|
||||
|
||||
org.mockito.ArgumentCaptor<HttpUriRequest> requests =
|
||||
org.mockito.ArgumentCaptor.forClass(HttpUriRequest.class);
|
||||
verify(http, times(4)).execute(requests.capture());
|
||||
JsonNode previousBody = requestBody(requests.getAllValues().get(2));
|
||||
JsonNode nextBody = requestBody(requests.getAllValues().get(3));
|
||||
assertEquals("2024-07-02 15:01:00", previousBody.path("end_time").asText());
|
||||
assertEquals("2024-07-02 15:01:00", nextBody.path("start_time").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fractionalLogicalWindowIsExpandedWithoutLosingEventsInsideTheWindow() throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
ObjectNode data = mapper.createObjectNode();
|
||||
data.putArray("logs")
|
||||
.add(flowAt(mapper, "before", "2024-07-02T15:00:00.568+08:00"))
|
||||
.add(flowAt(mapper, "at-start", "2024-07-02T15:00:00.569+08:00"))
|
||||
.add(flowAt(mapper, "before-end", "2024-07-02T15:00:15.568999+08:00"))
|
||||
.add(flowAt(mapper, "at-end", "2024-07-02T15:00:15.569+08:00"));
|
||||
data.put("total", 4);
|
||||
ObjectNode root = mapper.createObjectNode();
|
||||
root.put("result", 1);
|
||||
root.set("data", data);
|
||||
CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
CloseableHttpResponse challenge =
|
||||
response(200, "{\"result\":1,\"data\":{\"challenge\":\"c\"}}");
|
||||
CloseableHttpResponse verified =
|
||||
response(200, "{\"result\":1,\"data\":{\"token\":\"t\"}}");
|
||||
CloseableHttpResponse business = response(200, mapper.writeValueAsString(root));
|
||||
when(http.execute(any(HttpUriRequest.class)))
|
||||
.thenReturn(challenge, verified, business);
|
||||
DeviceCollectApiConfig config = config();
|
||||
config.setStreamKey(NdrServerApiAdapter.FLOW);
|
||||
Instant start = Instant.parse("2024-07-02T07:00:00.569Z");
|
||||
Instant end = Instant.parse("2024-07-02T07:00:15.569Z");
|
||||
|
||||
PullPage page = adapter(http).pull(new PullRequest(config, start, end, 100, 0));
|
||||
|
||||
assertEquals(4, page.getSourceRecordCount());
|
||||
assertEquals(2, page.getEvents().size());
|
||||
assertEquals("at-start", page.getEvents().get(0).getExternalEventId());
|
||||
assertEquals("before-end", page.getEvents().get(1).getExternalEventId());
|
||||
org.mockito.ArgumentCaptor<HttpUriRequest> requests =
|
||||
org.mockito.ArgumentCaptor.forClass(HttpUriRequest.class);
|
||||
verify(http, times(3)).execute(requests.capture());
|
||||
JsonNode body = requestBody(requests.getAllValues().get(2));
|
||||
assertEquals("2024-07-02 15:00:00", body.path("start_time").asText());
|
||||
assertEquals("2024-07-02 15:00:16", body.path("end_time").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void alertHttpPayloadGainsTcpCompatibleAliasesAndKeepsVendorFields() throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
@@ -472,6 +559,11 @@ class NdrServerApiAdapterTest {
|
||||
return new NdrServerApiAdapter(provider, new SafeBaseUrlValidator(p), new ObjectMapper(), http);
|
||||
}
|
||||
|
||||
private JsonNode requestBody(HttpUriRequest request) throws Exception {
|
||||
HttpEntityEnclosingRequest entityRequest = (HttpEntityEnclosingRequest) request;
|
||||
return new ObjectMapper().readTree(EntityUtils.toString(entityRequest.getEntity(), "UTF-8"));
|
||||
}
|
||||
|
||||
private ObjectNode flowLog(ObjectMapper mapper) {
|
||||
ObjectNode log = mapper.createObjectNode();
|
||||
log.put("_id", "flow-1");
|
||||
@@ -480,6 +572,13 @@ class NdrServerApiAdapterTest {
|
||||
return log;
|
||||
}
|
||||
|
||||
private ObjectNode flowAt(ObjectMapper mapper, String id, String timestamp) {
|
||||
ObjectNode log = mapper.createObjectNode();
|
||||
log.put("_id", id);
|
||||
log.put("timestamp", timestamp);
|
||||
return log;
|
||||
}
|
||||
|
||||
private ObjectNode alertLog(ObjectMapper mapper) {
|
||||
ObjectNode alert = mapper.createObjectNode();
|
||||
alert.put("index", "ndralert-20260904");
|
||||
|
||||
+269
@@ -0,0 +1,269 @@
|
||||
package com.haobang.apipull.adapter.quanxi;
|
||||
|
||||
import com.fasterxml.jackson.databind.*;
|
||||
import com.fasterxml.jackson.databind.node.*;
|
||||
import com.haobang.apipull.*;
|
||||
import com.haobang.apipull.adapter.*;
|
||||
import com.haobang.apipull.credential.*;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
|
||||
import com.haobang.apipull.security.SafeBaseUrlValidator;
|
||||
import java.io.*;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.nio.file.*;
|
||||
import java.time.*;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.apache.http.*;
|
||||
import org.apache.http.client.methods.*;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
|
||||
/** No sockets, Spring Boot, DB, Kafka or vendor credentials: all HTTP executions are mocked. */
|
||||
class QuanxiApiAdapterTest {
|
||||
static final ObjectMapper JSON = new ObjectMapper();
|
||||
static final Instant START = Instant.parse("2026-08-10T00:00:00Z");
|
||||
static final Instant END = START.plusSeconds(86400);
|
||||
static final String TOKEN = "synthetic-unit-test-token";
|
||||
|
||||
@Test void realFixtureBecomesTwentyIndependentOriginalDocumentsAndGoldenIds() throws Exception {
|
||||
Fixture f = new Fixture();
|
||||
PullPage page = f.pull();
|
||||
JsonNode expected = JSON.readTree(file("fixtures/expected-events.json").toFile());
|
||||
assertEquals(20, page.getEvents().size());
|
||||
assertEquals(2311, page.getTotal());
|
||||
assertEquals("2311:2311", page.getDatasetFingerprint());
|
||||
for (int i = 0; i < 20; i++) {
|
||||
PulledEvent event = page.getEvents().get(i);
|
||||
assertEquals(f.root.path("result").path("data").get(i), JSON.readTree(event.getRawMessage()));
|
||||
assertEquals(expected.get(i).path("event_id").asText(), event.getEventId());
|
||||
assertEquals(expected.get(i).path("event_time_ms").asLong(), event.getEventTime().toEpochMilli());
|
||||
assertEquals(expected.get(i).path("doc_id").asText(), event.getExternalEventId());
|
||||
assertFalse(JSON.readTree(event.getRawMessage()).has("http_req_body"));
|
||||
}
|
||||
assertEquals(page.getEvents().get(6).getEventTime(), page.getEvents().get(7).getEventTime());
|
||||
assertNotEquals(page.getEvents().get(6).getEventId(), page.getEvents().get(7).getEventId());
|
||||
}
|
||||
|
||||
@Test void protocolHeaderPaginationAndMillisecondsAreExact() throws Exception {
|
||||
Fixture f = new Fixture();
|
||||
f.pull(20);
|
||||
JsonNode body = f.request.get();
|
||||
assertEquals("AlarmService.GetAlarmDocument", body.path("method").asText());
|
||||
assertEquals("2.0", body.path("jsonrpc").asText());
|
||||
JsonNode p = body.path("params");
|
||||
assertEquals(20, p.path("offset").asLong());
|
||||
assertEquals(20, p.path("count").asInt());
|
||||
assertEquals(START.toEpochMilli(), p.path("time_range_start").asLong());
|
||||
assertEquals(END.toEpochMilli() - 1, p.path("time_range_end").asLong());
|
||||
assertFalse(p.has("ignore"));
|
||||
assertFalse(p.has("collapse_key"));
|
||||
assertFalse(p.has("agg_sort"));
|
||||
}
|
||||
|
||||
@Test void configuredParamsDoNotBecomeGuessedDefaults() throws Exception {
|
||||
Fixture f = new Fixture();
|
||||
ObjectNode opts = (ObjectNode) JSON.readTree(f.config.getExtraConfigJson());
|
||||
ObjectNode p = (ObjectNode) opts.get("queryParams");
|
||||
p.put("ignore", 0); p.put("collapse_key", "collapse_field");
|
||||
p.putObject("agg_sort").put("field", "maxtime").put("operator", "").put("ascending", false);
|
||||
opts.put("endInclusive", false); opts.put("paginationTotalField", "page_total");
|
||||
f.config.setExtraConfigJson(opts.toString());
|
||||
f.root.with("result").put("total", 9000).put("page_total", 2311);
|
||||
assertEquals(2311, f.pull().getTotal());
|
||||
assertEquals(END.toEpochMilli(), f.request.get().path("params").path("time_range_end").asLong());
|
||||
assertEquals(p.get("agg_sort"), f.request.get().path("params").get("agg_sort"));
|
||||
assertEquals(0, f.request.get().path("params").path("ignore").asInt());
|
||||
}
|
||||
|
||||
@Test void retriesAndObjectKeyOrderAreIdempotentButRevisionsAndNamespacesAreNot() throws Exception {
|
||||
Fixture f = new Fixture();
|
||||
String id = f.pull().getEvents().get(0).getEventId();
|
||||
assertEquals(id, f.pull().getEvents().get(0).getEventId());
|
||||
ObjectNode original = (ObjectNode) f.data().get(0);
|
||||
List<String> keys = new ArrayList<>(); original.fieldNames().forEachRemaining(keys::add);
|
||||
Collections.reverse(keys); ObjectNode reordered = JSON.createObjectNode();
|
||||
for (String key : keys) reordered.set(key, original.get(key));
|
||||
f.data().set(0, reordered);
|
||||
assertEquals(id, f.pull().getEvents().get(0).getEventId());
|
||||
reordered.with("collapse_info").put("count", "3");
|
||||
assertNotEquals(id, f.pull().getEvents().get(0).getEventId());
|
||||
f.data().set(0, original); f.config.setDeviceId(8);
|
||||
assertNotEquals(id, f.pull().getEvents().get(0).getEventId());
|
||||
f.config.setDeviceId(7); f.config.setCollectTaskId(12);
|
||||
assertNotEquals(id, f.pull().getEvents().get(0).getEventId());
|
||||
}
|
||||
|
||||
@Test void rpcErrorsAtHttp200NeverSucceedAndNeverEchoRemoteMessages() throws Exception {
|
||||
Fixture f = new Fixture();
|
||||
f.root.remove("result");
|
||||
f.root.putObject("error").put("code", -32001).put("message", TOKEN);
|
||||
AdapterException e = assertThrows(AdapterException.class, f::pull);
|
||||
assertEquals("RPC_ERROR", e.getCode()); assertFalse(e.toString().contains(TOKEN));
|
||||
assertNull(e.getCause());
|
||||
}
|
||||
|
||||
@Test void wrongVersionMissingOrMismatchedIdsAndMissingResultFail() throws Exception {
|
||||
for (String mode : Arrays.asList("version", "id", "result")) {
|
||||
Fixture f = new Fixture(); f.rewriteId = !mode.equals("id");
|
||||
if (mode.equals("version")) f.root.put("jsonrpc", "1.0");
|
||||
if (mode.equals("result")) f.root.remove("result");
|
||||
assertThrows(AdapterException.class, f::pull);
|
||||
}
|
||||
}
|
||||
|
||||
@Test void httpErrorsAreClassifiedWithoutRetryOrErrorBodyLeak() throws Exception {
|
||||
for (int status : new int[]{301, 401, 403, 429, 500}) {
|
||||
Fixture f = new Fixture(); f.status = status;
|
||||
AdapterException e = assertThrows(AdapterException.class, f::pull);
|
||||
assertEquals(Integer.valueOf(status), e.getHttpStatus());
|
||||
assertEquals(status == 401 ? "AUTH_HTTP_401" : status == 403 ? "AUTH_HTTP_403" : "HTTP_ERROR", e.getCode());
|
||||
verify(f.http, times(1)).execute(any(HttpUriRequest.class));
|
||||
}
|
||||
}
|
||||
|
||||
@Test void timeoutsDoNotLeakClientExceptions() throws Exception {
|
||||
Fixture f = new Fixture();
|
||||
when(f.http.execute(any(HttpUriRequest.class))).thenThrow(new SocketTimeoutException(TOKEN));
|
||||
AdapterException e = assertThrows(AdapterException.class, f::pull);
|
||||
assertEquals("HTTP_TIMEOUT", e.getCode()); assertFalse(e.toString().contains(TOKEN));
|
||||
assertNull(e.getCause());
|
||||
}
|
||||
|
||||
@Test void emptyArrayRequiresConsistentCountsAndExplicitData() throws Exception {
|
||||
Fixture f = new Fixture(); f.data().removeAll();
|
||||
assertEquals("PAGINATION_INCONSISTENT", assertThrows(AdapterException.class, f::pull).getCode());
|
||||
f.root.with("result").put("total", 0).put("page_total", 0);
|
||||
assertTrue(f.pull().getEvents().isEmpty());
|
||||
f.root.with("result").remove("data");
|
||||
assertThrows(AdapterException.class, f::pull);
|
||||
}
|
||||
|
||||
@Test void fractionalNegativeStringCountsAndOversizedPagesAreRejected() throws Exception {
|
||||
for (JsonNode value : Arrays.asList(JSON.readTree("1.5"), JSON.readTree("-1"), JSON.readTree("\"20\""))) {
|
||||
Fixture f = new Fixture(); f.root.with("result").set("total", value);
|
||||
assertThrows(AdapterException.class, f::pull);
|
||||
}
|
||||
Fixture f = new Fixture(); f.data().add(f.data().get(0));
|
||||
assertThrows(AdapterException.class, f::pull);
|
||||
}
|
||||
|
||||
@Test void sameDocTwiceCannotSatisfyPagination() throws Exception {
|
||||
Fixture f = new Fixture(); f.data().set(1, f.data().get(0));
|
||||
assertEquals("PAGINATION_DUPLICATE_DOCUMENT", assertThrows(AdapterException.class, f::pull).getCode());
|
||||
}
|
||||
|
||||
@Test void timestampIsMillisecondsNotSecondsMicrosOrFlowStart() throws Exception {
|
||||
for (long value : new long[]{1786377562L, 1786377562453000L, -1L}) {
|
||||
Fixture f = new Fixture(); ((ObjectNode) f.data().get(0)).put("timestamp", value);
|
||||
assertEquals("EVENT_TIME_INVALID", assertThrows(AdapterException.class, f::pull).getCode());
|
||||
}
|
||||
Fixture f = new Fixture();
|
||||
PulledEvent e = f.pull().getEvents().get(6);
|
||||
assertEquals(1786377127842L, e.getEventTime().toEpochMilli());
|
||||
assertNotEquals(f.data().get(6).path("flow_startts").asLong(), e.getEventTime().toEpochMilli());
|
||||
((ObjectNode) f.data().get(0)).put("timestamp", END.toEpochMilli());
|
||||
assertEquals("EVENT_OUTSIDE_WINDOW", assertThrows(AdapterException.class, f::pull).getCode());
|
||||
((ObjectNode) f.data().get(0)).put("timestamp", START.toEpochMilli());
|
||||
assertEquals(START, f.pull().getEvents().get(0).getEventTime());
|
||||
}
|
||||
|
||||
@Test void malformedJsonExcessiveDepthAndResponseBoundsFailClosed() throws Exception {
|
||||
for (String malformed : Arrays.asList("{", "{}{}", "{\"a\":1,\"a\":2}", nested(66))) {
|
||||
Fixture f = new Fixture(); f.rawOverride = malformed;
|
||||
assertThrows(AdapterException.class, f::pull);
|
||||
}
|
||||
Fixture f = new Fixture(); f.config.setMaxResponseBytes(1024L);
|
||||
assertEquals("RESPONSE_TOO_LARGE", assertThrows(AdapterException.class, f::pull).getCode());
|
||||
Fixture stream = new Fixture(); stream.unknownLength = true; stream.config.setMaxResponseBytes(1024L);
|
||||
assertEquals("RESPONSE_TOO_LARGE", assertThrows(AdapterException.class, stream::pull).getCode());
|
||||
}
|
||||
|
||||
@Test void unsafeOrUnconfiguredRequestsAreRejectedBeforeHttp() throws Exception {
|
||||
Fixture f = new Fixture(); f.config.setExtraConfigJson("{}");
|
||||
assertThrows(AdapterException.class, f::pull); verifyNoInteractions(f.http);
|
||||
f.config = config(); f.config.setBaseUrl("https://192.0.2.10/ndr/api");
|
||||
assertThrows(AdapterException.class, f::pull);
|
||||
f.config = config(); f.config.setBaseUrl("http://192.0.2.10/rpc");
|
||||
assertThrows(AdapterException.class, f::pull);
|
||||
f.config = config();
|
||||
assertThrows(AdapterException.class, () -> f.adapter.pull(new PullRequest(f.config, START, END, 20, -1)));
|
||||
assertThrows(AdapterException.class, () -> f.adapter.pull(new PullRequest(f.config, START.plusNanos(1), END, 20, 0)));
|
||||
verifyNoInteractions(f.http);
|
||||
}
|
||||
|
||||
@Test void existingFieldsAndVeryLargeBodiesAreNeverDuplicatedOrDecoded() throws Exception {
|
||||
Fixture f = new Fixture(); ObjectNode doc = (ObjectNode) f.data().get(6);
|
||||
doc.put("http_req_body", "already-present");
|
||||
doc.with("appbrief").with("http").put("req_body", repeat('x', 10001));
|
||||
JsonNode raw = JSON.readTree(f.pull().getEvents().get(6).getRawMessage());
|
||||
assertEquals(doc, raw);
|
||||
assertEquals("already-present", raw.path("http_req_body").asText());
|
||||
assertEquals(10001, raw.path("appbrief").path("http").path("req_body").asText().length());
|
||||
doc.put("payload", repeat('x', 1048577));
|
||||
assertThrows(AdapterException.class, f::pull);
|
||||
}
|
||||
|
||||
static String repeat(char c, int n) { char[] a = new char[n]; Arrays.fill(a, c); return new String(a); }
|
||||
static String nested(int n) { return repeat('[', n) + "0" + repeat(']', n); }
|
||||
static Path file(String name) {
|
||||
Path root = Paths.get("").toAbsolutePath();
|
||||
while (!Files.isDirectory(root.resolve("design/device-integration-docs/quanxi"))) root = root.getParent();
|
||||
return root.resolve("design/device-integration-docs/quanxi/" + name);
|
||||
}
|
||||
static DeviceCollectApiConfig config() {
|
||||
DeviceCollectApiConfig c = new DeviceCollectApiConfig();
|
||||
c.setDeviceId(7); c.setCollectTaskId(11); c.setAdapterType(QuanxiApiAdapter.TYPE);
|
||||
c.setStreamKey(QuanxiApiAdapter.STREAM); c.setBaseUrl("https://192.0.2.10/rpc");
|
||||
c.setCredentialRef("fixture"); c.setPageSize(20); c.setMaxPagesPerRun(100);
|
||||
c.setPollIntervalSeconds(10); c.setWindowSizeSeconds(60); c.setSafetyDelaySeconds(120);
|
||||
c.setLookbackSeconds(300); c.setConnectTimeoutMs(1000); c.setRequestTimeoutMs(1000);
|
||||
c.setMaxResponseBytes(16777216L);
|
||||
c.setExtraConfigJson("{\"recordMode\":\"DOCUMENT_SNAPSHOT\",\"paginationTotalField\":\"total\","
|
||||
+ "\"endInclusive\":true,\"queryParams\":{\"ignore\":null,\"collapse_key\":null,\"agg_sort\":null}}");
|
||||
return c;
|
||||
}
|
||||
static class Fixture {
|
||||
CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
ObjectNode root = (ObjectNode) JSON.readTree(file("fixtures/api-response.json").toFile());
|
||||
AtomicReference<JsonNode> request = new AtomicReference<>();
|
||||
DeviceCollectApiConfig config = config();
|
||||
int status = 200; boolean rewriteId = true; boolean unknownLength;
|
||||
String rawOverride;
|
||||
QuanxiApiAdapter adapter;
|
||||
Fixture() throws Exception {
|
||||
ApiPullProperties p = new ApiPullProperties(); p.setAllowedHosts("192.0.2.10");
|
||||
CredentialProvider credentials = (ref, type, device) -> new CredentialSnapshot(1, "fixture", type,
|
||||
Collections.singletonMap("api_token", new SecretValue(TOKEN.toCharArray())));
|
||||
adapter = new QuanxiApiAdapter(credentials, new SafeBaseUrlValidator(p), JSON, http);
|
||||
when(http.execute(any(HttpUriRequest.class))).thenAnswer(inv -> {
|
||||
HttpPost post = inv.getArgument(0);
|
||||
assertEquals("https://192.0.2.10/rpc", post.getURI().toString());
|
||||
assertEquals(TOKEN, post.getFirstHeader("API-Token").getValue());
|
||||
assertNull(post.getFirstHeader("Authorization"));
|
||||
assertFalse(post.getConfig().isRedirectsEnabled());
|
||||
assertEquals(1000, post.getConfig().getSocketTimeout());
|
||||
request.set(JSON.readTree(EntityUtils.toString(post.getEntity(), "UTF-8")));
|
||||
ObjectNode envelope = root.deepCopy();
|
||||
if (rewriteId) envelope.set("id", request.get().get("id"));
|
||||
CloseableHttpResponse response = mock(CloseableHttpResponse.class);
|
||||
StatusLine line = mock(StatusLine.class); when(line.getStatusCode()).thenReturn(status);
|
||||
when(response.getStatusLine()).thenReturn(line);
|
||||
String text = rawOverride == null ? envelope.toString() : rawOverride;
|
||||
if (unknownLength) {
|
||||
HttpEntity entity = mock(HttpEntity.class); when(entity.getContentLength()).thenReturn(-1L);
|
||||
when(entity.getContent()).thenReturn(new ByteArrayInputStream(text.getBytes("UTF-8")));
|
||||
when(response.getEntity()).thenReturn(entity);
|
||||
} else when(response.getEntity()).thenReturn(new StringEntity(text, "UTF-8"));
|
||||
return response;
|
||||
});
|
||||
}
|
||||
ArrayNode data() { return (ArrayNode) root.path("result").path("data"); }
|
||||
PullPage pull() { return pull(0); }
|
||||
PullPage pull(long offset) { return adapter.pull(new PullRequest(config, START, END, 20, offset)); }
|
||||
}
|
||||
}
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
package com.haobang.apipull.adapter.quanxi;
|
||||
|
||||
import com.common.entity.*;
|
||||
import com.common.mapper.*;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.apipull.*;
|
||||
import com.haobang.apipull.adapter.*;
|
||||
import com.haobang.apipull.adapter.ndr.NdrServerApiAdapter;
|
||||
import com.haobang.apipull.api.ApiPullDtos.*;
|
||||
import com.haobang.apipull.credential.*;
|
||||
import com.haobang.apipull.persistence.*;
|
||||
import com.haobang.apipull.security.SafeBaseUrlValidator;
|
||||
import com.haobang.config.AppProperties;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import static org.mockito.ArgumentMatchers.*;
|
||||
|
||||
class QuanxiManagementTest {
|
||||
@Test void productionConstructorsRegisterBothVendorsWithoutExternalSystems() {
|
||||
try (AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext()) {
|
||||
ApiPullProperties props = new ApiPullProperties(); props.setAllowedHosts("192.0.2.10");
|
||||
context.registerBean(CredentialProvider.class, () -> mock(CredentialProvider.class));
|
||||
context.registerBean(ObjectMapper.class, () -> new ObjectMapper());
|
||||
context.registerBean(SafeBaseUrlValidator.class, () -> new SafeBaseUrlValidator(props));
|
||||
context.register(QuanxiApiAdapter.class, NdrServerApiAdapter.class, AdapterRegistry.class);
|
||||
context.refresh();
|
||||
AdapterRegistry registry = context.getBean(AdapterRegistry.class);
|
||||
assertEquals(Collections.singleton(QuanxiApiAdapter.STREAM), registry.require("QUANXI_V1").streams());
|
||||
assertTrue(registry.require("NDRSERVER_V1").streams().contains("NDR_ALERT"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test void createsPausedTaskAndInitialCheckpointWithRealAdapterValidation() throws Exception {
|
||||
QuanxiApiAdapterTest.Fixture f = new QuanxiApiAdapterTest.Fixture();
|
||||
DeviceCollectTaskMapper tasks = mock(DeviceCollectTaskMapper.class);
|
||||
DeviceCollectApiMapper mapper = mock(DeviceCollectApiMapper.class);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = new DeviceDevice(); device.setId(7); device.setDeviceCollectId(3);
|
||||
device.setIp("192.0.2.10"); when(devices.getByIdSafely(7)).thenReturn(device);
|
||||
AppProperties app = new AppProperties(); app.setAppServiceDeviceCollectId(3);
|
||||
ApiPullProperties props = new ApiPullProperties(); props.setAllowedHosts("192.0.2.10");
|
||||
CredentialProvider credentials = (ref, type, id) -> new CredentialSnapshot(1, "fixture", type,
|
||||
Collections.singletonMap("api_token", new SecretValue("synthetic-token".toCharArray())));
|
||||
AtomicReference<DeviceCollectApiConfig> config = new AtomicReference<>();
|
||||
AtomicReference<DeviceCollectApiCheckpoint> checkpoint = new AtomicReference<>();
|
||||
when(tasks.insertApiPullTask(any())).thenAnswer(i -> { ((DeviceCollectTask)i.getArgument(0)).setId(11); return 1; });
|
||||
when(mapper.insertConfig(any())).thenAnswer(i -> { config.set(i.getArgument(0)); return 1; });
|
||||
when(mapper.insertCheckpoint(any())).thenAnswer(i -> { checkpoint.set(i.getArgument(0)); return 1; });
|
||||
when(mapper.selectConfig(11)).thenAnswer(i -> config.get());
|
||||
when(mapper.selectCheckpoint(11)).thenAnswer(i -> checkpoint.get());
|
||||
ApiPullTaskService service = new ApiPullTaskService(tasks, mapper, devices,
|
||||
new AdapterRegistry(Collections.singletonList(f.adapter)), credentials, new SafeBaseUrlValidator(props),
|
||||
mock(ApiPullScheduler.class), app, props, new ObjectMapper());
|
||||
CreateApiPullTaskRequest request = new CreateApiPullTaskRequest();
|
||||
request.setDeviceId(7); request.setTaskName("fixture-quanxi"); request.setAdapterType(QuanxiApiAdapter.TYPE);
|
||||
request.setStreamKey(QuanxiApiAdapter.STREAM); request.setBaseUrl(f.config.getBaseUrl());
|
||||
request.setCredentialRef("fixture"); request.setInitialStartTime(OffsetDateTime.parse("2026-08-10T00:00:00+08:00"));
|
||||
request.setExtraConfig(new ObjectMapper().readValue(f.config.getExtraConfigJson(), Map.class));
|
||||
ApiPullTaskResponse response = service.create(request);
|
||||
assertFalse(response.isEnabled()); assertEquals(11, response.getTaskId());
|
||||
assertEquals(QuanxiApiAdapter.TYPE, checkpoint.get().getAdapterType());
|
||||
assertEquals(request.getInitialStartTime(), response.getCommittedWindowEnd());
|
||||
verifyNoInteractions(f.http);
|
||||
}
|
||||
|
||||
@Test void managementRpcFailureDoesNotClaimAuthenticationSuccess() throws Exception {
|
||||
QuanxiApiAdapterTest.Fixture f = new QuanxiApiAdapterTest.Fixture();
|
||||
f.root.remove("result"); f.root.putObject("error").put("code", -32000);
|
||||
f.config.setConfigVersion(1L); f.config.setAdapterVersion(1);
|
||||
DeviceCollectApiMapper mapper = mock(DeviceCollectApiMapper.class);
|
||||
when(mapper.selectConfig(11)).thenReturn(f.config);
|
||||
ApiPullProperties props = new ApiPullProperties(); props.setAllowedHosts("192.0.2.10");
|
||||
CredentialProvider credentials = (ref, type, id) -> new CredentialSnapshot(1, "fixture", type, Collections.emptyMap());
|
||||
ApiPullTaskService service = new ApiPullTaskService(mock(DeviceCollectTaskMapper.class), mapper,
|
||||
mock(DeviceDeviceService.class), new AdapterRegistry(Collections.singletonList(f.adapter)),
|
||||
credentials, new SafeBaseUrlValidator(props), mock(ApiPullScheduler.class), new AppProperties(), props, new ObjectMapper());
|
||||
ValidateApiPullTaskResponse response = service.validate(11, 1L);
|
||||
assertFalse(response.isValid());
|
||||
Map<String, ValidationCheckResponse> checks = new HashMap<>();
|
||||
response.getChecks().forEach(c -> checks.put(c.getName(), c));
|
||||
assertEquals("FAIL", checks.get("MINIMAL_ALERT_QUERY").getStatus());
|
||||
assertEquals("RPC_ERROR", checks.get("MINIMAL_ALERT_QUERY").getErrorCode());
|
||||
assertNotEquals("PASS", checks.get("AUTHENTICATION").getStatus());
|
||||
assertFalse(checks.containsKey("MINIMAL_FLOW_QUERY"));
|
||||
}
|
||||
|
||||
@Test void tokenCreationRotationLoadAndSerializationHonorVendorSchema() throws Exception {
|
||||
DeviceApiCredentialMapper mapper = mock(DeviceApiCredentialMapper.class);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = new DeviceDevice(); device.setId(7); device.setDeviceCollectId(3);
|
||||
when(devices.getByIdSafely(7)).thenReturn(device);
|
||||
AdapterRegistry registry = mock(AdapterRegistry.class);
|
||||
AppProperties app = new AppProperties(); app.setAppServiceDeviceCollectId(3);
|
||||
AtomicReference<DeviceApiCredential> stored = new AtomicReference<>();
|
||||
when(mapper.insert(any())).thenAnswer(i -> { stored.set(i.getArgument(0)); return 1; });
|
||||
when(mapper.selectByRef("fixture")).thenAnswer(i -> stored.get());
|
||||
when(mapper.updateCas(any())).thenAnswer(i -> {
|
||||
DeviceApiCredential value = i.getArgument(0); value.setDeviceId(7); stored.set(value); return 1;
|
||||
});
|
||||
when(mapper.selectForLoad("fixture", 7, "QUANXI_V1")).thenAnswer(i -> stored.get());
|
||||
ApiCredentialService service = new ApiCredentialService(mapper, devices, registry, app);
|
||||
CreateApiCredentialRequest request = new CreateApiCredentialRequest();
|
||||
request.setDeviceId(7); request.setCredentialRef("fixture"); request.setCredentialVersion("v1");
|
||||
request.setAdapterType("QUANXI_V1"); request.setApiToken("synthetic-unit-secret");
|
||||
ApiCredentialResponse response = service.create(request);
|
||||
assertNull(stored.get().getAppKey()); assertNull(stored.get().getAppSecret());
|
||||
assertFalse(request.toString().contains("synthetic-unit-secret"));
|
||||
assertFalse(new ObjectMapper().writeValueAsString(request).contains("synthetic-unit-secret"));
|
||||
assertFalse(new ObjectMapper().writeValueAsString(response).contains("apiToken"));
|
||||
UpdateApiCredentialRequest update = new UpdateApiCredentialRequest();
|
||||
update.setConfigVersion(1); update.setCredentialVersion("v2"); update.setApiToken("synthetic-rotated");
|
||||
service.update("fixture", update);
|
||||
DatabaseCredentialProvider provider = new DatabaseCredentialProvider(mapper);
|
||||
CredentialSnapshot snapshot = provider.load("fixture", "QUANXI_V1", 7);
|
||||
assertArrayEquals("synthetic-rotated".toCharArray(), snapshot.require("api_token").copy());
|
||||
assertThrows(CredentialException.class, () -> snapshot.require("app_secret"));
|
||||
snapshot.close(); assertEquals('\0', snapshot.require("api_token").copy()[0]);
|
||||
assertThrows(CredentialException.class, () -> provider.load("fixture", "QUANXI_V1", 8));
|
||||
request.setAppKey("wrong-vendor-field");
|
||||
assertThrows(CredentialException.class, () -> service.create(request));
|
||||
update.setApiToken("bad\r\nheader");
|
||||
assertThrows(CredentialException.class, () -> service.update("fixture", update));
|
||||
assertThrows(CredentialException.class, () -> CredentialSchemas.validate("NDRSERVER_V1", null, null, "token"));
|
||||
assertThrows(CredentialException.class, () -> CredentialSchemas.validate("QUANXI_V1", null, null, ""));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user