feat: 新增设备 API 拉取采集及配套处理能力
- 接入 NDRServer 告警和流量日志，支持凭据与任务管理 - 支持分页拉取、窗口拆分、调度优先级及 WAL 持久化后推进断点 - 增加迟到告警规则重放，优化多条泛化规则选择和事件时间处理 - 补充回归测试、数据库升级脚本及部署文档
This commit is contained in:
王绍宏
+67
@@ -0,0 +1,67 @@
|
||||
package com.haobang.apipull;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.mapper.DeviceApiCredentialMapper;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.apipull.adapter.AdapterRegistry;
|
||||
import com.haobang.apipull.adapter.ApiPullAdapter;
|
||||
import com.haobang.apipull.api.ApiPullDtos.ApiCredentialResponse;
|
||||
import com.haobang.apipull.api.ApiPullDtos.CreateApiCredentialRequest;
|
||||
import com.haobang.apipull.persistence.DeviceApiCredential;
|
||||
import com.haobang.config.AppProperties;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class ApiCredentialServiceTest {
|
||||
@Test
|
||||
void storesPlaintextButNeverReturnsAppSecret() throws Exception {
|
||||
DeviceApiCredentialMapper mapper = mock(DeviceApiCredentialMapper.class);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
AdapterRegistry adapters = mock(AdapterRegistry.class);
|
||||
AppProperties app = new AppProperties();
|
||||
app.setAppServiceDeviceCollectId(7);
|
||||
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
device.setDeviceCollectId(7);
|
||||
when(devices.getByIdSafely(126)).thenReturn(device);
|
||||
when(adapters.require("NDRSERVER_V1")).thenReturn(mock(ApiPullAdapter.class));
|
||||
|
||||
AtomicReference<DeviceApiCredential> stored = new AtomicReference<>();
|
||||
when(mapper.insert(any(DeviceApiCredential.class)))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
stored.set(invocation.getArgument(0));
|
||||
return 1;
|
||||
});
|
||||
when(mapper.selectByRef("ndr-v1")).thenAnswer(invocation -> stored.get());
|
||||
|
||||
ApiCredentialService service = new ApiCredentialService(mapper, devices, adapters, app);
|
||||
ApiCredentialResponse response = service.create(request());
|
||||
|
||||
assertEquals("plain-secret", stored.get().getAppSecret());
|
||||
String responseJson = new ObjectMapper().writeValueAsString(response);
|
||||
assertFalse(responseJson.contains("app-key"));
|
||||
assertFalse(responseJson.contains("appKey"));
|
||||
assertFalse(responseJson.contains("plain-secret"));
|
||||
assertFalse(responseJson.contains("appSecret"));
|
||||
}
|
||||
|
||||
private CreateApiCredentialRequest request() {
|
||||
CreateApiCredentialRequest request = new CreateApiCredentialRequest();
|
||||
request.setCredentialRef("ndr-v1");
|
||||
request.setDeviceId(126);
|
||||
request.setAdapterType("NDRSERVER_V1");
|
||||
request.setCredentialVersion("20260901-v1");
|
||||
request.setAppKey("app-key");
|
||||
request.setAppSecret("plain-secret");
|
||||
return request;
|
||||
}
|
||||
}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
package com.haobang.apipull;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
import com.config.CacheConfig;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.apipull.api.ApiPullDtos.ApiPullTaskResponse;
|
||||
import com.haobang.apipull.api.ApiPullDtos.CreateApiPullTaskRequest;
|
||||
import java.io.InputStream;
|
||||
import org.apache.ibatis.builder.xml.XMLMapperBuilder;
|
||||
import org.apache.ibatis.session.Configuration;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class ApiPullContractTest {
|
||||
@Test
|
||||
void webJsonNeverPublishesJavaClassMetadata() throws Exception {
|
||||
ApiPullTaskResponse response = new ApiPullTaskResponse();
|
||||
response.setTaskId(1);
|
||||
response.setAdapterType("NDRSERVER_V1");
|
||||
String json = new CacheConfig().webObjectMapper().writeValueAsString(response);
|
||||
assertFalse(json.contains("@class"));
|
||||
assertFalse(json.contains("com.haobang"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void managementDtoRejectsUnknownSecretBearingFields() {
|
||||
String body =
|
||||
"{\"deviceId\":1,\"taskName\":\"x\",\"adapterType\":\"NDRSERVER_V1\",\"streamKey\":\"NDR_ALERT\","
|
||||
+ "\"baseUrl\":\"https://localhost/ndr/api\",\"credentialRef\":\"ref\",\"initialStartTime\":\"2026-08-31T10:00:00+08:00\","
|
||||
+ "\"extraConfig\":{},\"appSecret\":\"must-not-be-accepted\"}";
|
||||
assertThrows(
|
||||
Exception.class,
|
||||
() ->
|
||||
new ObjectMapper()
|
||||
.findAndRegisterModules()
|
||||
.readValue(body, CreateApiPullTaskRequest.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void mybatisMapperXmlIsSyntacticallyValidAndBindsEveryStatement() throws Exception {
|
||||
Configuration configuration = new Configuration();
|
||||
try (InputStream in =
|
||||
getClass().getClassLoader().getResourceAsStream("mapper/DeviceCollectApiMapper.xml")) {
|
||||
assertNotNull(in);
|
||||
new XMLMapperBuilder(
|
||||
in,
|
||||
configuration,
|
||||
"mapper/DeviceCollectApiMapper.xml",
|
||||
configuration.getSqlFragments())
|
||||
.parse();
|
||||
}
|
||||
try (InputStream in =
|
||||
getClass().getClassLoader().getResourceAsStream("mapper/DeviceApiCredentialMapper.xml")) {
|
||||
assertNotNull(in);
|
||||
new XMLMapperBuilder(
|
||||
in,
|
||||
configuration,
|
||||
"mapper/DeviceApiCredentialMapper.xml",
|
||||
configuration.getSqlFragments())
|
||||
.parse();
|
||||
}
|
||||
assertTrue(configuration.hasStatement("com.common.mapper.DeviceCollectApiMapper.acquireLease"));
|
||||
assertTrue(
|
||||
configuration.hasStatement("com.common.mapper.DeviceCollectApiMapper.commitCheckpoint"));
|
||||
assertTrue(
|
||||
configuration.hasStatement("com.common.mapper.DeviceCollectApiMapper.resetCheckpoint"));
|
||||
assertTrue(
|
||||
configuration.hasStatement("com.common.mapper.DeviceApiCredentialMapper.selectForLoad"));
|
||||
assertTrue(configuration.hasStatement("com.common.mapper.DeviceApiCredentialMapper.updateCas"));
|
||||
}
|
||||
}
|
||||
+316
@@ -0,0 +1,316 @@
|
||||
package com.haobang.apipull;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.common.mapper.DeviceCollectApiMapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.apipull.adapter.AdapterRegistry;
|
||||
import com.haobang.apipull.adapter.ApiPullAdapter;
|
||||
import com.haobang.apipull.adapter.PullPage;
|
||||
import com.haobang.apipull.adapter.PullRequest;
|
||||
import com.haobang.apipull.adapter.PulledEvent;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiRun;
|
||||
import com.haobang.wal.AppendAttempt;
|
||||
import com.haobang.wal.DurableReceipt;
|
||||
import com.haobang.wal.IngressEvent;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import com.haobang.wal.UnifiedIngressService;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class ApiPullRunnerTest {
|
||||
private static final Instant WINDOW_START = Instant.parse("2026-08-31T02:00:00Z");
|
||||
|
||||
@Test
|
||||
void checkpointWaitsForEveryDurableReceipt() throws Exception {
|
||||
RunnerFixture fixture = new RunnerFixture();
|
||||
CompletableFuture<DurableReceipt> durable = new CompletableFuture<>();
|
||||
CountDownLatch accepted = new CountDownLatch(1);
|
||||
|
||||
when(fixture.ingress.tryAppend(any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
accepted.countDown();
|
||||
return AppendAttempt.accepted(durable);
|
||||
});
|
||||
when(fixture.adapter.pull(any())).thenReturn(page(1L, 1, 0));
|
||||
|
||||
AtomicBoolean committed = new AtomicBoolean();
|
||||
when(fixture.mapper.commitCheckpoint(
|
||||
anyInt(),
|
||||
anyLong(),
|
||||
anyString(),
|
||||
anyLong(),
|
||||
anyLong(),
|
||||
anyString(),
|
||||
any(),
|
||||
any(),
|
||||
any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
committed.set(true);
|
||||
return 1;
|
||||
});
|
||||
|
||||
ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
Future<?> future =
|
||||
executor.submit(
|
||||
() -> fixture.runner.execute(fixture.config, fixture.checkpoint, fixture.run));
|
||||
|
||||
try {
|
||||
assertTrue(accepted.await(2, TimeUnit.SECONDS));
|
||||
Thread.sleep(100L);
|
||||
assertFalse(committed.get(), "事件仅入队、尚未完成 WAL force 时不能推进 checkpoint");
|
||||
|
||||
durable.complete(new DurableReceipt("event-0", 1L, 4, Instant.now()));
|
||||
future.get(2, TimeUnit.SECONDS);
|
||||
|
||||
assertTrue(committed.get());
|
||||
assertEquals("SUCCESS", fixture.run.getStatus());
|
||||
assertEquals(Long.valueOf(1L), fixture.run.getRecordsDurable());
|
||||
verify(fixture.adapter).pull(any(PullRequest.class));
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitsOversizedWindowBeforeWritingWalAndCommitsLeftSubwindow() {
|
||||
RunnerFixture fixture = new RunnerFixture();
|
||||
fixture.config.setPageSize(2);
|
||||
fixture.config.setMaxPagesPerRun(2);
|
||||
|
||||
/*
|
||||
* 计划的 60 秒窗口有 5 条，超过容量 2 × 2；拆为 30 秒后只有 3 条，
|
||||
* 可以用两页完成。父窗口探测页的两条记录不能进入 WAL。
|
||||
*/
|
||||
when(fixture.adapter.pull(any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
PullRequest request = invocation.getArgument(0);
|
||||
long windowSeconds =
|
||||
Duration.between(WINDOW_START, request.getQueryEnd()).getSeconds();
|
||||
if (windowSeconds == 60L) {
|
||||
return page(5L, 2, request.getOffset());
|
||||
}
|
||||
if (request.getOffset() == 0L) {
|
||||
return page(3L, 2, 0);
|
||||
}
|
||||
return page(3L, 1, request.getOffset());
|
||||
});
|
||||
|
||||
AtomicReference<LocalDateTime> committedEnd = new AtomicReference<>();
|
||||
when(fixture.mapper.commitCheckpoint(
|
||||
anyInt(),
|
||||
anyLong(),
|
||||
anyString(),
|
||||
anyLong(),
|
||||
anyLong(),
|
||||
anyString(),
|
||||
any(),
|
||||
any(),
|
||||
any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
committedEnd.set(invocation.getArgument(6));
|
||||
return 1;
|
||||
});
|
||||
|
||||
fixture.runner.execute(fixture.config, fixture.checkpoint, fixture.run);
|
||||
|
||||
assertEquals("SUCCESS", fixture.run.getStatus());
|
||||
assertEquals(Integer.valueOf(2), fixture.run.getPagesFetched());
|
||||
assertEquals(Long.valueOf(3L), fixture.run.getRecordsFetched());
|
||||
assertEquals(Long.valueOf(3L), fixture.run.getRecordsDurable());
|
||||
assertEquals(
|
||||
ApiPullTime.toDatabase(WINDOW_START.plusSeconds(30L)), committedEnd.get());
|
||||
verify(fixture.adapter, times(3)).pull(any(PullRequest.class));
|
||||
verify(fixture.ingress, times(3)).tryAppend(any(IngressEvent.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void schedulingSliceCanSplitBelowTheHardRunCapacity() {
|
||||
RunnerFixture fixture = new RunnerFixture();
|
||||
fixture.config.setPageSize(2);
|
||||
fixture.config.setMaxPagesPerRun(100);
|
||||
fixture.properties.setMaxPagesPerSlice(2);
|
||||
|
||||
when(fixture.adapter.pull(any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
PullRequest request = invocation.getArgument(0);
|
||||
long windowSeconds =
|
||||
Duration.between(WINDOW_START, request.getQueryEnd()).getSeconds();
|
||||
if (windowSeconds == 60L) {
|
||||
return page(5L, 2, request.getOffset());
|
||||
}
|
||||
if (request.getOffset() == 0L) {
|
||||
return page(3L, 2, 0);
|
||||
}
|
||||
return page(3L, 1, request.getOffset());
|
||||
});
|
||||
|
||||
fixture.runner.execute(fixture.config, fixture.checkpoint, fixture.run);
|
||||
|
||||
assertEquals("SUCCESS", fixture.run.getStatus());
|
||||
assertEquals(Long.valueOf(3L), fixture.run.getRecordsDurable());
|
||||
verify(fixture.adapter, times(3)).pull(any(PullRequest.class));
|
||||
verify(fixture.ingress, times(3)).tryAppend(any(IngressEvent.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void failsWithoutWritingWalWhenOneSecondWindowStillExceedsCapacity() {
|
||||
RunnerFixture fixture = new RunnerFixture();
|
||||
fixture.config.setPageSize(2);
|
||||
fixture.config.setMaxPagesPerRun(2);
|
||||
fixture.config.setWindowSizeSeconds(1);
|
||||
when(fixture.adapter.pull(any())).thenReturn(page(5L, 2, 0));
|
||||
|
||||
fixture.runner.execute(fixture.config, fixture.checkpoint, fixture.run);
|
||||
|
||||
assertEquals("FAILED", fixture.run.getStatus());
|
||||
assertEquals("MIN_WINDOW_CAPACITY_EXCEEDED", fixture.run.getErrorCode());
|
||||
assertEquals(Long.valueOf(0L), fixture.run.getRecordsFetched());
|
||||
assertEquals(Long.valueOf(0L), fixture.run.getRecordsDurable());
|
||||
verify(fixture.ingress, never()).tryAppend(any(IngressEvent.class));
|
||||
verify(fixture.mapper, never())
|
||||
.commitCheckpoint(
|
||||
anyInt(),
|
||||
anyLong(),
|
||||
anyString(),
|
||||
anyLong(),
|
||||
anyLong(),
|
||||
anyString(),
|
||||
any(),
|
||||
any(),
|
||||
any());
|
||||
}
|
||||
|
||||
private static PullPage page(long total, int count, long offset) {
|
||||
List<PulledEvent> events = new ArrayList<>();
|
||||
for (int index = 0; index < count; index++) {
|
||||
long eventNumber = offset + index;
|
||||
events.add(
|
||||
new PulledEvent(
|
||||
"event-" + eventNumber,
|
||||
"external-" + eventNumber,
|
||||
WINDOW_START.plusSeconds(10L),
|
||||
"{\"id\":" + eventNumber + "}"));
|
||||
}
|
||||
return new PullPage(events, total, 200, 20L, 10L, 1L);
|
||||
}
|
||||
|
||||
/** 集中准备 Runner 的基础依赖，让每个测试只突出需要验证的可靠性边界。 */
|
||||
private static final class RunnerFixture {
|
||||
private final DeviceCollectApiMapper mapper = mock(DeviceCollectApiMapper.class);
|
||||
private final ApiPullAdapter adapter = mock(ApiPullAdapter.class);
|
||||
private final AdapterRegistry registry = mock(AdapterRegistry.class);
|
||||
private final UnifiedIngressService ingress = mock(UnifiedIngressService.class);
|
||||
private final SyslogWalService wal = mock(SyslogWalService.class);
|
||||
private final DeviceCollectApiConfig config = config();
|
||||
private final DeviceCollectApiCheckpoint checkpoint = checkpoint();
|
||||
private final DeviceCollectApiRun run = run();
|
||||
private final ApiPullProperties properties = new ApiPullProperties();
|
||||
private final ApiPullRunner runner;
|
||||
|
||||
private RunnerFixture() {
|
||||
when(registry.require("NDRSERVER_V1")).thenReturn(adapter);
|
||||
when(adapter.minimumWindowSeconds()).thenReturn(1);
|
||||
when(mapper.selectConfig(config.getCollectTaskId())).thenReturn(config);
|
||||
when(mapper.releaseLease(anyInt(), anyString(), anyLong())).thenReturn(1);
|
||||
when(mapper.scheduleNext(anyInt(), anyLong(), anyInt())).thenReturn(1);
|
||||
when(mapper.commitCheckpoint(
|
||||
anyInt(),
|
||||
anyLong(),
|
||||
anyString(),
|
||||
anyLong(),
|
||||
anyLong(),
|
||||
anyString(),
|
||||
any(),
|
||||
any(),
|
||||
any()))
|
||||
.thenReturn(1);
|
||||
when(wal.prepareIngressEvent(any()))
|
||||
.thenAnswer(
|
||||
invocation ->
|
||||
((IngressEvent) invocation.getArgument(0)).withPointTimeNs(123L));
|
||||
when(ingress.tryAppend(any()))
|
||||
.thenAnswer(
|
||||
invocation ->
|
||||
AppendAttempt.accepted(
|
||||
CompletableFuture.completedFuture(
|
||||
new DurableReceipt("event", 1L, 4, Instant.now()))));
|
||||
|
||||
properties.setInstanceId("test-instance");
|
||||
properties.setLeaseSeconds(120);
|
||||
properties.setBackpressureTimeoutMs(1000L);
|
||||
properties.setSlowPageThresholdMs(100000L);
|
||||
runner =
|
||||
new ApiPullRunner(mapper, registry, ingress, wal, properties, new ObjectMapper());
|
||||
}
|
||||
}
|
||||
|
||||
private static DeviceCollectApiConfig config() {
|
||||
DeviceCollectApiConfig config = new DeviceCollectApiConfig();
|
||||
config.setCollectTaskId(2);
|
||||
config.setDeviceId(1);
|
||||
config.setAdapterType("NDRSERVER_V1");
|
||||
config.setAdapterVersion(1);
|
||||
config.setStreamKey("NDR_ALERT");
|
||||
config.setBaseUrl("https://localhost/ndr/api");
|
||||
config.setCredentialRef("secret");
|
||||
config.setEnabled(1);
|
||||
config.setConfigVersion(1L);
|
||||
config.setPageSize(100);
|
||||
config.setMaxPagesPerRun(10);
|
||||
config.setRequestTimeoutMs(1000);
|
||||
config.setPollIntervalSeconds(10);
|
||||
config.setWindowSizeSeconds(60);
|
||||
config.setSafetyDelaySeconds(0);
|
||||
config.setLookbackSeconds(0);
|
||||
config.setInitialStartTime(ApiPullTime.toDatabase(WINDOW_START));
|
||||
return config;
|
||||
}
|
||||
|
||||
private static DeviceCollectApiCheckpoint checkpoint() {
|
||||
DeviceCollectApiCheckpoint checkpoint = new DeviceCollectApiCheckpoint();
|
||||
checkpoint.setCollectTaskId(2);
|
||||
checkpoint.setVersion(1L);
|
||||
checkpoint.setFencingToken(3L);
|
||||
checkpoint.setCommittedWindowEnd(ApiPullTime.toDatabase(WINDOW_START));
|
||||
return checkpoint;
|
||||
}
|
||||
|
||||
private static DeviceCollectApiRun run() {
|
||||
DeviceCollectApiRun run = new DeviceCollectApiRun();
|
||||
run.setRunId(UUID.randomUUID().toString());
|
||||
run.setStatus("RUNNING");
|
||||
return run;
|
||||
}
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package com.haobang.apipull;
|
||||
|
||||
import com.common.mapper.DeviceCollectApiMapper;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;
|
||||
import static org.junit.jupiter.api.Assertions.*;import static org.mockito.ArgumentMatchers.*;import static org.mockito.Mockito.*;
|
||||
|
||||
class ApiPullSchedulerTest {
|
||||
@Test void sameHostConcurrencyOnePreventsSecondTaskFromAcquiringLease()throws Exception{
|
||||
ApiPullProperties p=new ApiPullProperties();p.setEnabled(true);p.setInstanceId("i1");p.setScanIntervalMs(5000);p.setLeaseSeconds(120);
|
||||
p.setGlobalConcurrency(2);p.setPerHostConcurrency(1);p.setSlowPageThresholdMs(1000);p.setBackpressureTimeoutMs(1000);
|
||||
DeviceCollectApiMapper mapper=mock(DeviceCollectApiMapper.class);ApiPullRunner runner=mock(ApiPullRunner.class);SyslogWalService wal=mock(SyslogWalService.class);
|
||||
when(wal.isDiskCritical()).thenReturn(false);when(wal.shouldPauseTcp()).thenReturn(false);
|
||||
DeviceCollectApiConfig first=config(1,"https://localhost/ndr/api");DeviceCollectApiConfig second=config(2,"https://localhost/ndr/api");
|
||||
when(mapper.selectConfig(1)).thenReturn(first);when(mapper.selectConfig(2)).thenReturn(second);when(mapper.acquireLease(eq(1),anyLong(),anyString(),anyInt())).thenReturn(1);
|
||||
DeviceCollectApiCheckpoint cp=new DeviceCollectApiCheckpoint();cp.setFencingToken(1L);when(mapper.selectCheckpoint(1)).thenReturn(cp);
|
||||
CountDownLatch entered=new CountDownLatch(1);CountDownLatch release=new CountDownLatch(1);
|
||||
doAnswer(i->{entered.countDown();release.await(2,TimeUnit.SECONDS);return null;}).when(runner).execute(eq(first),eq(cp),any());
|
||||
ApiPullScheduler scheduler=new ApiPullScheduler(p,mapper,runner,wal);scheduler.init();
|
||||
try{assertNotNull(scheduler.trigger(1));assertTrue(entered.await(1,TimeUnit.SECONDS));
|
||||
ApiPullException conflict=assertThrows(ApiPullException.class,()->scheduler.trigger(2));assertEquals("LEASE_CONFLICT",conflict.getCode());
|
||||
verify(mapper,never()).acquireLease(eq(2),anyLong(),anyString(),anyInt());}
|
||||
finally{release.countDown();scheduler.shutdown();}
|
||||
}
|
||||
private DeviceCollectApiConfig config(int id,String base){DeviceCollectApiConfig c=new DeviceCollectApiConfig();c.setCollectTaskId(id);c.setEnabled(1);c.setConfigVersion(1L);
|
||||
c.setBaseUrl(base);c.setPollIntervalSeconds(10);c.setDeviceId(id);c.setAdapterType("NDRSERVER_V1");c.setAdapterVersion(1);c.setStreamKey("NDR_ALERT");return c;}
|
||||
}
|
||||
+287
@@ -0,0 +1,287 @@
|
||||
package com.haobang.apipull;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.common.entity.DeviceCollectTask;
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.mapper.DeviceCollectApiMapper;
|
||||
import com.common.mapper.DeviceCollectTaskMapper;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.apipull.adapter.AdapterException;
|
||||
import com.haobang.apipull.adapter.AdapterRegistry;
|
||||
import com.haobang.apipull.adapter.ApiPullAdapter;
|
||||
import com.haobang.apipull.api.ApiPullDtos.ApiPullTaskResponse;
|
||||
import com.haobang.apipull.api.ApiPullDtos.CreateApiPullTaskRequest;
|
||||
import com.haobang.apipull.api.ApiPullDtos.UpdateApiPullTaskRequest;
|
||||
import com.haobang.apipull.api.ApiPullDtos.ValidateApiPullTaskResponse;
|
||||
import com.haobang.apipull.api.ApiPullDtos.ValidationCheckResponse;
|
||||
import com.haobang.apipull.credential.CredentialProvider;
|
||||
import com.haobang.apipull.credential.CredentialSnapshot;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
|
||||
import com.haobang.apipull.security.SafeBaseUrlValidator;
|
||||
import com.haobang.config.AppProperties;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.SelectKey;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
class ApiPullTaskServiceTest {
|
||||
|
||||
@Test
|
||||
void apiInsertSelectsDeviceCollectTaskSequenceBeforeInsert() throws Exception {
|
||||
java.lang.reflect.Method method =
|
||||
DeviceCollectTaskMapper.class.getMethod("insertApiPullTask", DeviceCollectTask.class);
|
||||
SelectKey selectKey = method.getAnnotation(SelectKey.class);
|
||||
Insert insert = method.getAnnotation(Insert.class);
|
||||
|
||||
assertTrue(selectKey.before());
|
||||
assertEquals("id", selectKey.keyProperty());
|
||||
assertTrue(
|
||||
String.join(" ", selectKey.statement()).contains("device_collect_task_id_seq.NEXTVAL"));
|
||||
assertTrue(String.join(" ", insert.value()).contains("VALUES (#{id}"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void createUsesExplicitSequenceBackedInsertInsteadOfJdbcGeneratedKeys() {
|
||||
DeviceCollectTaskMapper tasks = mock(DeviceCollectTaskMapper.class);
|
||||
DeviceCollectApiMapper apiMapper = mock(DeviceCollectApiMapper.class);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
AdapterRegistry adapters = mock(AdapterRegistry.class);
|
||||
ApiPullAdapter adapter = mock(ApiPullAdapter.class);
|
||||
ApiPullScheduler scheduler = mock(ApiPullScheduler.class);
|
||||
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
device.setIp("127.0.0.1");
|
||||
device.setOrganizationId(1);
|
||||
device.setDeviceCollectId(7);
|
||||
when(devices.getByIdSafely(126)).thenReturn(device);
|
||||
when(adapters.require("NDRSERVER_V1")).thenReturn(adapter);
|
||||
|
||||
when(tasks.insertApiPullTask(any(DeviceCollectTask.class)))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
DeviceCollectTask task = invocation.getArgument(0);
|
||||
// 模拟 @SelectKey 在 INSERT 前写回数据库序列值。
|
||||
task.setId(12);
|
||||
return 1;
|
||||
});
|
||||
|
||||
AtomicReference<DeviceCollectApiConfig> storedConfig = new AtomicReference<>();
|
||||
AtomicReference<DeviceCollectApiCheckpoint> storedCheckpoint = new AtomicReference<>();
|
||||
when(apiMapper.insertConfig(any(DeviceCollectApiConfig.class)))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
storedConfig.set(invocation.getArgument(0));
|
||||
return 1;
|
||||
});
|
||||
when(apiMapper.insertCheckpoint(any(DeviceCollectApiCheckpoint.class)))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
storedCheckpoint.set(invocation.getArgument(0));
|
||||
return 1;
|
||||
});
|
||||
when(apiMapper.selectConfig(12)).thenAnswer(invocation -> storedConfig.get());
|
||||
when(apiMapper.selectCheckpoint(12)).thenAnswer(invocation -> storedCheckpoint.get());
|
||||
|
||||
CredentialProvider credentials =
|
||||
(credentialRef, adapterType, deviceId) ->
|
||||
new CredentialSnapshot(1, "v1", adapterType, Collections.emptyMap());
|
||||
AppProperties app = new AppProperties();
|
||||
app.setAppServiceDeviceCollectId(7);
|
||||
ApiPullProperties apiProperties = new ApiPullProperties();
|
||||
apiProperties.setAllowedHosts("localhost");
|
||||
|
||||
ApiPullTaskService service =
|
||||
new ApiPullTaskService(
|
||||
tasks,
|
||||
apiMapper,
|
||||
devices,
|
||||
adapters,
|
||||
credentials,
|
||||
new SafeBaseUrlValidator(apiProperties),
|
||||
scheduler,
|
||||
app,
|
||||
apiProperties,
|
||||
new ObjectMapper());
|
||||
|
||||
ApiPullTaskResponse response = service.create(request());
|
||||
|
||||
assertEquals(12, response.getTaskId());
|
||||
assertEquals(12, storedConfig.get().getCollectTaskId());
|
||||
assertEquals(12, storedCheckpoint.get().getCollectTaskId());
|
||||
verify(tasks).insertApiPullTask(any(DeviceCollectTask.class));
|
||||
verify(tasks, never()).insert(any(DeviceCollectTask.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void querySchemaFailureDoesNotMasqueradeAsAuthenticationFailure() {
|
||||
DeviceCollectTaskMapper tasks = mock(DeviceCollectTaskMapper.class);
|
||||
DeviceCollectApiMapper apiMapper = mock(DeviceCollectApiMapper.class);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
AdapterRegistry adapters = mock(AdapterRegistry.class);
|
||||
ApiPullAdapter adapter = mock(ApiPullAdapter.class);
|
||||
ApiPullScheduler scheduler = mock(ApiPullScheduler.class);
|
||||
|
||||
DeviceCollectApiConfig config = new DeviceCollectApiConfig();
|
||||
config.setCollectTaskId(12);
|
||||
config.setDeviceId(126);
|
||||
config.setAdapterType("NDRSERVER_V1");
|
||||
config.setAdapterVersion(1);
|
||||
config.setStreamKey("NDR_ALERT");
|
||||
config.setBaseUrl("https://localhost/ndr/api");
|
||||
config.setCredentialRef("ndr-v1");
|
||||
config.setConfigVersion(2L);
|
||||
config.setSafetyDelaySeconds(120);
|
||||
when(apiMapper.selectConfig(12)).thenReturn(config);
|
||||
when(apiMapper.insertRun(any())).thenReturn(1);
|
||||
when(apiMapper.updateRun(any())).thenReturn(1);
|
||||
when(adapters.require("NDRSERVER_V1")).thenReturn(adapter);
|
||||
when(adapter.pull(any()))
|
||||
.thenThrow(new AdapterException("QUERY_RESPONSE_SCHEMA_ERROR", "NDR 响应缺少 data.alerts"));
|
||||
|
||||
CredentialProvider credentials =
|
||||
(credentialRef, adapterType, deviceId) ->
|
||||
new CredentialSnapshot(1, "v1", adapterType, Collections.emptyMap());
|
||||
AppProperties app = new AppProperties();
|
||||
ApiPullProperties apiProperties = new ApiPullProperties();
|
||||
apiProperties.setAllowedHosts("localhost");
|
||||
ApiPullTaskService service =
|
||||
new ApiPullTaskService(
|
||||
tasks,
|
||||
apiMapper,
|
||||
devices,
|
||||
adapters,
|
||||
credentials,
|
||||
new SafeBaseUrlValidator(apiProperties),
|
||||
scheduler,
|
||||
app,
|
||||
apiProperties,
|
||||
new ObjectMapper());
|
||||
|
||||
ValidateApiPullTaskResponse response = service.validate(12, 2);
|
||||
Map<String, ValidationCheckResponse> checks =
|
||||
response.getChecks().stream()
|
||||
.collect(Collectors.toMap(ValidationCheckResponse::getName, value -> value));
|
||||
|
||||
assertEquals("PASS", checks.get("TLS_HANDSHAKE").getStatus());
|
||||
assertEquals("PASS", checks.get("AUTHENTICATION").getStatus());
|
||||
assertEquals("PASS", checks.get("API_PERMISSION").getStatus());
|
||||
assertEquals("FAIL", checks.get("MINIMAL_ALERT_QUERY").getStatus());
|
||||
assertEquals("QUERY_RESPONSE_SCHEMA_ERROR", checks.get("MINIMAL_ALERT_QUERY").getErrorCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void updateFromOldClientKeepsExistingSchedulerPriority() {
|
||||
DeviceCollectTaskMapper tasks = mock(DeviceCollectTaskMapper.class);
|
||||
DeviceCollectApiMapper apiMapper = mock(DeviceCollectApiMapper.class);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
AdapterRegistry adapters = mock(AdapterRegistry.class);
|
||||
ApiPullAdapter adapter = mock(ApiPullAdapter.class);
|
||||
ApiPullScheduler scheduler = mock(ApiPullScheduler.class);
|
||||
|
||||
DeviceCollectApiConfig current = new DeviceCollectApiConfig();
|
||||
current.setCollectTaskId(12);
|
||||
current.setDeviceId(126);
|
||||
current.setAdapterType("NDRSERVER_V1");
|
||||
current.setAdapterVersion(1);
|
||||
current.setStreamKey("NDR_ALERT");
|
||||
current.setBaseUrl("https://localhost/ndr/api");
|
||||
current.setCredentialRef("ndr-v1");
|
||||
current.setSchedulerPriority(100);
|
||||
current.setConfigVersion(7L);
|
||||
current.setEnabled(0);
|
||||
current.setInitialStartTime(ApiPullTime.toDatabase(OffsetDateTime.parse("2026-09-01T16:00:00+08:00")));
|
||||
current.setExtraConfigJson("{}");
|
||||
when(apiMapper.selectConfig(12)).thenReturn(current);
|
||||
when(apiMapper.updateConfigCas(any(DeviceCollectApiConfig.class))).thenReturn(1);
|
||||
when(adapters.require("NDRSERVER_V1")).thenReturn(adapter);
|
||||
when(tasks.updateTaskName(12, "NDR 告警")).thenReturn(1);
|
||||
|
||||
DeviceCollectTask task = new DeviceCollectTask();
|
||||
task.setId(12);
|
||||
task.setMethod(ApiPullTaskService.API_PULL_METHOD);
|
||||
task.setTaskName("NDR 告警");
|
||||
when(tasks.selectById(12)).thenReturn(task);
|
||||
DeviceCollectApiCheckpoint checkpoint = new DeviceCollectApiCheckpoint();
|
||||
checkpoint.setCommittedWindowEnd(current.getInitialStartTime());
|
||||
when(apiMapper.selectCheckpoint(12)).thenReturn(checkpoint);
|
||||
|
||||
CredentialProvider credentialProvider =
|
||||
(credentialRef, adapterType, deviceId) ->
|
||||
new CredentialSnapshot(1, "v1", adapterType, Collections.emptyMap());
|
||||
ApiPullProperties apiProperties = new ApiPullProperties();
|
||||
apiProperties.setAllowedHosts("localhost");
|
||||
ApiPullTaskService service =
|
||||
new ApiPullTaskService(
|
||||
tasks,
|
||||
apiMapper,
|
||||
devices,
|
||||
adapters,
|
||||
credentialProvider,
|
||||
new SafeBaseUrlValidator(apiProperties),
|
||||
scheduler,
|
||||
new AppProperties(),
|
||||
apiProperties,
|
||||
new ObjectMapper());
|
||||
|
||||
UpdateApiPullTaskRequest update = new UpdateApiPullTaskRequest();
|
||||
update.setConfigVersion(7L);
|
||||
update.setTaskName("NDR 告警");
|
||||
update.setBaseUrl("https://localhost/ndr/api");
|
||||
update.setCredentialRef("ndr-v1");
|
||||
update.setPollIntervalSeconds(1);
|
||||
update.setPageSize(500);
|
||||
update.setMaxPagesPerRun(1000);
|
||||
// 模拟升级前管理端：请求中没有 schedulerPriority。
|
||||
update.setConnectTimeoutMs(5000);
|
||||
update.setRequestTimeoutMs(30000);
|
||||
update.setWindowSizeSeconds(60);
|
||||
update.setSafetyDelaySeconds(120);
|
||||
update.setLookbackSeconds(0);
|
||||
update.setMaxResponseBytes(16777216L);
|
||||
update.setExtraConfig(Collections.emptyMap());
|
||||
|
||||
service.update(12, update);
|
||||
|
||||
ArgumentCaptor<DeviceCollectApiConfig> updated =
|
||||
ArgumentCaptor.forClass(DeviceCollectApiConfig.class);
|
||||
verify(apiMapper).updateConfigCas(updated.capture());
|
||||
assertEquals(100, updated.getValue().getSchedulerPriority());
|
||||
}
|
||||
|
||||
private CreateApiPullTaskRequest request() {
|
||||
CreateApiPullTaskRequest request = new CreateApiPullTaskRequest();
|
||||
request.setDeviceId(126);
|
||||
request.setTaskName("NDR 告警");
|
||||
request.setAdapterType("NDRSERVER_V1");
|
||||
request.setStreamKey("NDR_ALERT");
|
||||
request.setBaseUrl("https://localhost/ndr/api");
|
||||
request.setCredentialRef("ndr-v1");
|
||||
request.setPollIntervalSeconds(1);
|
||||
request.setPageSize(100);
|
||||
request.setMaxPagesPerRun(1000);
|
||||
request.setConnectTimeoutMs(5000);
|
||||
request.setRequestTimeoutMs(30000);
|
||||
request.setWindowSizeSeconds(60);
|
||||
request.setSafetyDelaySeconds(120);
|
||||
request.setLookbackSeconds(300);
|
||||
request.setMaxResponseBytes(16777216L);
|
||||
request.setInitialStartTime(OffsetDateTime.parse("2020-09-01T14:30:00+08:00"));
|
||||
request.setExtraConfig(Collections.emptyMap());
|
||||
return request;
|
||||
}
|
||||
}
|
||||
+582
@@ -0,0 +1,582 @@
|
||||
package com.haobang.apipull.adapter.ndr;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.haobang.apipull.ApiPullProperties;
|
||||
import com.haobang.apipull.adapter.AdapterException;
|
||||
import com.haobang.apipull.adapter.PullPage;
|
||||
import com.haobang.apipull.adapter.PullRequest;
|
||||
import com.haobang.apipull.credential.*;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
|
||||
import com.haobang.apipull.security.SafeBaseUrlValidator;
|
||||
import java.time.Instant;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import org.apache.http.HttpEntityEnclosingRequest;
|
||||
import org.apache.http.StatusLine;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpUriRequest;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class NdrServerApiAdapterTest {
|
||||
@Test
|
||||
void challengeAnswerUsesExactUtf8AndLowercaseMd5() throws Exception {
|
||||
assertEquals(
|
||||
"3956ed0865d1cd94c99f8e01c3988788",
|
||||
NdrServerApiAdapter.answer("challenge", "secret".toCharArray()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void timestampAcceptsColonAndCompactOffsetsWithFractionalSeconds() {
|
||||
Instant colon = NdrServerApiAdapter.parseEventTime("2024-07-02T15:08:58.127356+08:00");
|
||||
Instant compact = NdrServerApiAdapter.parseEventTime("2024-07-02T15:08:58.127356+0800");
|
||||
assertEquals(colon, compact);
|
||||
assertEquals(127356000, colon.getNano());
|
||||
}
|
||||
|
||||
@Test
|
||||
void timestampWithoutOffsetUsesNdrLocalTimezone() {
|
||||
Instant parsed = NdrServerApiAdapter.parseEventTime("2026-09-01 16:00:01");
|
||||
|
||||
assertEquals(Instant.parse("2026-09-01T08:00:01Z"), parsed);
|
||||
}
|
||||
|
||||
@Test
|
||||
void uuidV5IsDeterministicAndHasCorrectVersion() throws Exception {
|
||||
UUID namespace = UUID.fromString("a8cc91a6-f674-5f2f-a33f-bbd3ccae422a");
|
||||
UUID a = NdrServerApiAdapter.uuidV5(namespace, "NDRSERVER_V1|1|NDR_ALERT|idx:1");
|
||||
UUID b = NdrServerApiAdapter.uuidV5(namespace, "NDRSERVER_V1|1|NDR_ALERT|idx:1");
|
||||
assertEquals(a, b);
|
||||
assertEquals(5, a.version());
|
||||
assertEquals(2, a.variant());
|
||||
}
|
||||
|
||||
@Test
|
||||
void challengeVerifyAndBusinessRequestUseRawAuthorizationToken() throws Exception {
|
||||
CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
CloseableHttpResponse challenge =
|
||||
response(200, "{\"result\":1,\"data\":{\"challenge\":\"challenge\"}}");
|
||||
CloseableHttpResponse verifyResponse =
|
||||
response(200, "{\"result\":1,\"data\":{\"token\":\"raw-token\"}}");
|
||||
CloseableHttpResponse business =
|
||||
response(
|
||||
200,
|
||||
"{\"result\":1,\"data\":{\"alerts\":[{\"index\":\"i\",\"id\":\"1\",\"timestamp\":\"2024-07-02T15:08:58.127356+08:00\"}],\"total\":1}}");
|
||||
when(http.execute(any(HttpUriRequest.class))).thenReturn(challenge, verifyResponse, business);
|
||||
NdrServerApiAdapter adapter = adapter(http);
|
||||
PullPage page =
|
||||
adapter.pull(
|
||||
new PullRequest(
|
||||
config(),
|
||||
Instant.parse("2024-07-02T07:00:00Z"),
|
||||
Instant.parse("2024-07-02T07:10:00Z"),
|
||||
100,
|
||||
0));
|
||||
assertEquals(1, page.getEvents().size());
|
||||
assertEquals("i:1", page.getEvents().get(0).getExternalEventId());
|
||||
org.mockito.ArgumentCaptor<HttpUriRequest> requests =
|
||||
org.mockito.ArgumentCaptor.forClass(HttpUriRequest.class);
|
||||
verify(http, times(3)).execute(requests.capture());
|
||||
assertNull(requests.getAllValues().get(0).getFirstHeader("Authorization"));
|
||||
assertEquals(
|
||||
"raw-token", requests.getAllValues().get(2).getFirstHeader("Authorization").getValue());
|
||||
assertFalse(
|
||||
requests
|
||||
.getAllValues()
|
||||
.get(2)
|
||||
.getFirstHeader("Authorization")
|
||||
.getValue()
|
||||
.startsWith("Bearer "));
|
||||
}
|
||||
|
||||
@Test
|
||||
void one401InvalidatesTokenAndRepeatsChallengeVerifyExactlyOnce() throws Exception {
|
||||
CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
CloseableHttpResponse r1 = response(200, "{\"result\":1,\"data\":{\"challenge\":\"c1\"}}");
|
||||
CloseableHttpResponse r2 = response(200, "{\"result\":1,\"data\":{\"token\":\"t1\"}}");
|
||||
CloseableHttpResponse r3 = response(401, "{}");
|
||||
CloseableHttpResponse r4 = response(200, "{\"result\":1,\"data\":{\"challenge\":\"c2\"}}");
|
||||
CloseableHttpResponse r5 = response(200, "{\"result\":1,\"data\":{\"token\":\"t2\"}}");
|
||||
CloseableHttpResponse r6 = response(200, "{\"result\":1,\"data\":{\"alerts\":[],\"total\":0}}");
|
||||
when(http.execute(any(HttpUriRequest.class))).thenReturn(r1, r2, r3, r4, r5, r6);
|
||||
PullPage page =
|
||||
adapter(http)
|
||||
.pull(
|
||||
new PullRequest(
|
||||
config(),
|
||||
Instant.parse("2024-07-02T07:00:00Z"),
|
||||
Instant.parse("2024-07-02T07:10:00Z"),
|
||||
100,
|
||||
0));
|
||||
assertEquals(0, page.getTotal());
|
||||
verify(http, times(6)).execute(any(HttpUriRequest.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingArrayIsAnEmptyPageOnlyWhenTotalIsZero() throws Exception {
|
||||
CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
CloseableHttpResponse challenge =
|
||||
response(200, "{\"result\":1,\"data\":{\"challenge\":\"c\"}}");
|
||||
CloseableHttpResponse verified = response(200, "{\"result\":1,\"data\":{\"token\":\"t\"}}");
|
||||
CloseableHttpResponse empty = response(200, "{\"result\":1,\"data\":{\"total\":0}}");
|
||||
when(http.execute(any(HttpUriRequest.class))).thenReturn(challenge, verified, empty);
|
||||
|
||||
PullPage page =
|
||||
adapter(http)
|
||||
.pull(
|
||||
new PullRequest(
|
||||
config(),
|
||||
Instant.parse("2024-07-02T07:00:00Z"),
|
||||
Instant.parse("2024-07-02T07:10:00Z"),
|
||||
100,
|
||||
0));
|
||||
|
||||
assertEquals(0, page.getTotal());
|
||||
assertTrue(page.getEvents().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void nullArrayIsAnEmptyPageOnlyWhenTotalIsZero() throws Exception {
|
||||
CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
CloseableHttpResponse challenge =
|
||||
response(200, "{\"result\":1,\"data\":{\"challenge\":\"c\"}}");
|
||||
CloseableHttpResponse verified = response(200, "{\"result\":1,\"data\":{\"token\":\"t\"}}");
|
||||
CloseableHttpResponse empty =
|
||||
response(200, "{\"result\":1,\"data\":{\"alerts\":null,\"total\":0}}");
|
||||
when(http.execute(any(HttpUriRequest.class))).thenReturn(challenge, verified, empty);
|
||||
|
||||
PullPage page =
|
||||
adapter(http)
|
||||
.pull(
|
||||
new PullRequest(
|
||||
config(),
|
||||
Instant.parse("2024-07-02T07:00:00Z"),
|
||||
Instant.parse("2024-07-02T07:10:00Z"),
|
||||
100,
|
||||
0));
|
||||
|
||||
assertEquals(0, page.getTotal());
|
||||
assertTrue(page.getEvents().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingArrayWithPositiveTotalRemainsASchemaFailure() throws Exception {
|
||||
CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
CloseableHttpResponse challenge =
|
||||
response(200, "{\"result\":1,\"data\":{\"challenge\":\"c\"}}");
|
||||
CloseableHttpResponse verified = response(200, "{\"result\":1,\"data\":{\"token\":\"t\"}}");
|
||||
CloseableHttpResponse invalid = response(200, "{\"result\":1,\"data\":{\"total\":1}}");
|
||||
when(http.execute(any(HttpUriRequest.class))).thenReturn(challenge, verified, invalid);
|
||||
|
||||
com.haobang.apipull.adapter.AdapterException failure =
|
||||
assertThrows(
|
||||
com.haobang.apipull.adapter.AdapterException.class,
|
||||
() ->
|
||||
adapter(http)
|
||||
.pull(
|
||||
new PullRequest(
|
||||
config(),
|
||||
Instant.parse("2024-07-02T07:00:00Z"),
|
||||
Instant.parse("2024-07-02T07:10:00Z"),
|
||||
100,
|
||||
0)));
|
||||
|
||||
assertEquals("QUERY_RESPONSE_SCHEMA_ERROR", failure.getCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void flowStreamUsesNetlogEndpointCompactOffsetAndUnderscoreId() throws Exception {
|
||||
CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
CloseableHttpResponse r1 = response(200, "{\"result\":1,\"data\":{\"challenge\":\"c\"}}");
|
||||
CloseableHttpResponse r2 = response(200, "{\"result\":1,\"data\":{\"token\":\"t\"}}");
|
||||
CloseableHttpResponse r3 =
|
||||
response(
|
||||
200,
|
||||
"{\"result\":1,\"data\":{\"logs\":[{\"_id\":\"flow-1\",\"timestamp\":\"2024-07-03T12:36:53.813498+0800\"}],\"total\":1}}");
|
||||
when(http.execute(any(HttpUriRequest.class))).thenReturn(r1, r2, r3);
|
||||
DeviceCollectApiConfig config = config();
|
||||
config.setStreamKey("NDR_FLOW_LOG");
|
||||
config.setPageSize(500);
|
||||
PullPage page =
|
||||
adapter(http)
|
||||
.pull(
|
||||
new PullRequest(
|
||||
config,
|
||||
Instant.parse("2024-07-03T04:00:00Z"),
|
||||
Instant.parse("2024-07-03T05:00:00Z"),
|
||||
500,
|
||||
0));
|
||||
assertEquals("flow-1", page.getEvents().get(0).getExternalEventId());
|
||||
org.mockito.ArgumentCaptor<HttpUriRequest> requests =
|
||||
org.mockito.ArgumentCaptor.forClass(HttpUriRequest.class);
|
||||
verify(http, times(3)).execute(requests.capture());
|
||||
assertEquals("/ndr/api/netlog/investigate", requests.getAllValues().get(2).getURI().getPath());
|
||||
HttpEntityEnclosingRequest businessRequest =
|
||||
(HttpEntityEnclosingRequest) requests.getAllValues().get(2);
|
||||
String requestBody = EntityUtils.toString(businessRequest.getEntity(), "UTF-8");
|
||||
assertEquals(500, new ObjectMapper().readTree(requestBody).path("limit").asInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void alertHttpPayloadGainsTcpCompatibleAliasesAndKeepsVendorFields() throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
ObjectNode alert = alertLog(mapper);
|
||||
ObjectNode payload = alert.putObject("payload");
|
||||
payload.put("http_req_head", "GET /robots.txt HTTP/1.1\r\nHost: example.test");
|
||||
payload.put("http_req_body", "request-body");
|
||||
payload.put("http_rsp_head", "HTTP/1.1 404 Not Found\r\nContent-Type: text/html");
|
||||
payload.put("http_rsp_body", "<html>not found</html>");
|
||||
|
||||
JsonNode raw = pullAlert(alert);
|
||||
|
||||
assertEquals(payload.get("http_req_head"), raw.get("http_req_header"));
|
||||
assertEquals(payload.get("http_req_body"), raw.get("http_req_body"));
|
||||
assertEquals(payload.get("http_rsp_head"), raw.get("http_resp_header"));
|
||||
assertEquals(payload.get("http_rsp_body"), raw.get("http_resp_body"));
|
||||
assertEquals(payload.get("http_req_head"), raw.get("request_headers"));
|
||||
assertEquals(payload.get("http_req_body"), raw.get("data"));
|
||||
assertEquals(payload.get("http_rsp_head"), raw.get("response_headers"));
|
||||
assertEquals(payload.get("http_rsp_body"), raw.get("http_response_body"));
|
||||
assertEquals(payload, raw.get("payload"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void alertDocumentedPayloadNamesGainTcpCompatibleAliases() throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
ObjectNode alert = alertLog(mapper);
|
||||
ObjectNode payload = alert.putObject("payload");
|
||||
payload.put("request_headers", "api-request-header");
|
||||
payload.put("request_body", "api-request-body");
|
||||
payload.put("response_headers", "api-response-header");
|
||||
payload.put("response_body", "api-response-body");
|
||||
|
||||
JsonNode raw = pullAlert(alert);
|
||||
|
||||
assertEquals("api-request-header", raw.path("http_req_header").asText());
|
||||
assertEquals("api-request-body", raw.path("http_req_body").asText());
|
||||
assertEquals("api-response-header", raw.path("http_resp_header").asText());
|
||||
assertEquals("api-response-body", raw.path("http_resp_body").asText());
|
||||
assertEquals("api-request-header", raw.path("request_headers").asText());
|
||||
assertEquals("api-request-body", raw.path("data").asText());
|
||||
assertEquals("api-response-header", raw.path("response_headers").asText());
|
||||
assertEquals("api-response-body", raw.path("http_response_body").asText());
|
||||
assertEquals(payload, raw.get("payload"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void alertLivePayloadGainsLegacyRuleInputsWithoutInventingContent() throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
ObjectNode alert = alertLog(mapper);
|
||||
ObjectNode payload = alert.putObject("payload");
|
||||
payload.put("http_req_head", "GET / HTTP/1.1\r\nUser-Agent: zgrab/0.x");
|
||||
payload.put("http_req_body", "");
|
||||
payload.put("http_rsp_head", " ");
|
||||
payload.put("http_rsp_body", "");
|
||||
|
||||
JsonNode raw = pullAlert(alert);
|
||||
|
||||
assertEquals(payload.get("http_req_head"), raw.get("request_headers"));
|
||||
assertEquals("", raw.path("data").asText());
|
||||
assertEquals(" ", raw.path("response_headers").asText());
|
||||
assertEquals("", raw.path("http_response_body").asText());
|
||||
assertEquals(payload, raw.get("payload"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void alertHttpAliasesNeverOverwriteFieldsAlreadyReturnedByDevice() throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
ObjectNode alert = alertLog(mapper);
|
||||
ObjectNode payload = alert.putObject("payload");
|
||||
payload.put("http_req_head", "api-request-header");
|
||||
payload.put("http_req_body", "api-request-body");
|
||||
payload.put("http_rsp_head", "api-response-header");
|
||||
payload.put("http_rsp_body", "api-response-body");
|
||||
alert.put("http_req_header", "device-request-header");
|
||||
alert.put("http_req_body", "device-request-body");
|
||||
alert.put("http_resp_header", "device-response-header");
|
||||
alert.put("http_resp_body", "device-response-body");
|
||||
alert.put("request_headers", "device-legacy-request-header");
|
||||
alert.put("data", "device-legacy-request-body");
|
||||
alert.put("response_headers", "device-legacy-response-header");
|
||||
alert.put("http_response_body", "device-legacy-response-body");
|
||||
|
||||
JsonNode raw = pullAlert(alert);
|
||||
|
||||
assertEquals("device-request-header", raw.path("http_req_header").asText());
|
||||
assertEquals("device-request-body", raw.path("http_req_body").asText());
|
||||
assertEquals("device-response-header", raw.path("http_resp_header").asText());
|
||||
assertEquals("device-response-body", raw.path("http_resp_body").asText());
|
||||
assertEquals("device-legacy-request-header", raw.path("request_headers").asText());
|
||||
assertEquals("device-legacy-request-body", raw.path("data").asText());
|
||||
assertEquals("device-legacy-response-header", raw.path("response_headers").asText());
|
||||
assertEquals("device-legacy-response-body", raw.path("http_response_body").asText());
|
||||
assertEquals("api-request-header", raw.path("payload").path("http_req_head").asText());
|
||||
assertEquals("api-request-body", raw.path("payload").path("http_req_body").asText());
|
||||
assertEquals("api-response-header", raw.path("payload").path("http_rsp_head").asText());
|
||||
assertEquals("api-response-body", raw.path("payload").path("http_rsp_body").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void alertMissingBodiesAreNotInventedAndOversizedBodiesRemainOnlyInPayload() throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
ObjectNode missing = alertLog(mapper);
|
||||
ObjectNode missingPayload = missing.putObject("payload");
|
||||
missingPayload.put("http_req_head", "request-header");
|
||||
missingPayload.put("http_rsp_head", "response-header");
|
||||
|
||||
JsonNode missingRaw = pullAlert(missing);
|
||||
|
||||
assertFalse(missingRaw.has("http_req_body"));
|
||||
assertFalse(missingRaw.has("http_resp_body"));
|
||||
assertFalse(missingRaw.has("data"));
|
||||
assertFalse(missingRaw.has("http_response_body"));
|
||||
assertEquals("request-header", missingRaw.path("http_req_header").asText());
|
||||
assertEquals("response-header", missingRaw.path("http_resp_header").asText());
|
||||
assertEquals("request-header", missingRaw.path("request_headers").asText());
|
||||
assertEquals("response-header", missingRaw.path("response_headers").asText());
|
||||
|
||||
char[] chars = new char[10001];
|
||||
Arrays.fill(chars, 'x');
|
||||
ObjectNode oversized = alertLog(mapper);
|
||||
ObjectNode oversizedPayload = oversized.putObject("payload");
|
||||
oversizedPayload.put("http_req_body", new String(chars));
|
||||
oversizedPayload.put("http_rsp_body", new String(chars));
|
||||
|
||||
JsonNode oversizedRaw = pullAlert(oversized);
|
||||
|
||||
assertEquals(10001, oversizedRaw.path("payload").path("http_req_body").asText().length());
|
||||
assertEquals(10001, oversizedRaw.path("payload").path("http_rsp_body").asText().length());
|
||||
assertFalse(oversizedRaw.has("http_req_body"));
|
||||
assertFalse(oversizedRaw.has("http_resp_body"));
|
||||
assertFalse(oversizedRaw.has("data"));
|
||||
assertFalse(oversizedRaw.has("http_response_body"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void flowHttpHeadersAndBodiesGainTcpCompatibleAliasesAndKeepVendorFields() throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
ObjectNode log = flowLog(mapper);
|
||||
log.put("request_headers", "GET / HTTP/1.1\\r\\nHost: example.test\\r\\n");
|
||||
log.put("request_body", "name=alice");
|
||||
log.put("response_headers", "HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\n");
|
||||
log.put("response_body", "ok");
|
||||
|
||||
JsonNode raw = pullFlowLog(log);
|
||||
|
||||
assertEquals(raw.get("request_headers"), raw.get("http_req_header"));
|
||||
assertEquals(raw.get("request_body"), raw.get("http_req_body"));
|
||||
assertEquals(raw.get("response_headers"), raw.get("http_resp_header"));
|
||||
assertEquals(raw.get("response_body"), raw.get("http_resp_body"));
|
||||
assertTrue(raw.has("request_headers"));
|
||||
assertTrue(raw.has("request_body"));
|
||||
assertTrue(raw.has("response_headers"));
|
||||
assertTrue(raw.has("response_body"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void flowHttpAliasesNeverOverwriteFieldsAlreadyReturnedByDevice() throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
ObjectNode log = flowLog(mapper);
|
||||
log.put("request_headers", "api-request-header");
|
||||
log.put("request_body", "api-request-body");
|
||||
log.put("response_headers", "api-response-header");
|
||||
log.put("response_body", "api-response-body");
|
||||
log.put("http_req_header", "device-request-header");
|
||||
log.put("http_req_body", "device-request-body");
|
||||
log.put("http_resp_header", "device-response-header");
|
||||
log.put("http_resp_body", "device-response-body");
|
||||
|
||||
JsonNode raw = pullFlowLog(log);
|
||||
|
||||
assertEquals("device-request-header", raw.path("http_req_header").asText());
|
||||
assertEquals("device-request-body", raw.path("http_req_body").asText());
|
||||
assertEquals("device-response-header", raw.path("http_resp_header").asText());
|
||||
assertEquals("device-response-body", raw.path("http_resp_body").asText());
|
||||
assertEquals("api-request-header", raw.path("request_headers").asText());
|
||||
assertEquals("api-request-body", raw.path("request_body").asText());
|
||||
assertEquals("api-response-header", raw.path("response_headers").asText());
|
||||
assertEquals("api-response-body", raw.path("response_body").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingBodiesAreNotInventedAndOversizedBodiesRemainOnlyInVendorFields() throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
ObjectNode missing = flowLog(mapper);
|
||||
missing.put("request_headers", "request-header");
|
||||
missing.put("response_headers", "response-header");
|
||||
|
||||
JsonNode missingRaw = pullFlowLog(missing);
|
||||
|
||||
assertFalse(missingRaw.has("http_req_body"));
|
||||
assertFalse(missingRaw.has("http_resp_body"));
|
||||
assertEquals("request-header", missingRaw.path("http_req_header").asText());
|
||||
assertEquals("response-header", missingRaw.path("http_resp_header").asText());
|
||||
|
||||
char[] chars = new char[10001];
|
||||
Arrays.fill(chars, 'x');
|
||||
ObjectNode oversized = flowLog(mapper);
|
||||
oversized.put("request_body", new String(chars));
|
||||
oversized.put("response_body", new String(chars));
|
||||
|
||||
JsonNode oversizedRaw = pullFlowLog(oversized);
|
||||
|
||||
assertEquals(10001, oversizedRaw.path("request_body").asText().length());
|
||||
assertEquals(10001, oversizedRaw.path("response_body").asText().length());
|
||||
assertFalse(oversizedRaw.has("http_req_body"));
|
||||
assertFalse(oversizedRaw.has("http_resp_body"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void flowStreamAllowsFiveHundredRecordPageSize() {
|
||||
DeviceCollectApiConfig config = config();
|
||||
config.setStreamKey("NDR_FLOW_LOG");
|
||||
config.setPageSize(500);
|
||||
|
||||
assertDoesNotThrow(() -> adapter(mock(CloseableHttpClient.class)).validateConfig(config));
|
||||
}
|
||||
|
||||
@Test
|
||||
void pageSizeAbovePlatformProtectionLimitIsRejected() {
|
||||
DeviceCollectApiConfig config = config();
|
||||
config.setStreamKey("NDR_FLOW_LOG");
|
||||
config.setPageSize(1001);
|
||||
|
||||
AdapterException failure =
|
||||
assertThrows(
|
||||
AdapterException.class,
|
||||
() -> adapter(mock(CloseableHttpClient.class)).validateConfig(config));
|
||||
|
||||
assertEquals("CONFIG_SCHEMA_ERROR", failure.getCode());
|
||||
}
|
||||
|
||||
private NdrServerApiAdapter adapter(CloseableHttpClient http) {
|
||||
ApiPullProperties p = new ApiPullProperties();
|
||||
p.setAllowedHosts("localhost");
|
||||
CredentialProvider provider =
|
||||
(ref, type, deviceId) -> {
|
||||
Map<String, SecretValue> values = new LinkedHashMap<>();
|
||||
values.put("app_key", new SecretValue("key".toCharArray()));
|
||||
values.put("app_secret", new SecretValue("secret".toCharArray()));
|
||||
return new CredentialSnapshot(1, "v1", type, values);
|
||||
};
|
||||
return new NdrServerApiAdapter(provider, new SafeBaseUrlValidator(p), new ObjectMapper(), http);
|
||||
}
|
||||
|
||||
private ObjectNode flowLog(ObjectMapper mapper) {
|
||||
ObjectNode log = mapper.createObjectNode();
|
||||
log.put("_id", "flow-1");
|
||||
log.put("timestamp", "2024-07-03T12:36:53.813498+0800");
|
||||
log.put("xdr_log_type", "http");
|
||||
return log;
|
||||
}
|
||||
|
||||
private ObjectNode alertLog(ObjectMapper mapper) {
|
||||
ObjectNode alert = mapper.createObjectNode();
|
||||
alert.put("index", "ndralert-20260904");
|
||||
alert.put("id", "alert-1");
|
||||
alert.put("timestamp", "2026-09-04 17:11:58");
|
||||
alert.put("app_proto", "http");
|
||||
return alert;
|
||||
}
|
||||
|
||||
private JsonNode pullAlert(ObjectNode alert) throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
CloseableHttpResponse challenge =
|
||||
response(200, "{\"result\":1,\"data\":{\"challenge\":\"c\"}}");
|
||||
CloseableHttpResponse verified =
|
||||
response(200, "{\"result\":1,\"data\":{\"token\":\"t\"}}");
|
||||
ObjectNode data = mapper.createObjectNode();
|
||||
data.putArray("alerts").add(alert);
|
||||
data.put("total", 1);
|
||||
ObjectNode root = mapper.createObjectNode();
|
||||
root.put("result", 1);
|
||||
root.set("data", data);
|
||||
CloseableHttpResponse business = response(200, mapper.writeValueAsString(root));
|
||||
when(http.execute(any(HttpUriRequest.class))).thenReturn(challenge, verified, business);
|
||||
|
||||
PullPage page =
|
||||
adapter(http)
|
||||
.pull(
|
||||
new PullRequest(
|
||||
config(),
|
||||
Instant.parse("2026-09-04T09:11:00Z"),
|
||||
Instant.parse("2026-09-04T09:12:00Z"),
|
||||
100,
|
||||
0));
|
||||
|
||||
return mapper.readTree(page.getEvents().get(0).getRawMessage());
|
||||
}
|
||||
|
||||
private JsonNode pullFlowLog(ObjectNode log) throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
CloseableHttpResponse challenge =
|
||||
response(200, "{\"result\":1,\"data\":{\"challenge\":\"c\"}}");
|
||||
CloseableHttpResponse verified =
|
||||
response(200, "{\"result\":1,\"data\":{\"token\":\"t\"}}");
|
||||
ObjectNode data = mapper.createObjectNode();
|
||||
data.putArray("logs").add(log);
|
||||
data.put("total", 1);
|
||||
ObjectNode root = mapper.createObjectNode();
|
||||
root.put("result", 1);
|
||||
root.set("data", data);
|
||||
CloseableHttpResponse business = response(200, mapper.writeValueAsString(root));
|
||||
when(http.execute(any(HttpUriRequest.class))).thenReturn(challenge, verified, business);
|
||||
DeviceCollectApiConfig config = config();
|
||||
config.setStreamKey("NDR_FLOW_LOG");
|
||||
|
||||
PullPage page =
|
||||
adapter(http)
|
||||
.pull(
|
||||
new PullRequest(
|
||||
config,
|
||||
Instant.parse("2024-07-03T04:00:00Z"),
|
||||
Instant.parse("2024-07-03T05:00:00Z"),
|
||||
100,
|
||||
0));
|
||||
|
||||
return mapper.readTree(page.getEvents().get(0).getRawMessage());
|
||||
}
|
||||
|
||||
private DeviceCollectApiConfig config() {
|
||||
DeviceCollectApiConfig c = new DeviceCollectApiConfig();
|
||||
c.setCollectTaskId(2);
|
||||
c.setDeviceId(1);
|
||||
c.setAdapterType("NDRSERVER_V1");
|
||||
c.setAdapterVersion(1);
|
||||
c.setStreamKey("NDR_ALERT");
|
||||
c.setBaseUrl("https://localhost/ndr/api");
|
||||
c.setCredentialRef("secret");
|
||||
c.setPageSize(100);
|
||||
c.setPollIntervalSeconds(10);
|
||||
c.setMaxPagesPerRun(100);
|
||||
c.setConnectTimeoutMs(1000);
|
||||
c.setRequestTimeoutMs(3000);
|
||||
c.setWindowSizeSeconds(60);
|
||||
c.setSafetyDelaySeconds(0);
|
||||
c.setLookbackSeconds(300);
|
||||
c.setMaxResponseBytes(1024 * 1024L);
|
||||
c.setExtraConfigJson("{}");
|
||||
return c;
|
||||
}
|
||||
|
||||
private CloseableHttpResponse response(int status, String body) throws Exception {
|
||||
CloseableHttpResponse r = mock(CloseableHttpResponse.class);
|
||||
StatusLine line = mock(StatusLine.class);
|
||||
when(line.getStatusCode()).thenReturn(status);
|
||||
when(r.getStatusLine()).thenReturn(line);
|
||||
when(r.getEntity()).thenReturn(new StringEntity(body, "UTF-8"));
|
||||
return r;
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package com.haobang.apipull.adapter.ndr;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.apipull.ApiPullProperties;
|
||||
import com.haobang.apipull.credential.CredentialProvider;
|
||||
import com.haobang.apipull.credential.CredentialSnapshot;
|
||||
import com.haobang.apipull.credential.SecretValue;
|
||||
import com.haobang.apipull.security.SafeBaseUrlValidator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
|
||||
|
||||
/** 验证 NDR 适配器在真实 Spring 构造器选择规则下可以完成装配。 */
|
||||
class NdrServerSpringWiringTest {
|
||||
|
||||
@Test
|
||||
void springSelectsTheProductionConstructor() {
|
||||
try (AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext()) {
|
||||
ApiPullProperties properties = new ApiPullProperties();
|
||||
properties.setAllowedHosts("localhost");
|
||||
|
||||
CredentialProvider credentialProvider =
|
||||
(credentialRef, adapterType, deviceId) -> {
|
||||
Map<String, SecretValue> values = new LinkedHashMap<>();
|
||||
values.put("app_key", new SecretValue("key".toCharArray()));
|
||||
values.put("app_secret", new SecretValue("secret".toCharArray()));
|
||||
return new CredentialSnapshot(1, "v1", adapterType, values);
|
||||
};
|
||||
|
||||
context.registerBean(CredentialProvider.class, () -> credentialProvider);
|
||||
context.registerBean(SafeBaseUrlValidator.class, () -> new SafeBaseUrlValidator(properties));
|
||||
context.registerBean(ObjectMapper.class, () -> new ObjectMapper());
|
||||
context.register(NdrServerApiAdapter.class);
|
||||
context.refresh();
|
||||
|
||||
assertNotNull(context.getBean(NdrServerApiAdapter.class));
|
||||
}
|
||||
}
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.haobang.apipull.credential;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.common.mapper.DeviceApiCredentialMapper;
|
||||
import com.haobang.apipull.persistence.DeviceApiCredential;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class DatabaseCredentialProviderTest {
|
||||
@Test
|
||||
void loadsCredentialOnlyForExactDeviceAndAdapterBinding() {
|
||||
DeviceApiCredentialMapper mapper = mock(DeviceApiCredentialMapper.class);
|
||||
DeviceApiCredential record = credential();
|
||||
when(mapper.selectForLoad("ndr-v1", 126, "NDRSERVER_V1")).thenReturn(record);
|
||||
|
||||
DatabaseCredentialProvider provider = new DatabaseCredentialProvider(mapper);
|
||||
try (CredentialSnapshot snapshot = provider.load("ndr-v1", "NDRSERVER_V1", 126)) {
|
||||
assertEquals("20260901-v1", snapshot.getCredentialVersion());
|
||||
assertArrayEquals("app-key".toCharArray(), snapshot.require("app_key").copy());
|
||||
assertEquals("[REDACTED]", snapshot.require("app_secret").toString());
|
||||
}
|
||||
|
||||
verify(mapper).selectForLoad("ndr-v1", 126, "NDRSERVER_V1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsMissingOrMismatchedCredentialWithoutLeakingValues() {
|
||||
DeviceApiCredentialMapper mapper = mock(DeviceApiCredentialMapper.class);
|
||||
DatabaseCredentialProvider provider = new DatabaseCredentialProvider(mapper);
|
||||
|
||||
CredentialException error =
|
||||
assertThrows(CredentialException.class, () -> provider.load("ndr-v1", "NDRSERVER_V1", 999));
|
||||
|
||||
assertEquals("CREDENTIAL_NOT_FOUND", error.getCode());
|
||||
}
|
||||
|
||||
private DeviceApiCredential credential() {
|
||||
DeviceApiCredential record = new DeviceApiCredential();
|
||||
record.setCredentialRef("ndr-v1");
|
||||
record.setDeviceId(126);
|
||||
record.setAdapterType("NDRSERVER_V1");
|
||||
record.setCredentialVersion("20260901-v1");
|
||||
record.setAppKey("app-key");
|
||||
record.setAppSecret("app-secret");
|
||||
return record;
|
||||
}
|
||||
}
|
||||
+20
-1
@@ -4,6 +4,8 @@ import com.common.entity.DeviceDevice;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.wal.SyslogIngressEvent;
|
||||
import com.haobang.wal.IngressEvent;
|
||||
import com.haobang.wal.IngressType;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import com.kafka.kafkaProducer;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
@@ -13,6 +15,8 @@ import org.junit.jupiter.api.io.TempDir;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Files;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.Instant;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
@@ -36,6 +40,22 @@ class SyslogWalDispatcherTest {
|
||||
if (wal != null) wal.shutdown();
|
||||
}
|
||||
|
||||
@Test
|
||||
void apiV4ResolvesByDeviceIdAndForwardsDistinctTimes() throws Exception {
|
||||
AppProperties properties=properties(tempDirectory);wal=new SyslogWalService(properties,new ObjectMapper());wal.initialize();
|
||||
IngressEvent event=new IngressEvent("20260831-api",Instant.parse("2026-08-31T02:01:02.123Z"),
|
||||
Instant.parse("2026-08-31T02:00:58.127Z"),IngressType.API_PULL,"HTTPS",126,2001,
|
||||
"NDRSERVER_V1","idx:1","ndr.example",443,"JSON",StandardCharsets.UTF_8,"{}",null);
|
||||
event=wal.prepareIngressEvent(event);wal.tryAppendUnified(event);waitUntil(()->wal.getAppendedCount()==1L);
|
||||
DeviceDevice device=new DeviceDevice();device.setId(126);DeviceContextResolver resolver=mock(DeviceContextResolver.class);
|
||||
when(resolver.resolveDeviceId(126)).thenReturn(DeviceContextResolver.Resolution.registered(device));
|
||||
kafkaProducer producer=mock(kafkaProducer.class);when(producer.messagePush(any(DeviceDevice.class),anyString(),anyString(),anyString(),anyString(),any()))
|
||||
.thenReturn(CompletableFuture.completedFuture(null));
|
||||
dispatcher=new SyslogWalDispatcher(wal,resolver,producer,properties);dispatcher.start();waitUntil(()->wal.getCheckpointSequence()==1L);
|
||||
verify(resolver).resolveDeviceId(126);verify(resolver,never()).resolve(anyString(),anyString());
|
||||
verify(producer).messagePush(eq(device),eq("{}"),eq("20260831100102123"),eq("20260831100058127"),eq("20260831-api"),eq(event.getPointTimeNs()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void outOfOrderKafkaAckAdvancesOnlyContiguousCheckpoint() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
@@ -123,7 +143,6 @@ class SyslogWalDispatcherTest {
|
||||
value.setSyslogWalCheckpointFlushRecords(1);
|
||||
value.setSyslogWalCheckpointFlushIntervalMs(5);
|
||||
value.setSyslogWalDiskHighWatermarkPercent(98);
|
||||
value.setSyslogWalDiskCriticalWatermarkPercent(99);
|
||||
value.setSyslogWalMinFreeBytes(0);
|
||||
value.setSyslogCharset("UTF-8");
|
||||
value.setSyslogDispatchWorkerThreads(2);
|
||||
|
||||
-1
@@ -20,7 +20,6 @@ import java.util.List;
|
||||
"syslog.server.enabled=false",
|
||||
"syslog.wal.directory=${java.io.tmpdir}/syslog-serve-device-test-${random.uuid}",
|
||||
"syslog.wal.disk-high-watermark-percent=98",
|
||||
"syslog.wal.disk-critical-watermark-percent=99",
|
||||
"syslog.wal.min-free-bytes=0"
|
||||
})
|
||||
@ActiveProfiles("dev")
|
||||
|
||||
-1
@@ -7,7 +7,6 @@ import org.springframework.boot.test.context.SpringBootTest;
|
||||
"syslog.server.enabled=false",
|
||||
"syslog.wal.directory=${java.io.tmpdir}/syslog-serve-context-${random.uuid}",
|
||||
"syslog.wal.disk-high-watermark-percent=98",
|
||||
"syslog.wal.disk-critical-watermark-percent=99",
|
||||
"syslog.wal.min-free-bytes=0"
|
||||
})
|
||||
class SysjavacollectApplicationTests {
|
||||
|
||||
+123
-1
@@ -11,8 +11,14 @@ import org.junit.jupiter.api.io.TempDir;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
@@ -21,6 +27,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class SyslogWalServiceTest {
|
||||
private static final long GIB = 1024L * 1024L * 1024L;
|
||||
|
||||
@TempDir Path tempDirectory;
|
||||
private SyslogWalService service;
|
||||
|
||||
@@ -29,6 +37,67 @@ class SyslogWalServiceTest {
|
||||
if (service != null) service.shutdown();
|
||||
}
|
||||
|
||||
@Test
|
||||
void forceFailureCompletesAcceptedReceiptExceptionallyAndClosesAdmission() throws Exception {
|
||||
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper(),
|
||||
(channel, metadata) -> { throw new java.io.IOException("injected force failure"); });
|
||||
service.initialize();
|
||||
IngressEvent event = service.prepareIngressEvent(new IngressEvent("20260831-force-fail", Instant.now(),
|
||||
Instant.parse("2026-08-31T02:00:58.127Z"), IngressType.API_PULL, "HTTPS", 1001, 2001,
|
||||
"NDRSERVER_V1", "idx:failed", "ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{}", null));
|
||||
AppendAttempt accepted = service.tryAppendUnified(event);
|
||||
assertEquals(AdmissionStatus.ACCEPTED, accepted.getStatus());
|
||||
assertThrows(java.util.concurrent.ExecutionException.class,
|
||||
() -> accepted.getDurableStage().toCompletableFuture().get(3, TimeUnit.SECONDS));
|
||||
assertNotNull(service.getFatalFailure());
|
||||
assertEquals(AdmissionStatus.UNAVAILABLE, service.tryAppendUnified(event).getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
void apiPullWritesV4AndReceiptCompletesOnlyAfterForce() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setSyslogWalFlushRecords(100);
|
||||
properties.setSyslogWalFlushIntervalMs(300);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
Instant eventTime = Instant.parse("2026-08-31T02:00:58.127Z");
|
||||
IngressEvent event = new IngressEvent("20260831-api-event", Instant.now(), eventTime,
|
||||
IngressType.API_PULL, "HTTPS", 1001, 2001, "NDRSERVER_V1", "idx:42",
|
||||
"ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{\"id\":42}", null);
|
||||
event = service.prepareIngressEvent(event);
|
||||
AppendAttempt attempt = service.tryAppendUnified(event);
|
||||
assertEquals(AdmissionStatus.ACCEPTED, attempt.getStatus());
|
||||
CompletableFuture<DurableReceipt> future = attempt.getDurableStage().toCompletableFuture();
|
||||
assertFalse(future.isDone(), "入队或 write 不能提前完成 durable receipt");
|
||||
|
||||
DurableReceipt receipt = future.get(3, TimeUnit.SECONDS);
|
||||
assertEquals(4, receipt.getWalSchemaVersion());
|
||||
assertEquals("20260831-api-event", receipt.getEventId());
|
||||
assertTrue(receipt.getWalSequence() > 0);
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord restored = cursor.next(1000L);
|
||||
assertEquals(4, restored.getSchemaVersion());
|
||||
assertEquals("API_PULL", restored.getIngressType());
|
||||
assertEquals(Integer.valueOf(1001), restored.getDeviceId());
|
||||
assertEquals(Integer.valueOf(2001), restored.getCollectTaskId());
|
||||
assertEquals("NDRSERVER_V1", restored.getAdapterType());
|
||||
assertEquals("idx:42", restored.getExternalEventId());
|
||||
assertEquals("2026-08-31T10:00:58.127", restored.getEventTime());
|
||||
assertEquals(eventTime.toEpochMilli(), Math.floorDiv(restored.getPointTimeNs(), 1_000_000L));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectedAppendNeverExposesDurableStage() {
|
||||
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
|
||||
IngressEvent event = new IngressEvent("20260831-api-event", Instant.now(), Instant.now(),
|
||||
IngressType.API_PULL, "HTTPS", 1001, 2001, "NDRSERVER_V1", "42",
|
||||
"ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{}", 1L);
|
||||
AppendAttempt attempt = service.tryAppendUnified(event);
|
||||
assertEquals(AdmissionStatus.UNAVAILABLE, attempt.getStatus());
|
||||
assertEquals(null, attempt.getDurableStage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void forceBeforeCursorAndPersistSchemaV2() throws Exception {
|
||||
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
|
||||
@@ -167,6 +236,60 @@ class SyslogWalServiceTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void largeDiskUsesTenGiBAsSmallerRemainingCapacityLimit() {
|
||||
long twoTiB = 2L * 1024L * GIB;
|
||||
|
||||
assertEquals(10L * GIB,
|
||||
SyslogWalService.calculateDiskStopThresholdBytes(twoTiB, 90, 10L * GIB));
|
||||
}
|
||||
|
||||
@Test
|
||||
void smallDiskUsesRemainingTenPercentAsSmallerCapacityLimit() {
|
||||
long fiftyGiB = 50L * GIB;
|
||||
|
||||
assertEquals(5L * GIB,
|
||||
SyslogWalService.calculateDiskStopThresholdBytes(fiftyGiB, 90, 10L * GIB));
|
||||
}
|
||||
|
||||
@Test
|
||||
void diskStopsAtInclusiveEffectiveCapacityBoundary() {
|
||||
long threshold = 10L * GIB;
|
||||
|
||||
assertFalse(SyslogWalService.isDiskStopRequired(threshold + 1L, threshold));
|
||||
assertTrue(SyslogWalService.isDiskStopRequired(threshold, threshold));
|
||||
assertTrue(SyslogWalService.isDiskStopRequired(threshold - 1L, threshold));
|
||||
}
|
||||
|
||||
@Test
|
||||
void completedCleanupIsRateLimitedToThirtyMinutes() {
|
||||
long lastCleanupNanos = 1_000L;
|
||||
|
||||
assertFalse(SyslogWalService.isCompletedCleanupDue(lastCleanupNanos,
|
||||
lastCleanupNanos + SyslogWalService.COMPLETED_CLEANUP_INTERVAL_NANOS - 1L));
|
||||
assertTrue(SyslogWalService.isCompletedCleanupDue(lastCleanupNanos,
|
||||
lastCleanupNanos + SyslogWalService.COMPLETED_CLEANUP_INTERVAL_NANOS));
|
||||
}
|
||||
|
||||
@Test
|
||||
void startupImmediatelyDeletesOnlyExpiredCompletedSegments() throws Exception {
|
||||
Path completed = tempDirectory.resolve("completed");
|
||||
Files.createDirectories(completed);
|
||||
Path expired = completed.resolve("syslog-wal-00000000000000000001-20260820-000000000.jsonl");
|
||||
Path retained = completed.resolve("syslog-wal-00000000000000000002-20260827-000000000.jsonl");
|
||||
Files.write(expired, new byte[] { '\n' });
|
||||
Files.write(retained, new byte[] { '\n' });
|
||||
Files.setLastModifiedTime(expired, FileTime.from(Instant.now().minus(4L, ChronoUnit.DAYS)));
|
||||
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setSyslogWalRetentionDays(2);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
|
||||
assertFalse(Files.exists(expired));
|
||||
assertTrue(Files.exists(retained));
|
||||
}
|
||||
|
||||
@Test
|
||||
void restartResumesAfterPersistedCheckpointWithoutChangingEventIdentity() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
@@ -299,7 +422,6 @@ class SyslogWalServiceTest {
|
||||
value.setSyslogWalCheckpointFlushRecords(1);
|
||||
value.setSyslogWalCheckpointFlushIntervalMs(5);
|
||||
value.setSyslogWalDiskHighWatermarkPercent(98);
|
||||
value.setSyslogWalDiskCriticalWatermarkPercent(99);
|
||||
value.setSyslogWalMinFreeBytes(0);
|
||||
value.setSyslogCharset("UTF-8");
|
||||
return value;
|
||||
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.config.AppProperties;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Collections;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
|
||||
import org.springframework.core.env.MapPropertySource;
|
||||
|
||||
/** 验证 WAL 服务在真实 Spring 构造器选择和生命周期规则下可以完成装配。 */
|
||||
class SyslogWalSpringWiringTest {
|
||||
|
||||
@TempDir Path tempDirectory;
|
||||
|
||||
@Test
|
||||
void springSelectsTheProductionConstructor() {
|
||||
try (AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext()) {
|
||||
context
|
||||
.getEnvironment()
|
||||
.getPropertySources()
|
||||
.addFirst(
|
||||
new MapPropertySource(
|
||||
"wal-wiring-test",
|
||||
Collections.singletonMap("syslog.wal.directory", tempDirectory.toString())));
|
||||
context.registerBean(AppProperties.class, this::walProperties);
|
||||
context.registerBean(ObjectMapper.class, () -> new ObjectMapper());
|
||||
context.register(SyslogWalService.class);
|
||||
context.refresh();
|
||||
|
||||
assertNotNull(context.getBean(SyslogWalService.class));
|
||||
}
|
||||
}
|
||||
|
||||
private AppProperties walProperties() {
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setSyslogWalEnabled(true);
|
||||
properties.setSyslogWalDirectory(tempDirectory.toString());
|
||||
properties.setSyslogWalQueueCapacity(16);
|
||||
properties.setSyslogWalHighWatermarkPercent(70);
|
||||
properties.setSyslogWalLowWatermarkPercent(40);
|
||||
properties.setSyslogWalRetryIntervalMs(5);
|
||||
properties.setSyslogWalDiskRetryIntervalMs(50);
|
||||
properties.setSyslogWalFlushRecords(1);
|
||||
properties.setSyslogWalFlushIntervalMs(5);
|
||||
properties.setSyslogWalSegmentMaxBytes(1024 * 1024);
|
||||
properties.setSyslogWalRetentionDays(0);
|
||||
properties.setSyslogWalShutdownTimeoutMs(5000);
|
||||
properties.setSyslogWalCheckpointFlushRecords(1);
|
||||
properties.setSyslogWalCheckpointFlushIntervalMs(5);
|
||||
properties.setSyslogWalDiskHighWatermarkPercent(98);
|
||||
properties.setSyslogWalMinFreeBytes(0);
|
||||
properties.setSyslogCharset("UTF-8");
|
||||
return properties;
|
||||
}
|
||||
}
|
||||
+15
-1
@@ -52,7 +52,7 @@ class KafkaProducerEventIdTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void kafkaBusinessHeaderAddsOnlyPointTimeAndKeepsEventIdName() {
|
||||
void kafkaBusinessHeaderAddsEventTimeEqualToReceiveTimeForTcpAndKeepsEventIdName() {
|
||||
DeviceDevice device = new DeviceDevice(); device.setId(126); device.setName("device"); device.setVendor("vendor");
|
||||
AppProperties properties = new AppProperties(); properties.setAppServiceDataType("json"); properties.setAppServiceDeviceCollectId(1);
|
||||
String payload = DeviceInfoUtil.getFullLogString(device, "raw", "20260814000000000",
|
||||
@@ -60,10 +60,24 @@ class KafkaProducerEventIdTest {
|
||||
assertTrue(payload.contains("event_id=20260814-event"));
|
||||
assertTrue(payload.contains("point_time_ns=1786636800000000123"));
|
||||
assertTrue(payload.contains("receive_time=20260814000000000"));
|
||||
assertTrue(payload.contains("event_time=20260814000000000"));
|
||||
assertFalse(payload.contains("schema_version"));
|
||||
assertFalse(payload.contains("event_id_ns"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void kafkaBusinessHeaderCanCarryDifferentReceiveAndEventTimesForFutureApiPull() {
|
||||
DeviceDevice device = new DeviceDevice(); device.setId(126); device.setName("device"); device.setVendor("vendor");
|
||||
AppProperties properties = new AppProperties(); properties.setAppServiceDataType("json"); properties.setAppServiceDeviceCollectId(1);
|
||||
|
||||
String payload = DeviceInfoUtil.getFullLogString(device, "raw",
|
||||
"20260831100000000", "20260830143520123",
|
||||
"20260830-event", 1_788_079_320_123_000_001L, properties);
|
||||
|
||||
assertTrue(payload.contains("receive_time=20260831100000000"));
|
||||
assertTrue(payload.contains("event_time=20260830143520123"));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private kafkaProducer producer(String strategy) {
|
||||
AppProperties properties = new AppProperties();
|
||||
|
||||
Reference in New Issue
Block a user