nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 新增设备 API 拉取采集及配套处理能力

Browse Source 
- 接入 NDRServer 告警和流量日志，支持凭据与任务管理
- 支持分页拉取、窗口拆分、调度优先级及 WAL 持久化后推进断点
- 增加迟到告警规则重放，优化多条泛化规则选择和事件时间处理
- 补充回归测试、数据库升级脚本及部署文档
This commit is contained in:
王绍宏
2026-09-06 16:46:21 +08:00
parent b61ec2e37c
commit 00c0a1fb7f
141 changed files with 11318 additions and 467 deletions
Download Patch File Download Diff File Expand all files Collapse all files
haobang-security-dm/syslog-serve/src/test/java/com/haobang/wal/SyslogWalServiceTest.java
+123 -1
View File
@@ -11,8 +11,14 @@ import org.junit.jupiter.api.io.TempDir;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileTime;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -21,6 +27,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertThrows;
class SyslogWalServiceTest {
private static final long GIB = 1024L * 1024L * 1024L;
@TempDir Path tempDirectory;
private SyslogWalService service;
@@ -29,6 +37,67 @@ class SyslogWalServiceTest {
if (service != null) service.shutdown();
}
@Test
void forceFailureCompletesAcceptedReceiptExceptionallyAndClosesAdmission() throws Exception {
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper(),
(channel, metadata) -> { throw new java.io.IOException("injected force failure"); });
service.initialize();
IngressEvent event = service.prepareIngressEvent(new IngressEvent("20260831-force-fail", Instant.now(),
Instant.parse("2026-08-31T02:00:58.127Z"), IngressType.API_PULL, "HTTPS", 1001, 2001,
"NDRSERVER_V1", "idx:failed", "ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{}", null));
AppendAttempt accepted = service.tryAppendUnified(event);
assertEquals(AdmissionStatus.ACCEPTED, accepted.getStatus());
assertThrows(java.util.concurrent.ExecutionException.class,
() -> accepted.getDurableStage().toCompletableFuture().get(3, TimeUnit.SECONDS));
assertNotNull(service.getFatalFailure());
assertEquals(AdmissionStatus.UNAVAILABLE, service.tryAppendUnified(event).getStatus());
}
@Test
void apiPullWritesV4AndReceiptCompletesOnlyAfterForce() throws Exception {
AppProperties properties = properties(tempDirectory);
properties.setSyslogWalFlushRecords(100);
properties.setSyslogWalFlushIntervalMs(300);
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
Instant eventTime = Instant.parse("2026-08-31T02:00:58.127Z");
IngressEvent event = new IngressEvent("20260831-api-event", Instant.now(), eventTime,
IngressType.API_PULL, "HTTPS", 1001, 2001, "NDRSERVER_V1", "idx:42",
"ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{\"id\":42}", null);
event = service.prepareIngressEvent(event);
AppendAttempt attempt = service.tryAppendUnified(event);
assertEquals(AdmissionStatus.ACCEPTED, attempt.getStatus());
CompletableFuture<DurableReceipt> future = attempt.getDurableStage().toCompletableFuture();
assertFalse(future.isDone(), "入队或 write 不能提前完成 durable receipt");
DurableReceipt receipt = future.get(3, TimeUnit.SECONDS);
assertEquals(4, receipt.getWalSchemaVersion());
assertEquals("20260831-api-event", receipt.getEventId());
assertTrue(receipt.getWalSequence() > 0);
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
SyslogWalRecord restored = cursor.next(1000L);
assertEquals(4, restored.getSchemaVersion());
assertEquals("API_PULL", restored.getIngressType());
assertEquals(Integer.valueOf(1001), restored.getDeviceId());
assertEquals(Integer.valueOf(2001), restored.getCollectTaskId());
assertEquals("NDRSERVER_V1", restored.getAdapterType());
assertEquals("idx:42", restored.getExternalEventId());
assertEquals("2026-08-31T10:00:58.127", restored.getEventTime());
assertEquals(eventTime.toEpochMilli(), Math.floorDiv(restored.getPointTimeNs(), 1_000_000L));
}
}
@Test
void rejectedAppendNeverExposesDurableStage() {
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
IngressEvent event = new IngressEvent("20260831-api-event", Instant.now(), Instant.now(),
IngressType.API_PULL, "HTTPS", 1001, 2001, "NDRSERVER_V1", "42",
"ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{}", 1L);
AppendAttempt attempt = service.tryAppendUnified(event);
assertEquals(AdmissionStatus.UNAVAILABLE, attempt.getStatus());
assertEquals(null, attempt.getDurableStage());
}
@Test
void forceBeforeCursorAndPersistSchemaV2() throws Exception {
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
@@ -167,6 +236,60 @@ class SyslogWalServiceTest {
}
}
@Test
void largeDiskUsesTenGiBAsSmallerRemainingCapacityLimit() {
long twoTiB = 2L * 1024L * GIB;
assertEquals(10L * GIB,
SyslogWalService.calculateDiskStopThresholdBytes(twoTiB, 90, 10L * GIB));
}
@Test
void smallDiskUsesRemainingTenPercentAsSmallerCapacityLimit() {
long fiftyGiB = 50L * GIB;
assertEquals(5L * GIB,
SyslogWalService.calculateDiskStopThresholdBytes(fiftyGiB, 90, 10L * GIB));
}
@Test
void diskStopsAtInclusiveEffectiveCapacityBoundary() {
long threshold = 10L * GIB;
assertFalse(SyslogWalService.isDiskStopRequired(threshold + 1L, threshold));
assertTrue(SyslogWalService.isDiskStopRequired(threshold, threshold));
assertTrue(SyslogWalService.isDiskStopRequired(threshold - 1L, threshold));
}
@Test
void completedCleanupIsRateLimitedToThirtyMinutes() {
long lastCleanupNanos = 1_000L;
assertFalse(SyslogWalService.isCompletedCleanupDue(lastCleanupNanos,
lastCleanupNanos + SyslogWalService.COMPLETED_CLEANUP_INTERVAL_NANOS - 1L));
assertTrue(SyslogWalService.isCompletedCleanupDue(lastCleanupNanos,
lastCleanupNanos + SyslogWalService.COMPLETED_CLEANUP_INTERVAL_NANOS));
}
@Test
void startupImmediatelyDeletesOnlyExpiredCompletedSegments() throws Exception {
Path completed = tempDirectory.resolve("completed");
Files.createDirectories(completed);
Path expired = completed.resolve("syslog-wal-00000000000000000001-20260820-000000000.jsonl");
Path retained = completed.resolve("syslog-wal-00000000000000000002-20260827-000000000.jsonl");
Files.write(expired, new byte[] { '\n' });
Files.write(retained, new byte[] { '\n' });
Files.setLastModifiedTime(expired, FileTime.from(Instant.now().minus(4L, ChronoUnit.DAYS)));
AppProperties properties = properties(tempDirectory);
properties.setSyslogWalRetentionDays(2);
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
assertFalse(Files.exists(expired));
assertTrue(Files.exists(retained));
}
@Test
void restartResumesAfterPersistedCheckpointWithoutChangingEventIdentity() throws Exception {
AppProperties properties = properties(tempDirectory);
@@ -299,7 +422,6 @@ class SyslogWalServiceTest {
value.setSyslogWalCheckpointFlushRecords(1);
value.setSyslogWalCheckpointFlushIntervalMs(5);
value.setSyslogWalDiskHighWatermarkPercent(98);
value.setSyslogWalDiskCriticalWatermarkPercent(99);
value.setSyslogWalMinFreeBytes(0);
value.setSyslogCharset("UTF-8");
return value;
haobang-security-dm/syslog-serve/src/test/java/com/haobang/wal/SyslogWalSpringWiringTest.java
+59
View File
@@ -0,0 +1,59 @@
package com.haobang.wal;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.haobang.config.AppProperties;
import java.nio.file.Path;
import java.util.Collections;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.core.env.MapPropertySource;
/** 验证 WAL 服务在真实 Spring 构造器选择和生命周期规则下可以完成装配。 */
class SyslogWalSpringWiringTest {
@TempDir Path tempDirectory;
@Test
void springSelectsTheProductionConstructor() {
try (AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext()) {
context
.getEnvironment()
.getPropertySources()
.addFirst(
new MapPropertySource(
"wal-wiring-test",
Collections.singletonMap("syslog.wal.directory", tempDirectory.toString())));
context.registerBean(AppProperties.class, this::walProperties);
context.registerBean(ObjectMapper.class, () -> new ObjectMapper());
context.register(SyslogWalService.class);
context.refresh();
assertNotNull(context.getBean(SyslogWalService.class));
}
}
private AppProperties walProperties() {
AppProperties properties = new AppProperties();
properties.setSyslogWalEnabled(true);
properties.setSyslogWalDirectory(tempDirectory.toString());
properties.setSyslogWalQueueCapacity(16);
properties.setSyslogWalHighWatermarkPercent(70);
properties.setSyslogWalLowWatermarkPercent(40);
properties.setSyslogWalRetryIntervalMs(5);
properties.setSyslogWalDiskRetryIntervalMs(50);
properties.setSyslogWalFlushRecords(1);
properties.setSyslogWalFlushIntervalMs(5);
properties.setSyslogWalSegmentMaxBytes(1024 * 1024);
properties.setSyslogWalRetentionDays(0);
properties.setSyslogWalShutdownTimeoutMs(5000);
properties.setSyslogWalCheckpointFlushRecords(1);
properties.setSyslogWalCheckpointFlushIntervalMs(5);
properties.setSyslogWalDiskHighWatermarkPercent(98);
properties.setSyslogWalMinFreeBytes(0);
properties.setSyslogCharset("UTF-8");
return properties;
}
}