nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(syslog): 优化大日志通道并增加异常连续消费兜底

Browse Source 
- 128 KiB 及以上日志默认进入大日志 Topic
- InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存
- 截断内容追加 HB_XDR_TRUNCATED 标记
- InfluxDB 写入失败时继续执行规则解析和达梦写入
- 未知或系统性异常优先写入普通/大日志 DLT
- DLT 写入失败时记录特殊告警并提交 offset，避免阻塞后续消息
- DLT 失败等待时间默认限制为 1 秒
- 增加连续性放行、DLT 成功及失败监控指标
- 更新 Docker Compose、v3 升级文档及相关测试
This commit is contained in:
王绍宏
2026-09-12 19:24:58 +08:00
parent 0e9e6656db
commit c868e3befd
16 changed files with 342 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java
+67 -1
View File
@@ -8,12 +8,14 @@ import com.common.service.DeviceCollectTaskService;
import com.common.service.DeviceDeviceService;
import com.common.service.DmStandardizedBatchWriter;
import com.common.service.SyslogNonNormalMessageService;
import com.influx.raw.RawInfluxMetrics;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.kafka.support.Acknowledgment;
import java.time.Instant;
import java.time.LocalDateTime;
@@ -21,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
@@ -31,13 +34,17 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
/** 验证恢复并行后仍保留“全部可靠终态才允许返回”的处理边界。 */
/** 验证批内并行、常规可靠路由及连续性优先的最终放行边界。 */
class SysLogProcessorParallelTest {
private ExecutorService executor;
@@ -47,6 +54,7 @@ class SysLogProcessorParallelTest {
private SyslogFailureRouter failureRouter;
private DmStandardizedBatchWriter batchWriter;
private SyslogNonNormalMessageService nonNormalMessageService;
private RawInfluxMetrics rawInfluxMetrics;
@BeforeEach
void setUp() {
@@ -57,6 +65,7 @@ class SysLogProcessorParallelTest {
failureRouter = mock(SyslogFailureRouter.class);
batchWriter = mock(DmStandardizedBatchWriter.class);
nonNormalMessageService = mock(SyslogNonNormalMessageService.class);
rawInfluxMetrics = mock(RawInfluxMetrics.class);
ReflectionTestUtils.setField(processor, "taskExecutor", executor);
ReflectionTestUtils.setField(processor, "processTimeoutMs", 5000L);
ReflectionTestUtils.setField(processor, "logNormalProcessor", normalProcessor);
@@ -66,6 +75,8 @@ class SysLogProcessorParallelTest {
ReflectionTestUtils.setField(processor, "nonNormalWriterExecutor", nonNormalExecutor);
ReflectionTestUtils.setField(processor, "nonNormalMaxConcurrencyRetries", 3);
ReflectionTestUtils.setField(processor, "nonNormalConcurrencyRetryBackoffMs", 0L);
ReflectionTestUtils.setField(processor, "failureContinuityEnabled", true);
ReflectionTestUtils.setField(processor, "rawInfluxMetrics", rawInfluxMetrics);
when(batchWriter.write(any())).thenReturn(new DmStandardizedBatchWriter.WriteReport<>());
when(normalProcessor.captureRuleSnapshot(any())).thenReturn(
new BatchRuleSnapshot(Collections.emptyMap(), System.currentTimeMillis()));
@@ -232,6 +243,61 @@ class SysLogProcessorParallelTest {
assertEquals(3, metrics.filteredCount());
}
@Test
void unexpectedBatchFailureIsRecordedAndOffsetIsAcknowledged() {
SysLogProcessor spyProcessor = spy(processor);
RuntimeException sourceFailure = new IllegalStateException("unknown processing failure");
doThrow(sourceFailure).when(spyProcessor)
.processBatch(anyList(), eq(false), eq("RAW"), any(Executor.class));
when(failureRouter.resolve(any())).thenAnswer(invocation -> {
ConsumerRecord<String, String> record = invocation.getArgument(0);
return new SyslogFailureRouter.MessageContext(record.value(), "event-" + record.offset(),
0, null, Instant.now(), record.topic(), null);
});
when(failureRouter.routeContinuityDlt(any(), any(), eq(sourceFailure), eq("BATCH_RAW")))
.thenReturn(SyslogFailureRouter.TerminalState.DLT_PUBLISHED);
Acknowledgment ack = mock(Acknowledgment.class);
spyProcessor.consumeAndParallelProcess(records(2), ack);
verify(failureRouter, times(2))
.routeContinuityDlt(any(), any(), eq(sourceFailure), eq("BATCH_RAW"));
verify(rawInfluxMetrics).continuityDltPublished(2L);
verify(ack).acknowledge();
}
@Test
void dltPublishFailureIsLoggedAndOffsetIsStillAcknowledged() {
SysLogProcessor spyProcessor = spy(processor);
RuntimeException sourceFailure = new IllegalStateException("unknown processing failure");
doThrow(sourceFailure).when(spyProcessor)
.processBatch(anyList(), eq(false), eq("RAW"), any(Executor.class));
when(failureRouter.resolve(any())).thenAnswer(invocation -> {
ConsumerRecord<String, String> record = invocation.getArgument(0);
return new SyslogFailureRouter.MessageContext(record.value(), "event-" + record.offset(),
0, null, Instant.now(), record.topic(), null);
});
when(failureRouter.routeContinuityDlt(any(), any(), eq(sourceFailure), eq("BATCH_RAW")))
.thenThrow(new SyslogFailureRouter.FailurePublishException(
"broker unavailable", new IllegalStateException("send failed")));
Acknowledgment ack = mock(Acknowledgment.class);
spyProcessor.consumeAndParallelProcess(records(2), ack);
verify(failureRouter)
.routeContinuityDlt(any(), any(), eq(sourceFailure), eq("BATCH_RAW"));
verify(rawInfluxMetrics).continuityDltPublishFailed(2L);
verify(ack).acknowledge();
}
private List<ConsumerRecord<String, String>> records(int count) {
List<ConsumerRecord<String, String>> records = new ArrayList<>();
for (int i = 0; i < count; i++) {
records.add(new ConsumerRecord<>("raw", 0, i, "key-" + i, "ciphertext-" + i));
}
return records;
}
private List<SysLogProcessor.PreparedMessage> messages(int count) {
List<SysLogProcessor.PreparedMessage> messages = new ArrayList<>();
for (int i = 0; i < count; i++) {
haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SyslogFailureRouterReliabilityTest.java
+35
View File
@@ -117,6 +117,41 @@ class SyslogFailureRouterReliabilityTest {
() -> router.route(record, router.resolve(record), new SocketException("reset")));
}
@Test
void continuityFailureIsWrittenDirectlyToDltWithOriginalPayload() throws Exception {
SettableListenableFuture<SendResult<String, String>> acknowledged = new SettableListenableFuture<>();
acknowledged.set(null);
when(template.send(eq("raw.DLT"), anyString(), anyString())).thenReturn(acknowledged);
ArgumentCaptor<String> body = ArgumentCaptor.forClass(String.class);
SyslogFailureRouter.TerminalState state = router.routeContinuityDlt(
record, router.resolve(record), new InfluxException("influx unavailable"), "RAW_STORAGE");
assertEquals(SyslogFailureRouter.TerminalState.DLT_PUBLISHED, state);
verify(template).send(eq("raw.DLT"), anyString(), body.capture());
SyslogFailureRouter.FailureEnvelope envelope = new ObjectMapper().findAndRegisterModules()
.readValue(body.getValue(), SyslogFailureRouter.FailureEnvelope.class);
assertEquals("SYSLOG_CONTINUITY_BYPASS", envelope.getErrorCode());
assertEquals("RAW_STORAGE", envelope.getFailedStage());
assertEquals("ciphertext", envelope.getOriginalPayload());
assertEquals("raw", envelope.getOriginalTopic());
assertEquals(42L, envelope.getOriginalOffset());
}
@Test
void continuityFailureFromLargeLaneUsesLargeDlt() {
ConsumerRecord<String, String> largeRecord = new ConsumerRecord<>(
"raw.large", 1, 7L, "event", "large-ciphertext");
SettableListenableFuture<SendResult<String, String>> acknowledged = new SettableListenableFuture<>();
acknowledged.set(null);
when(template.send(eq("raw.large.DLT"), anyString(), anyString())).thenReturn(acknowledged);
router.routeContinuityDlt(largeRecord, router.resolve(largeRecord),
new IllegalStateException("unknown"), "BATCH_RAW_LARGE");
verify(template).send(eq("raw.large.DLT"), anyString(), anyString());
}
@Test
void influxDependencyFailurePausesWithoutDltStorm() {
SyslogFailureRouter.TerminalState state = router.route(