feat(syslog): 增加积压采样免采样窗口
- 新增 start-lag-ms 配置，默认15分钟内全量保留 - 在15分钟至1小时之间按10级逐步降低保留率 - 延迟达到1小时后使用普通10%、非标20%的最低保留率 - 校验起始延迟必须小于最低保留率阈值 - 状态接口新增 start_lag_ms 配置展示 - 将采样策略版本升级为 v2 - 更新 Docker Compose 和 v4 升级配置说明 - 补充采样时间边界及非法配置单元测试
This commit is contained in:
王绍宏
+126
@@ -0,0 +1,126 @@
|
||||
package com.Modules.NormalData;
|
||||
|
||||
import com.config.BacklogSamplingProperties;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class BacklogSamplingDeciderTest {
|
||||
private BacklogSamplingProperties properties;
|
||||
private BacklogSamplingDecider decider;
|
||||
private final Instant now = Instant.parse("2026-09-12T12:00:00Z");
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
properties = new BacklogSamplingProperties();
|
||||
ReflectionTestUtils.setField(properties, "enabled", true);
|
||||
ReflectionTestUtils.setField(properties, "startLagMs", 900_000L);
|
||||
ReflectionTestUtils.setField(properties, "lagThresholdMs", 3_600_000L);
|
||||
ReflectionTestUtils.setField(properties, "normalRetainBps", 0);
|
||||
ReflectionTestUtils.setField(properties, "nonNormalRetainBps", 0);
|
||||
ReflectionTestUtils.setField(properties, "degradeSteps", 10);
|
||||
ReflectionTestUtils.setField(properties, "policyVersion", "test-v1");
|
||||
decider = new BacklogSamplingDecider(properties);
|
||||
}
|
||||
|
||||
@Test
|
||||
void recentMessageIsFullyRetained() {
|
||||
BacklogSamplingDecision decision = decider.decide("event", LocalDateTime.of(2026, 9, 12, 19, 58),
|
||||
false, true, false, now);
|
||||
|
||||
assertFalse(decision.isDegraded());
|
||||
assertTrue(decision.isKeepNormal());
|
||||
assertTrue(decision.isKeepRaw());
|
||||
}
|
||||
|
||||
@Test
|
||||
void oldNormalMessageCanDropBothDmRowAndRawPoint() {
|
||||
BacklogSamplingDecision decision = decider.decide("event", LocalDateTime.of(2026, 9, 12, 18, 0),
|
||||
false, true, false, now);
|
||||
|
||||
assertTrue(decision.isDegraded());
|
||||
assertFalse(decision.isKeepNormal());
|
||||
assertFalse(decision.isKeepRaw());
|
||||
}
|
||||
|
||||
@Test
|
||||
void oldAlarmAlwaysKeepsRawEvenAtZeroNormalRate() {
|
||||
BacklogSamplingDecision decision = decider.decide("event", LocalDateTime.of(2026, 9, 12, 18, 0),
|
||||
true, false, false, now);
|
||||
|
||||
assertTrue(decision.isDegraded());
|
||||
assertTrue(decision.isKeepRaw());
|
||||
}
|
||||
|
||||
@Test
|
||||
void stableHashReturnsSameDecisionForReplay() {
|
||||
boolean first = BacklogSamplingDecider.sampled("event-42", "NORMAL", 3210, "v7");
|
||||
boolean second = BacklogSamplingDecider.sampled("event-42", "NORMAL", 3210, "v7");
|
||||
|
||||
assertTrue(first == second);
|
||||
}
|
||||
|
||||
@Test
|
||||
void disabledPolicyRetainsOldMessagesInFull() {
|
||||
ReflectionTestUtils.setField(properties, "enabled", false);
|
||||
|
||||
BacklogSamplingDecision decision = decider.decide("event",
|
||||
LocalDateTime.of(2000, 1, 1, 0, 0), false, true, true, now);
|
||||
|
||||
assertFalse(decision.isDegraded());
|
||||
assertTrue(decision.isKeepNormal());
|
||||
assertTrue(decision.isKeepNonNormal());
|
||||
assertTrue(decision.isKeepRaw());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fifteenMinuteGraceAndRemainingWindowAreSplitIntoTenEqualSteps() {
|
||||
ReflectionTestUtils.setField(properties, "normalRetainBps", 1000);
|
||||
ReflectionTestUtils.setField(properties, "nonNormalRetainBps", 2000);
|
||||
|
||||
BacklogSamplingDecision fifteenMinutes = decider.decide("event-15m",
|
||||
LocalDateTime.of(2026, 9, 12, 19, 45), false, true, false, now);
|
||||
BacklogSamplingDecision nineteenMinutes = decider.decide("event-19m",
|
||||
LocalDateTime.of(2026, 9, 12, 19, 41), false, true, false, now);
|
||||
BacklogSamplingDecision twentyMinutes = decider.decide("event-20m",
|
||||
LocalDateTime.of(2026, 9, 12, 19, 40), false, true, false, now);
|
||||
BacklogSamplingDecision thirtyEightMinutes = decider.decide("event-38m",
|
||||
LocalDateTime.of(2026, 9, 12, 19, 22), false, true, false, now);
|
||||
BacklogSamplingDecision oneHour = decider.decide("event-60m",
|
||||
LocalDateTime.of(2026, 9, 12, 19, 0), false, true, false, now);
|
||||
|
||||
assertFalse(fifteenMinutes.isDegraded());
|
||||
assertFalse(nineteenMinutes.isDegraded());
|
||||
assertEquals(9100, twentyMinutes.getNormalRetainBps());
|
||||
assertEquals(5500, thirtyEightMinutes.getNormalRetainBps());
|
||||
assertEquals(1000, oneHour.getNormalRetainBps());
|
||||
assertEquals(2000, oneHour.getNonNormalRetainBps());
|
||||
}
|
||||
|
||||
@Test
|
||||
void currentStepUsesGraceWindowAndReachesMinimumAtThreshold() {
|
||||
assertEquals(0, BacklogSamplingDecider.currentStep(900_000L, 900_000L, 3_600_000L, 10));
|
||||
assertEquals(0, BacklogSamplingDecider.currentStep(1_169_999L, 900_000L, 3_600_000L, 10));
|
||||
assertEquals(1, BacklogSamplingDecider.currentStep(1_170_000L, 900_000L, 3_600_000L, 10));
|
||||
assertEquals(9, BacklogSamplingDecider.currentStep(3_599_999L, 900_000L, 3_600_000L, 10));
|
||||
assertEquals(10, BacklogSamplingDecider.currentStep(3_600_000L, 900_000L, 3_600_000L, 10));
|
||||
}
|
||||
|
||||
@Test
|
||||
void startLagMustBeLowerThanMinimumRateThreshold() {
|
||||
ReflectionTestUtils.setField(properties, "startLagMs", 3_600_000L);
|
||||
|
||||
IllegalArgumentException failure = assertThrows(
|
||||
IllegalArgumentException.class, properties::validate);
|
||||
|
||||
assertTrue(failure.getMessage().contains("start-lag-ms"));
|
||||
}
|
||||
}
|
||||
+108
@@ -8,6 +8,9 @@ import com.common.service.DeviceCollectTaskService;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.common.service.DmStandardizedBatchWriter;
|
||||
import com.common.service.SyslogNonNormalMessageService;
|
||||
import com.common.service.SyslogNormalDataService;
|
||||
import com.common.service.BacklogRetentionDelta;
|
||||
import com.config.BacklogSamplingProperties;
|
||||
import com.influx.raw.RawInfluxMetrics;
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
@@ -55,6 +58,7 @@ class SysLogProcessorParallelTest {
|
||||
private DmStandardizedBatchWriter batchWriter;
|
||||
private SyslogNonNormalMessageService nonNormalMessageService;
|
||||
private RawInfluxMetrics rawInfluxMetrics;
|
||||
private SyslogNormalDataService normalDataService;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
@@ -66,6 +70,7 @@ class SysLogProcessorParallelTest {
|
||||
batchWriter = mock(DmStandardizedBatchWriter.class);
|
||||
nonNormalMessageService = mock(SyslogNonNormalMessageService.class);
|
||||
rawInfluxMetrics = mock(RawInfluxMetrics.class);
|
||||
normalDataService = mock(SyslogNormalDataService.class);
|
||||
ReflectionTestUtils.setField(processor, "taskExecutor", executor);
|
||||
ReflectionTestUtils.setField(processor, "processTimeoutMs", 5000L);
|
||||
ReflectionTestUtils.setField(processor, "logNormalProcessor", normalProcessor);
|
||||
@@ -77,6 +82,9 @@ class SysLogProcessorParallelTest {
|
||||
ReflectionTestUtils.setField(processor, "nonNormalConcurrencyRetryBackoffMs", 0L);
|
||||
ReflectionTestUtils.setField(processor, "failureContinuityEnabled", true);
|
||||
ReflectionTestUtils.setField(processor, "rawInfluxMetrics", rawInfluxMetrics);
|
||||
ReflectionTestUtils.setField(processor, "syslogNormalDataService", normalDataService);
|
||||
when(normalDataService.isAlarmData(any())).thenAnswer(invocation ->
|
||||
((java.util.Map<?, ?>) invocation.getArgument(0)).get("event_level") != null);
|
||||
when(batchWriter.write(any())).thenReturn(new DmStandardizedBatchWriter.WriteReport<>());
|
||||
when(normalProcessor.captureRuleSnapshot(any())).thenReturn(
|
||||
new BatchRuleSnapshot(Collections.emptyMap(), System.currentTimeMillis()));
|
||||
@@ -290,6 +298,92 @@ class SysLogProcessorParallelTest {
|
||||
verify(ack).acknowledge();
|
||||
}
|
||||
|
||||
@Test
|
||||
void influxFailureNeverBlocksMainFlowEvenWhenGeneralContinuityIsDisabled() {
|
||||
SysLogProcessor spyProcessor = spy(processor);
|
||||
ReflectionTestUtils.setField(spyProcessor, "failureContinuityEnabled", false);
|
||||
RuntimeException influxFailure = new IllegalStateException("influx unavailable");
|
||||
doThrow(influxFailure).when(spyProcessor)
|
||||
.persistRawBatch(anyList(), any(SysLogProcessor.BatchMetrics.class));
|
||||
when(failureRouter.routeContinuityDlt(any(), any(), eq(influxFailure), eq("RAW_STORAGE")))
|
||||
.thenReturn(SyslogFailureRouter.TerminalState.DLT_PUBLISHED);
|
||||
List<SysLogProcessor.PreparedMessage> messages = messages(2);
|
||||
|
||||
List<SysLogProcessor.PreparedMessage> result = spyProcessor.persistRawWithoutBlockingMainFlow(
|
||||
messages, new SysLogProcessor.BatchMetrics(), "RAW");
|
||||
|
||||
assertEquals(messages, result);
|
||||
verify(failureRouter, times(2))
|
||||
.routeContinuityDlt(any(), any(), eq(influxFailure), eq("RAW_STORAGE"));
|
||||
verify(rawInfluxMetrics).rawWriteBypassed(2L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void influxAndDltDoubleFailureStillReturnsToMainFlow() {
|
||||
SysLogProcessor spyProcessor = spy(processor);
|
||||
RuntimeException influxFailure = new IllegalStateException("influx unavailable");
|
||||
doThrow(influxFailure).when(spyProcessor)
|
||||
.persistRawBatch(anyList(), any(SysLogProcessor.BatchMetrics.class));
|
||||
when(failureRouter.routeContinuityDlt(any(), any(), eq(influxFailure), eq("RAW_STORAGE")))
|
||||
.thenThrow(new SyslogFailureRouter.FailurePublishException(
|
||||
"broker unavailable", new IllegalStateException("send failed")));
|
||||
List<SysLogProcessor.PreparedMessage> messages = messages(2);
|
||||
|
||||
List<SysLogProcessor.PreparedMessage> result = spyProcessor.persistRawWithoutBlockingMainFlow(
|
||||
messages, new SysLogProcessor.BatchMetrics(), "RAW");
|
||||
|
||||
assertEquals(messages, result);
|
||||
verify(failureRouter).routeContinuityDlt(any(), any(), eq(influxFailure), eq("RAW_STORAGE"));
|
||||
verify(rawInfluxMetrics).continuityDltPublishFailed(2L);
|
||||
verify(rawInfluxMetrics).rawWriteBypassed(2L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void backlogSamplingKeepsAlarmAndDropsNormalAndNonNormalFromRawPlan() {
|
||||
BacklogSamplingProperties properties = new BacklogSamplingProperties();
|
||||
ReflectionTestUtils.setField(properties, "enabled", true);
|
||||
ReflectionTestUtils.setField(properties, "lagThresholdMs", 1L);
|
||||
ReflectionTestUtils.setField(properties, "normalRetainBps", 0);
|
||||
ReflectionTestUtils.setField(properties, "nonNormalRetainBps", 0);
|
||||
ReflectionTestUtils.setField(properties, "degradeSteps", 10);
|
||||
ReflectionTestUtils.setField(properties, "policyVersion", "test-v1");
|
||||
ReflectionTestUtils.setField(processor, "backlogSamplingDecider",
|
||||
new BacklogSamplingDecider(properties));
|
||||
|
||||
List<SysLogProcessor.PreparedMessage> source = oldMessages(3);
|
||||
java.util.Map<String, Object> alarm = normalizedRow();
|
||||
alarm.put("event_level", "HIGH");
|
||||
java.util.Map<String, Object> normal = normalizedRow();
|
||||
SyslogNonNormalMessage nonNormal = new SyslogNonNormalMessage();
|
||||
nonNormal.setId("non-normal");
|
||||
List<SysLogProcessor.PreparedNormalization> computed = new ArrayList<>();
|
||||
computed.add(new SysLogProcessor.PreparedNormalization(source.get(0),
|
||||
LogNormalProcessor.NormalizationResult.success(Collections.singletonList(alarm)), null));
|
||||
computed.add(new SysLogProcessor.PreparedNormalization(source.get(1),
|
||||
LogNormalProcessor.NormalizationResult.success(Collections.singletonList(normal)), null));
|
||||
computed.add(new SysLogProcessor.PreparedNormalization(source.get(2),
|
||||
LogNormalProcessor.NormalizationResult.filtered(Collections.singletonList(nonNormal)), null));
|
||||
|
||||
SysLogProcessor.SamplingPlan plan = processor.buildSamplingPlan(computed);
|
||||
|
||||
assertEquals(1, plan.rawMessageCount(), "只有告警原文必须保留");
|
||||
assertEquals(3, plan.auditMessageCount());
|
||||
long alarmSaved = 0L;
|
||||
long normalDropped = 0L;
|
||||
long nonNormalDropped = 0L;
|
||||
long rawDropped = 0L;
|
||||
for (BacklogRetentionDelta delta : plan.auditDeltas()) {
|
||||
alarmSaved += delta.getAlarmRowsSaved();
|
||||
normalDropped += delta.getNormalRowsDropped();
|
||||
nonNormalDropped += delta.getNonNormalRowsDropped();
|
||||
rawDropped += delta.getRawDropped();
|
||||
}
|
||||
assertEquals(1L, alarmSaved);
|
||||
assertEquals(1L, normalDropped);
|
||||
assertEquals(1L, nonNormalDropped);
|
||||
assertEquals(2L, rawDropped);
|
||||
}
|
||||
|
||||
private List<ConsumerRecord<String, String>> records(int count) {
|
||||
List<ConsumerRecord<String, String>> records = new ArrayList<>();
|
||||
for (int i = 0; i < count; i++) {
|
||||
@@ -314,6 +408,20 @@ class SysLogProcessorParallelTest {
|
||||
return messages;
|
||||
}
|
||||
|
||||
private List<SysLogProcessor.PreparedMessage> oldMessages(int count) {
|
||||
List<SysLogProcessor.PreparedMessage> old = new ArrayList<>();
|
||||
for (int i = 0; i < count; i++) {
|
||||
ConsumerRecord<String, String> record =
|
||||
new ConsumerRecord<>("raw", i % 2, i, "key-" + i, "ciphertext");
|
||||
SyslogFailureRouter.MessageContext context = new SyslogFailureRouter.MessageContext(
|
||||
"ciphertext", "old-event-" + i, 0, null, Instant.EPOCH);
|
||||
old.add(new SysLogProcessor.PreparedMessage(record, context, "message-" + i,
|
||||
"old-event-" + i, null, LocalDateTime.of(2000, 1, 1, 0, 0),
|
||||
7L, 3L, System.currentTimeMillis()));
|
||||
}
|
||||
return old;
|
||||
}
|
||||
|
||||
private java.util.Map<String, Object> normalizedRow() {
|
||||
java.util.Map<String, Object> row = new java.util.HashMap<>();
|
||||
row.put("id", java.util.UUID.randomUUID().toString());
|
||||
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
package com.common.service;
|
||||
|
||||
import com.common.mapper.BacklogRetentionAuditMapper;
|
||||
import com.config.BacklogSamplingProperties;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
import org.springframework.transaction.TransactionStatus;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class BacklogSamplingAuditServiceTest {
|
||||
|
||||
@Test
|
||||
void aggregatesSameHourTopicPartitionAndPolicyWithoutMutatingInput() {
|
||||
BacklogRetentionDelta first = delta(1, 1, 0, 1, 0);
|
||||
BacklogRetentionDelta second = delta(1, 0, 1, 0, 1);
|
||||
|
||||
List<BacklogRetentionDelta> result = BacklogSamplingAuditService.aggregate(
|
||||
Arrays.asList(first, second));
|
||||
|
||||
assertEquals(1, result.size());
|
||||
assertEquals(2L, result.get(0).getMessageTotal());
|
||||
assertEquals(1L, result.get(0).getNormalRowsSaved());
|
||||
assertEquals(1L, result.get(0).getNormalRowsDropped());
|
||||
assertEquals(1L, result.get(0).getRawSaved());
|
||||
assertEquals(1L, result.get(0).getRawDropped());
|
||||
assertEquals(1L, first.getMessageTotal(), "聚合不能修改调用方的逐消息审计对象");
|
||||
}
|
||||
|
||||
@Test
|
||||
void auditFailureReturnsFalseAndRecordsFullRetentionFallback() {
|
||||
BacklogRetentionAuditMapper mapper = mock(BacklogRetentionAuditMapper.class);
|
||||
when(mapper.upsert(any())).thenThrow(new IllegalStateException("table unavailable"));
|
||||
PlatformTransactionManager transactionManager = transactionManager();
|
||||
BacklogSamplingMetrics metrics = new BacklogSamplingMetrics();
|
||||
BacklogSamplingAuditService service = new BacklogSamplingAuditService(
|
||||
mapper, properties(), metrics, transactionManager);
|
||||
|
||||
boolean audited = service.recordBeforeDropping(Arrays.asList(
|
||||
delta(2, 1, 1, 1, 1)));
|
||||
|
||||
assertFalse(audited);
|
||||
Map<String, Object> snapshot = metrics.snapshot();
|
||||
assertEquals(1L, snapshot.get("audit_write_failures_total"));
|
||||
assertEquals(2L, snapshot.get("audit_fallback_messages_total"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void successfulAuditCommitsAndRecordsSamplingMetrics() {
|
||||
BacklogRetentionAuditMapper mapper = mock(BacklogRetentionAuditMapper.class);
|
||||
when(mapper.upsert(any())).thenReturn(1);
|
||||
PlatformTransactionManager transactionManager = transactionManager();
|
||||
BacklogSamplingMetrics metrics = new BacklogSamplingMetrics();
|
||||
BacklogSamplingAuditService service = new BacklogSamplingAuditService(
|
||||
mapper, properties(), metrics, transactionManager);
|
||||
|
||||
assertTrue(service.recordBeforeDropping(Arrays.asList(
|
||||
delta(1, 0, 1, 0, 1))));
|
||||
|
||||
verify(mapper).upsert(any());
|
||||
assertEquals(1L, metrics.snapshot().get("degraded_messages_total"));
|
||||
assertEquals(1L, metrics.snapshot().get("normal_rows_dropped_total"));
|
||||
}
|
||||
|
||||
private BacklogSamplingProperties properties() {
|
||||
BacklogSamplingProperties properties = new BacklogSamplingProperties();
|
||||
ReflectionTestUtils.setField(properties, "auditTimeoutSeconds", 5);
|
||||
ReflectionTestUtils.setField(properties, "auditRetryBackoffMs", 60_000L);
|
||||
return properties;
|
||||
}
|
||||
|
||||
private PlatformTransactionManager transactionManager() {
|
||||
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
|
||||
TransactionStatus status = mock(TransactionStatus.class);
|
||||
when(transactionManager.getTransaction(any(TransactionDefinition.class))).thenReturn(status);
|
||||
return transactionManager;
|
||||
}
|
||||
|
||||
private BacklogRetentionDelta delta(long messages, long normalSaved, long normalDropped,
|
||||
long rawSaved, long rawDropped) {
|
||||
return new BacklogRetentionDelta(LocalDateTime.of(2026, 9, 12, 18, 0),
|
||||
"raw", 0, "v1", 1000, 10000,
|
||||
messages, 0, 0, 1, normalSaved, normalDropped,
|
||||
0, 0, 0, 1, rawSaved, rawDropped, 3_600_000L, 3_600_000L);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user