feat: 新增设备 API 拉取采集及配套处理能力
- 接入 NDRServer 告警和流量日志，支持凭据与任务管理 - 支持分页拉取、窗口拆分、调度优先级及 WAL 持久化后推进断点 - 增加迟到告警规则重放，优化多条泛化规则选择和事件时间处理 - 补充回归测试、数据库升级脚本及部署文档
This commit is contained in:
王绍宏
+4
@@ -59,6 +59,8 @@ class LogNormalProcessorCompiledEquivalenceTest {
|
||||
|
||||
assertEquals(legacyResult.getOutcome(), compiledResult.getOutcome());
|
||||
assertEquals(legacyResult.getRows(), compiledResult.getRows());
|
||||
assertEquals(1, compiledResult.getRows().size());
|
||||
assertEquals(31L, compiledResult.getRows().get(0).get("normalize_rule_id"));
|
||||
assertEquals(1, metrics.getParseMisses());
|
||||
assertEquals(1, metrics.getParseHits());
|
||||
assertEquals(2, metrics.getCompiledMisses());
|
||||
@@ -118,6 +120,8 @@ class LogNormalProcessorCompiledEquivalenceTest {
|
||||
properties.getCompiledRule().setShadowCompareEnabled(false);
|
||||
properties.getMessageParseCache().setEnabled(parseCache);
|
||||
properties.getBatchEnrichmentCache().setEnabled(compiled);
|
||||
properties.getRuleSelection().setEnabled(true);
|
||||
properties.getRuleSelection().setShadowEnabled(false);
|
||||
return properties;
|
||||
}
|
||||
|
||||
|
||||
+24
-7
@@ -33,29 +33,46 @@ class LogNormalProcessorReceiveTimeTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void initialReceiveTimeSetsBothPartitionAndCreationTime() {
|
||||
void eventTimeSetsPartitionAndReceiveTimeSetsCreationTime() {
|
||||
LocalDateTime eventTime = LocalDateTime.of(2026, 8, 10, 20, 11, 24, 736_717_000);
|
||||
LocalDateTime receiveTime = LocalDateTime.of(2026, 8, 11, 3, 55, 57, 109_000_000);
|
||||
Map<String, Object> columns = new HashMap<>();
|
||||
|
||||
LogNormalProcessor.applyInitialReceiveTime(columns, receiveTime);
|
||||
LogNormalProcessor.applyIngressTimes(columns, eventTime, receiveTime);
|
||||
|
||||
assertEquals(receiveTime, columns.get("log_time"));
|
||||
assertEquals(eventTime, columns.get("log_time"));
|
||||
assertEquals(receiveTime, columns.get("created_at"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ruleProducedLogTimeCannotChangeReceiveDayPartition() {
|
||||
void eventTimeOverridesRuleValueAndSelectsEventDayPartition() {
|
||||
LocalDateTime eventTime = LocalDateTime.of(2026, 8, 10, 20, 11, 24, 736_717_000);
|
||||
LocalDateTime receiveTime = LocalDateTime.of(2026, 8, 11, 12, 30, 44, 199_000_000);
|
||||
Map<String, Object> columns = new HashMap<>();
|
||||
columns.put("log_time", eventTime);
|
||||
columns.put("log_time", receiveTime);
|
||||
|
||||
LogNormalProcessor.applyInitialReceiveTime(columns, receiveTime);
|
||||
LogNormalProcessor.applyIngressTimes(columns, eventTime, receiveTime);
|
||||
|
||||
assertEquals(receiveTime, columns.get("log_time"));
|
||||
assertEquals(eventTime, columns.get("log_time"));
|
||||
assertEquals(receiveTime, columns.get("created_at"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingEventTimeFallsBackToReceiveTimeForLegacyMessages() {
|
||||
LocalDateTime receiveTime = LocalDateTime.of(2026, 8, 11, 12, 30, 44, 199_000_000);
|
||||
|
||||
assertEquals(receiveTime, LogNormalProcessor.resolveEventAt(null, receiveTime));
|
||||
assertEquals(receiveTime, LogNormalProcessor.resolveEventAt("", receiveTime));
|
||||
}
|
||||
|
||||
@Test
|
||||
void explicitEventTimeIsParsedForHistoricalApiPull() {
|
||||
LocalDateTime receiveTime = LocalDateTime.of(2026, 9, 2, 9, 18, 38);
|
||||
|
||||
assertEquals(LocalDateTime.of(2026, 9, 1, 16, 17),
|
||||
LogNormalProcessor.resolveEventAt("20260901161700000", receiveTime));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deterministicResultIdIsStableAndKeepsUuidLength() {
|
||||
String first = LogNormalProcessor.deterministicResultId("20260811-event", 7L, "legacy");
|
||||
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
package com.Modules.NormalData;
|
||||
|
||||
import com.Modules.NormalData.compiled.BatchEnrichmentContext;
|
||||
import com.Modules.NormalData.compiled.BatchRuleSnapshot;
|
||||
import com.Modules.NormalData.compiled.CompiledRuleCache;
|
||||
import com.Modules.NormalData.compiled.CompiledRuleCompiler;
|
||||
import com.Modules.NormalData.compiled.RuleExecutionMetrics;
|
||||
import com.Modules.NormalData.compiled.RuleSource;
|
||||
import com.common.entity.SyslogNonNormalMessage;
|
||||
import com.common.mapper.DmColumnMapper;
|
||||
import com.common.mapper.DmNormalizeRuleMapper;
|
||||
import com.common.service.DeviceCollectTaskService;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.common.service.DmColumnService;
|
||||
import com.common.service.DmNormalizeRuleService;
|
||||
import com.common.service.LogDataCompleteService;
|
||||
import com.common.service.LogDataFilterService;
|
||||
import com.common.service.SyslogNonNormalMessageService;
|
||||
import com.common.service.SyslogNormalDataService;
|
||||
import com.config.AppProperties;
|
||||
import com.config.KafkaConsumerProperties;
|
||||
import com.config.NormalizationOptimizationProperties;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class LogNormalProcessorRuleSelectionTest {
|
||||
|
||||
@Test
|
||||
void explicitRuleProducesTheOnlyRowAndOverridesHistoricalRule() {
|
||||
RuleSource historical = jsonRule(10L, "historical", "", mapping("allow"), "");
|
||||
RuleSource explicit = jsonRule(20L, "explicit",
|
||||
"\"match_params\":{\"enabled\":true,\"priority\":100,"
|
||||
+ "\"all\":[{\"field\":\"type\",\"op\":\"eq\","
|
||||
+ "\"value\":\"firewall\"}]},",
|
||||
mapping("deny"), "");
|
||||
Fixture fixture = fixture(Arrays.asList(historical, explicit), config(false));
|
||||
|
||||
LogNormalProcessor.NormalizationResult result = fixture.prepare(
|
||||
"{\"type\":\"firewall\",\"status\":\"x\"}");
|
||||
|
||||
assertEquals(LogNormalProcessor.ProcessingOutcome.SUCCESS, result.getOutcome());
|
||||
assertEquals(1, result.getRows().size());
|
||||
assertEquals(20L, result.getRows().get(0).get("normalize_rule_id"));
|
||||
assertEquals("deny", result.getRows().get(0).get("action"));
|
||||
verify(fixture.nonNormalService, never()).saveMessage(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedCandidateDoesNotCreateNonNormalWhenAnotherRuleWins() {
|
||||
RuleSource failedRegex = new RuleSource(10L, "regex", "regex", "regex", "{"
|
||||
+ "\"regexp\":\"never=(\\\\S+)\","
|
||||
+ "\"mappers\":[{\"origin_field\":\"1\",\"dest_field\":\"value\","
|
||||
+ "\"action\":{\"type\":\"equal\",\"param\":{}}}]}", true, "0");
|
||||
RuleSource valid = jsonRule(20L, "json", "", equalMapping(), "");
|
||||
Fixture fixture = fixture(Arrays.asList(failedRegex, valid), config(true));
|
||||
|
||||
LogNormalProcessor.NormalizationResult result = fixture.prepare(
|
||||
"{\"type\":\"firewall\",\"status\":\"ok\"}");
|
||||
|
||||
assertEquals(1, result.getRows().size());
|
||||
assertEquals(20L, result.getRows().get(0).get("normalize_rule_id"));
|
||||
verify(fixture.nonNormalService, never()).saveMessage(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void allCandidatesFailAndOnlyOneNonNormalRecordIsSaved() {
|
||||
RuleSource first = regexRule(10L, "first", "first=(\\S+)");
|
||||
RuleSource second = regexRule(20L, "second", "second=(\\S+)");
|
||||
Fixture fixture = fixture(Arrays.asList(first, second), config(true));
|
||||
|
||||
LogNormalProcessor.NormalizationResult result = fixture.prepare("nothing matches");
|
||||
|
||||
assertEquals(LogNormalProcessor.ProcessingOutcome.FILTERED, result.getOutcome());
|
||||
verify(fixture.nonNormalService).saveMessage(any(SyslogNonNormalMessage.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonNormalSaveFailureRemainsRetryable() {
|
||||
Fixture fixture = fixture(Arrays.asList(
|
||||
regexRule(10L, "first", "first=(\\S+)"),
|
||||
regexRule(20L, "second", "second=(\\S+)")), config(true));
|
||||
when(fixture.nonNormalService.saveMessage(any())).thenReturn(false);
|
||||
|
||||
assertThrows(RuntimeException.class, () -> fixture.prepare("nothing matches"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void winnerFilterIsTerminalAndDoesNotTryTheRunnerUp() {
|
||||
String filter = "\"filters_params\":{\"expressions\":[{"
|
||||
+ "\"filters_relation\":\"and\",\"expressions\":[{"
|
||||
+ "\"column_name\":\"action\",\"op\":\"=\",\"value\":\"deny\"}]}]},";
|
||||
RuleSource historical = jsonRule(10L, "historical", "", equalMapping(), "");
|
||||
RuleSource explicit = jsonRule(20L, "explicit",
|
||||
"\"match_params\":{\"enabled\":true,\"all\":[{"
|
||||
+ "\"field\":\"type\",\"op\":\"eq\","
|
||||
+ "\"value\":\"firewall\"}]},",
|
||||
equalMapping(), filter);
|
||||
Fixture fixture = fixture(Arrays.asList(historical, explicit), config(true));
|
||||
|
||||
LogNormalProcessor.NormalizationResult result = fixture.prepare(
|
||||
"{\"type\":\"firewall\",\"status\":\"deny\"}");
|
||||
|
||||
assertEquals(LogNormalProcessor.ProcessingOutcome.FILTERED, result.getOutcome());
|
||||
assertTrue(result.getRows().isEmpty());
|
||||
verify(fixture.nonNormalService, never()).saveMessage(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void strictAmbiguitySavesNonNormalWhenEqualCandidatesProduceDifferentBusinessResults() {
|
||||
RuleSource allow = jsonRule(10L, "allow", "", mapping("allow"), "");
|
||||
RuleSource deny = jsonRule(20L, "deny", "", mapping("deny"), "");
|
||||
NormalizationOptimizationProperties properties = config(true);
|
||||
properties.getRuleSelection().setAmbiguousFallback("STRICT");
|
||||
Fixture fixture = fixture(Arrays.asList(allow, deny), properties);
|
||||
|
||||
LogNormalProcessor.NormalizationResult result = fixture.prepare(
|
||||
"{\"type\":\"firewall\",\"status\":\"x\"}");
|
||||
|
||||
assertEquals(LogNormalProcessor.ProcessingOutcome.FILTERED, result.getOutcome());
|
||||
assertTrue(result.getRows().isEmpty());
|
||||
ArgumentCaptor<SyslogNonNormalMessage> captor =
|
||||
ArgumentCaptor.forClass(SyslogNonNormalMessage.class);
|
||||
verify(fixture.nonNormalService).saveMessage(captor.capture());
|
||||
assertEquals("多规则归属歧义", captor.getValue().getReason());
|
||||
assertTrue(captor.getValue().getReasonDetail().contains("10"));
|
||||
assertTrue(captor.getValue().getReasonDetail().contains("20"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void strictAmbiguityStillUsesLowestIdWhenBusinessResultsAreEquivalent() {
|
||||
RuleSource highId = jsonRule(20L, "same-high", "", mapping("allow"), "");
|
||||
RuleSource lowId = jsonRule(10L, "same-low", "", mapping("allow"), "");
|
||||
NormalizationOptimizationProperties properties = config(true);
|
||||
properties.getRuleSelection().setAmbiguousFallback("STRICT");
|
||||
Fixture fixture = fixture(Arrays.asList(highId, lowId), properties);
|
||||
|
||||
LogNormalProcessor.NormalizationResult result = fixture.prepare(
|
||||
"{\"type\":\"firewall\",\"status\":\"x\"}");
|
||||
|
||||
assertEquals(LogNormalProcessor.ProcessingOutcome.SUCCESS, result.getOutcome());
|
||||
assertEquals(1, result.getRows().size());
|
||||
assertEquals(10L, result.getRows().get(0).get("normalize_rule_id"));
|
||||
verify(fixture.nonNormalService, never()).saveMessage(any());
|
||||
}
|
||||
|
||||
private static NormalizationOptimizationProperties config(boolean compiled) {
|
||||
NormalizationOptimizationProperties properties = new NormalizationOptimizationProperties();
|
||||
properties.getCompiledRule().setEnabled(compiled);
|
||||
properties.getCompiledRule().setShadowCompareEnabled(false);
|
||||
properties.getMessageParseCache().setEnabled(true);
|
||||
properties.getRuleSelection().setEnabled(true);
|
||||
properties.getRuleSelection().setShadowEnabled(false);
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static RuleSource jsonRule(long id, String name, String prefix,
|
||||
String mapper, String extra) {
|
||||
String content = "{" + prefix
|
||||
+ "\"cropper_params\":{\"head_key\":\"{\",\"head_offset\":0,"
|
||||
+ "\"tail_key\":\"\",\"tail_offset\":0},"
|
||||
+ "\"mappers\":[" + mapper + "]," + extra
|
||||
+ "\"complete_columns\":[]}";
|
||||
return new RuleSource(id, name, name, "json", content, true, "0");
|
||||
}
|
||||
|
||||
private static RuleSource regexRule(long id, String name, String regexp) {
|
||||
String content = "{\"regexp\":\"" + regexp.replace("\\", "\\\\") + "\","
|
||||
+ "\"mappers\":[{\"origin_field\":\"1\",\"dest_field\":\"value\","
|
||||
+ "\"action\":{\"type\":\"equal\",\"param\":{}}}]}";
|
||||
return new RuleSource(id, name, name, "regex", content, true, "0");
|
||||
}
|
||||
|
||||
private static String equalMapping() {
|
||||
return "{\"origin_field\":\"status\",\"dest_field\":\"action\","
|
||||
+ "\"action\":{\"type\":\"equal\",\"param\":{}}}";
|
||||
}
|
||||
|
||||
private static String mapping(String value) {
|
||||
return "{\"origin_field\":\"status\",\"dest_field\":\"action\","
|
||||
+ "\"action\":{\"type\":\"mapping\",\"param\":{\"mapping\":{"
|
||||
+ "\"x\":\"" + value + "\"}}}}";
|
||||
}
|
||||
|
||||
private static Fixture fixture(List<RuleSource> rules,
|
||||
NormalizationOptimizationProperties properties) {
|
||||
Fixture fixture = new Fixture();
|
||||
fixture.taskService = mock(DeviceCollectTaskService.class);
|
||||
fixture.deviceService = mock(DeviceDeviceService.class);
|
||||
fixture.nonNormalService = mock(SyslogNonNormalMessageService.class);
|
||||
when(fixture.nonNormalService.saveMessage(any())).thenReturn(true);
|
||||
CompiledRuleCompiler compiler = new CompiledRuleCompiler(new ObjectMapper());
|
||||
fixture.processor = new LogNormalProcessor(mock(DmNormalizeRuleService.class),
|
||||
mock(DmColumnService.class), mock(SyslogNormalDataService.class),
|
||||
fixture.nonNormalService, new LogDataFilterService(), new LogDataCompleteService(),
|
||||
fixture.taskService, fixture.deviceService, mock(DmColumnMapper.class),
|
||||
mock(DmNormalizeRuleMapper.class), mock(KafkaConsumerProperties.class),
|
||||
mock(AppProperties.class), new CompiledRuleCache(compiler, properties), properties);
|
||||
Map<Long, List<RuleSource>> byDevice = new LinkedHashMap<>();
|
||||
byDevice.put(7L, rules);
|
||||
fixture.snapshot = new BatchRuleSnapshot(byDevice, System.currentTimeMillis());
|
||||
return fixture;
|
||||
}
|
||||
|
||||
private static final class Fixture {
|
||||
private LogNormalProcessor processor;
|
||||
private SyslogNonNormalMessageService nonNormalService;
|
||||
private DeviceCollectTaskService taskService;
|
||||
private DeviceDeviceService deviceService;
|
||||
private BatchRuleSnapshot snapshot;
|
||||
|
||||
private LogNormalProcessor.NormalizationResult prepare(String body) {
|
||||
String message = "[receive_time=20260821150000000 event_id=event-selection device_id=7 "
|
||||
+ "device_name=fw vendor=vendor data_type=json device_collect_id=3]" + body;
|
||||
return processor.prepare(message, "event-selection", "raw",
|
||||
LocalDateTime.of(2026, 8, 21, 15, 0), snapshot,
|
||||
new BatchEnrichmentContext(false, taskService, deviceService),
|
||||
new RuleExecutionMetrics());
|
||||
}
|
||||
}
|
||||
}
|
||||
+35
-13
@@ -1,31 +1,53 @@
|
||||
package com.Modules.NormalData;
|
||||
|
||||
import com.influx.raw.PointTimeAllocator;
|
||||
import com.influx.raw.PointTimeResolver;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class SysLogProcessorTimeTest {
|
||||
|
||||
@Test
|
||||
void receiveTimeDeterminesBothPointTimestampAndMeasurementDate() {
|
||||
void eventTimeDeterminesBothPointTimestampAndMeasurementDate() {
|
||||
SysLogProcessor.ResolvedInfluxTime resolved = SysLogProcessor.resolveInfluxTime(
|
||||
"20260811035557109", Instant.parse("2026-08-09T00:00:00Z"));
|
||||
"20260810035557109", "20260811035557109");
|
||||
|
||||
assertEquals(Instant.parse("2026-08-10T19:55:57.109Z").toEpochMilli(),
|
||||
assertEquals(java.time.Instant.parse("2026-08-09T19:55:57.109Z").toEpochMilli(),
|
||||
resolved.getTimestampMs());
|
||||
assertEquals("20260811", resolved.getMeasurementDate());
|
||||
assertEquals(LocalDateTime.of(2026, 8, 11, 3, 55, 57, 109_000_000),
|
||||
resolved.getReceivedAt());
|
||||
assertEquals("20260810", resolved.getMeasurementDate());
|
||||
assertEquals(java.time.LocalDateTime.of(2026, 8, 10, 3, 55, 57, 109_000_000),
|
||||
resolved.getEventAt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidReceiveTimeIsRejectedInsteadOfUsingConsumerTime() {
|
||||
Instant firstReceivedAt = Instant.parse("2026-08-10T19:55:57.109Z");
|
||||
void legacyMessageWithoutEventTimeFallsBackToReceiveTime() {
|
||||
SysLogProcessor.ResolvedInfluxTime resolved = SysLogProcessor.resolveInfluxTime(
|
||||
null, "20260811035557109");
|
||||
|
||||
org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
|
||||
() -> SysLogProcessor.resolveInfluxTime("invalid", firstReceivedAt));
|
||||
assertEquals(java.time.Instant.parse("2026-08-10T19:55:57.109Z").toEpochMilli(),
|
||||
resolved.getTimestampMs());
|
||||
assertEquals("20260811", resolved.getMeasurementDate());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidEventTimeIsRejectedInsteadOfUsingReceiveTime() {
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> SysLogProcessor.resolveInfluxTime("invalid", "20260811035557109"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void historicalPointTimeIsValidatedAgainstEventTimeRatherThanNextDayReceiveTime() {
|
||||
SysLogProcessor.ResolvedInfluxTime event = SysLogProcessor.resolveInfluxTime(
|
||||
"20260901161700123", "20260902091800456");
|
||||
long pointTimeNs = event.getTimestampMs() * PointTimeAllocator.NANOS_PER_MILLI + 789L;
|
||||
|
||||
assertDoesNotThrow(() -> PointTimeResolver.validate(pointTimeNs, event.getTimestampMs()));
|
||||
|
||||
long receiveTimeMs = java.time.Instant.parse("2026-09-02T01:18:00.456Z").toEpochMilli();
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> PointTimeResolver.validate(pointTimeNs, receiveTimeMs));
|
||||
}
|
||||
}
|
||||
|
||||
+200
@@ -0,0 +1,200 @@
|
||||
package com.Modules.NormalData.compiled;
|
||||
|
||||
import com.Modules.NormalData.metrics.NormalizationRuleMetrics;
|
||||
import com.config.NormalizationOptimizationProperties;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class RuleSelectionEngineTest {
|
||||
private NormalizationOptimizationProperties properties;
|
||||
private NormalizationRuleMetrics metrics;
|
||||
private CompiledRuleCache cache;
|
||||
private RuleSelectionEngine engine;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
properties = new NormalizationOptimizationProperties();
|
||||
properties.getRuleSelection().setEnabled(true);
|
||||
properties.getRuleSelection().setShadowEnabled(false);
|
||||
metrics = new NormalizationRuleMetrics();
|
||||
cache = new CompiledRuleCache(new CompiledRuleCompiler(new ObjectMapper()), properties, metrics);
|
||||
engine = new RuleSelectionEngine(cache, properties, metrics);
|
||||
}
|
||||
|
||||
@Test
|
||||
void explicitMatchWinsBeforeHistoricalAutomaticRule() {
|
||||
RuleSource explicit = rule(20L, "title,client", "\"match_params\":{"
|
||||
+ "\"enabled\":true,\"priority\":10,"
|
||||
+ "\"all\":[{\"field\":\"title\",\"op\":\"eq\",\"value\":\"HFish\"}],"
|
||||
+ "\"any\":[{\"source\":\"raw\",\"op\":\"contains\",\"value\":\"threat\"}],"
|
||||
+ "\"none\":[{\"field\":\"honeypot_id\",\"op\":\"exists\"}]},");
|
||||
RuleSource historical = rule(10L, "title,client", "");
|
||||
Map<String, Object> fields = fields("title", "HFish", "client", "agent");
|
||||
|
||||
RuleSelectionEngine.Decision<String> result = engine.select(Arrays.asList(
|
||||
candidate(historical, fields), candidate(explicit, fields)), "HFish threat", false);
|
||||
|
||||
assertTrue(result.isSelected());
|
||||
assertEquals(20L, result.getWinner().getSource().getId());
|
||||
assertEquals(RuleSelectionEngine.SelectionMode.EXPLICIT, result.getMode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedExplicitRuleCannotFallBackToAutomaticScoring() {
|
||||
RuleSource explicit = rule(1L, "title,special", "\"match_params\":{"
|
||||
+ "\"enabled\":true,\"priority\":999,"
|
||||
+ "\"all\":[{\"field\":\"title\",\"op\":\"eq\",\"value\":\"other\"}]},");
|
||||
RuleSource historical = rule(2L, "title", "");
|
||||
Map<String, Object> fields = fields("title", "actual", "special", "yes");
|
||||
|
||||
RuleSelectionEngine.Decision<String> result = engine.select(Arrays.asList(
|
||||
candidate(explicit, fields), candidate(historical, fields)), "actual", false);
|
||||
|
||||
assertEquals(2L, result.getWinner().getSource().getId());
|
||||
assertEquals(RuleSelectionEngine.SelectionMode.AUTOMATIC, result.getMode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void explicitPriorityOnlyAppliesAfterConditionsMatch() {
|
||||
RuleSource low = rule(10L, "kind", explicitEq("kind", "event", 10));
|
||||
RuleSource high = rule(20L, "kind", explicitEq("kind", "event", 100));
|
||||
|
||||
RuleSelectionEngine.Decision<String> result = engine.select(Arrays.asList(
|
||||
candidate(low, fields("kind", "event")),
|
||||
candidate(high, fields("kind", "event"))), "event", false);
|
||||
|
||||
assertEquals(20L, result.getWinner().getSource().getId());
|
||||
assertFalse(result.isAmbiguous());
|
||||
}
|
||||
|
||||
@Test
|
||||
void weightedOwnershipPreventsTinyCommonRuleFromBeatingSpecificRule() {
|
||||
RuleSource small = rule(10L, "src_ip,time", "");
|
||||
RuleSource large = rule(20L, "src_ip,time,u1,u2,u3,u4,u5,u6,u7,u8", "");
|
||||
Map<String, Object> parsed = fields("src_ip", "10.0.0.1", "time", "now",
|
||||
"u1", 1, "u2", 2, "u3", 3, "u4", 4, "u5", 5, "u6", 6, "u7", 7);
|
||||
|
||||
RuleSelectionEngine.Decision<String> result = engine.select(Arrays.asList(
|
||||
candidate(small, parsed), candidate(large, parsed)), "raw", false);
|
||||
|
||||
assertEquals(20L, result.getWinner().getSource().getId());
|
||||
assertEquals(RuleSelectionEngine.SelectionMode.AUTOMATIC, result.getMode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void completelyEqualCandidatesUseLowestRuleIdAndExposeAmbiguity() {
|
||||
RuleSource highId = rule(40L, "src_ip,type", "");
|
||||
RuleSource lowId = rule(30L, "src_ip,type", "");
|
||||
Map<String, Object> parsed = fields("src_ip", "10.0.0.1", "type", "deny");
|
||||
|
||||
RuleSelectionEngine.Decision<String> result = engine.select(Arrays.asList(
|
||||
candidate(highId, parsed), candidate(lowId, parsed)), "raw", false);
|
||||
|
||||
assertEquals(30L, result.getWinner().getSource().getId());
|
||||
assertTrue(result.isAmbiguous());
|
||||
assertEquals(2, result.getTiedCandidates().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void allExplicitRulesFailingProducesNoCandidate() {
|
||||
RuleSource first = rule(1L, "kind", explicitEq("kind", "a", 0));
|
||||
RuleSource second = rule(2L, "kind", explicitEq("kind", "b", 0));
|
||||
|
||||
RuleSelectionEngine.Decision<String> result = engine.select(Arrays.asList(
|
||||
candidate(first, fields("kind", "c")),
|
||||
candidate(second, fields("kind", "c"))), "c", false);
|
||||
|
||||
assertFalse(result.isSelected());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidMatchConfigurationIsRejectedDuringCompilation() {
|
||||
RuleSource invalid = rule(1L, "kind", "\"match_params\":{"
|
||||
+ "\"enabled\":true,\"all\":[{\"field\":\"kind\","
|
||||
+ "\"op\":\"in\",\"value\":\"not-an-array\"}]},");
|
||||
|
||||
assertThrows(CompiledRuleCompiler.RuleCompilationException.class,
|
||||
() -> cache.getMatchProfile(invalid));
|
||||
}
|
||||
|
||||
@Test
|
||||
void explicitProfileSupportsAllDocumentedOperatorsAndGroupSemantics() {
|
||||
RuleSource source = rule(7L, "a", "\"match_params\":{\"enabled\":true,"
|
||||
+ "\"all\":["
|
||||
+ "{\"field\":\"a\",\"op\":\"exists\"},"
|
||||
+ "{\"field\":\"a\",\"op\":\"not_blank\"},"
|
||||
+ "{\"field\":\"a\",\"op\":\"eq\",\"value\":\"abc\"},"
|
||||
+ "{\"field\":\"a\",\"op\":\"ne\",\"value\":\"zzz\"},"
|
||||
+ "{\"field\":\"a\",\"op\":\"in\",\"value\":[\"abc\",\"def\"]},"
|
||||
+ "{\"field\":\"a\",\"op\":\"contains\",\"value\":\"b\"},"
|
||||
+ "{\"field\":\"a\",\"op\":\"regexp\",\"value\":\"^a.*\"},"
|
||||
+ "{\"field\":\"missing\",\"op\":\"not_exists\"},"
|
||||
+ "{\"source\":\"raw\",\"op\":\"not_blank\"},"
|
||||
+ "{\"source\":\"raw\",\"op\":\"contains\",\"value\":\"CEF:\"},"
|
||||
+ "{\"source\":\"raw\",\"op\":\"regexp\",\"value\":\"CEF:.*abc\"}],"
|
||||
+ "\"any\":[{\"field\":\"a\",\"op\":\"eq\",\"value\":\"no\"},"
|
||||
+ "{\"field\":\"a\",\"op\":\"eq\",\"value\":\"abc\"}],"
|
||||
+ "\"none\":[{\"field\":\"blocked\",\"op\":\"exists\"}]},");
|
||||
|
||||
RuleMatchProfile profile = cache.getMatchProfile(source);
|
||||
|
||||
assertTrue(profile.evaluate(fields("a", "abc"), "CEF: vendor abc").isMatched());
|
||||
assertFalse(profile.evaluate(fields("a", "abc", "blocked", true),
|
||||
"CEF: vendor abc").isMatched());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rawSourceRejectsFieldOnlyOperations() {
|
||||
RuleSource invalid = rule(8L, "a", "\"match_params\":{\"enabled\":true,"
|
||||
+ "\"all\":[{\"source\":\"raw\",\"op\":\"exists\"}]},");
|
||||
|
||||
assertThrows(CompiledRuleCompiler.RuleCompilationException.class,
|
||||
() -> cache.getMatchProfile(invalid));
|
||||
}
|
||||
|
||||
private static String explicitEq(String field, String value, int priority) {
|
||||
return "\"match_params\":{\"enabled\":true,\"priority\":" + priority
|
||||
+ ",\"all\":[{\"field\":\"" + field + "\",\"op\":\"eq\","
|
||||
+ "\"value\":\"" + value + "\"}]},";
|
||||
}
|
||||
|
||||
private static RuleSource rule(long id, String origins, String prefix) {
|
||||
StringBuilder mappers = new StringBuilder();
|
||||
if (!origins.isEmpty()) {
|
||||
for (String field : origins.split(",")) {
|
||||
if (mappers.length() > 0) mappers.append(',');
|
||||
mappers.append("{\"origin_field\":\"").append(field)
|
||||
.append("\",\"dest_field\":\"").append(field)
|
||||
.append("\",\"action\":{\"type\":\"equal\",\"param\":{}}}");
|
||||
}
|
||||
}
|
||||
String content = "{" + prefix + "\"mappers\":[" + mappers + "]}";
|
||||
return new RuleSource(id, "rule-" + id, "rule-" + id,
|
||||
"json", content, true, "0");
|
||||
}
|
||||
|
||||
private static RuleSelectionEngine.Candidate<String> candidate(
|
||||
RuleSource source, Map<String, Object> fields) {
|
||||
return new RuleSelectionEngine.Candidate<>(source, fields, "payload-" + source.getId());
|
||||
}
|
||||
|
||||
private static Map<String, Object> fields(Object... values) {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
for (int i = 0; i < values.length; i += 2) {
|
||||
result.put(values[i].toString(), values[i + 1]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
+6
@@ -19,6 +19,9 @@ class NormalizationRuleMetricsTest {
|
||||
metrics.enrichment("device", "hit");
|
||||
metrics.invalidation("after_commit");
|
||||
metrics.messageParse("miss");
|
||||
metrics.ruleSelection("selected");
|
||||
metrics.ruleSelection("explicit_selected");
|
||||
metrics.ruleSelectionObservation(3, 2.0d / 3.0d, 0.5d);
|
||||
metrics.ruleJsonParse();
|
||||
metrics.compiledCacheGauge(3, 1);
|
||||
|
||||
@@ -28,6 +31,9 @@ class NormalizationRuleMetricsTest {
|
||||
assertTrue(snapshot.containsKey("rule_source_cache_get_total"));
|
||||
assertTrue(snapshot.containsKey("enrichment_cache_get_total"));
|
||||
assertTrue(snapshot.containsKey("rule_invalidation_total"));
|
||||
assertEquals(1L, ((Map<String, Long>) snapshot.get("rule_selection_total")).get("selected"));
|
||||
assertEquals(3L, snapshot.get("rule_selection_candidate_count_total"));
|
||||
assertEquals(2.0d / 3.0d, (Double) snapshot.get("rule_selection_match_rate_avg"), 0.000001d);
|
||||
assertEquals(1L, snapshot.get("rule_json_parse_total"));
|
||||
}
|
||||
}
|
||||
|
||||
+16
@@ -13,6 +13,7 @@ import org.springframework.test.util.ReflectionTestUtils;
|
||||
import java.util.Arrays;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
@@ -62,6 +63,21 @@ class DmNormalizeRuleServiceUpdateTest {
|
||||
verify(publisher, never()).publishEvent(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidMatchParamsAreRejectedBeforeRuleUpdate() {
|
||||
when(mapper.selectById(11L)).thenReturn(validRule());
|
||||
DmNormalizeRule patch = new DmNormalizeRule();
|
||||
patch.setId(11L);
|
||||
patch.setRuleContent("{\"match_params\":{\"enabled\":true,\"all\":[{"
|
||||
+ "\"field\":\"type\",\"op\":\"in\",\"value\":\"bad\"}]},"
|
||||
+ "\"mappers\":[]}");
|
||||
|
||||
assertThrows(CompiledRuleCompiler.RuleCompilationException.class,
|
||||
() -> service.updateById(patch));
|
||||
verify(mapper, never()).updateById(any());
|
||||
verify(publisher, never()).publishEvent(any());
|
||||
}
|
||||
|
||||
private static DmNormalizeRule validRule() {
|
||||
DmNormalizeRule rule = new DmNormalizeRule();
|
||||
rule.setId(11L);
|
||||
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
package com.common.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
class LateAlarmBucketWriterTest {
|
||||
|
||||
@Test
|
||||
void groupsRowsByDeviceAndEventMinute() {
|
||||
JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class);
|
||||
LateAlarmBucketWriter writer = new LateAlarmBucketWriter(jdbcTemplate);
|
||||
ReflectionTestUtils.setField(writer, "enabled", true);
|
||||
|
||||
writer.record(
|
||||
Arrays.asList(
|
||||
row(126, LocalDateTime.of(2026, 9, 4, 10, 1, 2), 3),
|
||||
row(126, LocalDateTime.of(2026, 9, 4, 10, 1, 59), 4),
|
||||
row(126, LocalDateTime.of(2026, 9, 4, 10, 2, 1), 5)));
|
||||
|
||||
verify(jdbcTemplate, times(2))
|
||||
.update(
|
||||
anyString(),
|
||||
any(),
|
||||
any(),
|
||||
any(),
|
||||
any(),
|
||||
any(),
|
||||
any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsAlarmRowsWithoutStableTimes() {
|
||||
LateAlarmBucketWriter writer = new LateAlarmBucketWriter(mock(JdbcTemplate.class));
|
||||
ReflectionTestUtils.setField(writer, "enabled", true);
|
||||
Map<String, Object> invalid = new HashMap<>();
|
||||
invalid.put("device_id", 126);
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> writer.record(Arrays.asList(invalid)));
|
||||
}
|
||||
|
||||
private Map<String, Object> row(int deviceId, LocalDateTime eventTime, int receiveSecond) {
|
||||
Map<String, Object> row = new HashMap<>();
|
||||
row.put("device_id", deviceId);
|
||||
row.put("log_time", eventTime);
|
||||
row.put("created_at", LocalDateTime.of(2026, 9, 5, 1, 0, receiveSecond));
|
||||
return row;
|
||||
}
|
||||
}
|
||||
+4
@@ -22,6 +22,7 @@ class NormalizationCachePropertiesTest {
|
||||
assertFalse(properties.isJetcacheEnabled());
|
||||
assertFalse(properties.getRuleSourceCache().isEnabled());
|
||||
assertFalse(properties.getCompiledRule().isEnabled());
|
||||
assertEquals(true, properties.getRuleSelection().isEnabled());
|
||||
assertEquals(1000, properties.getCompiledRule().getShadowTimeoutMs());
|
||||
assertEquals(30, properties.getRuleSourceCache().getLocalTtlSeconds());
|
||||
assertEquals(300, properties.getRuleSourceCache().getRemoteTtlSeconds());
|
||||
@@ -58,6 +59,8 @@ class NormalizationCachePropertiesTest {
|
||||
"normalization.batch-enrichment-cache.enabled"), resource);
|
||||
assertEquals("true", values.getProperty(
|
||||
"normalization.message-parse-cache.enabled"), resource);
|
||||
assertEquals("true", values.getProperty(
|
||||
"normalization.rule-selection.enabled"), resource);
|
||||
|
||||
NormalizationOptimizationProperties bound = new Binder(
|
||||
new MapConfigurationPropertySource(values))
|
||||
@@ -72,6 +75,7 @@ class NormalizationCachePropertiesTest {
|
||||
assertEquals(true, bound.getCompiledRule().isRuleDetailLogEnabled(), resource);
|
||||
assertEquals(true, bound.getBatchEnrichmentCache().isEnabled(), resource);
|
||||
assertEquals(true, bound.getMessageParseCache().isEnabled(), resource);
|
||||
assertEquals(true, bound.getRuleSelection().isEnabled(), resource);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+6
-6
@@ -29,9 +29,9 @@ class InfluxRawV2ContractTest {
|
||||
@Test void configurationRejectsChangingPersistentBucketCount() {
|
||||
RawStorageProperties properties=new RawStorageProperties();
|
||||
ReflectionTestUtils.setField(properties,"writeSchemaValue","V2");
|
||||
ReflectionTestUtils.setField(properties,"readModeValue","V2_ONLY");
|
||||
ReflectionTestUtils.setField(properties,"readModeValue","V1_V2");
|
||||
ReflectionTestUtils.setField(properties,"v1Bucket","yelangbucket");
|
||||
ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket");
|
||||
ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket_v2");
|
||||
ReflectionTestUtils.setField(properties,"uuidBucketCount",1024);
|
||||
ReflectionTestUtils.setField(properties,"queryMaxRows",10000);
|
||||
ReflectionTestUtils.setField(properties,"queryMaxResponseBytes",10485760L);
|
||||
@@ -62,9 +62,9 @@ class InfluxRawV2ContractTest {
|
||||
}
|
||||
|
||||
@Test void v2HasUuidFieldButNeverUuidTagAndUsesNanoseconds() {
|
||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
|
||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
|
||||
InfluxRawPointFactory.RawPoint raw=new InfluxRawPointFactory(properties).create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5","message","20260814000000000","126","1","agent-01-syslog-topic",ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L);
|
||||
String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket",raw.getBucket());
|
||||
String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket_v2",raw.getBucket());
|
||||
}
|
||||
|
||||
@Test void onlyMissingPointTimeFallsBack() {
|
||||
@@ -83,7 +83,7 @@ class InfluxRawV2ContractTest {
|
||||
}
|
||||
|
||||
@Test void mixedLegacyAndNewMessagesAllProduceDistinctV2PointsAndRetryIsIdempotent() {
|
||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
|
||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
|
||||
RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); InfluxRawPointFactory factory=new InfluxRawPointFactory(properties); ReceiveTimeParser.ParsedReceiveTime parsed=ReceiveTimeParser.parse("20260814000000000");
|
||||
long first=resolver.resolve(false,null,parsed.getEpochMillis(),true); long second=resolver.resolve(false,null,parsed.getEpochMillis(),true); assertNotEquals(first,second);
|
||||
String id="20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"; String originalTopic="agent-01-syslog-topic";
|
||||
@@ -100,7 +100,7 @@ class InfluxRawV2ContractTest {
|
||||
|
||||
@Test void uuidCandidateFluxComputesBucketEscapesUuidAndSelectsEarliestGlobally() {
|
||||
String uuid="20260814-a\" |> die() //";
|
||||
String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket",
|
||||
String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket_v2",
|
||||
"syslog_security_20260814",uuid,Instant.parse("2026-08-13T16:00:00Z"),
|
||||
Instant.parse("2026-08-14T16:00:00Z"));
|
||||
assertTrue(query.contains("r.event_bucket == \""+InfluxEventBucket.bucketOf(uuid)+"\""));
|
||||
|
||||
Reference in New Issue
Block a user