nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(syslog): 完成高吞吐可靠链路与统计存储优化

Browse Source 
- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程
- 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标
- 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读
- 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换
- 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
王绍宏
2026-08-20 00:18:33 +08:00
parent a3852fc0f9
commit 29445c6825
303 changed files with 20419 additions and 4481 deletions
Download Patch File Download Diff File Expand all files Collapse all files
haobang-security-dm/syslog-consumer-rule/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java
+111
View File
@@ -0,0 +1,111 @@
package com.influx.raw;
import com.influx.InfluxDBClient;
import com.common.service.InfluxSyslogService;
import com.influxdb.client.domain.WritePrecision;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import java.time.Instant;
import java.util.HashSet;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
class InfluxRawV2ContractTest {
@Test void fixedBucketVectorsMatchProtocol() {
assertEquals("1269", InfluxEventBucket.bucketOf("20260814-00000000-0000-0000-0000-000000000000"));
assertEquals("2085", InfluxEventBucket.bucketOf("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"));
assertThrows(IllegalArgumentException.class, () -> InfluxEventBucket.bucketOf(" uuid"));
}
@Test void configurationRejectsChangingPersistentBucketCount() {
RawStorageProperties properties=new RawStorageProperties();
ReflectionTestUtils.setField(properties,"writeSchemaValue","V2");
ReflectionTestUtils.setField(properties,"readModeValue","V1_V2");
ReflectionTestUtils.setField(properties,"v1Bucket","yelangbucket");
ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket_v2");
ReflectionTestUtils.setField(properties,"uuidBucketCount",1024);
ReflectionTestUtils.setField(properties,"queryMaxRows",10000);
ReflectionTestUtils.setField(properties,"queryMaxResponseBytes",10485760L);
assertThrows(IllegalArgumentException.class,properties::validate);
}
@Test void firstMillionSuffixesAreUniqueAndThenCycle() {
PointTimeAllocator allocator = new PointTimeAllocator(); long millis = 1_786_686_912_123L;
Set<Long> suffixes = new HashSet<>(1_333_334);
for (int i=0;i<1_000_000;i++) { long value=allocator.allocate(millis); assertEquals(millis,Math.floorDiv(value,1_000_000L)); assertTrue(suffixes.add(Math.floorMod(value,1_000_000L))); }
assertEquals(0L, Math.floorMod(allocator.allocate(millis),1_000_000L));
}
@Test void concurrentAllocationUsesAtomicSequence() throws Exception {
PointTimeAllocator allocator=new PointTimeAllocator(); Set<Long> values=ConcurrentHashMap.newKeySet(); ExecutorService pool=Executors.newFixedThreadPool(8);
for(int i=0;i<100_000;i++) pool.submit(()->values.add(allocator.allocate(1_786_686_912_123L)));
pool.shutdown(); assertTrue(pool.awaitTermination(20, TimeUnit.SECONDS)); assertEquals(100_000,values.size());
}
@Test void overflowIsNotSilentlyReplacedWithCurrentTime() {
assertThrows(ArithmeticException.class,()->new PointTimeAllocator().allocate(Long.MAX_VALUE));
}
@Test void receiveTimeAlwaysUsesShanghaiAndOneParsedValue() {
TimeZone original=TimeZone.getDefault();
try { TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")); ReceiveTimeParser.ParsedReceiveTime parsed=ReceiveTimeParser.parse("20260814000000000"); assertEquals(Instant.parse("2026-08-13T16:00:00Z").toEpochMilli(),parsed.getEpochMillis()); assertEquals("20260814",parsed.getMeasurementDate()); }
finally { TimeZone.setDefault(original); }
}
@Test void v2HasUuidFieldButNeverUuidTagAndUsesNanoseconds() {
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
InfluxRawPointFactory.RawPoint raw=new InfluxRawPointFactory(properties).create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5","message","20260814000000000","126","1","agent-01-syslog-topic",ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L);
String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket_v2",raw.getBucket());
}
@Test void onlyMissingPointTimeFallsBack() {
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); long millis=ReceiveTimeParser.parse("20260814000000000").getEpochMillis();
assertNotNull(resolver.resolve(false,null,millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,"-1",millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,"not-number",millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,Long.toString(millis*1_000_000L+1_000_000L),millis,true)); assertEquals(1L,metrics.snapshot().get("legacy_missing_point_time_total"));
}
@Test void disabledFallbackStillCountsMissingAndFailure() {
RawStorageProperties properties=mock(RawStorageProperties.class);
RawInfluxMetrics metrics=new RawInfluxMetrics();
PointTimeResolver resolver=new PointTimeResolver(properties,metrics);
assertThrows(IllegalArgumentException.class,
()->resolver.resolve(false,null,1_786_636_800_000L,true));
assertEquals(1L,metrics.snapshot().get("legacy_missing_point_time_total"));
assertEquals(1L,metrics.snapshot().get("legacy_point_time_fallback_failed_total"));
}
@Test void mixedLegacyAndNewMessagesAllProduceDistinctV2PointsAndRetryIsIdempotent() {
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); InfluxRawPointFactory factory=new InfluxRawPointFactory(properties); ReceiveTimeParser.ParsedReceiveTime parsed=ReceiveTimeParser.parse("20260814000000000");
long first=resolver.resolve(false,null,parsed.getEpochMillis(),true); long second=resolver.resolve(false,null,parsed.getEpochMillis(),true); assertNotEquals(first,second);
String id="20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"; String originalTopic="agent-01-syslog-topic";
String firstLine=factory.create(id,"message","20260814000000000","126","1",originalTopic,parsed,first).getPoint().toLineProtocol();
String retryLine=factory.create(id,"message","20260814000000000","126","1",originalTopic,parsed,first).getPoint().toLineProtocol();
String secondLine=factory.create("20260814-00000000-0000-0000-0000-000000000000","message2","20260814000000000","126","1",originalTopic,parsed,second).getPoint().toLineProtocol();
assertEquals(firstLine,retryLine); assertNotEquals(firstLine,secondLine); assertTrue(firstLine.contains("topic=agent-01-syslog-topic")); assertFalse(firstLine.contains("RETRY"));
}
@Test void fluxEscapingPreventsLiteralAndControlCharacterInjection() {
assertEquals("a\\\"b\\\\c", InfluxDBClient.escapeFluxString("a\"b\\c"));
assertThrows(IllegalArgumentException.class,()->InfluxDBClient.escapeFluxString("x\u0001y"));
}
@Test void uuidCandidateFluxComputesBucketEscapesUuidAndSelectsEarliestGlobally() {
String uuid="20260814-a\" |> die() //";
String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket_v2",
"syslog_security_20260814",uuid,Instant.parse("2026-08-13T16:00:00Z"),
Instant.parse("2026-08-14T16:00:00Z"));
assertTrue(query.contains("r.event_bucket == \""+InfluxEventBucket.bucketOf(uuid)+"\""));
assertTrue(query.contains("r._value == \"20260814-a\\\" |> die() //\""));
assertTrue(query.indexOf("|> group()") < query.indexOf("|> sort("));
assertTrue(query.indexOf("|> sort(") < query.indexOf("|> limit(n: 2)"));
}
}