nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(upgrade): 增加 升级脚本和部署说明

Browse Source 
- 新增达梦数据库结构、幂等索引、统计基线及校验脚本
- 新增 Docker Kafka Retry、DLT、Replay Topic 升级脚本
- 更新生产 Compose，启用 DIRECT 并关闭旧增量捕获和投影
- 补充数据库、Kafka、应用启动及验证说明
- 增加日志和本地生成文件忽略规则
This commit is contained in:
王绍宏
2026-08-24 15:09:36 +08:00
parent 29445c6825
commit f53661931f
15 changed files with 694 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-prod.yaml
+6 -5
View File
@@ -21,9 +21,10 @@ services:
# 环境配置
- SPRING_PROFILES_ACTIVE=prod
- TZ=Asia/Shanghai
# 首次启动保持影子模式：维护增量投影，但正式字段仍由旧全量任务更新。
- STATS_INCREMENTAL_CAPTURE_ENABLED=true
- STATS_INCREMENTAL_PROJECTION_ENABLED=true
# 生产环境只使用 DIRECT：事实与统计同事务写入，不再写旧 change-log/pending。
- STATS_WRITE_MODE=DIRECT
- STATS_INCREMENTAL_CAPTURE_ENABLED=false
- STATS_INCREMENTAL_PROJECTION_ENABLED=false
- STATS_INCREMENTAL_READ_ENABLED=true
- STATS_LEGACY_FULL_SCAN_ENABLED=false
- STATS_INGRESS_METADATA_ENABLED=true
@@ -77,9 +78,9 @@ services:
- INFLUXDB_ORG=influxdb
- INFLUXDB_BUCKET=yelangbucket
- INFLUXDB_RAW_WRITE_SCHEMA=V2
- INFLUXDB_RAW_READ_MODE=V1_V2
- INFLUXDB_RAW_READ_MODE=V2_ONLY
- INFLUXDB_RAW_V1_BUCKET=yelangbucket
- INFLUXDB_RAW_V2_BUCKET=yelangbucket_v2
- INFLUXDB_RAW_V2_BUCKET=yelangbucket
- INFLUXDB_RAW_UUID_BUCKET_COUNT=4096
- INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED=true
- INFLUXDB_RAW_QUERY_MAX_ROWS=10000
haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/RawStorageProperties.java
+2 -2
View File
@@ -12,9 +12,9 @@ public class RawStorageProperties {
public static final int UUID_BUCKET_COUNT = 4096;
@Value("${influxdb.raw.write-schema:V1}") private String writeSchemaValue;
@Value("${influxdb.raw.read-mode:V1_ONLY}") private String readModeValue;
@Value("${influxdb.raw.read-mode:V2_ONLY}") private String readModeValue;
@Value("${influxdb.raw.v1-bucket:${influxdb.bucket:yelangbucket}}") private String v1Bucket;
@Value("${influxdb.raw.v2-bucket:yelangbucket_v2}") private String v2Bucket;
@Value("${influxdb.raw.v2-bucket:yelangbucket}") private String v2Bucket;
@Value("${influxdb.raw.uuid-bucket-count:4096}") private int uuidBucketCount;
@Value("${influxdb.raw.legacy-point-time-fallback-enabled:true}") private boolean legacyFallbackEnabled;
@Value("${influxdb.raw.query-max-rows:10000}") private int queryMaxRows;
haobang-security-dm/syslog-consumer/src/main/resources/application.properties
+2 -2
View File
@@ -28,9 +28,9 @@ influxdb.retry.attempts=3
influxdb.retry.delay=1000
# 原始 Syslog V2 分阶段开关。兼容部署先保持 V1/V1_ONLY，切换时统一写 V2。
influxdb.raw.write-schema=${INFLUXDB_RAW_WRITE_SCHEMA:V2}
influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V1_V2}
influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V2_ONLY}
influxdb.raw.v1-bucket=${INFLUXDB_RAW_V1_BUCKET:${influxdb.bucket}}
influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket_v2}
influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket}
influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
haobang-security-dm/syslog-consumer/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java
+6 -6
View File
@@ -29,9 +29,9 @@ class InfluxRawV2ContractTest {
@Test void configurationRejectsChangingPersistentBucketCount() {
RawStorageProperties properties=new RawStorageProperties();
ReflectionTestUtils.setField(properties,"writeSchemaValue","V2");
ReflectionTestUtils.setField(properties,"readModeValue","V1_V2");
ReflectionTestUtils.setField(properties,"readModeValue","V2_ONLY");
ReflectionTestUtils.setField(properties,"v1Bucket","yelangbucket");
ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket_v2");
ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket");
ReflectionTestUtils.setField(properties,"uuidBucketCount",1024);
ReflectionTestUtils.setField(properties,"queryMaxRows",10000);
ReflectionTestUtils.setField(properties,"queryMaxResponseBytes",10485760L);
@@ -62,9 +62,9 @@ class InfluxRawV2ContractTest {
}
@Test void v2HasUuidFieldButNeverUuidTagAndUsesNanoseconds() {
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
InfluxRawPointFactory.RawPoint raw=new InfluxRawPointFactory(properties).create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5","message","20260814000000000","126","1","agent-01-syslog-topic",ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L);
String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket_v2",raw.getBucket());
String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket",raw.getBucket());
}
@Test void onlyMissingPointTimeFallsBack() {
@@ -83,7 +83,7 @@ class InfluxRawV2ContractTest {
}
@Test void mixedLegacyAndNewMessagesAllProduceDistinctV2PointsAndRetryIsIdempotent() {
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); InfluxRawPointFactory factory=new InfluxRawPointFactory(properties); ReceiveTimeParser.ParsedReceiveTime parsed=ReceiveTimeParser.parse("20260814000000000");
long first=resolver.resolve(false,null,parsed.getEpochMillis(),true); long second=resolver.resolve(false,null,parsed.getEpochMillis(),true); assertNotEquals(first,second);
String id="20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"; String originalTopic="agent-01-syslog-topic";
@@ -100,7 +100,7 @@ class InfluxRawV2ContractTest {
@Test void uuidCandidateFluxComputesBucketEscapesUuidAndSelectsEarliestGlobally() {
String uuid="20260814-a\" |> die() //";
String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket_v2",
String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket",
"syslog_security_20260814",uuid,Instant.parse("2026-08-13T16:00:00Z"),
Instant.parse("2026-08-14T16:00:00Z"));
assertTrue(query.contains("r.event_bucket == \""+InfluxEventBucket.bucketOf(uuid)+"\""));