feat(upgrade): 增加 升级脚本和部署说明
- 新增达梦数据库结构、幂等索引、统计基线及校验脚本 - 新增 Docker Kafka Retry、DLT、Replay Topic 升级脚本 - 更新生产 Compose，启用 DIRECT 并关闭旧增量捕获和投影 - 补充数据库、Kafka、应用启动及验证说明 - 增加日志和本地生成文件忽略规则
This commit is contained in:
王绍宏
@@ -1,9 +1,9 @@
|
|||||||
2026-08-15 04:48:58.158 [main] INFO o.s.b.t.c.SpringBootTestContextBootstrapper - Neither @ContextConfiguration nor @ContextHierarchy found for test class [com.haobang.syslog.SysjavacollectApplicationTests], using SpringBootContextLoader
|
2026-08-20 00:07:55.926 [main] INFO o.s.b.t.c.SpringBootTestContextBootstrapper - Neither @ContextConfiguration nor @ContextHierarchy found for test class [com.haobang.syslog.SysjavacollectApplicationTests], using SpringBootContextLoader
|
||||||
2026-08-15 04:48:58.162 [main] INFO o.s.t.c.s.AbstractContextLoader - Could not detect default resource locations for test class [com.haobang.syslog.SysjavacollectApplicationTests]: no resource found for suffixes {-context.xml, Context.groovy}.
|
2026-08-20 00:07:55.931 [main] INFO o.s.t.c.s.AbstractContextLoader - Could not detect default resource locations for test class [com.haobang.syslog.SysjavacollectApplicationTests]: no resource found for suffixes {-context.xml, Context.groovy}.
|
||||||
2026-08-15 04:48:58.162 [main] INFO o.s.t.c.s.AnnotationConfigContextLoaderUtils - Could not detect default configuration classes for test class [com.haobang.syslog.SysjavacollectApplicationTests]: SysjavacollectApplicationTests does not declare any static, non-private, non-final, nested classes annotated with @Configuration.
|
2026-08-20 00:07:55.931 [main] INFO o.s.t.c.s.AnnotationConfigContextLoaderUtils - Could not detect default configuration classes for test class [com.haobang.syslog.SysjavacollectApplicationTests]: SysjavacollectApplicationTests does not declare any static, non-private, non-final, nested classes annotated with @Configuration.
|
||||||
2026-08-15 04:48:58.225 [main] INFO o.s.b.t.c.SpringBootTestContextBootstrapper - Found @SpringBootConfiguration com.haobang.syslog.syslogClientApplication for test class com.haobang.syslog.SysjavacollectApplicationTests
|
2026-08-20 00:07:56.000 [main] INFO o.s.b.t.c.SpringBootTestContextBootstrapper - Found @SpringBootConfiguration com.haobang.syslog.syslogClientApplication for test class com.haobang.syslog.SysjavacollectApplicationTests
|
||||||
2026-08-15 04:48:58.288 [main] INFO o.s.b.t.c.SpringBootTestContextBootstrapper - Loaded default TestExecutionListener class names from location [META-INF/spring.factories]: [org.springframework.boot.test.mock.mockito.MockitoTestExecutionListener, org.springframework.boot.test.mock.mockito.ResetMocksTestExecutionListener, org.springframework.boot.test.autoconfigure.restdocs.RestDocsTestExecutionListener, org.springframework.boot.test.autoconfigure.web.client.MockRestServiceServerResetTestExecutionListener, org.springframework.boot.test.autoconfigure.web.servlet.MockMvcPrintOnlyOnFailureTestExecutionListener, org.springframework.boot.test.autoconfigure.web.servlet.WebDriverTestExecutionListener, org.springframework.boot.test.autoconfigure.webservices.client.MockWebServiceServerTestExecutionListener, org.springframework.test.context.web.ServletTestExecutionListener, org.springframework.test.context.support.DirtiesContextBeforeModesTestExecutionListener, org.springframework.test.context.event.ApplicationEventsTestExecutionListener, org.springframework.test.context.support.DependencyInjectionTestExecutionListener, org.springframework.test.context.support.DirtiesContextTestExecutionListener, org.springframework.test.context.transaction.TransactionalTestExecutionListener, org.springframework.test.context.jdbc.SqlScriptsTestExecutionListener, org.springframework.test.context.event.EventPublishingTestExecutionListener]
|
2026-08-20 00:07:56.089 [main] INFO o.s.b.t.c.SpringBootTestContextBootstrapper - Loaded default TestExecutionListener class names from location [META-INF/spring.factories]: [org.springframework.boot.test.mock.mockito.MockitoTestExecutionListener, org.springframework.boot.test.mock.mockito.ResetMocksTestExecutionListener, org.springframework.boot.test.autoconfigure.restdocs.RestDocsTestExecutionListener, org.springframework.boot.test.autoconfigure.web.client.MockRestServiceServerResetTestExecutionListener, org.springframework.boot.test.autoconfigure.web.servlet.MockMvcPrintOnlyOnFailureTestExecutionListener, org.springframework.boot.test.autoconfigure.web.servlet.WebDriverTestExecutionListener, org.springframework.boot.test.autoconfigure.webservices.client.MockWebServiceServerTestExecutionListener, org.springframework.test.context.web.ServletTestExecutionListener, org.springframework.test.context.support.DirtiesContextBeforeModesTestExecutionListener, org.springframework.test.context.event.ApplicationEventsTestExecutionListener, org.springframework.test.context.support.DependencyInjectionTestExecutionListener, org.springframework.test.context.support.DirtiesContextTestExecutionListener, org.springframework.test.context.transaction.TransactionalTestExecutionListener, org.springframework.test.context.jdbc.SqlScriptsTestExecutionListener, org.springframework.test.context.event.EventPublishingTestExecutionListener]
|
||||||
2026-08-15 04:48:58.294 [main] INFO o.s.b.t.c.SpringBootTestContextBootstrapper - Using TestExecutionListeners: [org.springframework.test.context.support.DirtiesContextBeforeModesTestExecutionListener@22f59fa, org.springframework.test.context.event.ApplicationEventsTestExecutionListener@78fa769e, org.springframework.boot.test.mock.mockito.MockitoTestExecutionListener@16612a51, org.springframework.boot.test.autoconfigure.SpringBootDependencyInjectionTestExecutionListener@54e041a4, org.springframework.test.context.support.DirtiesContextTestExecutionListener@2c78324b, org.springframework.test.context.event.EventPublishingTestExecutionListener@79defdc, org.springframework.boot.test.mock.mockito.ResetMocksTestExecutionListener@18317edc, org.springframework.boot.test.autoconfigure.restdocs.RestDocsTestExecutionListener@4e0ae11f, org.springframework.boot.test.autoconfigure.web.client.MockRestServiceServerResetTestExecutionListener@238d68ff, org.springframework.boot.test.autoconfigure.web.servlet.MockMvcPrintOnlyOnFailureTestExecutionListener@4b86805d, org.springframework.boot.test.autoconfigure.web.servlet.WebDriverTestExecutionListener@5852c06f, org.springframework.boot.test.autoconfigure.webservices.client.MockWebServiceServerTestExecutionListener@4149c063]
|
2026-08-20 00:07:56.098 [main] INFO o.s.b.t.c.SpringBootTestContextBootstrapper - Using TestExecutionListeners: [org.springframework.test.context.support.DirtiesContextBeforeModesTestExecutionListener@7fd7a283, org.springframework.test.context.event.ApplicationEventsTestExecutionListener@22f59fa, org.springframework.boot.test.mock.mockito.MockitoTestExecutionListener@78fa769e, org.springframework.boot.test.autoconfigure.SpringBootDependencyInjectionTestExecutionListener@16612a51, org.springframework.test.context.support.DirtiesContextTestExecutionListener@54e041a4, org.springframework.test.context.event.EventPublishingTestExecutionListener@2c78324b, org.springframework.boot.test.mock.mockito.ResetMocksTestExecutionListener@79defdc, org.springframework.boot.test.autoconfigure.restdocs.RestDocsTestExecutionListener@18317edc, org.springframework.boot.test.autoconfigure.web.client.MockRestServiceServerResetTestExecutionListener@4e0ae11f, org.springframework.boot.test.autoconfigure.web.servlet.MockMvcPrintOnlyOnFailureTestExecutionListener@238d68ff, org.springframework.boot.test.autoconfigure.web.servlet.WebDriverTestExecutionListener@4b86805d, org.springframework.boot.test.autoconfigure.webservices.client.MockWebServiceServerTestExecutionListener@5852c06f]
|
||||||
2026-08-15 04:48:58.433 [main] INFO c.h.s.SysjavacollectApplicationTests - Starting SysjavacollectApplicationTests using Java 1.8.0_392 on wangshaohongdeMacBook-Pro.local with PID 34847 (started by wangshaohong in /Users/wangshaohong/WorkSpace/fufeng/ai-security-xdr/haobang-security-dm/syslog-client)
|
2026-08-20 00:07:56.280 [main] INFO c.h.s.SysjavacollectApplicationTests - Starting SysjavacollectApplicationTests using Java 1.8.0_392 on wangshaohongdeMacBook-Pro.local with PID 23533 (started by wangshaohong in /Users/wangshaohong/WorkSpace/fufeng/ai-security-xdr-orig/ai-security-xdr/haobang-security-dm/syslog-client)
|
||||||
2026-08-15 04:48:58.434 [main] INFO c.h.s.SysjavacollectApplicationTests - No active profile set, falling back to 1 default profile: "default"
|
2026-08-20 00:07:56.280 [main] INFO c.h.s.SysjavacollectApplicationTests - No active profile set, falling back to 1 default profile: "default"
|
||||||
2026-08-15 04:48:58.712 [main] INFO c.h.s.SysjavacollectApplicationTests - Started SysjavacollectApplicationTests in 0.397 seconds (JVM running for 0.904)
|
2026-08-20 00:07:56.613 [main] INFO c.h.s.SysjavacollectApplicationTests - Started SysjavacollectApplicationTests in 0.489 seconds (JVM running for 1.098)
|
||||||
|
|||||||
+5
-4
@@ -20,8 +20,9 @@ services:
|
|||||||
# 环境配置
|
# 环境配置
|
||||||
- SPRING_PROFILES_ACTIVE=prod
|
- SPRING_PROFILES_ACTIVE=prod
|
||||||
- TZ=Asia/Shanghai
|
- TZ=Asia/Shanghai
|
||||||
# 全停升级且已完成增量统计 DDL/基线后，所有 rule 节点统一开启捕获。
|
# 生产环境只使用 DIRECT：事实与统计同事务写入，不再写旧 change-log/pending。
|
||||||
- STATS_INCREMENTAL_CAPTURE_ENABLED=true
|
- STATS_WRITE_MODE=DIRECT
|
||||||
|
- STATS_INCREMENTAL_CAPTURE_ENABLED=false
|
||||||
- STATS_INGRESS_METADATA_ENABLED=true
|
- STATS_INGRESS_METADATA_ENABLED=true
|
||||||
- STATS_INCREMENTAL_READ_ENABLED=true
|
- STATS_INCREMENTAL_READ_ENABLED=true
|
||||||
- STATS_DEVICE_RECEIVE_CHANGE_CAPTURE_ENABLED=false
|
- STATS_DEVICE_RECEIVE_CHANGE_CAPTURE_ENABLED=false
|
||||||
@@ -71,9 +72,9 @@ services:
|
|||||||
- INFLUXDB_ORG=influxdb
|
- INFLUXDB_ORG=influxdb
|
||||||
- INFLUXDB_BUCKET=yelangbucket
|
- INFLUXDB_BUCKET=yelangbucket
|
||||||
- INFLUXDB_RAW_WRITE_SCHEMA=V2
|
- INFLUXDB_RAW_WRITE_SCHEMA=V2
|
||||||
- INFLUXDB_RAW_READ_MODE=V1_V2
|
- INFLUXDB_RAW_READ_MODE=V2_ONLY
|
||||||
- INFLUXDB_RAW_V1_BUCKET=yelangbucket
|
- INFLUXDB_RAW_V1_BUCKET=yelangbucket
|
||||||
- INFLUXDB_RAW_V2_BUCKET=yelangbucket_v2
|
- INFLUXDB_RAW_V2_BUCKET=yelangbucket
|
||||||
- INFLUXDB_RAW_UUID_BUCKET_COUNT=4096
|
- INFLUXDB_RAW_UUID_BUCKET_COUNT=4096
|
||||||
- INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED=true
|
- INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED=true
|
||||||
- INFLUXDB_RAW_QUERY_MAX_ROWS=10000
|
- INFLUXDB_RAW_QUERY_MAX_ROWS=10000
|
||||||
|
|||||||
+2
-2
@@ -12,9 +12,9 @@ public class RawStorageProperties {
|
|||||||
public static final int UUID_BUCKET_COUNT = 4096;
|
public static final int UUID_BUCKET_COUNT = 4096;
|
||||||
|
|
||||||
@Value("${influxdb.raw.write-schema:V1}") private String writeSchemaValue;
|
@Value("${influxdb.raw.write-schema:V1}") private String writeSchemaValue;
|
||||||
@Value("${influxdb.raw.read-mode:V1_ONLY}") private String readModeValue;
|
@Value("${influxdb.raw.read-mode:V2_ONLY}") private String readModeValue;
|
||||||
@Value("${influxdb.raw.v1-bucket:${influxdb.bucket:yelangbucket}}") private String v1Bucket;
|
@Value("${influxdb.raw.v1-bucket:${influxdb.bucket:yelangbucket}}") private String v1Bucket;
|
||||||
@Value("${influxdb.raw.v2-bucket:yelangbucket_v2}") private String v2Bucket;
|
@Value("${influxdb.raw.v2-bucket:yelangbucket}") private String v2Bucket;
|
||||||
@Value("${influxdb.raw.uuid-bucket-count:4096}") private int uuidBucketCount;
|
@Value("${influxdb.raw.uuid-bucket-count:4096}") private int uuidBucketCount;
|
||||||
@Value("${influxdb.raw.legacy-point-time-fallback-enabled:true}") private boolean legacyFallbackEnabled;
|
@Value("${influxdb.raw.legacy-point-time-fallback-enabled:true}") private boolean legacyFallbackEnabled;
|
||||||
@Value("${influxdb.raw.query-max-rows:10000}") private int queryMaxRows;
|
@Value("${influxdb.raw.query-max-rows:10000}") private int queryMaxRows;
|
||||||
|
|||||||
@@ -27,9 +27,9 @@ influxdb.retry.attempts=3
|
|||||||
influxdb.retry.delay=1000
|
influxdb.retry.delay=1000
|
||||||
# 原始 Syslog V2 分阶段开关。兼容部署先保持 V1/V1_ONLY，切换时统一写 V2。
|
# 原始 Syslog V2 分阶段开关。兼容部署先保持 V1/V1_ONLY，切换时统一写 V2。
|
||||||
influxdb.raw.write-schema=${INFLUXDB_RAW_WRITE_SCHEMA:V2}
|
influxdb.raw.write-schema=${INFLUXDB_RAW_WRITE_SCHEMA:V2}
|
||||||
influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V1_V2}
|
influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V2_ONLY}
|
||||||
influxdb.raw.v1-bucket=${INFLUXDB_RAW_V1_BUCKET:${influxdb.bucket}}
|
influxdb.raw.v1-bucket=${INFLUXDB_RAW_V1_BUCKET:${influxdb.bucket}}
|
||||||
influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket_v2}
|
influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket}
|
||||||
influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
|
influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
|
||||||
influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
|
influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
|
||||||
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
|
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
|
||||||
|
|||||||
+6
-6
@@ -29,9 +29,9 @@ class InfluxRawV2ContractTest {
|
|||||||
@Test void configurationRejectsChangingPersistentBucketCount() {
|
@Test void configurationRejectsChangingPersistentBucketCount() {
|
||||||
RawStorageProperties properties=new RawStorageProperties();
|
RawStorageProperties properties=new RawStorageProperties();
|
||||||
ReflectionTestUtils.setField(properties,"writeSchemaValue","V2");
|
ReflectionTestUtils.setField(properties,"writeSchemaValue","V2");
|
||||||
ReflectionTestUtils.setField(properties,"readModeValue","V1_V2");
|
ReflectionTestUtils.setField(properties,"readModeValue","V2_ONLY");
|
||||||
ReflectionTestUtils.setField(properties,"v1Bucket","yelangbucket");
|
ReflectionTestUtils.setField(properties,"v1Bucket","yelangbucket");
|
||||||
ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket_v2");
|
ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket");
|
||||||
ReflectionTestUtils.setField(properties,"uuidBucketCount",1024);
|
ReflectionTestUtils.setField(properties,"uuidBucketCount",1024);
|
||||||
ReflectionTestUtils.setField(properties,"queryMaxRows",10000);
|
ReflectionTestUtils.setField(properties,"queryMaxRows",10000);
|
||||||
ReflectionTestUtils.setField(properties,"queryMaxResponseBytes",10485760L);
|
ReflectionTestUtils.setField(properties,"queryMaxResponseBytes",10485760L);
|
||||||
@@ -62,9 +62,9 @@ class InfluxRawV2ContractTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test void v2HasUuidFieldButNeverUuidTagAndUsesNanoseconds() {
|
@Test void v2HasUuidFieldButNeverUuidTagAndUsesNanoseconds() {
|
||||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
|
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
|
||||||
InfluxRawPointFactory.RawPoint raw=new InfluxRawPointFactory(properties).create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5","message","20260814000000000","126","1","agent-01-syslog-topic",ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L);
|
InfluxRawPointFactory.RawPoint raw=new InfluxRawPointFactory(properties).create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5","message","20260814000000000","126","1","agent-01-syslog-topic",ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L);
|
||||||
String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket_v2",raw.getBucket());
|
String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket",raw.getBucket());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test void onlyMissingPointTimeFallsBack() {
|
@Test void onlyMissingPointTimeFallsBack() {
|
||||||
@@ -83,7 +83,7 @@ class InfluxRawV2ContractTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test void mixedLegacyAndNewMessagesAllProduceDistinctV2PointsAndRetryIsIdempotent() {
|
@Test void mixedLegacyAndNewMessagesAllProduceDistinctV2PointsAndRetryIsIdempotent() {
|
||||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
|
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
|
||||||
RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); InfluxRawPointFactory factory=new InfluxRawPointFactory(properties); ReceiveTimeParser.ParsedReceiveTime parsed=ReceiveTimeParser.parse("20260814000000000");
|
RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); InfluxRawPointFactory factory=new InfluxRawPointFactory(properties); ReceiveTimeParser.ParsedReceiveTime parsed=ReceiveTimeParser.parse("20260814000000000");
|
||||||
long first=resolver.resolve(false,null,parsed.getEpochMillis(),true); long second=resolver.resolve(false,null,parsed.getEpochMillis(),true); assertNotEquals(first,second);
|
long first=resolver.resolve(false,null,parsed.getEpochMillis(),true); long second=resolver.resolve(false,null,parsed.getEpochMillis(),true); assertNotEquals(first,second);
|
||||||
String id="20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"; String originalTopic="agent-01-syslog-topic";
|
String id="20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"; String originalTopic="agent-01-syslog-topic";
|
||||||
@@ -100,7 +100,7 @@ class InfluxRawV2ContractTest {
|
|||||||
|
|
||||||
@Test void uuidCandidateFluxComputesBucketEscapesUuidAndSelectsEarliestGlobally() {
|
@Test void uuidCandidateFluxComputesBucketEscapesUuidAndSelectsEarliestGlobally() {
|
||||||
String uuid="20260814-a\" |> die() //";
|
String uuid="20260814-a\" |> die() //";
|
||||||
String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket_v2",
|
String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket",
|
||||||
"syslog_security_20260814",uuid,Instant.parse("2026-08-13T16:00:00Z"),
|
"syslog_security_20260814",uuid,Instant.parse("2026-08-13T16:00:00Z"),
|
||||||
Instant.parse("2026-08-14T16:00:00Z"));
|
Instant.parse("2026-08-14T16:00:00Z"));
|
||||||
assertTrue(query.contains("r.event_bucket == \""+InfluxEventBucket.bucketOf(uuid)+"\""));
|
assertTrue(query.contains("r.event_bucket == \""+InfluxEventBucket.bucketOf(uuid)+"\""));
|
||||||
|
|||||||
+6
-5
@@ -21,9 +21,10 @@ services:
|
|||||||
# 环境配置
|
# 环境配置
|
||||||
- SPRING_PROFILES_ACTIVE=prod
|
- SPRING_PROFILES_ACTIVE=prod
|
||||||
- TZ=Asia/Shanghai
|
- TZ=Asia/Shanghai
|
||||||
# 首次启动保持影子模式：维护增量投影，但正式字段仍由旧全量任务更新。
|
# 生产环境只使用 DIRECT：事实与统计同事务写入，不再写旧 change-log/pending。
|
||||||
- STATS_INCREMENTAL_CAPTURE_ENABLED=true
|
- STATS_WRITE_MODE=DIRECT
|
||||||
- STATS_INCREMENTAL_PROJECTION_ENABLED=true
|
- STATS_INCREMENTAL_CAPTURE_ENABLED=false
|
||||||
|
- STATS_INCREMENTAL_PROJECTION_ENABLED=false
|
||||||
- STATS_INCREMENTAL_READ_ENABLED=true
|
- STATS_INCREMENTAL_READ_ENABLED=true
|
||||||
- STATS_LEGACY_FULL_SCAN_ENABLED=false
|
- STATS_LEGACY_FULL_SCAN_ENABLED=false
|
||||||
- STATS_INGRESS_METADATA_ENABLED=true
|
- STATS_INGRESS_METADATA_ENABLED=true
|
||||||
@@ -77,9 +78,9 @@ services:
|
|||||||
- INFLUXDB_ORG=influxdb
|
- INFLUXDB_ORG=influxdb
|
||||||
- INFLUXDB_BUCKET=yelangbucket
|
- INFLUXDB_BUCKET=yelangbucket
|
||||||
- INFLUXDB_RAW_WRITE_SCHEMA=V2
|
- INFLUXDB_RAW_WRITE_SCHEMA=V2
|
||||||
- INFLUXDB_RAW_READ_MODE=V1_V2
|
- INFLUXDB_RAW_READ_MODE=V2_ONLY
|
||||||
- INFLUXDB_RAW_V1_BUCKET=yelangbucket
|
- INFLUXDB_RAW_V1_BUCKET=yelangbucket
|
||||||
- INFLUXDB_RAW_V2_BUCKET=yelangbucket_v2
|
- INFLUXDB_RAW_V2_BUCKET=yelangbucket
|
||||||
- INFLUXDB_RAW_UUID_BUCKET_COUNT=4096
|
- INFLUXDB_RAW_UUID_BUCKET_COUNT=4096
|
||||||
- INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED=true
|
- INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED=true
|
||||||
- INFLUXDB_RAW_QUERY_MAX_ROWS=10000
|
- INFLUXDB_RAW_QUERY_MAX_ROWS=10000
|
||||||
|
|||||||
+2
-2
@@ -12,9 +12,9 @@ public class RawStorageProperties {
|
|||||||
public static final int UUID_BUCKET_COUNT = 4096;
|
public static final int UUID_BUCKET_COUNT = 4096;
|
||||||
|
|
||||||
@Value("${influxdb.raw.write-schema:V1}") private String writeSchemaValue;
|
@Value("${influxdb.raw.write-schema:V1}") private String writeSchemaValue;
|
||||||
@Value("${influxdb.raw.read-mode:V1_ONLY}") private String readModeValue;
|
@Value("${influxdb.raw.read-mode:V2_ONLY}") private String readModeValue;
|
||||||
@Value("${influxdb.raw.v1-bucket:${influxdb.bucket:yelangbucket}}") private String v1Bucket;
|
@Value("${influxdb.raw.v1-bucket:${influxdb.bucket:yelangbucket}}") private String v1Bucket;
|
||||||
@Value("${influxdb.raw.v2-bucket:yelangbucket_v2}") private String v2Bucket;
|
@Value("${influxdb.raw.v2-bucket:yelangbucket}") private String v2Bucket;
|
||||||
@Value("${influxdb.raw.uuid-bucket-count:4096}") private int uuidBucketCount;
|
@Value("${influxdb.raw.uuid-bucket-count:4096}") private int uuidBucketCount;
|
||||||
@Value("${influxdb.raw.legacy-point-time-fallback-enabled:true}") private boolean legacyFallbackEnabled;
|
@Value("${influxdb.raw.legacy-point-time-fallback-enabled:true}") private boolean legacyFallbackEnabled;
|
||||||
@Value("${influxdb.raw.query-max-rows:10000}") private int queryMaxRows;
|
@Value("${influxdb.raw.query-max-rows:10000}") private int queryMaxRows;
|
||||||
|
|||||||
@@ -28,9 +28,9 @@ influxdb.retry.attempts=3
|
|||||||
influxdb.retry.delay=1000
|
influxdb.retry.delay=1000
|
||||||
# 原始 Syslog V2 分阶段开关。兼容部署先保持 V1/V1_ONLY，切换时统一写 V2。
|
# 原始 Syslog V2 分阶段开关。兼容部署先保持 V1/V1_ONLY，切换时统一写 V2。
|
||||||
influxdb.raw.write-schema=${INFLUXDB_RAW_WRITE_SCHEMA:V2}
|
influxdb.raw.write-schema=${INFLUXDB_RAW_WRITE_SCHEMA:V2}
|
||||||
influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V1_V2}
|
influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V2_ONLY}
|
||||||
influxdb.raw.v1-bucket=${INFLUXDB_RAW_V1_BUCKET:${influxdb.bucket}}
|
influxdb.raw.v1-bucket=${INFLUXDB_RAW_V1_BUCKET:${influxdb.bucket}}
|
||||||
influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket_v2}
|
influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket}
|
||||||
influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
|
influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
|
||||||
influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
|
influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
|
||||||
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
|
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
|
||||||
|
|||||||
+6
-6
@@ -29,9 +29,9 @@ class InfluxRawV2ContractTest {
|
|||||||
@Test void configurationRejectsChangingPersistentBucketCount() {
|
@Test void configurationRejectsChangingPersistentBucketCount() {
|
||||||
RawStorageProperties properties=new RawStorageProperties();
|
RawStorageProperties properties=new RawStorageProperties();
|
||||||
ReflectionTestUtils.setField(properties,"writeSchemaValue","V2");
|
ReflectionTestUtils.setField(properties,"writeSchemaValue","V2");
|
||||||
ReflectionTestUtils.setField(properties,"readModeValue","V1_V2");
|
ReflectionTestUtils.setField(properties,"readModeValue","V2_ONLY");
|
||||||
ReflectionTestUtils.setField(properties,"v1Bucket","yelangbucket");
|
ReflectionTestUtils.setField(properties,"v1Bucket","yelangbucket");
|
||||||
ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket_v2");
|
ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket");
|
||||||
ReflectionTestUtils.setField(properties,"uuidBucketCount",1024);
|
ReflectionTestUtils.setField(properties,"uuidBucketCount",1024);
|
||||||
ReflectionTestUtils.setField(properties,"queryMaxRows",10000);
|
ReflectionTestUtils.setField(properties,"queryMaxRows",10000);
|
||||||
ReflectionTestUtils.setField(properties,"queryMaxResponseBytes",10485760L);
|
ReflectionTestUtils.setField(properties,"queryMaxResponseBytes",10485760L);
|
||||||
@@ -62,9 +62,9 @@ class InfluxRawV2ContractTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test void v2HasUuidFieldButNeverUuidTagAndUsesNanoseconds() {
|
@Test void v2HasUuidFieldButNeverUuidTagAndUsesNanoseconds() {
|
||||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
|
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
|
||||||
InfluxRawPointFactory.RawPoint raw=new InfluxRawPointFactory(properties).create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5","message","20260814000000000","126","1","agent-01-syslog-topic",ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L);
|
InfluxRawPointFactory.RawPoint raw=new InfluxRawPointFactory(properties).create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5","message","20260814000000000","126","1","agent-01-syslog-topic",ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L);
|
||||||
String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket_v2",raw.getBucket());
|
String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket",raw.getBucket());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test void onlyMissingPointTimeFallsBack() {
|
@Test void onlyMissingPointTimeFallsBack() {
|
||||||
@@ -83,7 +83,7 @@ class InfluxRawV2ContractTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test void mixedLegacyAndNewMessagesAllProduceDistinctV2PointsAndRetryIsIdempotent() {
|
@Test void mixedLegacyAndNewMessagesAllProduceDistinctV2PointsAndRetryIsIdempotent() {
|
||||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
|
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
|
||||||
RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); InfluxRawPointFactory factory=new InfluxRawPointFactory(properties); ReceiveTimeParser.ParsedReceiveTime parsed=ReceiveTimeParser.parse("20260814000000000");
|
RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); InfluxRawPointFactory factory=new InfluxRawPointFactory(properties); ReceiveTimeParser.ParsedReceiveTime parsed=ReceiveTimeParser.parse("20260814000000000");
|
||||||
long first=resolver.resolve(false,null,parsed.getEpochMillis(),true); long second=resolver.resolve(false,null,parsed.getEpochMillis(),true); assertNotEquals(first,second);
|
long first=resolver.resolve(false,null,parsed.getEpochMillis(),true); long second=resolver.resolve(false,null,parsed.getEpochMillis(),true); assertNotEquals(first,second);
|
||||||
String id="20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"; String originalTopic="agent-01-syslog-topic";
|
String id="20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"; String originalTopic="agent-01-syslog-topic";
|
||||||
@@ -100,7 +100,7 @@ class InfluxRawV2ContractTest {
|
|||||||
|
|
||||||
@Test void uuidCandidateFluxComputesBucketEscapesUuidAndSelectsEarliestGlobally() {
|
@Test void uuidCandidateFluxComputesBucketEscapesUuidAndSelectsEarliestGlobally() {
|
||||||
String uuid="20260814-a\" |> die() //";
|
String uuid="20260814-a\" |> die() //";
|
||||||
String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket_v2",
|
String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket",
|
||||||
"syslog_security_20260814",uuid,Instant.parse("2026-08-13T16:00:00Z"),
|
"syslog_security_20260814",uuid,Instant.parse("2026-08-13T16:00:00Z"),
|
||||||
Instant.parse("2026-08-14T16:00:00Z"));
|
Instant.parse("2026-08-14T16:00:00Z"));
|
||||||
assertTrue(query.contains("r.event_bucket == \""+InfluxEventBucket.bucketOf(uuid)+"\""));
|
assertTrue(query.contains("r.event_bucket == \""+InfluxEventBucket.bucketOf(uuid)+"\""));
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
# 系统升级说明（简约版）
|
||||||
|
|
||||||
|
本目录用于把旧备份版本升级到当前版本，只需要按下面顺序操作。
|
||||||
|
|
||||||
|
## 1. 升级前
|
||||||
|
|
||||||
|
1. 备份达梦数据库，保留旧版 JAR 和配置。
|
||||||
|
2. 记录 Kafka 原始 Topic、当前正在使用的消费组和 offset。
|
||||||
|
3. 停止 `syslog-consumer-rule`、`syslog-consumer` 全部实例。
|
||||||
|
|
||||||
|
> 不要清空 Kafka，不要重置 offset，不要删除数据库历史数据。
|
||||||
|
|
||||||
|
## 2. 升级数据库
|
||||||
|
|
||||||
|
DBA 按顺序执行：
|
||||||
|
|
||||||
|
```text
|
||||||
|
database/01-schema-upgrade.sql
|
||||||
|
database/02-idempotency-indexes.sql
|
||||||
|
database/03-direct-stats-baseline.sql
|
||||||
|
database/04-verify.sql
|
||||||
|
```
|
||||||
|
|
||||||
|
执行 `03`、`04` 前，把脚本中的日期占位符全部替换为升级当天：
|
||||||
|
|
||||||
|
```text
|
||||||
|
BUSINESS_DATE -> 2026-08-24
|
||||||
|
YYYYMMDD -> 20260824
|
||||||
|
```
|
||||||
|
|
||||||
|
日期仅为示例，以实际升级日为准。任一 SQL 报错就停止，不要从头反复执行。
|
||||||
|
|
||||||
|
## 3. 升级 Kafka（Docker）
|
||||||
|
|
||||||
|
先在 Kafka 宿主机确认容器名：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker ps --format '{{.Names}}' | grep -i kafka
|
||||||
|
```
|
||||||
|
|
||||||
|
进入本升级目录的上级目录（能够看到 `upgrade-doc`），执行：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export KAFKA_CONTAINER=kafka_ycgy-bt-kafka-1
|
||||||
|
|
||||||
|
docker exec -i \
|
||||||
|
-e KAFKA_BIN=/opt/kafka/bin \
|
||||||
|
-e BOOTSTRAP=localhost:9092 \
|
||||||
|
-e RAW_TOPIC=agent-01-syslog-topic \
|
||||||
|
"$KAFKA_CONTAINER" \
|
||||||
|
bash -s < upgrade-doc/kafka/kafka-upgrade.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
如果现场容器名不同，只修改 `KAFKA_CONTAINER`。如果 Kafka 容器内的命令目录或端口不同，再调整 `KAFKA_BIN`、`BOOTSTRAP`。
|
||||||
|
|
||||||
|
执行结束出现以下内容表示成功：
|
||||||
|
|
||||||
|
```text
|
||||||
|
PASS: Kafka Topic 已创建/确认；脚本未删除 Topic，也未修改消费组 offset。
|
||||||
|
```
|
||||||
|
|
||||||
|
脚本只新增以下 5 个 Topic，不会删除 Topic，也不会修改消费组 offset：
|
||||||
|
|
||||||
|
```text
|
||||||
|
agent-01-syslog-topic.RETRY.1M
|
||||||
|
agent-01-syslog-topic.RETRY.10M
|
||||||
|
agent-01-syslog-topic.RETRY.1H
|
||||||
|
agent-01-syslog-topic.DLT
|
||||||
|
agent-01-syslog-topic.REPLAY
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. 应用配置
|
||||||
|
|
||||||
|
`syslog-consumer-rule`：
|
||||||
|
|
||||||
|
```text
|
||||||
|
SYSLOG_RAW_CONSUMER_ENABLED=true
|
||||||
|
STATS_WRITE_MODE=DIRECT
|
||||||
|
STATS_INCREMENTAL_CAPTURE_ENABLED=false
|
||||||
|
STATS_INCREMENTAL_READ_ENABLED=true
|
||||||
|
STATS_INGRESS_METADATA_ENABLED=true
|
||||||
|
SPRING_KAFKA_CONSUMER_GROUP_ID=现场原来的活动消费组
|
||||||
|
```
|
||||||
|
|
||||||
|
`syslog-consumer`：
|
||||||
|
|
||||||
|
```text
|
||||||
|
SYSLOG_RAW_CONSUMER_ENABLED=false
|
||||||
|
STATS_WRITE_MODE=DIRECT
|
||||||
|
STATS_INCREMENTAL_CAPTURE_ENABLED=false
|
||||||
|
STATS_INCREMENTAL_PROJECTION_ENABLED=false
|
||||||
|
STATS_INCREMENTAL_READ_ENABLED=true
|
||||||
|
STATS_LEGACY_FULL_SCAN_ENABLED=false
|
||||||
|
STATS_INGRESS_METADATA_ENABLED=true
|
||||||
|
```
|
||||||
|
|
||||||
|
本次升级只使用 `DIRECT`，禁止再启用以下旧模式开关：
|
||||||
|
|
||||||
|
```text
|
||||||
|
STATS_INCREMENTAL_CAPTURE_ENABLED=true
|
||||||
|
STATS_INCREMENTAL_PROJECTION_ENABLED=true
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. 启动和确认
|
||||||
|
|
||||||
|
按顺序启动：
|
||||||
|
|
||||||
|
1. `syslog-consumer`
|
||||||
|
2. 一台 `syslog-consumer-rule` 灰度实例
|
||||||
|
3. 其余 `syslog-consumer-rule` 实例
|
||||||
|
4. `syslog-serve`
|
||||||
|
|
||||||
|
确认 Kafka 消费延迟持续下降、达梦数据持续写入、Retry/DLT 没有异常增长后，升级完成。
|
||||||
|
|
||||||
|
如需回滚，只回滚应用 JAR 和配置；数据库新增表、字段、索引及 Kafka 新 Topic 保留，不要清库或重置 offset。
|
||||||
@@ -0,0 +1,224 @@
|
|||||||
|
-- haobang-security-dm：旧备份基线 -> 当前版本结构升级。
|
||||||
|
-- 前置：已完成备份，并停止所有目标表写入者。
|
||||||
|
-- 本文件为一次性 DDL。发生任一错误立即停止，禁止整文件盲目重跑。
|
||||||
|
|
||||||
|
-- 1. Retry/DLT 可检索索引和人工回放对象。
|
||||||
|
CREATE TABLE syslog_failure_record (
|
||||||
|
id VARCHAR2(36) NOT NULL,
|
||||||
|
event_id VARCHAR2(80) NOT NULL,
|
||||||
|
device_id VARCHAR2(64),
|
||||||
|
original_topic VARCHAR2(255) NOT NULL,
|
||||||
|
original_partition INT NOT NULL,
|
||||||
|
original_offset BIGINT NOT NULL,
|
||||||
|
current_topic VARCHAR2(255),
|
||||||
|
first_received_at TIMESTAMP,
|
||||||
|
failed_stage VARCHAR2(64) NOT NULL,
|
||||||
|
error_code VARCHAR2(128),
|
||||||
|
error_type VARCHAR2(512),
|
||||||
|
error_message VARCHAR2(2048),
|
||||||
|
attempt_count INT DEFAULT 0 NOT NULL,
|
||||||
|
next_retry_at TIMESTAMP,
|
||||||
|
rule_id VARCHAR2(128),
|
||||||
|
rule_version VARCHAR2(64),
|
||||||
|
consumer_version VARCHAR2(64),
|
||||||
|
point_time_ns BIGINT,
|
||||||
|
original_payload CLOB NOT NULL,
|
||||||
|
status VARCHAR2(32) DEFAULT 'OPEN' NOT NULL,
|
||||||
|
replay_batch_id VARCHAR2(36),
|
||||||
|
replay_count INT DEFAULT 0 NOT NULL,
|
||||||
|
last_replay_at TIMESTAMP,
|
||||||
|
created_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
CONSTRAINT pk_syslog_failure_record PRIMARY KEY(id),
|
||||||
|
CONSTRAINT uk_syslog_failure_origin
|
||||||
|
UNIQUE(original_topic,original_partition,original_offset,failed_stage)
|
||||||
|
);
|
||||||
|
|
||||||
|
COMMENT ON TABLE syslog_failure_record IS 'Syslog Retry/DLT 异常消息可检索索引，Kafka DLT 为原始事实源';
|
||||||
|
COMMENT ON COLUMN syslog_failure_record.event_id IS '接入生成并在 Retry/Replay 中保持不变的事件标识';
|
||||||
|
COMMENT ON COLUMN syslog_failure_record.point_time_ns IS 'InfluxDB V2 Point 纳秒时间戳；旧 DLT 记录允许为空';
|
||||||
|
COMMENT ON COLUMN syslog_failure_record.original_payload IS '原始加密载荷，禁止接口明文返回';
|
||||||
|
COMMENT ON COLUMN syslog_failure_record.status IS 'OPEN/REPLAYED/RESOLVED 等补偿状态';
|
||||||
|
|
||||||
|
CREATE INDEX idx_syslog_failure_event
|
||||||
|
ON syslog_failure_record(event_id);
|
||||||
|
|
||||||
|
CREATE INDEX idx_syslog_failure_filter
|
||||||
|
ON syslog_failure_record(device_id,error_code,rule_version,first_received_at);
|
||||||
|
|
||||||
|
CREATE TABLE syslog_replay_batch (
|
||||||
|
id VARCHAR2(36) NOT NULL,
|
||||||
|
status VARCHAR2(32) NOT NULL,
|
||||||
|
operator_name VARCHAR2(128) NOT NULL,
|
||||||
|
reason VARCHAR2(1000) NOT NULL,
|
||||||
|
rate_per_second INT NOT NULL,
|
||||||
|
total_count INT NOT NULL,
|
||||||
|
success_count INT DEFAULT 0 NOT NULL,
|
||||||
|
failed_count INT DEFAULT 0 NOT NULL,
|
||||||
|
duplicate_count INT DEFAULT 0 NOT NULL,
|
||||||
|
created_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
completed_at TIMESTAMP,
|
||||||
|
CONSTRAINT pk_syslog_replay_batch PRIMARY KEY(id)
|
||||||
|
);
|
||||||
|
|
||||||
|
COMMENT ON TABLE syslog_replay_batch IS '人工补偿回放批次及操作审计';
|
||||||
|
|
||||||
|
CREATE TABLE syslog_replay_item (
|
||||||
|
id VARCHAR2(36) NOT NULL,
|
||||||
|
replay_batch_id VARCHAR2(36) NOT NULL,
|
||||||
|
failure_id VARCHAR2(36) NOT NULL,
|
||||||
|
status VARCHAR2(32) NOT NULL,
|
||||||
|
error_message VARCHAR2(1000),
|
||||||
|
created_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
CONSTRAINT pk_syslog_replay_item PRIMARY KEY(id),
|
||||||
|
CONSTRAINT uk_syslog_replay_item UNIQUE(replay_batch_id,failure_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
COMMENT ON TABLE syslog_replay_item IS '回放批次逐条状态';
|
||||||
|
|
||||||
|
CREATE INDEX idx_syslog_replay_pending
|
||||||
|
ON syslog_replay_item(replay_batch_id,status,created_at);
|
||||||
|
|
||||||
|
-- 2. 标准化、规则告警和算法告警幂等字段。
|
||||||
|
-- 只增加可空列；历史数据保持 NULL，不执行无法可靠还原版本/窗口的全表 datafix。
|
||||||
|
ALTER TABLE syslog_normal_data ADD rule_version VARCHAR2(64);
|
||||||
|
ALTER TABLE syslog_normal_alarm ADD rule_version VARCHAR2(64);
|
||||||
|
|
||||||
|
COMMENT ON COLUMN syslog_normal_data.rule_version IS '标准化结果使用的规则版本；确定性 id 的组成部分';
|
||||||
|
COMMENT ON COLUMN syslog_normal_alarm.rule_version IS '标准化结果使用的规则版本；确定性 id 的组成部分';
|
||||||
|
|
||||||
|
ALTER TABLE alarm ADD rule_id VARCHAR2(128);
|
||||||
|
ALTER TABLE alarm ADD rule_version VARCHAR2(64);
|
||||||
|
ALTER TABLE alarm ADD window_start TIMESTAMP;
|
||||||
|
ALTER TABLE alarm ADD group_key_hash VARCHAR2(64);
|
||||||
|
|
||||||
|
COMMENT ON COLUMN alarm.rule_id IS '生成告警的关联分析规则标识';
|
||||||
|
COMMENT ON COLUMN alarm.rule_version IS '生成告警的关联分析规则版本';
|
||||||
|
COMMENT ON COLUMN alarm.window_start IS '聚合告警业务窗口起点';
|
||||||
|
COMMENT ON COLUMN alarm.group_key_hash IS '规范化分组结果的稳定哈希';
|
||||||
|
|
||||||
|
ALTER TABLE alarm_visit ADD algorithm_id VARCHAR2(128);
|
||||||
|
ALTER TABLE alarm_visit ADD algorithm_version VARCHAR2(64);
|
||||||
|
ALTER TABLE alarm_visit ADD event_id VARCHAR2(80);
|
||||||
|
ALTER TABLE alarm_visit ADD result_type VARCHAR2(64);
|
||||||
|
|
||||||
|
COMMENT ON COLUMN alarm_visit.algorithm_id IS '生成结果的算法标识';
|
||||||
|
COMMENT ON COLUMN alarm_visit.algorithm_version IS '生成结果的算法版本';
|
||||||
|
COMMENT ON COLUMN alarm_visit.event_id IS '算法输入源事件的稳定 event_id';
|
||||||
|
COMMENT ON COLUMN alarm_visit.result_type IS '算法返回结果类型';
|
||||||
|
|
||||||
|
-- 3. 规则/算法窗口状态和多实例租约。
|
||||||
|
CREATE TABLE analysis_checkpoint (
|
||||||
|
id VARCHAR2(36) NOT NULL,
|
||||||
|
task_type VARCHAR2(32) NOT NULL,
|
||||||
|
task_id VARCHAR2(128) NOT NULL,
|
||||||
|
task_version VARCHAR2(64) NOT NULL,
|
||||||
|
window_start TIMESTAMP NOT NULL,
|
||||||
|
window_end TIMESTAMP NOT NULL,
|
||||||
|
status VARCHAR2(32) NOT NULL,
|
||||||
|
attempt_count INT DEFAULT 0 NOT NULL,
|
||||||
|
input_count BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
output_count BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
error_code VARCHAR2(128),
|
||||||
|
error_message VARCHAR2(2000),
|
||||||
|
owner_id VARCHAR2(255),
|
||||||
|
fencing_token BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
lease_until TIMESTAMP,
|
||||||
|
started_at TIMESTAMP,
|
||||||
|
completed_at TIMESTAMP,
|
||||||
|
created_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
CONSTRAINT pk_analysis_checkpoint PRIMARY KEY(id),
|
||||||
|
CONSTRAINT uk_analysis_checkpoint_window
|
||||||
|
UNIQUE(task_type,task_id,task_version,window_start,window_end),
|
||||||
|
CONSTRAINT ck_analysis_checkpoint_status
|
||||||
|
CHECK(status IN ('PENDING','RUNNING','COMPLETED','FAILED','RETRYING'))
|
||||||
|
);
|
||||||
|
|
||||||
|
COMMENT ON TABLE analysis_checkpoint IS '规则和算法独立窗口状态、连续成功水位与多实例租约';
|
||||||
|
|
||||||
|
CREATE INDEX idx_analysis_checkpoint_due
|
||||||
|
ON analysis_checkpoint(task_type,status,lease_until,window_end);
|
||||||
|
|
||||||
|
-- 4. 当前最终 DIRECT 统计正式表。
|
||||||
|
-- 旧备份基线没有中间版 change-log 对象，本升级直接建立最终小型聚合模型。
|
||||||
|
CREATE TABLE device_daily_stats (
|
||||||
|
stat_date DATE NOT NULL,
|
||||||
|
device_id BIGINT NOT NULL,
|
||||||
|
parse_count BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
non_log_count BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
last_receive_time TIMESTAMP,
|
||||||
|
projection_seq BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
CONSTRAINT pk_device_daily_stats PRIMARY KEY(stat_date,device_id),
|
||||||
|
CONSTRAINT ck_device_daily_stats_count CHECK(parse_count>=0 AND non_log_count>=0)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE normalize_rule_daily_stats (
|
||||||
|
stat_date DATE NOT NULL,
|
||||||
|
normalize_rule_id BIGINT NOT NULL,
|
||||||
|
max_log_time TIMESTAMP,
|
||||||
|
projection_seq BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
CONSTRAINT pk_normalize_rule_daily_stats PRIMARY KEY(stat_date,normalize_rule_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE device_collect_stats (
|
||||||
|
device_collect_id BIGINT NOT NULL,
|
||||||
|
first_success_time TIMESTAMP,
|
||||||
|
last_success_time TIMESTAMP,
|
||||||
|
last_fail_time TIMESTAMP,
|
||||||
|
projection_seq BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
CONSTRAINT pk_device_collect_stats PRIMARY KEY(device_collect_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
COMMENT ON TABLE device_daily_stats IS 'DIRECT 按业务日期和设备维护的物理行计数及入口时间';
|
||||||
|
COMMENT ON TABLE normalize_rule_daily_stats IS 'DIRECT 按业务日期和规则维护的最大日志时间';
|
||||||
|
COMMENT ON TABLE device_collect_stats IS 'DIRECT 维护采集任务首次成功及最后成功/失败时间';
|
||||||
|
|
||||||
|
-- 5. DIRECT 短期影子表和人工重建状态。
|
||||||
|
CREATE TABLE device_daily_stats_direct_shadow (
|
||||||
|
stat_date DATE NOT NULL,
|
||||||
|
device_id BIGINT NOT NULL,
|
||||||
|
parse_count BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
non_log_count BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
last_receive_time TIMESTAMP,
|
||||||
|
projection_seq BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
CONSTRAINT pk_device_daily_direct_shadow PRIMARY KEY(stat_date,device_id),
|
||||||
|
CONSTRAINT ck_device_daily_direct_count CHECK(parse_count>=0 AND non_log_count>=0)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE normalize_rule_daily_stats_direct_shadow (
|
||||||
|
stat_date DATE NOT NULL,
|
||||||
|
normalize_rule_id BIGINT NOT NULL,
|
||||||
|
max_log_time TIMESTAMP,
|
||||||
|
projection_seq BIGINT DEFAULT 0 NOT NULL,
|
||||||
|
updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
CONSTRAINT pk_rule_daily_direct_shadow PRIMARY KEY(stat_date,normalize_rule_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE stats_direct_control (
|
||||||
|
control_name VARCHAR2(64) NOT NULL,
|
||||||
|
rebuild_required SMALLINT DEFAULT 0 NOT NULL,
|
||||||
|
updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||||
|
CONSTRAINT pk_stats_direct_control PRIMARY KEY(control_name),
|
||||||
|
CONSTRAINT ck_stats_direct_rebuild CHECK(rebuild_required IN (0,1))
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO stats_direct_control(control_name,rebuild_required,updated_at)
|
||||||
|
VALUES ('SYSLOG_STATS',0,SYSDATE);
|
||||||
|
|
||||||
|
COMMENT ON TABLE device_daily_stats_direct_shadow IS 'DIRECT 短期对账设备影子表，正式 DIRECT 不读取';
|
||||||
|
COMMENT ON TABLE normalize_rule_daily_stats_direct_shadow IS 'DIRECT 短期对账规则影子表，正式 DIRECT 不读取';
|
||||||
|
COMMENT ON TABLE stats_direct_control IS 'DIRECT 人工降级和重建状态';
|
||||||
|
|
||||||
|
-- 6. 今日计数日切保护字段。
|
||||||
|
ALTER TABLE device_device ADD today_stats_date DATE;
|
||||||
|
|
||||||
|
COMMENT ON COLUMN device_device.today_stats_date IS 'today_parse_count/today_non_log_count 对应的上海业务日期';
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
-- haobang-security-dm 幂等索引升级。
|
||||||
|
-- 前置：01-schema-upgrade.sql 成功，所有相关旧应用仍保持停止，DBA 已评估索引空间和维护窗口。
|
||||||
|
-- 任一 CREATE 报错立即停止；禁止通过删除历史数据绕过重复问题。
|
||||||
|
|
||||||
|
-- 两张标准化表现场均按 LOG_TIME 做 RANGE 分区。
|
||||||
|
-- 当前 DM8 现场语法在不指定 GLOBAL 时建立分区索引；唯一键包含完整分区键，因此形成本地唯一索引。
|
||||||
|
-- 新版 id 是 event_id + normalize_rule_id + rule_version 的确定性 UUID，长度仍为 36 字符。
|
||||||
|
CREATE UNIQUE INDEX uk_normal_data_log_id
|
||||||
|
ON syslog_normal_data(log_time,id);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX uk_normal_alarm_log_id
|
||||||
|
ON syslog_normal_alarm(log_time,id);
|
||||||
|
|
||||||
|
-- 规则聚合窗口和算法单事件结果需要跨日幂等，使用全局非分区唯一索引。
|
||||||
|
-- 历史新增字段保持 NULL，索引只保护新版本写入且幂等字段完整的数据。
|
||||||
|
CREATE UNIQUE INDEX uk_alarm_rule_window_group
|
||||||
|
ON alarm(rule_id,rule_version,window_start,group_key_hash) GLOBAL;
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX uk_alarm_visit_algorithm_event
|
||||||
|
ON alarm_visit(algorithm_id,algorithm_version,event_id,result_type) GLOBAL;
|
||||||
|
|
||||||
|
-- 结果门禁：前两个必须 UNIQUE/PARTITIONED=YES/VALID；后两个必须 UNIQUE/PARTITIONED=NO/VALID。
|
||||||
|
SELECT index_name,table_name,uniqueness,partitioned,status
|
||||||
|
FROM user_indexes
|
||||||
|
WHERE index_name IN (
|
||||||
|
'UK_NORMAL_DATA_LOG_ID','UK_NORMAL_ALARM_LOG_ID',
|
||||||
|
'UK_ALARM_RULE_WINDOW_GROUP','UK_ALARM_VISIT_ALGORITHM_EVENT'
|
||||||
|
)
|
||||||
|
ORDER BY table_name,index_name;
|
||||||
|
|
||||||
|
-- 旧全局宽字符串索引不属于目标结构。若现场已存在，立即停止并单独评审，禁止在本文件中自动 DROP。
|
||||||
|
SELECT index_name,table_name,uniqueness,partitioned,status
|
||||||
|
FROM user_indexes
|
||||||
|
WHERE index_name IN ('UK_NORMAL_DATA_EVENT_RULE','UK_NORMAL_ALARM_EVENT_RULE')
|
||||||
|
ORDER BY table_name,index_name;
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
-- DIRECT 统计初始化（仅执行一次）。
|
||||||
|
-- 执行前：停止 syslog-consumer-rule、syslog-consumer，完成 01、02；
|
||||||
|
-- 将 BUSINESS_DATE 替换为升级日（如 2026-08-24），YYYYMMDD 替换为 20260824。
|
||||||
|
-- 任一语句报错立即停止并 ROLLBACK；全部成功后再 COMMIT。
|
||||||
|
|
||||||
|
ROLLBACK;
|
||||||
|
SET TRANSACTION ISOLATION LEVEL SERIALIZABLE;
|
||||||
|
|
||||||
|
-- 1. 初始化当天设备统计。
|
||||||
|
INSERT INTO device_daily_stats(
|
||||||
|
stat_date,device_id,parse_count,non_log_count,
|
||||||
|
last_receive_time,projection_seq,updated_at
|
||||||
|
)
|
||||||
|
SELECT TO_DATE('BUSINESS_DATE','YYYY-MM-DD'),device_id,
|
||||||
|
SUM(parse_count),SUM(non_log_count),MAX(last_receive_time),0,SYSDATE
|
||||||
|
FROM (
|
||||||
|
SELECT device_id,COUNT(*) parse_count,0 non_log_count,
|
||||||
|
CAST(NULL AS TIMESTAMP) last_receive_time
|
||||||
|
FROM syslog_normal_data_p_YYYYMMDD
|
||||||
|
WHERE created_at>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND created_at<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
|
||||||
|
AND device_id IS NOT NULL
|
||||||
|
GROUP BY device_id
|
||||||
|
|
||||||
|
UNION ALL
|
||||||
|
|
||||||
|
SELECT device_id,COUNT(*),0,CAST(NULL AS TIMESTAMP)
|
||||||
|
FROM syslog_normal_alarm_p_YYYYMMDD
|
||||||
|
WHERE created_at>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND created_at<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
|
||||||
|
AND device_id IS NOT NULL
|
||||||
|
GROUP BY device_id
|
||||||
|
|
||||||
|
UNION ALL
|
||||||
|
|
||||||
|
SELECT device_id,0,COUNT(*),CAST(NULL AS TIMESTAMP)
|
||||||
|
FROM syslog_non_normal_message_p_YYYYMMDD
|
||||||
|
WHERE created_at>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND created_at<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
|
||||||
|
AND device_id IS NOT NULL
|
||||||
|
GROUP BY device_id
|
||||||
|
|
||||||
|
UNION ALL
|
||||||
|
|
||||||
|
SELECT id,0,0,last_receive_time
|
||||||
|
FROM device_device
|
||||||
|
WHERE id IS NOT NULL
|
||||||
|
AND last_receive_time>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND last_receive_time<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
|
||||||
|
) facts
|
||||||
|
GROUP BY device_id;
|
||||||
|
|
||||||
|
-- 2. 初始化当天规则统计。
|
||||||
|
INSERT INTO normalize_rule_daily_stats(
|
||||||
|
stat_date,normalize_rule_id,max_log_time,projection_seq,updated_at
|
||||||
|
)
|
||||||
|
SELECT TO_DATE('BUSINESS_DATE','YYYY-MM-DD'),normalize_rule_id,
|
||||||
|
MAX(log_time),0,SYSDATE
|
||||||
|
FROM (
|
||||||
|
SELECT normalize_rule_id,log_time
|
||||||
|
FROM syslog_normal_data_p_YYYYMMDD
|
||||||
|
WHERE created_at>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND created_at<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
|
||||||
|
AND normalize_rule_id IS NOT NULL
|
||||||
|
|
||||||
|
UNION ALL
|
||||||
|
|
||||||
|
SELECT normalize_rule_id,log_time
|
||||||
|
FROM syslog_normal_alarm_p_YYYYMMDD
|
||||||
|
WHERE created_at>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND created_at<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
|
||||||
|
AND normalize_rule_id IS NOT NULL
|
||||||
|
) facts
|
||||||
|
GROUP BY normalize_rule_id;
|
||||||
|
|
||||||
|
-- 3. 初始化采集任务统计。
|
||||||
|
INSERT INTO device_collect_stats(
|
||||||
|
device_collect_id,first_success_time,last_success_time,
|
||||||
|
last_fail_time,projection_seq,updated_at
|
||||||
|
)
|
||||||
|
SELECT id,first_time,last_success_time,last_failed_time,0,SYSDATE
|
||||||
|
FROM device_collect_task;
|
||||||
|
|
||||||
|
-- 4. 回填设备今日计数。
|
||||||
|
UPDATE device_device
|
||||||
|
SET today_parse_count=0,
|
||||||
|
today_non_log_count=0,
|
||||||
|
today_stats_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD'),
|
||||||
|
updated_at=SYSDATE;
|
||||||
|
|
||||||
|
UPDATE device_device d
|
||||||
|
SET today_parse_count=(
|
||||||
|
SELECT s.parse_count
|
||||||
|
FROM device_daily_stats s
|
||||||
|
WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND s.device_id=d.id
|
||||||
|
),
|
||||||
|
today_non_log_count=(
|
||||||
|
SELECT s.non_log_count
|
||||||
|
FROM device_daily_stats s
|
||||||
|
WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND s.device_id=d.id
|
||||||
|
),
|
||||||
|
updated_at=SYSDATE
|
||||||
|
WHERE EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM device_daily_stats s
|
||||||
|
WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND s.device_id=d.id
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 5. 回填规则最后命中时间，只允许时间向前推进。
|
||||||
|
UPDATE dm_normalize_rule r
|
||||||
|
SET first_data_saved_at=CASE
|
||||||
|
WHEN first_data_saved_at IS NULL OR first_data_saved_at<(
|
||||||
|
SELECT s.max_log_time
|
||||||
|
FROM normalize_rule_daily_stats s
|
||||||
|
WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND s.normalize_rule_id=r.id
|
||||||
|
) THEN (
|
||||||
|
SELECT s.max_log_time
|
||||||
|
FROM normalize_rule_daily_stats s
|
||||||
|
WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND s.normalize_rule_id=r.id
|
||||||
|
)
|
||||||
|
ELSE first_data_saved_at
|
||||||
|
END,
|
||||||
|
updated_at=SYSDATE
|
||||||
|
WHERE r.del_flag='0'
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM normalize_rule_daily_stats s
|
||||||
|
WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
AND s.normalize_rule_id=r.id
|
||||||
|
AND s.max_log_time IS NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
UPDATE stats_direct_control
|
||||||
|
SET rebuild_required=0,updated_at=SYSDATE
|
||||||
|
WHERE control_name='SYSLOG_STATS';
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
-- 将 BUSINESS_DATE、YYYYMMDD 替换为升级当天后执行。
|
||||||
|
-- 结果要求见每段注释；不符合时不要启动全部业务实例。
|
||||||
|
|
||||||
|
-- 1. 应返回 10 行。
|
||||||
|
SELECT table_name
|
||||||
|
FROM user_tables
|
||||||
|
WHERE table_name IN (
|
||||||
|
'SYSLOG_FAILURE_RECORD','SYSLOG_REPLAY_BATCH','SYSLOG_REPLAY_ITEM',
|
||||||
|
'ANALYSIS_CHECKPOINT','DEVICE_DAILY_STATS','NORMALIZE_RULE_DAILY_STATS',
|
||||||
|
'DEVICE_COLLECT_STATS','DEVICE_DAILY_STATS_DIRECT_SHADOW',
|
||||||
|
'NORMALIZE_RULE_DAILY_STATS_DIRECT_SHADOW','STATS_DIRECT_CONTROL'
|
||||||
|
)
|
||||||
|
ORDER BY table_name;
|
||||||
|
|
||||||
|
-- 2. POINT_TIME_NS 必须存在；其余升级字段也应全部列出。
|
||||||
|
SELECT table_name,column_name
|
||||||
|
FROM user_tab_columns
|
||||||
|
WHERE (table_name='SYSLOG_FAILURE_RECORD' AND column_name='POINT_TIME_NS')
|
||||||
|
OR (table_name IN ('SYSLOG_NORMAL_DATA','SYSLOG_NORMAL_ALARM') AND column_name='RULE_VERSION')
|
||||||
|
OR (table_name='ALARM' AND column_name IN ('RULE_ID','RULE_VERSION','WINDOW_START','GROUP_KEY_HASH'))
|
||||||
|
OR (table_name='ALARM_VISIT' AND column_name IN ('ALGORITHM_ID','ALGORITHM_VERSION','EVENT_ID','RESULT_TYPE'))
|
||||||
|
OR (table_name='DEVICE_DEVICE' AND column_name='TODAY_STATS_DATE')
|
||||||
|
ORDER BY table_name,column_name;
|
||||||
|
|
||||||
|
-- 3. 应返回 4 行且全部为 UNIQUE、VALID；前两个 PARTITIONED=YES，后两个为 NO。
|
||||||
|
SELECT index_name,table_name,uniqueness,partitioned,status
|
||||||
|
FROM user_indexes
|
||||||
|
WHERE index_name IN (
|
||||||
|
'UK_NORMAL_DATA_LOG_ID','UK_NORMAL_ALARM_LOG_ID',
|
||||||
|
'UK_ALARM_RULE_WINDOW_GROUP','UK_ALARM_VISIT_ALGORITHM_EVENT'
|
||||||
|
)
|
||||||
|
ORDER BY index_name;
|
||||||
|
|
||||||
|
-- 4. 三个物理事实表及三张统计表行数，用于现场留档。
|
||||||
|
SELECT 'SYSLOG_NORMAL_DATA_P_YYYYMMDD' AS item,COUNT(*) AS row_count
|
||||||
|
FROM syslog_normal_data_p_YYYYMMDD
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'SYSLOG_NORMAL_ALARM_P_YYYYMMDD',COUNT(*)
|
||||||
|
FROM syslog_normal_alarm_p_YYYYMMDD
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'SYSLOG_NON_NORMAL_MESSAGE_P_YYYYMMDD',COUNT(*)
|
||||||
|
FROM syslog_non_normal_message_p_YYYYMMDD
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'DEVICE_DAILY_STATS',COUNT(*)
|
||||||
|
FROM device_daily_stats
|
||||||
|
WHERE stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'NORMALIZE_RULE_DAILY_STATS',COUNT(*)
|
||||||
|
FROM normalize_rule_daily_stats
|
||||||
|
WHERE stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'DEVICE_COLLECT_STATS',COUNT(*)
|
||||||
|
FROM device_collect_stats;
|
||||||
|
|
||||||
|
-- 5. 必须返回 SYSLOG_STATS / 0。
|
||||||
|
SELECT control_name,rebuild_required,updated_at
|
||||||
|
FROM stats_direct_control
|
||||||
|
WHERE control_name='SYSLOG_STATS';
|
||||||
|
|
||||||
|
-- 6. 以下两个查询都必须返回 0 行。
|
||||||
|
SELECT d.id
|
||||||
|
FROM device_device d
|
||||||
|
LEFT JOIN device_daily_stats s
|
||||||
|
ON s.device_id=d.id
|
||||||
|
AND s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
WHERE d.today_stats_date IS NULL
|
||||||
|
OR d.today_stats_date<>TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
|
||||||
|
OR COALESCE(d.today_parse_count,0)<>COALESCE(s.parse_count,0)
|
||||||
|
OR COALESCE(d.today_non_log_count,0)<>COALESCE(s.non_log_count,0);
|
||||||
|
|
||||||
|
SELECT control_name
|
||||||
|
FROM stats_direct_control
|
||||||
|
WHERE control_name='SYSLOG_STATS' AND rebuild_required<>0;
|
||||||
+65
@@ -0,0 +1,65 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
: "${KAFKA_BIN:?请设置 KAFKA_BIN，例如 /opt/kafka/bin}"
|
||||||
|
: "${BOOTSTRAP:?请设置 BOOTSTRAP，例如 127.0.0.1:9092}"
|
||||||
|
|
||||||
|
RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
|
||||||
|
TOPICS_CMD="${KAFKA_BIN}/kafka-topics.sh"
|
||||||
|
CONFIGS_CMD="${KAFKA_BIN}/kafka-configs.sh"
|
||||||
|
|
||||||
|
if [[ ! -x "${TOPICS_CMD}" ]] || [[ ! -x "${CONFIGS_CMD}" ]]; then
|
||||||
|
echo "ERROR: ${KAFKA_BIN} 下缺少 kafka-topics.sh 或 kafka-configs.sh" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
COMMON_ARGS=(--bootstrap-server "${BOOTSTRAP}")
|
||||||
|
if [[ -n "${COMMAND_CONFIG:-}" ]]; then
|
||||||
|
COMMON_ARGS+=(--command-config "${COMMAND_CONFIG}")
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "检查原始 Topic：${RAW_TOPIC}"
|
||||||
|
RAW_INFO=$("${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}")
|
||||||
|
printf '%s\n' "${RAW_INFO}"
|
||||||
|
|
||||||
|
PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk '
|
||||||
|
/PartitionCount:/ {
|
||||||
|
for (i=1; i<=NF; i++) if ($i=="PartitionCount:") { print $(i+1); exit }
|
||||||
|
}
|
||||||
|
')
|
||||||
|
REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
|
||||||
|
/ReplicationFactor:/ {
|
||||||
|
for (i=1; i<=NF; i++) if ($i=="ReplicationFactor:") { print $(i+1); exit }
|
||||||
|
}
|
||||||
|
')
|
||||||
|
|
||||||
|
if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
|
||||||
|
echo "ERROR: 无法读取原始 Topic 的分区数或副本数" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
SUFFIXES=("RETRY.1M" "RETRY.10M" "RETRY.1H" "DLT" "REPLAY")
|
||||||
|
RETENTIONS=(259200000 604800000 604800000 7776000000 604800000)
|
||||||
|
|
||||||
|
for i in 0 1 2 3 4; do
|
||||||
|
TOPIC="${RAW_TOPIC}.${SUFFIXES[$i]}"
|
||||||
|
echo "创建或确认 Topic：${TOPIC}"
|
||||||
|
"${TOPICS_CMD}" "${COMMON_ARGS[@]}" \
|
||||||
|
--create --if-not-exists \
|
||||||
|
--topic "${TOPIC}" \
|
||||||
|
--partitions "${PARTITIONS}" \
|
||||||
|
--replication-factor "${REPLICAS}" \
|
||||||
|
--config "retention.ms=${RETENTIONS[$i]}"
|
||||||
|
|
||||||
|
"${CONFIGS_CMD}" "${COMMON_ARGS[@]}" \
|
||||||
|
--alter --entity-type topics --entity-name "${TOPIC}" \
|
||||||
|
--add-config "retention.ms=${RETENTIONS[$i]}"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "检查升级结果"
|
||||||
|
for SUFFIX in "${SUFFIXES[@]}"; do
|
||||||
|
"${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}.${SUFFIX}"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "PASS: Kafka Topic 已创建/确认；脚本未删除 Topic，也未修改消费组 offset。"
|
||||||
Reference in New Issue
Block a user