feat(syslog): 完成高吞吐可靠链路与统计存储优化

- 新增接入 WAL、检查点、背压恢复及可靠分发机制,完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程
- 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存,支持缓存失效广播、影子对账和执行指标
- 升级 InfluxDB V2 原始日志模型,统一接收时间与 Point 时间,支持按日 measurement、UUID 分桶及 V1/V2 双读
- 重构达梦标准化批量写入与统计链路,支持事务内直写、增量投影、幂等控制、窗口水位和日期切换
- 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
王绍宏
2026-08-20 00:18:33 +08:00
parent a3852fc0f9
commit 29445c6825
303 changed files with 20419 additions and 4481 deletions
@@ -53,15 +53,18 @@ mybatis-plus.type-handlers-package=com.Modules.etl.handler
# kafka Configuration
spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092
spring.kafka.consumer.group-id=agent-01-syslog-group
# 与现有生产 Compose 保持同一消费组,避免 JAR 直启和容器启动使用不同 offset。
spring.kafka.consumer.group-id=agent-01-syslog-group-dm
spring.kafka.consumer.auto-offset-reset=latest
spring.kafka.consumer.enable-auto-commit=false
spring.kafka.consumer.auto-commit-interval=1000
spring.kafka.consumer.topic=agent-01-syslog-topic
# 当前生产活跃的标准化消费节点;多实例必须保持同一 group-id。
syslog.raw-consumer.enabled=true
spring.kafka.consumer.max-poll-records=1000
spring.kafka.consumer.properties.max.poll.interval.ms=300000
spring.kafka.consumer.max-poll-records=500
spring.kafka.consumer.properties.max.poll.interval.ms=900000
spring.kafka.consumer.properties.session.timeout.ms=45000
spring.kafka.consumer.properties.partition.assignment.strategy=org.apache.kafka.clients.consumer.RoundRobinAssignor
spring.kafka.consumer.fetch-min-size= 1048576
@@ -106,7 +109,7 @@ app.processor.thread-pool.max-pool-size=20
app.processor.thread-pool.queue-capacity=2000
app.processor.thread-pool.keep-alive-seconds=60
app.processor.batch-size=100
app.processor.process-timeout-ms=30000
app.processor.process-timeout-ms=120000
# ETL配置
etl.batch.page-size=1000
@@ -125,4 +128,22 @@ spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL
spring.datasource.hikari.validation-timeout=5000
spring.datasource.hikari.leak-detection-threshold=30000
spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule
spring.datasource.hikari.auto-commit=false
spring.datasource.hikari.auto-commit=false
normalization.jetcache.enabled=true
normalization.rule-source-cache.enabled=true
normalization.compiled-rule.enabled=true
normalization.compiled-rule.shadow-compare-enabled=false
normalization.compiled-rule.shadow-sample-rate=0
normalization.compiled-rule.local-limit=10000
normalization.compiled-rule.expire-after-access-minutes=30
normalization.compiled-rule.failure-cache-seconds=10
normalization.compiled-rule.shadow-timeout-ms=1000
normalization.compiled-rule.rule-detail-log-enabled=true
normalization.batch-enrichment-cache.enabled=true
normalization.message-parse-cache.enabled=true
normalization.rule-source-cache.local-ttl-seconds=30
normalization.rule-source-cache.remote-ttl-seconds=300
normalization.rule-source-cache.empty-ttl-seconds=10
normalization.enrichment-cache.local-ttl-seconds=60
normalization.enrichment-cache.remote-ttl-seconds=3600