feat(syslog): 完成高吞吐可靠链路与统计存储优化
- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
王绍宏
+100
-4
@@ -25,6 +25,15 @@ influxdb.batch.size=1000
|
||||
influxdb.flush.interval=1000
|
||||
influxdb.retry.attempts=3
|
||||
influxdb.retry.delay=1000
|
||||
# 原始 Syslog V2 分阶段开关。兼容部署先保持 V1/V1_ONLY，切换时统一写 V2。
|
||||
influxdb.raw.write-schema=${INFLUXDB_RAW_WRITE_SCHEMA:V2}
|
||||
influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V1_V2}
|
||||
influxdb.raw.v1-bucket=${INFLUXDB_RAW_V1_BUCKET:${influxdb.bucket}}
|
||||
influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket_v2}
|
||||
influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
|
||||
influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
|
||||
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
|
||||
influxdb.raw.query-max-response-bytes=${INFLUXDB_RAW_QUERY_MAX_RESPONSE_BYTES:10485760}
|
||||
# InfluxDB 2.7 连接超时配置
|
||||
influxdb.connection.timeout=30s
|
||||
influxdb.connection.read-timeout=30s
|
||||
@@ -58,10 +67,31 @@ spring.kafka.consumer.auto-offset-reset=latest
|
||||
spring.kafka.consumer.enable-auto-commit=false
|
||||
spring.kafka.consumer.auto-commit-interval=1000
|
||||
spring.kafka.consumer.topic=test-topic
|
||||
# consumer-rule 是当前活跃的 Kafka 原始日志标准化节点，默认启动监听。
|
||||
syslog.raw-consumer.enabled=true
|
||||
# Raw Topic 当前有 3 个分区，使用独立工厂显式创建 3 个 consumer；可通过环境变量灰度调整。
|
||||
syslog.raw-listener.concurrency=${SYSLOG_RAW_LISTENER_CONCURRENCY:3}
|
||||
# P0：Retry/DLT 发布与原始消费使用同一 Kafka 集群；Topic 名称全部可由环境变量覆盖。
|
||||
spring.kafka.bootstrap-servers=${spring.kafka.consumer.bootstrap-servers}
|
||||
spring.kafka.producer.acks=all
|
||||
spring.kafka.producer.properties.enable.idempotence=true
|
||||
spring.kafka.producer.properties.delivery.timeout.ms=120000
|
||||
spring.kafka.producer.properties.request.timeout.ms=30000
|
||||
syslog.reliability.retry-1m-topic=${spring.kafka.consumer.topic}.RETRY.1M
|
||||
syslog.reliability.retry-10m-topic=${spring.kafka.consumer.topic}.RETRY.10M
|
||||
syslog.reliability.retry-1h-topic=${spring.kafka.consumer.topic}.RETRY.1H
|
||||
syslog.reliability.dlt-topic=${spring.kafka.consumer.topic}.DLT
|
||||
syslog.reliability.replay-topic=${spring.kafka.consumer.topic}.REPLAY
|
||||
syslog.reliability.publish-timeout-ms=30000
|
||||
syslog.reliability.consumer-version=${APP_VERSION:1.0.0}
|
||||
# Retry/Replay 使用独立消费组和线程，未到期重试不会占用 Raw listener。
|
||||
syslog.reliability.consumer-group-id=${spring.kafka.consumer.group-id}-retry
|
||||
syslog.reliability.consumer.enabled=true
|
||||
syslog.reliability.listener.concurrency=1
|
||||
|
||||
|
||||
spring.kafka.consumer.max-poll-records=1000
|
||||
spring.kafka.consumer.properties.max.poll.interval.ms=300000
|
||||
spring.kafka.consumer.max-poll-records=500
|
||||
spring.kafka.consumer.properties.max.poll.interval.ms=900000
|
||||
spring.kafka.consumer.properties.session.timeout.ms=45000
|
||||
|
||||
spring.kafka.consumer.fetch-min-size= 1048576
|
||||
@@ -106,7 +136,21 @@ app.processor.thread-pool.max-pool-size=20
|
||||
app.processor.thread-pool.queue-capacity=2000
|
||||
app.processor.thread-pool.keep-alive-seconds=60
|
||||
app.processor.batch-size=100
|
||||
app.processor.process-timeout-ms=30000
|
||||
app.processor.process-timeout-ms=120000
|
||||
|
||||
# 达梦标准化表默认启用批量写入；现场需要紧急回退时可显式切换为 SINGLE。
|
||||
# BATCH 模式按目标表、log_time 日期和动态列集合分组，失败时只对数据类错误有限二分；
|
||||
# 连接、超时或提交结果未知会立即停止并保留 Kafka offset，禁止放大为逐条请求风暴。
|
||||
dm.standardized-write.mode=BATCH
|
||||
dm.standardized-write.batch-size=100
|
||||
dm.standardized-write.writer-concurrency=2
|
||||
dm.standardized-write.writer-queue-capacity=100
|
||||
dm.standardized-write.transaction-timeout-seconds=45
|
||||
dm.standardized-write.kafka-batch-deadline-ms=90000
|
||||
dm.standardized-write.max-split-depth=10
|
||||
dm.standardized-write.max-fallback-operations=100
|
||||
dm.standardized-write.circuit-breaker-failure-threshold=3
|
||||
dm.standardized-write.circuit-breaker-backoff-ms=30000
|
||||
|
||||
# ETL配置
|
||||
etl.batch.page-size=1000
|
||||
@@ -125,4 +169,56 @@ spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL
|
||||
spring.datasource.hikari.validation-timeout=5000
|
||||
spring.datasource.hikari.leak-detection-threshold=30000
|
||||
spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule
|
||||
spring.datasource.hikari.auto-commit=false
|
||||
spring.datasource.hikari.auto-commit=false
|
||||
# 统计写入采用事实同事务直写，不再为每条结果写 change_log + pending。
|
||||
# 首次部署 DIRECT 前必须先执行 08/09/10/11 脚本并按设计文档排空旧 pending。
|
||||
stats.write-mode=DIRECT
|
||||
stats.incremental.capture-enabled=false
|
||||
stats.ingress-metadata.enabled=true
|
||||
stats.incremental.read-enabled=true
|
||||
stats.incremental.transaction-timeout-seconds=45
|
||||
stats.device-receive.change-capture-enabled=false
|
||||
stats.direct.device-table=device_daily_stats
|
||||
stats.direct.rule-table=normalize_rule_daily_stats
|
||||
stats.direct.shadow-device-table=device_daily_stats_direct_shadow
|
||||
stats.direct.shadow-rule-table=normalize_rule_daily_stats_direct_shadow
|
||||
stats.direct.control-table=stats_direct_control
|
||||
stats.direct.today-stats-date-enabled=true
|
||||
stats.direct.max-concurrency-retries=1
|
||||
stats.direct.concurrency-retry-backoff-ms=20
|
||||
stats.direct.transaction-log-enabled=false
|
||||
stats.direct.day-rollover-check-cron=5 * * * * ?
|
||||
stats.business-zone=Asia/Shanghai
|
||||
stats.fact-only.confirm-rebuild-required=false
|
||||
|
||||
# 标准化规则编译与批次缓存灰度开关
|
||||
normalization.jetcache.enabled=true
|
||||
normalization.rule-source-cache.enabled=true
|
||||
normalization.compiled-rule.enabled=true
|
||||
normalization.compiled-rule.shadow-compare-enabled=false
|
||||
normalization.compiled-rule.shadow-sample-rate=0
|
||||
normalization.compiled-rule.local-limit=10000
|
||||
normalization.compiled-rule.expire-after-access-minutes=30
|
||||
normalization.compiled-rule.failure-cache-seconds=10
|
||||
normalization.compiled-rule.shadow-timeout-ms=1000
|
||||
normalization.compiled-rule.rule-detail-log-enabled=true
|
||||
normalization.batch-enrichment-cache.enabled=true
|
||||
normalization.message-parse-cache.enabled=true
|
||||
normalization.rule-source-cache.local-ttl-seconds=30
|
||||
normalization.rule-source-cache.remote-ttl-seconds=300
|
||||
normalization.rule-source-cache.empty-ttl-seconds=10
|
||||
normalization.enrichment-cache.local-ttl-seconds=60
|
||||
normalization.enrichment-cache.remote-ttl-seconds=3600
|
||||
|
||||
# JetCache Redis URI 由 JetCacheRedisEnvironmentPostProcessor 从 spring.redis.* 派生。
|
||||
jetcache.statIntervalMinutes=15
|
||||
jetcache.areaInCacheName=false
|
||||
jetcache.penetrationProtect=true
|
||||
jetcache.enableMethodCache=true
|
||||
jetcache.local.default.type=caffeine
|
||||
jetcache.local.default.keyConvertor=fastjson2
|
||||
jetcache.remote.default.type=redis.lettuce
|
||||
jetcache.remote.default.keyConvertor=fastjson2
|
||||
jetcache.remote.default.valueEncoder=KRYO5
|
||||
jetcache.remote.default.valueDecoder=KRYO5
|
||||
jetcache.remote.default.broadcastChannel=xdr:norm:v1:cache-invalidate
|
||||
|
||||
Reference in New Issue
Block a user