fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
- 将设备、规则和采集任务业务统计改为专用 Kafka Topic 异步物化 - 使用维度 ID 作为 Kafka key,保证同一统计维度串行更新 - 物化过程读取小型统计表精确值,确保重复消费不会重复计数 - 增加提交后发布、失败重试及定时小表对账补偿 - 固定日统计聚合键写入顺序,完善达梦死锁识别和事务重试 - JSON 解析失败直接转为非标日志,避免进入 DLT 重试链路 - 非标日志使用独立异步队列,不再阻塞正常消息 Kafka ACK - Kafka 升级脚本新增 STATS.MATERIALIZE Topic - 增加统计物化状态指标、配置示例及单元测试
This commit is contained in:
@@ -160,6 +160,8 @@ dm.standardized-write.circuit-breaker-failure-threshold=3
|
||||
dm.standardized-write.circuit-breaker-backoff-ms=30000
|
||||
dm.non-normal-write.batch-size=${DM_NON_NORMAL_WRITE_BATCH_SIZE:500}
|
||||
dm.non-normal-write.queue-capacity=${DM_NON_NORMAL_WRITE_QUEUE_CAPACITY:500}
|
||||
dm.non-normal-write.max-concurrency-retries=${DM_NON_NORMAL_WRITE_MAX_CONCURRENCY_RETRIES:3}
|
||||
dm.non-normal-write.concurrency-retry-backoff-ms=${DM_NON_NORMAL_WRITE_CONCURRENCY_RETRY_BACKOFF_MS:50}
|
||||
|
||||
# ETL配置
|
||||
etl.batch.page-size=1000
|
||||
@@ -193,12 +195,26 @@ stats.direct.shadow-device-table=device_daily_stats_direct_shadow
|
||||
stats.direct.shadow-rule-table=normalize_rule_daily_stats_direct_shadow
|
||||
stats.direct.control-table=stats_direct_control
|
||||
stats.direct.today-stats-date-enabled=true
|
||||
stats.direct.max-concurrency-retries=1
|
||||
stats.direct.concurrency-retry-backoff-ms=20
|
||||
stats.direct.max-concurrency-retries=${STATS_DIRECT_MAX_CONCURRENCY_RETRIES:3}
|
||||
stats.direct.concurrency-retry-backoff-ms=${STATS_DIRECT_CONCURRENCY_RETRY_BACKOFF_MS:50}
|
||||
stats.direct.transaction-log-enabled=false
|
||||
stats.direct.day-rollover-check-cron=5 * * * * ?
|
||||
stats.business-zone=Asia/Shanghai
|
||||
stats.fact-only.confirm-rebuild-required=false
|
||||
# 事实事务只维护小型精确统计表;提交后按维度脏键异步刷新业务展示字段。
|
||||
# 三个实例必须配置相同 topic/group-id,Topic 分区数建议与原始 Topic 相同。
|
||||
stats.materialization.enabled=${STATS_MATERIALIZATION_ENABLED:true}
|
||||
stats.materialization.topic=${STATS_MATERIALIZATION_TOPIC:${spring.kafka.consumer.topic}.STATS.MATERIALIZE}
|
||||
stats.materialization.group-id=${STATS_MATERIALIZATION_GROUP_ID:${spring.kafka.consumer.group-id}-stats-materialize}
|
||||
# 每个实例一个物化 consumer;现场三个实例合计三个,足以处理低基数脏键并减少达梦并发。
|
||||
stats.materialization.listener-concurrency=${STATS_MATERIALIZATION_LISTENER_CONCURRENCY:1}
|
||||
stats.materialization.publish-interval-ms=${STATS_MATERIALIZATION_PUBLISH_INTERVAL_MS:200}
|
||||
stats.materialization.publish-timeout-ms=${STATS_MATERIALIZATION_PUBLISH_TIMEOUT_MS:10000}
|
||||
stats.materialization.publish-max-batch-size=${STATS_MATERIALIZATION_PUBLISH_MAX_BATCH_SIZE:1000}
|
||||
stats.materialization.max-concurrency-retries=${STATS_MATERIALIZATION_MAX_CONCURRENCY_RETRIES:3}
|
||||
stats.materialization.concurrency-retry-backoff-ms=${STATS_MATERIALIZATION_CONCURRENCY_RETRY_BACKOFF_MS:50}
|
||||
stats.materialization.reconcile-initial-delay-ms=${STATS_MATERIALIZATION_RECONCILE_INITIAL_DELAY_MS:30000}
|
||||
stats.materialization.reconcile-interval-ms=${STATS_MATERIALIZATION_RECONCILE_INTERVAL_MS:60000}
|
||||
|
||||
# 标准化规则编译与批次缓存灰度开关
|
||||
normalization.jetcache.enabled=true
|
||||
|
||||
Reference in New Issue
Block a user