feat(syslog): 完成高吞吐可靠链路与统计存储优化
- 新增接入 WAL、检查点、背压恢复及可靠分发机制,完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存,支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型,统一接收时间与 Point 时间,支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路,支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
+24
-4
@@ -55,6 +55,8 @@ mybatis-plus.type-handlers-package=com.Modules.etl.handler
|
||||
|
||||
# kafka Configuration
|
||||
spring.kafka.consumer.bootstrap-servers=10.11.2.142:9092
|
||||
# [SYSLOG-ALARM][TOPO] 若 syslog-consumer 同时使用此 topic + group-id,两服务会成为同组成员并竞争分区,
|
||||
# 每条记录只由其中一个成员处理。若职责不同,应使用独立 group-id,或让本服务消费 normalized Topic。
|
||||
spring.kafka.consumer.group-id=agent-syslog-group
|
||||
spring.kafka.consumer.auto-offset-reset=latest
|
||||
spring.kafka.consumer.enable-auto-commit=false
|
||||
@@ -62,8 +64,8 @@ spring.kafka.consumer.auto-commit-interval=1000
|
||||
spring.kafka.consumer.topic=agent-syslog-topic
|
||||
|
||||
|
||||
spring.kafka.consumer.max-poll-records=1000
|
||||
spring.kafka.consumer.properties.max.poll.interval.ms=300000
|
||||
spring.kafka.consumer.max-poll-records=500
|
||||
spring.kafka.consumer.properties.max.poll.interval.ms=900000
|
||||
spring.kafka.consumer.properties.session.timeout.ms=45000
|
||||
|
||||
#spring.kafka.consumer.key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
|
||||
@@ -108,7 +110,7 @@ app.processor.thread-pool.max-pool-size=20
|
||||
app.processor.thread-pool.queue-capacity=2000
|
||||
app.processor.thread-pool.keep-alive-seconds=60
|
||||
app.processor.batch-size=100
|
||||
app.processor.process-timeout-ms=30000
|
||||
app.processor.process-timeout-ms=120000
|
||||
|
||||
# 配置 Elasticsearch
|
||||
# Elasticsearch连接地址
|
||||
@@ -147,4 +149,22 @@ spring.datasource.hikari.auto-commit=false
|
||||
# 关联分析规则配置
|
||||
analysis.realtime.enabled= true
|
||||
# 检查间隔(秒) - 默认10秒
|
||||
analysis.realtime.check-interval-seconds: 10
|
||||
analysis.realtime.check-interval-seconds: 10
|
||||
|
||||
normalization.jetcache.enabled=true
|
||||
normalization.rule-source-cache.enabled=true
|
||||
normalization.compiled-rule.enabled=true
|
||||
normalization.compiled-rule.shadow-compare-enabled=false
|
||||
normalization.compiled-rule.shadow-sample-rate=0
|
||||
normalization.compiled-rule.local-limit=10000
|
||||
normalization.compiled-rule.expire-after-access-minutes=30
|
||||
normalization.compiled-rule.failure-cache-seconds=10
|
||||
normalization.compiled-rule.shadow-timeout-ms=1000
|
||||
normalization.compiled-rule.rule-detail-log-enabled=true
|
||||
normalization.batch-enrichment-cache.enabled=true
|
||||
normalization.message-parse-cache.enabled=true
|
||||
normalization.rule-source-cache.local-ttl-seconds=30
|
||||
normalization.rule-source-cache.remote-ttl-seconds=300
|
||||
normalization.rule-source-cache.empty-ttl-seconds=10
|
||||
normalization.enrichment-cache.local-ttl-seconds=60
|
||||
normalization.enrichment-cache.remote-ttl-seconds=3600
|
||||
|
||||
Reference in New Issue
Block a user