feat(syslog): 优化大日志通道并增加异常连续消费兜底
- 128 KiB 及以上日志默认进入大日志 Topic - InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存 - 截断内容追加 HB_XDR_TRUNCATED 标记 - InfluxDB 写入失败时继续执行规则解析和达梦写入 - 未知或系统性异常优先写入普通/大日志 DLT - DLT 写入失败时记录特殊告警并提交 offset,避免阻塞后续消息 - DLT 失败等待时间默认限制为 1 秒 - 增加连续性放行、DLT 成功及失败监控指标 - 更新 Docker Compose、v3 升级文档及相关测试
This commit is contained in:
@@ -35,7 +35,7 @@ influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME
|
||||
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
|
||||
influxdb.raw.query-max-response-bytes=${INFLUXDB_RAW_QUERY_MAX_RESPONSE_BYTES:10485760}
|
||||
# 0 表示不限制;启用后最终 message(含截断标记)不超过该 UTF-8 字节数。
|
||||
influxdb.raw.message-max-utf8-bytes=${INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES:1048576}
|
||||
influxdb.raw.message-max-utf8-bytes=${INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES:262144}
|
||||
# Influx 同步写按 Line Protocol 估算字节切批,避免少量大日志形成超大 HTTP 请求。
|
||||
influxdb.raw.write-batch-max-bytes=${INFLUXDB_RAW_WRITE_BATCH_MAX_BYTES:8388608}
|
||||
# InfluxDB 2.7 连接超时配置
|
||||
@@ -79,6 +79,10 @@ spring.kafka.consumer.large-topic=${KAFKA_CONSUMER_LARGE_TOPIC:${spring.kafka.co
|
||||
syslog.raw-consumer.enabled=true
|
||||
# Raw Topic 当前有 3 个分区,使用独立工厂显式创建 3 个 consumer;可通过环境变量灰度调整。
|
||||
syslog.raw-listener.concurrency=${SYSLOG_RAW_LISTENER_CONCURRENCY:3}
|
||||
# 连续性优先:特殊/系统性异常优先写 DLT;即使 DLT 失败也记录告警并推进原 offset,避免阻塞分区。
|
||||
syslog.failure-continuity.enabled=${SYSLOG_FAILURE_CONTINUITY_ENABLED:true}
|
||||
# DLT 故障时最多快速等待 1 秒;首次失败后整批直接记录并放行。
|
||||
syslog.failure-continuity.dlt-publish-timeout-ms=${SYSLOG_FAILURE_CONTINUITY_DLT_PUBLISH_TIMEOUT_MS:1000}
|
||||
# 大日志使用独立 Topic、消费组和 ConsumerFactory;建议生产环境独立 JVM 部署。
|
||||
syslog.large-listener.enabled=${SYSLOG_LARGE_LISTENER_ENABLED:true}
|
||||
syslog.large-listener.group-id=${SYSLOG_LARGE_LISTENER_GROUP_ID:${spring.kafka.consumer.group-id}-large}
|
||||
@@ -180,7 +184,7 @@ dm.standardized-write.batch-size=${DM_STANDARDIZED_WRITE_BATCH_SIZE:500}
|
||||
dm.standardized-write.writer-concurrency=2
|
||||
dm.standardized-write.writer-queue-capacity=100
|
||||
# 标准化大字段和非标原文均在达梦持久化边界截断;0 表示不限制。
|
||||
dm.normal.large-field-max-utf8-bytes=${DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES:65536}
|
||||
dm.normal.large-field-max-utf8-bytes=${DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES:262144}
|
||||
dm.normal.large-fields=${DM_NORMAL_LARGE_FIELDS:http_req_body,http_resp_body,http_req_header,http_resp_header,detail,description,cmdline,shell_cmdline,parent_cmdline,mail_message}
|
||||
dm.non-normal.large-field-max-utf8-bytes=${DM_NON_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES:262144}
|
||||
dm.standardized-write.transaction-timeout-seconds=45
|
||||
|
||||
Reference in New Issue
Block a user