feat(syslog): 优化大日志通道并增加异常连续消费兜底

- 128 KiB 及以上日志默认进入大日志 Topic
- InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存
- 截断内容追加 HB_XDR_TRUNCATED 标记
- InfluxDB 写入失败时继续执行规则解析和达梦写入
- 未知或系统性异常优先写入普通/大日志 DLT
- DLT 写入失败时记录特殊告警并提交 offset，避免阻塞后续消息
- DLT 失败等待时间默认限制为 1 秒
- 增加连续性放行、DLT 成功及失败监控指标
- 更新 Docker Compose、v3 升级文档及相关测试

haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties

@@ -34,6 +34,10 @@ influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
 influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
 influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
 influxdb.raw.query-max-response-bytes=${INFLUXDB_RAW_QUERY_MAX_RESPONSE_BYTES:10485760}
+# 0 表示不限制；启用后最终 message（含截断标记）不超过该 UTF-8 字节数。
+influxdb.raw.message-max-utf8-bytes=${INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES:1048576}
+# Influx 同步写按 Line Protocol 估算字节切批，避免少量大日志形成超大 HTTP 请求。
+influxdb.raw.write-batch-max-bytes=${INFLUXDB_RAW_WRITE_BATCH_MAX_BYTES:8388608}
 # InfluxDB 2.7 连接超时配置
 influxdb.connection.timeout=30s
 influxdb.connection.read-timeout=30s
@@ -70,16 +74,31 @@ spring.kafka.consumer.auto-offset-reset=latest
 spring.kafka.consumer.enable-auto-commit=false
 spring.kafka.consumer.auto-commit-interval=1000
 spring.kafka.consumer.topic=test-topic
+spring.kafka.consumer.large-topic=${KAFKA_CONSUMER_LARGE_TOPIC:${spring.kafka.consumer.topic}.large}
 # consumer-rule 是当前活跃的 Kafka 原始日志标准化节点，默认启动监听。
 syslog.raw-consumer.enabled=true
 # Raw Topic 当前有 3 个分区，使用独立工厂显式创建 3 个 consumer；可通过环境变量灰度调整。
 syslog.raw-listener.concurrency=${SYSLOG_RAW_LISTENER_CONCURRENCY:3}
+# 大日志使用独立 Topic、消费组和 ConsumerFactory；建议生产环境独立 JVM 部署。
+syslog.large-listener.enabled=${SYSLOG_LARGE_LISTENER_ENABLED:true}
+syslog.large-listener.group-id=${SYSLOG_LARGE_LISTENER_GROUP_ID:${spring.kafka.consumer.group-id}-large}
+syslog.large-listener.concurrency=${SYSLOG_LARGE_LISTENER_CONCURRENCY:1}
+syslog.large-listener.max-poll-records=${SYSLOG_LARGE_LISTENER_MAX_POLL_RECORDS:5}
+syslog.large-listener.max-partition-fetch-bytes=${SYSLOG_LARGE_LISTENER_MAX_PARTITION_FETCH_BYTES:33554432}
+syslog.large-listener.fetch-max-bytes=${SYSLOG_LARGE_LISTENER_FETCH_MAX_BYTES:67108864}
+syslog.large-listener.processing-batch-max-bytes=${SYSLOG_LARGE_PROCESSING_BATCH_MAX_BYTES:8388608}
+# 大日志规则计算不复用普通日志线程池，过载时由 Kafka offset 背压。
+syslog.large-processor.thread-pool.core-pool-size=${SYSLOG_LARGE_PROCESSOR_CORE_POOL_SIZE:1}
+syslog.large-processor.thread-pool.max-pool-size=${SYSLOG_LARGE_PROCESSOR_MAX_POOL_SIZE:2}
+syslog.large-processor.thread-pool.queue-capacity=${SYSLOG_LARGE_PROCESSOR_QUEUE_CAPACITY:10}
 # P0：Retry/DLT 发布与原始消费使用同一 Kafka 集群；Topic 名称全部可由环境变量覆盖。
 spring.kafka.bootstrap-servers=${spring.kafka.consumer.bootstrap-servers}
 spring.kafka.producer.acks=all
 spring.kafka.producer.properties.enable.idempotence=true
 spring.kafka.producer.properties.delivery.timeout.ms=120000
 spring.kafka.producer.properties.request.timeout.ms=30000
+# Retry/DLT 信封也可能携带大日志密文，不能保留 Kafka 默认 1MiB。
+spring.kafka.producer.properties.max.request.size=${KAFKA_RELIABILITY_MAX_REQUEST_SIZE:33554432}
 syslog.reliability.retry-1m-topic=${spring.kafka.consumer.topic}.RETRY.1M
 syslog.reliability.retry-10m-topic=${spring.kafka.consumer.topic}.RETRY.10M
 syslog.reliability.retry-1h-topic=${spring.kafka.consumer.topic}.RETRY.1H
@@ -91,6 +110,14 @@ syslog.reliability.consumer-version=${APP_VERSION:1.0.0}
 syslog.reliability.consumer-group-id=${spring.kafka.consumer.group-id}-retry
 syslog.reliability.consumer.enabled=true
 syslog.reliability.listener.concurrency=1
+# 大日志 Retry/DLT 不回流普通重试通道，避免再次形成队头阻塞。
+syslog.large-reliability.retry-1m-topic=${spring.kafka.consumer.large-topic}.RETRY.1M
+syslog.large-reliability.retry-10m-topic=${spring.kafka.consumer.large-topic}.RETRY.10M
+syslog.large-reliability.retry-1h-topic=${spring.kafka.consumer.large-topic}.RETRY.1H
+syslog.large-reliability.dlt-topic=${spring.kafka.consumer.large-topic}.DLT
+syslog.large-reliability.replay-topic=${spring.kafka.consumer.large-topic}.REPLAY
+syslog.large-reliability.consumer-group-id=${spring.kafka.consumer.group-id}-large-retry
+syslog.large-reliability.consumer.enabled=true
 
 
 spring.kafka.consumer.max-poll-records=500
@@ -152,6 +179,10 @@ dm.standardized-write.mode=BATCH
 dm.standardized-write.batch-size=${DM_STANDARDIZED_WRITE_BATCH_SIZE:500}
 dm.standardized-write.writer-concurrency=2
 dm.standardized-write.writer-queue-capacity=100
+# 标准化大字段和非标原文均在达梦持久化边界截断；0 表示不限制。
+dm.normal.large-field-max-utf8-bytes=${DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES:65536}
+dm.normal.large-fields=${DM_NORMAL_LARGE_FIELDS:http_req_body,http_resp_body,http_req_header,http_resp_header,detail,description,cmdline,shell_cmdline,parent_cmdline,mail_message}
+dm.non-normal.large-field-max-utf8-bytes=${DM_NON_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES:262144}
 dm.standardized-write.transaction-timeout-seconds=45
 dm.standardized-write.kafka-batch-deadline-ms=90000
 dm.standardized-write.max-split-depth=10
@@ -181,7 +212,7 @@ spring.datasource.hikari.validation-timeout=5000
 spring.datasource.hikari.leak-detection-threshold=30000
 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule
 spring.datasource.hikari.auto-commit=false
-# 统计写入采用事实同事务直写，不再为每条结果写 change_log + pending。
+# DIRECT 不再写逐事实 change_log；默认只在事实事务追加一条批次 Outbox，日统计异步投影。
 # 首次部署 DIRECT 前必须先执行 08/09/10/11 脚本并按设计文档排空旧 pending。
 stats.write-mode=DIRECT
 stats.incremental.capture-enabled=false
@@ -197,11 +228,11 @@ stats.direct.control-table=stats_direct_control
 stats.direct.today-stats-date-enabled=true
 stats.direct.max-concurrency-retries=${STATS_DIRECT_MAX_CONCURRENCY_RETRIES:3}
 stats.direct.concurrency-retry-backoff-ms=${STATS_DIRECT_CONCURRENCY_RETRY_BACKOFF_MS:50}
-stats.direct.transaction-log-enabled=false
+stats.direct.transaction-log-enabled=${STATS_DIRECT_TRANSACTION_LOG_ENABLED:false}
 stats.direct.day-rollover-check-cron=5 * * * * ?
 stats.business-zone=Asia/Shanghai
 stats.fact-only.confirm-rebuild-required=false
-# 事实事务只维护小型精确统计表；提交后按维度脏键异步刷新业务展示字段。
+# 日统计投影完成后按维度脏键异步刷新业务展示字段。
 # 三个实例必须配置相同 topic/group-id，Topic 分区数建议与原始 Topic 相同。
 stats.materialization.enabled=${STATS_MATERIALIZATION_ENABLED:true}
 stats.materialization.topic=${STATS_MATERIALIZATION_TOPIC:${spring.kafka.consumer.topic}.STATS.MATERIALIZE}
@@ -216,6 +247,25 @@ stats.materialization.concurrency-retry-backoff-ms=${STATS_MATERIALIZATION_CONCU
 stats.materialization.reconcile-initial-delay-ms=${STATS_MATERIALIZATION_RECONCILE_INITIAL_DELAY_MS:30000}
 stats.materialization.reconcile-interval-ms=${STATS_MATERIALIZATION_RECONCILE_INTERVAL_MS:60000}
 
+# Fact transactions append one aggregated durable event instead of locking the daily projection rows.
+# A dedicated single-lane Kafka consumer applies device/rule/collector projection deltas and then
+# emits the existing idempotent business-materialization dirty keys.
+stats.projection-async.enabled=${STATS_PROJECTION_ASYNC_ENABLED:true}
+stats.projection-async.outbox-table=${STATS_PROJECTION_OUTBOX_TABLE:stats_projection_outbox}
+stats.projection-async.applied-table=${STATS_PROJECTION_APPLIED_TABLE:stats_projection_applied_event}
+stats.projection-async.topic=${STATS_PROJECTION_TOPIC:${spring.kafka.consumer.topic}.STATS.PROJECTION}
+stats.projection-async.group-id=${STATS_PROJECTION_GROUP_ID:${spring.kafka.consumer.group-id}-stats-projection}
+stats.projection-async.relay-interval-ms=${STATS_PROJECTION_RELAY_INTERVAL_MS:200}
+stats.projection-async.relay-batch-size=${STATS_PROJECTION_RELAY_BATCH_SIZE:100}
+stats.projection-async.publish-timeout-ms=${STATS_PROJECTION_PUBLISH_TIMEOUT_MS:10000}
+stats.projection-async.retry-backoff-ms=${STATS_PROJECTION_RETRY_BACKOFF_MS:1000}
+stats.projection-async.redelivery-interval-ms=${STATS_PROJECTION_REDELIVERY_INTERVAL_MS:60000}
+stats.projection-async.max-concurrency-retries=${STATS_PROJECTION_MAX_CONCURRENCY_RETRIES:3}
+stats.projection-async.concurrency-retry-backoff-ms=${STATS_PROJECTION_CONCURRENCY_RETRY_BACKOFF_MS:50}
+stats.projection-async.retention-days=${STATS_PROJECTION_RETENTION_DAYS:14}
+stats.projection-async.cleanup-initial-delay-ms=${STATS_PROJECTION_CLEANUP_INITIAL_DELAY_MS:3600000}
+stats.projection-async.cleanup-interval-ms=${STATS_PROJECTION_CLEANUP_INTERVAL_MS:3600000}
+
 # 标准化规则编译与批次缓存灰度开关
 normalization.jetcache.enabled=true
 normalization.rule-source-cache.enabled=true