feat(syslog): 优化大日志通道并增加异常连续消费兜底
- 128 KiB 及以上日志默认进入大日志 Topic - InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存 - 截断内容追加 HB_XDR_TRUNCATED 标记 - InfluxDB 写入失败时继续执行规则解析和达梦写入 - 未知或系统性异常优先写入普通/大日志 DLT - DLT 写入失败时记录特殊告警并提交 offset,避免阻塞后续消息 - DLT 失败等待时间默认限制为 1 秒 - 增加连续性放行、DLT 成功及失败监控指标 - 更新 Docker Compose、v3 升级文档及相关测试
This commit is contained in:
+20
-1
@@ -19,9 +19,17 @@ public class AppProperties {
|
||||
@Value("${syslog.udp.port:515}")
|
||||
private int syslogUdpPort;
|
||||
|
||||
@Value("${syslog.max.frame.length:65536}")
|
||||
@Value("${syslog.max.frame.length:6291456}")
|
||||
private int syslogMaxFrameLength;
|
||||
|
||||
/** 所有接入方式共用的单条原文字节硬上限;0 表示不限制。 */
|
||||
@Value("${syslog.message.max-bytes:6291456}")
|
||||
private long syslogMessageMaxBytes;
|
||||
|
||||
/** 超过该值投递大日志 Topic;0 表示关闭分流。 */
|
||||
@Value("${syslog.large-message.threshold-bytes:262144}")
|
||||
private long syslogLargeMessageThresholdBytes;
|
||||
|
||||
@Value("${syslog.buffer.size:1024}")
|
||||
private int syslogBufferSize;
|
||||
|
||||
@@ -50,6 +58,10 @@ public class AppProperties {
|
||||
@Value("${syslog.wal.queue-capacity:10000}")
|
||||
private int syslogWalQueueCapacity;
|
||||
|
||||
/** 队列与待 force 记录共用的内存字节预算;0 表示不限制。 */
|
||||
@Value("${syslog.wal.max-pending-bytes:268435456}")
|
||||
private long syslogWalMaxPendingBytes;
|
||||
|
||||
@Value("${syslog.wal.high-watermark-percent:70}")
|
||||
private int syslogWalHighWatermarkPercent;
|
||||
|
||||
@@ -114,6 +126,10 @@ public class AppProperties {
|
||||
@Value("${syslog.dispatch.max-inflight:5000}")
|
||||
private int syslogDispatchMaxInflight;
|
||||
|
||||
/** Dispatcher 正文在途字节预算;0 表示不限制。 */
|
||||
@Value("${syslog.dispatch.max-inflight-bytes:268435456}")
|
||||
private long syslogDispatchMaxInflightBytes;
|
||||
|
||||
@Value("${syslog.dispatch.retry-backoff-ms:1000}")
|
||||
private long syslogDispatchRetryBackoffMs;
|
||||
|
||||
@@ -166,6 +182,9 @@ public class AppProperties {
|
||||
@Value("${spring.kafka.producer.topic:agent-syslog-topic}")
|
||||
private String kafkaProducerTopic;
|
||||
|
||||
@Value("${spring.kafka.producer.large-topic:agent-syslog-topic.large}")
|
||||
private String kafkaProducerLargeTopic;
|
||||
|
||||
@Value("${spring.kafka.producer.ack-timeout-ms:130000}")
|
||||
private long kafkaProducerAckTimeoutMs;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user