feat(syslog): 完成高吞吐可靠链路与统计存储优化
- 新增接入 WAL、检查点、背压恢复及可靠分发机制,完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存,支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型,统一接收时间与 Point 时间,支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路,支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
+108
-1
@@ -5,7 +5,7 @@ import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 应用配置属性 - 替代静态 AppConfig(syslog-serve 模块)
|
||||
* syslog-serve 统一应用配置属性
|
||||
* 使用 Spring Boot 标准 @Value 机制,支持 docker-compose 环境变量覆盖
|
||||
*/
|
||||
@Data
|
||||
@@ -25,9 +25,110 @@ public class AppProperties {
|
||||
@Value("${syslog.buffer.size:1024}")
|
||||
private int syslogBufferSize;
|
||||
|
||||
@Value("${syslog.tcp.backlog:4096}")
|
||||
private int syslogTcpBacklog;
|
||||
|
||||
@Value("${syslog.tcp.receive-buffer-bytes:4194304}")
|
||||
private int syslogTcpReceiveBufferBytes;
|
||||
|
||||
@Value("${syslog.udp.receive-buffer-bytes:16777216}")
|
||||
private int syslogUdpReceiveBufferBytes;
|
||||
|
||||
@Value("${syslog.charset:GBK}")
|
||||
private String syslogCharset;
|
||||
|
||||
// ========== 接入 WAL 配置 ==========
|
||||
/** WAL 是 Kafka 接管前的可靠性边界,正式链路不允许关闭。 */
|
||||
@Value("${syslog.wal.enabled:true}")
|
||||
private boolean syslogWalEnabled;
|
||||
|
||||
/** 必须挂载到宿主机持久化磁盘;兼容原归档目录环境变量。 */
|
||||
@Value("${syslog.wal.directory:${SYSLOG_WAL_DIRECTORY:${SYSLOG_RAW_ARCHIVE_DIRECTORY:/data/syslog-wal}}}")
|
||||
private String syslogWalDirectory;
|
||||
|
||||
/** Netty 到 WAL 单写线程之间的有界队列。 */
|
||||
@Value("${syslog.wal.queue-capacity:10000}")
|
||||
private int syslogWalQueueCapacity;
|
||||
|
||||
@Value("${syslog.wal.high-watermark-percent:70}")
|
||||
private int syslogWalHighWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.low-watermark-percent:40}")
|
||||
private int syslogWalLowWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.retry-interval-ms:10}")
|
||||
private long syslogWalRetryIntervalMs;
|
||||
|
||||
@Value("${syslog.wal.disk-retry-interval-ms:1000}")
|
||||
private long syslogWalDiskRetryIntervalMs;
|
||||
|
||||
/** 累计记录数或时间达到任一条件即 force,Future 只在 force 后视为持久化。 */
|
||||
@Value("${syslog.wal.flush-records:200}")
|
||||
private int syslogWalFlushRecords;
|
||||
|
||||
@Value("${syslog.wal.flush-interval-ms:200}")
|
||||
private long syslogWalFlushIntervalMs;
|
||||
|
||||
@Value("${syslog.wal.segment-max-bytes:268435456}")
|
||||
private long syslogWalSegmentMaxBytes;
|
||||
|
||||
/** 已完成段的保留天数;0 表示不自动删除。 */
|
||||
@Value("${syslog.wal.retention-days:0}")
|
||||
private int syslogWalRetentionDays;
|
||||
|
||||
@Value("${syslog.wal.shutdown-timeout-ms:30000}")
|
||||
private long syslogWalShutdownTimeoutMs;
|
||||
|
||||
/** ACK 游标允许批量刷盘;崩溃时最多重复投递该批,event_id 保证下游幂等。 */
|
||||
@Value("${syslog.wal.checkpoint-flush-records:200}")
|
||||
private int syslogWalCheckpointFlushRecords;
|
||||
|
||||
@Value("${syslog.wal.checkpoint-flush-interval-ms:200}")
|
||||
private long syslogWalCheckpointFlushIntervalMs;
|
||||
|
||||
@Value("${syslog.wal.disk-high-watermark-percent:80}")
|
||||
private int syslogWalDiskHighWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.disk-critical-watermark-percent:90}")
|
||||
private int syslogWalDiskCriticalWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.min-free-bytes:10737418240}")
|
||||
private long syslogWalMinFreeBytes;
|
||||
|
||||
/** 兼容部署阶段关闭;开启后新接入事件必须以 WAL schema v3 固化 point_time_ns。 */
|
||||
@Value("${syslog.point-time.write-enabled:false}")
|
||||
private boolean syslogPointTimeWriteEnabled;
|
||||
|
||||
// ========== WAL Dispatcher 配置 ==========
|
||||
@Value("${syslog.dispatch.worker-threads:4}")
|
||||
private int syslogDispatchWorkerThreads;
|
||||
|
||||
@Value("${syslog.dispatch.max-inflight:5000}")
|
||||
private int syslogDispatchMaxInflight;
|
||||
|
||||
@Value("${syslog.dispatch.retry-backoff-ms:1000}")
|
||||
private long syslogDispatchRetryBackoffMs;
|
||||
|
||||
@Value("${syslog.dispatch.shutdown-timeout-ms:30000}")
|
||||
private long syslogDispatchShutdownTimeoutMs;
|
||||
|
||||
@Value("${syslog.device-context.cache-ttl-ms:300000}")
|
||||
private long syslogDeviceContextCacheTtlMs;
|
||||
|
||||
@Value("${syslog.status-log-interval-ms:60000}")
|
||||
private long syslogStatusLogIntervalMs;
|
||||
|
||||
// ========== 旧 v1 原文归档精确导入(默认关闭) ==========
|
||||
@Value("${syslog.wal.legacy-import.enabled:false}")
|
||||
private boolean syslogWalLegacyImportEnabled;
|
||||
|
||||
@Value("${syslog.wal.legacy-import.archive-directory:}")
|
||||
private String syslogWalLegacyImportArchiveDirectory;
|
||||
|
||||
/** 一行可写一个 event_id,也可直接粘贴包含 event_id 的拒绝日志。 */
|
||||
@Value("${syslog.wal.legacy-import.event-id-file:}")
|
||||
private String syslogWalLegacyImportEventIdFile;
|
||||
|
||||
// ========== App Service 配置(探针设备信息)==========
|
||||
@Value("${app.service.device_id:}")
|
||||
private String appServiceDeviceId;
|
||||
@@ -47,6 +148,9 @@ public class AppProperties {
|
||||
@Value("${app.service.device_collect_id:1}")
|
||||
private int appServiceDeviceCollectId;
|
||||
|
||||
@Value("${app.service.device_collect_name:DevCollect-01}")
|
||||
private String appServiceDeviceCollectName;
|
||||
|
||||
// ========== Kafka Producer 配置 ==========
|
||||
@Value("${spring.kafka.producer.bootstrap-servers:localhost:9092}")
|
||||
private String kafkaProducerBootstrap;
|
||||
@@ -54,6 +158,9 @@ public class AppProperties {
|
||||
@Value("${spring.kafka.producer.topic:agent-syslog-topic}")
|
||||
private String kafkaProducerTopic;
|
||||
|
||||
@Value("${spring.kafka.producer.ack-timeout-ms:130000}")
|
||||
private long kafkaProducerAckTimeoutMs;
|
||||
|
||||
// ========== SM4 加密配置 ==========
|
||||
@Value("${syslog.sm4.generateKey:}")
|
||||
private String sm4Key;
|
||||
|
||||
Reference in New Issue
Block a user