c868e3befd
- 128 KiB 及以上日志默认进入大日志 Topic - InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存 - 截断内容追加 HB_XDR_TRUNCATED 标记 - InfluxDB 写入失败时继续执行规则解析和达梦写入 - 未知或系统性异常优先写入普通/大日志 DLT - DLT 写入失败时记录特殊告警并提交 offset,避免阻塞后续消息 - DLT 失败等待时间默认限制为 1 秒 - 增加连续性放行、DLT 成功及失败监控指标 - 更新 Docker Compose、v3 升级文档及相关测试
195 lines
7.0 KiB
Java
195 lines
7.0 KiB
Java
package com.haobang.config;
|
||
|
||
import lombok.Data;
|
||
import org.springframework.beans.factory.annotation.Value;
|
||
import org.springframework.stereotype.Component;
|
||
|
||
/**
|
||
* syslog-serve 统一应用配置属性
|
||
* 使用 Spring Boot 标准 @Value 机制,支持 docker-compose 环境变量覆盖
|
||
*/
|
||
@Data
|
||
@Component
|
||
public class AppProperties {
|
||
|
||
// ========== Syslog 配置 ==========
|
||
@Value("${syslog.tcp.port:514}")
|
||
private int syslogTcpPort;
|
||
|
||
@Value("${syslog.udp.port:515}")
|
||
private int syslogUdpPort;
|
||
|
||
@Value("${syslog.max.frame.length:6291456}")
|
||
private int syslogMaxFrameLength;
|
||
|
||
/** 所有接入方式共用的单条原文字节硬上限;0 表示不限制。 */
|
||
@Value("${syslog.message.max-bytes:6291456}")
|
||
private long syslogMessageMaxBytes;
|
||
|
||
/** 达到该值投递大日志 Topic;0 表示关闭分流。 */
|
||
@Value("${syslog.large-message.threshold-bytes:131072}")
|
||
private long syslogLargeMessageThresholdBytes;
|
||
|
||
@Value("${syslog.buffer.size:1024}")
|
||
private int syslogBufferSize;
|
||
|
||
@Value("${syslog.tcp.backlog:4096}")
|
||
private int syslogTcpBacklog;
|
||
|
||
@Value("${syslog.tcp.receive-buffer-bytes:4194304}")
|
||
private int syslogTcpReceiveBufferBytes;
|
||
|
||
@Value("${syslog.udp.receive-buffer-bytes:16777216}")
|
||
private int syslogUdpReceiveBufferBytes;
|
||
|
||
@Value("${syslog.charset:GBK}")
|
||
private String syslogCharset;
|
||
|
||
// ========== 接入 WAL 配置 ==========
|
||
/** WAL 是 Kafka 接管前的可靠性边界,正式链路不允许关闭。 */
|
||
@Value("${syslog.wal.enabled:true}")
|
||
private boolean syslogWalEnabled;
|
||
|
||
/** 必须挂载到宿主机持久化磁盘;兼容原归档目录环境变量。 */
|
||
@Value("${syslog.wal.directory:${SYSLOG_WAL_DIRECTORY:${SYSLOG_RAW_ARCHIVE_DIRECTORY:/data/syslog-wal}}}")
|
||
private String syslogWalDirectory;
|
||
|
||
/** Netty 到 WAL 单写线程之间的有界队列。 */
|
||
@Value("${syslog.wal.queue-capacity:10000}")
|
||
private int syslogWalQueueCapacity;
|
||
|
||
/** 队列与待 force 记录共用的内存字节预算;0 表示不限制。 */
|
||
@Value("${syslog.wal.max-pending-bytes:268435456}")
|
||
private long syslogWalMaxPendingBytes;
|
||
|
||
@Value("${syslog.wal.high-watermark-percent:70}")
|
||
private int syslogWalHighWatermarkPercent;
|
||
|
||
@Value("${syslog.wal.low-watermark-percent:40}")
|
||
private int syslogWalLowWatermarkPercent;
|
||
|
||
@Value("${syslog.wal.retry-interval-ms:10}")
|
||
private long syslogWalRetryIntervalMs;
|
||
|
||
@Value("${syslog.wal.disk-retry-interval-ms:1000}")
|
||
private long syslogWalDiskRetryIntervalMs;
|
||
|
||
/** 累计记录数或时间达到任一条件即 force,Future 只在 force 后视为持久化。 */
|
||
@Value("${syslog.wal.flush-records:200}")
|
||
private int syslogWalFlushRecords;
|
||
|
||
@Value("${syslog.wal.flush-interval-ms:200}")
|
||
private long syslogWalFlushIntervalMs;
|
||
|
||
@Value("${syslog.wal.segment-max-bytes:268435456}")
|
||
private long syslogWalSegmentMaxBytes;
|
||
|
||
/**
|
||
* 已完成并确认投递的 WAL 分段保留天数,默认 2 天;0 表示不自动删除。
|
||
* 启动时立即清理,运行期间随 checkpoint 推进最多每 30 分钟清理一次。
|
||
*/
|
||
@Value("${syslog.wal.retention-days:2}")
|
||
private int syslogWalRetentionDays;
|
||
|
||
@Value("${syslog.wal.shutdown-timeout-ms:30000}")
|
||
private long syslogWalShutdownTimeoutMs;
|
||
|
||
/** ACK 游标允许批量刷盘;崩溃时最多重复投递该批,event_id 保证下游幂等。 */
|
||
@Value("${syslog.wal.checkpoint-flush-records:200}")
|
||
private int syslogWalCheckpointFlushRecords;
|
||
|
||
@Value("${syslog.wal.checkpoint-flush-interval-ms:200}")
|
||
private long syslogWalCheckpointFlushIntervalMs;
|
||
|
||
/**
|
||
* WAL 所在文件系统的使用率保护线。系统会把该百分比换算为剩余容量,
|
||
* 再与 syslog.wal.min-free-bytes 比较,取较小值作为实际停止接入线。
|
||
*/
|
||
@Value("${syslog.wal.disk-high-watermark-percent:90}")
|
||
private int syslogWalDiskHighWatermarkPercent;
|
||
|
||
/**
|
||
* 实际停止接入线的容量上限,默认 10GiB。大磁盘最多预留 10GiB,
|
||
* 小磁盘则按使用率保护线对应的更小剩余容量处理。
|
||
*/
|
||
@Value("${syslog.wal.min-free-bytes:10737418240}")
|
||
private long syslogWalMinFreeBytes;
|
||
|
||
/** 兼容部署阶段关闭;开启后新接入事件必须以 WAL schema v3 固化 point_time_ns。 */
|
||
@Value("${syslog.point-time.write-enabled:false}")
|
||
private boolean syslogPointTimeWriteEnabled;
|
||
|
||
// ========== WAL Dispatcher 配置 ==========
|
||
@Value("${syslog.dispatch.worker-threads:4}")
|
||
private int syslogDispatchWorkerThreads;
|
||
|
||
@Value("${syslog.dispatch.max-inflight:5000}")
|
||
private int syslogDispatchMaxInflight;
|
||
|
||
/** Dispatcher 正文在途字节预算;0 表示不限制。 */
|
||
@Value("${syslog.dispatch.max-inflight-bytes:268435456}")
|
||
private long syslogDispatchMaxInflightBytes;
|
||
|
||
@Value("${syslog.dispatch.retry-backoff-ms:1000}")
|
||
private long syslogDispatchRetryBackoffMs;
|
||
|
||
@Value("${syslog.dispatch.shutdown-timeout-ms:30000}")
|
||
private long syslogDispatchShutdownTimeoutMs;
|
||
|
||
@Value("${syslog.device-context.cache-ttl-ms:300000}")
|
||
private long syslogDeviceContextCacheTtlMs;
|
||
|
||
@Value("${syslog.status-log-interval-ms:60000}")
|
||
private long syslogStatusLogIntervalMs;
|
||
|
||
// ========== 旧 v1 原文归档精确导入(默认关闭) ==========
|
||
@Value("${syslog.wal.legacy-import.enabled:false}")
|
||
private boolean syslogWalLegacyImportEnabled;
|
||
|
||
@Value("${syslog.wal.legacy-import.archive-directory:}")
|
||
private String syslogWalLegacyImportArchiveDirectory;
|
||
|
||
/** 一行可写一个 event_id,也可直接粘贴包含 event_id 的拒绝日志。 */
|
||
@Value("${syslog.wal.legacy-import.event-id-file:}")
|
||
private String syslogWalLegacyImportEventIdFile;
|
||
|
||
// ========== App Service 配置(探针设备信息)==========
|
||
@Value("${app.service.device_id:}")
|
||
private String appServiceDeviceId;
|
||
|
||
@Value("${app.service.device_name:}")
|
||
private String appServiceDeviceName;
|
||
|
||
@Value("${app.service.vendor:}")
|
||
private String appServiceVendor;
|
||
|
||
@Value("${app.service.product_name:}")
|
||
private String appServiceProductName;
|
||
|
||
@Value("${app.service.data_type:json}")
|
||
private String appServiceDataType;
|
||
|
||
@Value("${app.service.device_collect_id:1}")
|
||
private int appServiceDeviceCollectId;
|
||
|
||
@Value("${app.service.device_collect_name:DevCollect-01}")
|
||
private String appServiceDeviceCollectName;
|
||
|
||
// ========== Kafka Producer 配置 ==========
|
||
@Value("${spring.kafka.producer.bootstrap-servers:localhost:9092}")
|
||
private String kafkaProducerBootstrap;
|
||
|
||
@Value("${spring.kafka.producer.topic:agent-syslog-topic}")
|
||
private String kafkaProducerTopic;
|
||
|
||
@Value("${spring.kafka.producer.large-topic:agent-syslog-topic.large}")
|
||
private String kafkaProducerLargeTopic;
|
||
|
||
@Value("${spring.kafka.producer.ack-timeout-ms:130000}")
|
||
private long kafkaProducerAckTimeoutMs;
|
||
|
||
// ========== SM4 加密配置 ==========
|
||
@Value("${syslog.sm4.generateKey:}")
|
||
private String sm4Key;
|
||
}
|