feat(syslog): 完成高吞吐可靠链路与统计存储优化
- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
王绍宏
@@ -1,147 +0,0 @@
|
||||
package com.haobang.config;
|
||||
|
||||
import com.typesafe.config.Config;
|
||||
import com.typesafe.config.ConfigFactory;
|
||||
import java.io.File;
|
||||
import com.typesafe.config.ConfigValueFactory;
|
||||
import java.nio.charset.Charset;
|
||||
|
||||
import java.util.Map;
|
||||
public class AppConfig {
|
||||
|
||||
private static final Config config;
|
||||
|
||||
static {
|
||||
// 加载配置文件
|
||||
File configFile = new File("application.properties");
|
||||
Config loadedConfig;
|
||||
|
||||
if (configFile.exists()) {
|
||||
loadedConfig = ConfigFactory.parseFile(configFile);
|
||||
} else {
|
||||
loadedConfig = ConfigFactory.load("application.properties");
|
||||
}
|
||||
|
||||
// 解析环境变量占位符
|
||||
config = resolveEnvironmentVariables(loadedConfig);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 解析环境变量占位符
|
||||
*/
|
||||
private static Config resolveEnvironmentVariables(Config originalConfig) {
|
||||
Config resolvedConfig = originalConfig;
|
||||
|
||||
// 遍历所有配置项，查找需要解析的占位符
|
||||
for (Map.Entry<String, com.typesafe.config.ConfigValue> entry : originalConfig.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
String value = originalConfig.getString(key);
|
||||
|
||||
if (value.contains("${")) {
|
||||
String resolvedValue = resolvePlaceholder(value);
|
||||
resolvedConfig = resolvedConfig.withValue(
|
||||
key,
|
||||
ConfigValueFactory.fromAnyRef(resolvedValue)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return resolvedConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析单个占位符值
|
||||
* 格式: ${ENV_VAR:default_value}
|
||||
*/
|
||||
private static String resolvePlaceholder(String value) {
|
||||
if (!value.startsWith("${") || !value.endsWith("}")) {
|
||||
return value;
|
||||
}
|
||||
|
||||
String placeholder = value.substring(2, value.length() - 1);
|
||||
String[] parts = placeholder.split(":");
|
||||
|
||||
if (parts.length == 0) {
|
||||
return value; // 无效格式，返回原值
|
||||
}
|
||||
|
||||
String envVarName = parts[0].trim();
|
||||
String defaultValue = parts.length > 1 ? parts[1].trim() : "";
|
||||
|
||||
// 1. 从系统环境变量获取
|
||||
String envValue = System.getenv(envVarName);
|
||||
if (envValue != null && !envValue.trim().isEmpty()) {
|
||||
return envValue.trim();
|
||||
}
|
||||
|
||||
// 2. 从系统属性获取 (java -D参数)
|
||||
String sysValue = System.getProperty(envVarName);
|
||||
if (sysValue != null && !sysValue.trim().isEmpty()) {
|
||||
return sysValue.trim();
|
||||
}
|
||||
|
||||
// 3. 返回默认值
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
// Syslog 配置
|
||||
public static int getSyslogTcpPort() {
|
||||
return config.getInt("syslog.tcp.port");
|
||||
}
|
||||
|
||||
public static int getSyslogUdpPort() {
|
||||
return config.getInt("syslog.udp.port");
|
||||
}
|
||||
|
||||
public static int getSyslogMaxFrameLength() {
|
||||
return config.getInt("syslog.max.frame.length");
|
||||
}
|
||||
|
||||
public static int getSyslogBufferSize() {
|
||||
return config.getInt("syslog.buffer.size");
|
||||
}
|
||||
|
||||
/**
|
||||
* syslog 消息字符编码，默认 GBK（国内安全设备普遍使用 GBK）
|
||||
* 配置示例: syslog.charset=GBK
|
||||
*/
|
||||
public static Charset getSyslogCharset() {
|
||||
if (config.hasPath("syslog.charset")) {
|
||||
return Charset.forName(config.getString("syslog.charset"));
|
||||
}
|
||||
return Charset.forName("GBK");
|
||||
}
|
||||
|
||||
// app service 配置
|
||||
public static String getAppServieDeviceId() {
|
||||
return config.getString("app.service.device_id");
|
||||
}
|
||||
public static String getAppServieDeviceName() { return config.getString("app.service.device_name");
|
||||
}
|
||||
public static String getAppServieVendor() {
|
||||
return config.getString("app.service.vendor");
|
||||
}
|
||||
public static String getAppServieProductName() {
|
||||
return config.getString("app.service.product_name");
|
||||
}
|
||||
public static String getAppServieDataType() {
|
||||
return config.getString("app.service.data_type");
|
||||
}
|
||||
// kafka 配置
|
||||
public static String getKafkaProducerBootstrap() {
|
||||
return config.getString("spring.kafka.producer.bootstrap-servers");
|
||||
}
|
||||
public static String getKafkaProducerTopic() {
|
||||
return config.getString("spring.kafka.producer.topic");
|
||||
}
|
||||
|
||||
public static int getDeviceCollectId() {
|
||||
return config.getInt("app.service.device_collect_id");
|
||||
}
|
||||
|
||||
public static String getSM4Key() {
|
||||
return config.getString("syslog.sm4.generateKey");
|
||||
}
|
||||
|
||||
}
|
||||
+108
-1
@@ -5,7 +5,7 @@ import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 应用配置属性 - 替代静态 AppConfig（syslog-serve 模块）
|
||||
* syslog-serve 统一应用配置属性
|
||||
* 使用 Spring Boot 标准 @Value 机制，支持 docker-compose 环境变量覆盖
|
||||
*/
|
||||
@Data
|
||||
@@ -25,9 +25,110 @@ public class AppProperties {
|
||||
@Value("${syslog.buffer.size:1024}")
|
||||
private int syslogBufferSize;
|
||||
|
||||
@Value("${syslog.tcp.backlog:4096}")
|
||||
private int syslogTcpBacklog;
|
||||
|
||||
@Value("${syslog.tcp.receive-buffer-bytes:4194304}")
|
||||
private int syslogTcpReceiveBufferBytes;
|
||||
|
||||
@Value("${syslog.udp.receive-buffer-bytes:16777216}")
|
||||
private int syslogUdpReceiveBufferBytes;
|
||||
|
||||
@Value("${syslog.charset:GBK}")
|
||||
private String syslogCharset;
|
||||
|
||||
// ========== 接入 WAL 配置 ==========
|
||||
/** WAL 是 Kafka 接管前的可靠性边界，正式链路不允许关闭。 */
|
||||
@Value("${syslog.wal.enabled:true}")
|
||||
private boolean syslogWalEnabled;
|
||||
|
||||
/** 必须挂载到宿主机持久化磁盘；兼容原归档目录环境变量。 */
|
||||
@Value("${syslog.wal.directory:${SYSLOG_WAL_DIRECTORY:${SYSLOG_RAW_ARCHIVE_DIRECTORY:/data/syslog-wal}}}")
|
||||
private String syslogWalDirectory;
|
||||
|
||||
/** Netty 到 WAL 单写线程之间的有界队列。 */
|
||||
@Value("${syslog.wal.queue-capacity:10000}")
|
||||
private int syslogWalQueueCapacity;
|
||||
|
||||
@Value("${syslog.wal.high-watermark-percent:70}")
|
||||
private int syslogWalHighWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.low-watermark-percent:40}")
|
||||
private int syslogWalLowWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.retry-interval-ms:10}")
|
||||
private long syslogWalRetryIntervalMs;
|
||||
|
||||
@Value("${syslog.wal.disk-retry-interval-ms:1000}")
|
||||
private long syslogWalDiskRetryIntervalMs;
|
||||
|
||||
/** 累计记录数或时间达到任一条件即 force，Future 只在 force 后视为持久化。 */
|
||||
@Value("${syslog.wal.flush-records:200}")
|
||||
private int syslogWalFlushRecords;
|
||||
|
||||
@Value("${syslog.wal.flush-interval-ms:200}")
|
||||
private long syslogWalFlushIntervalMs;
|
||||
|
||||
@Value("${syslog.wal.segment-max-bytes:268435456}")
|
||||
private long syslogWalSegmentMaxBytes;
|
||||
|
||||
/** 已完成段的保留天数；0 表示不自动删除。 */
|
||||
@Value("${syslog.wal.retention-days:0}")
|
||||
private int syslogWalRetentionDays;
|
||||
|
||||
@Value("${syslog.wal.shutdown-timeout-ms:30000}")
|
||||
private long syslogWalShutdownTimeoutMs;
|
||||
|
||||
/** ACK 游标允许批量刷盘；崩溃时最多重复投递该批，event_id 保证下游幂等。 */
|
||||
@Value("${syslog.wal.checkpoint-flush-records:200}")
|
||||
private int syslogWalCheckpointFlushRecords;
|
||||
|
||||
@Value("${syslog.wal.checkpoint-flush-interval-ms:200}")
|
||||
private long syslogWalCheckpointFlushIntervalMs;
|
||||
|
||||
@Value("${syslog.wal.disk-high-watermark-percent:80}")
|
||||
private int syslogWalDiskHighWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.disk-critical-watermark-percent:90}")
|
||||
private int syslogWalDiskCriticalWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.min-free-bytes:10737418240}")
|
||||
private long syslogWalMinFreeBytes;
|
||||
|
||||
/** 兼容部署阶段关闭；开启后新接入事件必须以 WAL schema v3 固化 point_time_ns。 */
|
||||
@Value("${syslog.point-time.write-enabled:false}")
|
||||
private boolean syslogPointTimeWriteEnabled;
|
||||
|
||||
// ========== WAL Dispatcher 配置 ==========
|
||||
@Value("${syslog.dispatch.worker-threads:4}")
|
||||
private int syslogDispatchWorkerThreads;
|
||||
|
||||
@Value("${syslog.dispatch.max-inflight:5000}")
|
||||
private int syslogDispatchMaxInflight;
|
||||
|
||||
@Value("${syslog.dispatch.retry-backoff-ms:1000}")
|
||||
private long syslogDispatchRetryBackoffMs;
|
||||
|
||||
@Value("${syslog.dispatch.shutdown-timeout-ms:30000}")
|
||||
private long syslogDispatchShutdownTimeoutMs;
|
||||
|
||||
@Value("${syslog.device-context.cache-ttl-ms:300000}")
|
||||
private long syslogDeviceContextCacheTtlMs;
|
||||
|
||||
@Value("${syslog.status-log-interval-ms:60000}")
|
||||
private long syslogStatusLogIntervalMs;
|
||||
|
||||
// ========== 旧 v1 原文归档精确导入（默认关闭） ==========
|
||||
@Value("${syslog.wal.legacy-import.enabled:false}")
|
||||
private boolean syslogWalLegacyImportEnabled;
|
||||
|
||||
@Value("${syslog.wal.legacy-import.archive-directory:}")
|
||||
private String syslogWalLegacyImportArchiveDirectory;
|
||||
|
||||
/** 一行可写一个 event_id，也可直接粘贴包含 event_id 的拒绝日志。 */
|
||||
@Value("${syslog.wal.legacy-import.event-id-file:}")
|
||||
private String syslogWalLegacyImportEventIdFile;
|
||||
|
||||
// ========== App Service 配置（探针设备信息）==========
|
||||
@Value("${app.service.device_id:}")
|
||||
private String appServiceDeviceId;
|
||||
@@ -47,6 +148,9 @@ public class AppProperties {
|
||||
@Value("${app.service.device_collect_id:1}")
|
||||
private int appServiceDeviceCollectId;
|
||||
|
||||
@Value("${app.service.device_collect_name:DevCollect-01}")
|
||||
private String appServiceDeviceCollectName;
|
||||
|
||||
// ========== Kafka Producer 配置 ==========
|
||||
@Value("${spring.kafka.producer.bootstrap-servers:localhost:9092}")
|
||||
private String kafkaProducerBootstrap;
|
||||
@@ -54,6 +158,9 @@ public class AppProperties {
|
||||
@Value("${spring.kafka.producer.topic:agent-syslog-topic}")
|
||||
private String kafkaProducerTopic;
|
||||
|
||||
@Value("${spring.kafka.producer.ack-timeout-ms:130000}")
|
||||
private long kafkaProducerAckTimeoutMs;
|
||||
|
||||
// ========== SM4 加密配置 ==========
|
||||
@Value("${syslog.sm4.generateKey:}")
|
||||
private String sm4Key;
|
||||
|
||||
Reference in New Issue
Block a user