feat(syslog): 优化WAL配置并在入站阶段丢弃无效设备日志

- 默认在进入WAL前丢弃未注册、重复或未分配到当前探针的Syslog日志
- 使用后台设备内存快照,避免Netty网络线程访问Redis或达梦
- 增加Terminal审计开关,允许恢复原有逐条审计文件模式
- 历史WAL中的无效设备记录直接推进checkpoint
- 增加无效设备丢弃、快照状态及刷新失败监控指标
- 生产WAL队列调整为20000条
- WAL待写预算和分段大小调整为2GiB
- 同步生产配置、Docker Compose及升级文档
- 增加无效设备前置过滤和Terminal开关测试

chore(syslog): 统一生产环境 WAL 挂载目录

- 将生产环境 WAL 默认目录调整为 /app/logs/syslog-wal
- Docker Compose 显式设置 SYSLOG_WAL_DIRECTORY
- WAL 与应用日志共用 /app/logs 宿主机挂载
- 移除旧的 /data/syslog-archive 独立挂载
- 宿主机 WAL 路径统一为 /home/syslog/logs/syslog-wal
This commit is contained in:
王绍宏
2026-09-15 22:10:46 +08:00
parent 5a86a4f950
commit faa9ffe2d9
19 changed files with 553 additions and 32 deletions
@@ -47,6 +47,7 @@ public class SyslogWalDispatcher {
private final Map<Long, SyslogWalRecord> completedOutOfOrder = new HashMap<>();
private final LongAdder kafkaAcked = new LongAdder();
private final LongAdder terminalFiltered = new LongAdder();
private final LongAdder terminalDropped = new LongAdder();
private final LongAdder retryCount = new LongAdder();
private final LongAdder largeDispatched = new LongAdder();
private final AtomicLong inflight = new AtomicLong();
@@ -88,10 +89,15 @@ public class SyslogWalDispatcher {
scheduler.scheduleWithFixedDelay(this::logStatus,
properties.getSyslogStatusLogIntervalMs(), properties.getSyslogStatusLogIntervalMs(),
TimeUnit.MILLISECONDS);
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, maxInflightBytes={}, checkpointSequence={}",
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, maxInflightBytes={}, "
+ "terminalRecordEnabled={}, checkpointSequence={}",
properties.getSyslogDispatchWorkerThreads(), properties.getSyslogDispatchMaxInflight(),
properties.getSyslogDispatchMaxInflightBytes(),
properties.isSyslogDispatchTerminalRecordEnabled(),
walService.getCheckpointSequence());
if (!properties.isSyslogDispatchTerminalRecordEnabled()) {
log.warn("无效设备日志直接丢弃已启用: 新 Syslog 正文不进 WAL,历史记录不创建 Terminal 文件");
}
}
private void readLoop() {
@@ -147,7 +153,11 @@ public class SyslogWalDispatcher {
if (!resolution.isRegistered()) {
try {
walService.recordTerminal(record, resolution.getType().name(), resolution.getReason());
if (properties.isSyslogDispatchTerminalRecordEnabled()) {
walService.recordTerminal(record, resolution.getType().name(), resolution.getReason());
} else {
terminalDropped.increment();
}
terminalFiltered.increment();
markComplete(record);
} catch (Throwable failure) {
@@ -252,14 +262,16 @@ public class SyslogWalDispatcher {
private void logStatus() {
if (!running.get()) return;
log.info("Syslog WAL状态: networkReceived={}, appended={}, admissionOversized={}, walPendingApprox={}, walPendingBytes={}, walQueue={}/{}, durableSequence={}, "
+ "checkpointSequence={}, kafkaInflight={}, kafkaInflightBytes={}, kafkaAcked={}, largeDispatched={}, terminalFiltered={}, "
+ "checkpointSequence={}, kafkaInflight={}, kafkaInflightBytes={}, kafkaAcked={}, largeDispatched={}, "
+ "terminalRecordEnabled={}, terminalFiltered={}, terminalDropped={}, "
+ "retry={}, tcpPauseTotal={}, udpDropped={}, deviceCacheSize={}, deviceCacheHits={}, "
+ "deviceCacheMisses={}",
SyslogMessageHandler.getNetworkReceivedCount(), walService.getAppendedCount(),
walService.getOversizedCount(), walService.getPendingRecordsApprox(), walService.getPendingAppendBytes(),
walService.getQueueSize(), walService.getQueueCapacity(), walService.getDurableSequence(),
walService.getCheckpointSequence(), inflight.get(), byteBudget == null ? 0L : byteBudget.used(),
kafkaAcked.sum(), largeDispatched.sum(), terminalFiltered.sum(),
kafkaAcked.sum(), largeDispatched.sum(), properties.isSyslogDispatchTerminalRecordEnabled(),
terminalFiltered.sum(), terminalDropped.sum(),
retryCount.sum(), SyslogMessageHandler.getTcpPauseCount(),
SyslogMessageHandler.getUdpDroppedCount(), deviceResolver.getCacheSize(),
deviceResolver.getHitCount(), deviceResolver.getMissCount());
@@ -318,6 +330,8 @@ public class SyslogWalDispatcher {
public long getInflight() { return inflight.get(); }
public long getKafkaAcked() { return kafkaAcked.sum(); }
public long getTerminalFiltered() { return terminalFiltered.sum(); }
public long getTerminalDropped() { return terminalDropped.sum(); }
public boolean isTerminalRecordEnabled() { return properties.isSyslogDispatchTerminalRecordEnabled(); }
public long getRetryCount() { return retryCount.sum(); }
public long getLargeDispatched() { return largeDispatched.sum(); }
public long getInflightBytes() { return byteBudget == null ? 0L : byteBudget.used(); }