feat(syslog): 优化WAL配置并在入站阶段丢弃无效设备日志
- 默认在进入WAL前丢弃未注册、重复或未分配到当前探针的Syslog日志 - 使用后台设备内存快照,避免Netty网络线程访问Redis或达梦 - 增加Terminal审计开关,允许恢复原有逐条审计文件模式 - 历史WAL中的无效设备记录直接推进checkpoint - 增加无效设备丢弃、快照状态及刷新失败监控指标 - 生产WAL队列调整为20000条 - WAL待写预算和分段大小调整为2GiB - 同步生产配置、Docker Compose及升级文档 - 增加无效设备前置过滤和Terminal开关测试 chore(syslog): 统一生产环境 WAL 挂载目录 - 将生产环境 WAL 默认目录调整为 /app/logs/syslog-wal - Docker Compose 显式设置 SYSLOG_WAL_DIRECTORY - WAL 与应用日志共用 /app/logs 宿主机挂载 - 移除旧的 /data/syslog-archive 独立挂载 - 宿主机 WAL 路径统一为 /home/syslog/logs/syslog-wal
This commit is contained in:
+18
-4
@@ -47,6 +47,7 @@ public class SyslogWalDispatcher {
|
||||
private final Map<Long, SyslogWalRecord> completedOutOfOrder = new HashMap<>();
|
||||
private final LongAdder kafkaAcked = new LongAdder();
|
||||
private final LongAdder terminalFiltered = new LongAdder();
|
||||
private final LongAdder terminalDropped = new LongAdder();
|
||||
private final LongAdder retryCount = new LongAdder();
|
||||
private final LongAdder largeDispatched = new LongAdder();
|
||||
private final AtomicLong inflight = new AtomicLong();
|
||||
@@ -88,10 +89,15 @@ public class SyslogWalDispatcher {
|
||||
scheduler.scheduleWithFixedDelay(this::logStatus,
|
||||
properties.getSyslogStatusLogIntervalMs(), properties.getSyslogStatusLogIntervalMs(),
|
||||
TimeUnit.MILLISECONDS);
|
||||
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, maxInflightBytes={}, checkpointSequence={}",
|
||||
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, maxInflightBytes={}, "
|
||||
+ "terminalRecordEnabled={}, checkpointSequence={}",
|
||||
properties.getSyslogDispatchWorkerThreads(), properties.getSyslogDispatchMaxInflight(),
|
||||
properties.getSyslogDispatchMaxInflightBytes(),
|
||||
properties.isSyslogDispatchTerminalRecordEnabled(),
|
||||
walService.getCheckpointSequence());
|
||||
if (!properties.isSyslogDispatchTerminalRecordEnabled()) {
|
||||
log.warn("无效设备日志直接丢弃已启用: 新 Syslog 正文不进 WAL,历史记录不创建 Terminal 文件");
|
||||
}
|
||||
}
|
||||
|
||||
private void readLoop() {
|
||||
@@ -147,7 +153,11 @@ public class SyslogWalDispatcher {
|
||||
|
||||
if (!resolution.isRegistered()) {
|
||||
try {
|
||||
walService.recordTerminal(record, resolution.getType().name(), resolution.getReason());
|
||||
if (properties.isSyslogDispatchTerminalRecordEnabled()) {
|
||||
walService.recordTerminal(record, resolution.getType().name(), resolution.getReason());
|
||||
} else {
|
||||
terminalDropped.increment();
|
||||
}
|
||||
terminalFiltered.increment();
|
||||
markComplete(record);
|
||||
} catch (Throwable failure) {
|
||||
@@ -252,14 +262,16 @@ public class SyslogWalDispatcher {
|
||||
private void logStatus() {
|
||||
if (!running.get()) return;
|
||||
log.info("Syslog WAL状态: networkReceived={}, appended={}, admissionOversized={}, walPendingApprox={}, walPendingBytes={}, walQueue={}/{}, durableSequence={}, "
|
||||
+ "checkpointSequence={}, kafkaInflight={}, kafkaInflightBytes={}, kafkaAcked={}, largeDispatched={}, terminalFiltered={}, "
|
||||
+ "checkpointSequence={}, kafkaInflight={}, kafkaInflightBytes={}, kafkaAcked={}, largeDispatched={}, "
|
||||
+ "terminalRecordEnabled={}, terminalFiltered={}, terminalDropped={}, "
|
||||
+ "retry={}, tcpPauseTotal={}, udpDropped={}, deviceCacheSize={}, deviceCacheHits={}, "
|
||||
+ "deviceCacheMisses={}",
|
||||
SyslogMessageHandler.getNetworkReceivedCount(), walService.getAppendedCount(),
|
||||
walService.getOversizedCount(), walService.getPendingRecordsApprox(), walService.getPendingAppendBytes(),
|
||||
walService.getQueueSize(), walService.getQueueCapacity(), walService.getDurableSequence(),
|
||||
walService.getCheckpointSequence(), inflight.get(), byteBudget == null ? 0L : byteBudget.used(),
|
||||
kafkaAcked.sum(), largeDispatched.sum(), terminalFiltered.sum(),
|
||||
kafkaAcked.sum(), largeDispatched.sum(), properties.isSyslogDispatchTerminalRecordEnabled(),
|
||||
terminalFiltered.sum(), terminalDropped.sum(),
|
||||
retryCount.sum(), SyslogMessageHandler.getTcpPauseCount(),
|
||||
SyslogMessageHandler.getUdpDroppedCount(), deviceResolver.getCacheSize(),
|
||||
deviceResolver.getHitCount(), deviceResolver.getMissCount());
|
||||
@@ -318,6 +330,8 @@ public class SyslogWalDispatcher {
|
||||
public long getInflight() { return inflight.get(); }
|
||||
public long getKafkaAcked() { return kafkaAcked.sum(); }
|
||||
public long getTerminalFiltered() { return terminalFiltered.sum(); }
|
||||
public long getTerminalDropped() { return terminalDropped.sum(); }
|
||||
public boolean isTerminalRecordEnabled() { return properties.isSyslogDispatchTerminalRecordEnabled(); }
|
||||
public long getRetryCount() { return retryCount.sum(); }
|
||||
public long getLargeDispatched() { return largeDispatched.sum(); }
|
||||
public long getInflightBytes() { return byteBudget == null ? 0L : byteBudget.used(); }
|
||||
|
||||
Reference in New Issue
Block a user