nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(syslog): 优化WAL配置并在入站阶段丢弃无效设备日志

Browse Source 
- 默认在进入WAL前丢弃未注册、重复或未分配到当前探针的Syslog日志
- 使用后台设备内存快照，避免Netty网络线程访问Redis或达梦
- 增加Terminal审计开关，允许恢复原有逐条审计文件模式
- 历史WAL中的无效设备记录直接推进checkpoint
- 增加无效设备丢弃、快照状态及刷新失败监控指标
- 生产WAL队列调整为20000条
- WAL待写预算和分段大小调整为2GiB
- 同步生产配置、Docker Compose及升级文档
- 增加无效设备前置过滤和Terminal开关测试

chore(syslog): 统一生产环境 WAL 挂载目录

- 将生产环境 WAL 默认目录调整为 /app/logs/syslog-wal
- Docker Compose 显式设置 SYSLOG_WAL_DIRECTORY
- WAL 与应用日志共用 /app/logs 宿主机挂载
- 移除旧的 /data/syslog-archive 独立挂载
- 宿主机 WAL 路径统一为 /home/syslog/logs/syslog-wal
This commit is contained in:
王绍宏
2026-09-15 22:10:46 +08:00
parent 5a86a4f950
commit faa9ffe2d9
19 changed files with 553 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java
+4
View File
@@ -136,6 +136,10 @@ public class AppProperties {
@Value("${syslog.dispatch.shutdown-timeout-ms:30000}")
private long syslogDispatchShutdownTimeoutMs;
/** false 时 Syslog 无效设备正文在 WAL 前丢弃，历史无效记录也不创建 Terminal 审计文件。 */
@Value("${syslog.dispatch.terminal-record-enabled:false}")
private boolean syslogDispatchTerminalRecordEnabled;
@Value("${syslog.device-context.cache-ttl-ms:300000}")
private long syslogDeviceContextCacheTtlMs;
haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/DeviceContextResolver.java
+113 -2
View File
@@ -9,9 +9,17 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.time.LocalDateTime;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.LongAdder;
/**
@@ -31,6 +39,9 @@ public class DeviceContextResolver {
private final Object[] locks = new Object[64];
private final LongAdder hits = new LongAdder();
private final LongAdder misses = new LongAdder();
private final LongAdder ingressSnapshotRefreshFailed = new LongAdder();
private volatile IngressSnapshot ingressSnapshot = IngressSnapshot.unavailable();
private ScheduledExecutorService ingressSnapshotRefresher;
public DeviceContextResolver(DeviceDeviceService deviceService, DeviceUnknownService unknownService,
AppProperties properties) {
@@ -40,6 +51,77 @@ public class DeviceContextResolver {
for (int i = 0; i < locks.length; i++) locks[i] = new Object();
}
/** 网络线程只读取该快照，设备表加载固定在独立后台线程，禁止阻塞 Netty EventLoop。 */
@PostConstruct
public void initializeIngressSnapshot() {
if (properties.isSyslogDispatchTerminalRecordEnabled()) return;
long refreshIntervalMs = Math.max(1L, properties.getSyslogDeviceContextCacheTtlMs());
ingressSnapshotRefresher = Executors.newSingleThreadScheduledExecutor(runnable -> {
Thread thread = new Thread(runnable, "syslog-ingress-device-snapshot");
thread.setDaemon(true);
return thread;
});
// 不阻塞 Spring/Netty 启动；首次快照就绪前消息保守进入 WAL。
ingressSnapshotRefresher.scheduleWithFixedDelay(
this::refreshIngressSnapshot, 0L, refreshIntervalMs, TimeUnit.MILLISECONDS);
}
public void refreshIngressSnapshot() {
if (properties.isSyslogDispatchTerminalRecordEnabled()) return;
try {
List<DeviceDevice> devices = deviceService.getActiveDevices();
if (devices == null) throw new IllegalStateException("活动设备列表为空引用");
Map<String, List<DeviceDevice>> grouped = new HashMap<>();
for (DeviceDevice device : devices) {
if (device == null || device.getIp() == null || device.getIp().trim().isEmpty()) continue;
grouped.computeIfAbsent(device.getIp(), ignored -> new ArrayList<>()).add(device);
}
Map<String, Resolution> resolutions = new HashMap<>();
for (Map.Entry<String, List<DeviceDevice>> entry : grouped.entrySet()) {
List<DeviceDevice> sameIp = entry.getValue();
Resolution resolution = sameIp.size() == 1
? validateAssignment(sameIp.get(0))
: Resolution.terminal(Type.DUPLICATE_DEVICE, null,
"来源 IP 存在多条设备记录: " + entry.getKey());
resolutions.put(entry.getKey(), resolution);
}
ingressSnapshot = IngressSnapshot.ready(resolutions);
log.info("Syslog 入站设备内存快照已刷新: devices={}, sourceIps={}",
devices.size(), resolutions.size());
} catch (Throwable failure) {
ingressSnapshotRefreshFailed.increment();
log.error("Syslog 入站设备内存快照刷新失败，继续使用上一版本；无可用快照时消息保守进入 WAL", failure);
}
}
@PreDestroy
public void shutdownIngressSnapshotRefresher() {
if (ingressSnapshotRefresher != null) ingressSnapshotRefresher.shutdownNow();
}
/** 纯内存判断；绝不访问 Redis、达梦或 device_unknown。 */
public IngressStatus resolveIngress(String sourceIp, String protocol) {
IngressSnapshot snapshot = ingressSnapshot;
if (!snapshot.ready) return IngressStatus.UNAVAILABLE;
Resolution resolution = snapshot.resolutions.get(sourceIp);
if (resolution == null) {
logIngressInvalidOnce(snapshot, Type.UNKNOWN_DEVICE, sourceIp, protocol,
"来源 IP 未注册: " + sourceIp);
return IngressStatus.INVALID;
}
if (resolution.isRegistered()) return IngressStatus.REGISTERED;
logIngressInvalidOnce(snapshot, resolution.type, sourceIp, protocol, resolution.reason);
return IngressStatus.INVALID;
}
private void logIngressInvalidOnce(IngressSnapshot snapshot, Type type, String sourceIp,
String protocol, String reason) {
if (snapshot.loggedInvalidIps.putIfAbsent(sourceIp, Boolean.TRUE) == null) {
log.warn("Syslog 无效设备日志将直接丢弃: type={}, sourceIp={}, protocol={}, reason={}",
type, sourceIp, protocol, reason);
}
}
public Resolution resolve(String sourceIp, String protocol) {
long now = System.currentTimeMillis();
CacheEntry cached = cache.get(sourceIp);
@@ -78,8 +160,14 @@ public class DeviceContextResolver {
cache.put(sourceIp, new CacheEntry(resolution,
now + properties.getSyslogDeviceContextCacheTtlMs()));
if (resolution.type != Type.REGISTERED) {
log.warn("Syslog 设备接入终态: type={}, sourceIp={}, reason={}",
resolution.type, sourceIp, resolution.reason);
if (properties.isSyslogDispatchTerminalRecordEnabled()) {
log.warn("Syslog 无效设备日志将记录 Terminal 审计文件: type={}, sourceIp={}, reason={}",
resolution.type, sourceIp, resolution.reason);
} else {
// 无效结果会缓存；同一 IP 只在首次判定或缓存刷新时打印，避免逐条日志再次形成 IO 瓶颈。
log.warn("Syslog 无效设备日志将直接丢弃: type={}, sourceIp={}, reason={}",
resolution.type, sourceIp, resolution.reason);
}
}
return resolution;
}
@@ -134,6 +222,9 @@ public class DeviceContextResolver {
public long getHitCount() { return hits.sum(); }
public long getMissCount() { return misses.sum(); }
public int getCacheSize() { return cache.size() + deviceIdCache.size(); }
public boolean isIngressSnapshotReady() { return ingressSnapshot.ready; }
public int getIngressSnapshotSize() { return ingressSnapshot.resolutions.size(); }
public long getIngressSnapshotRefreshFailedCount() { return ingressSnapshotRefreshFailed.sum(); }
/** 未知设备仍保留原业务登记能力，但最多每个本地缓存 TTL 执行一次，不占用网络线程。 */
private void recordUnknown(String sourceIp, String protocol) {
@@ -165,6 +256,26 @@ public class DeviceContextResolver {
}
public enum Type { REGISTERED, UNKNOWN_DEVICE, DUPLICATE_DEVICE, NOT_ASSIGNED }
public enum IngressStatus { REGISTERED, INVALID, UNAVAILABLE }
private static final class IngressSnapshot {
private final boolean ready;
private final Map<String, Resolution> resolutions;
private final ConcurrentHashMap<String, Boolean> loggedInvalidIps = new ConcurrentHashMap<>();
private IngressSnapshot(boolean ready, Map<String, Resolution> resolutions) {
this.ready = ready;
this.resolutions = java.util.Collections.unmodifiableMap(new HashMap<>(resolutions));
}
private static IngressSnapshot unavailable() {
return new IngressSnapshot(false, java.util.Collections.emptyMap());
}
private static IngressSnapshot ready(Map<String, Resolution> resolutions) {
return new IngressSnapshot(true, resolutions);
}
}
public static final class Resolution {
private final Type type;
haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/SyslogWalDispatcher.java
+18 -4
View File
@@ -47,6 +47,7 @@ public class SyslogWalDispatcher {
private final Map<Long, SyslogWalRecord> completedOutOfOrder = new HashMap<>();
private final LongAdder kafkaAcked = new LongAdder();
private final LongAdder terminalFiltered = new LongAdder();
private final LongAdder terminalDropped = new LongAdder();
private final LongAdder retryCount = new LongAdder();
private final LongAdder largeDispatched = new LongAdder();
private final AtomicLong inflight = new AtomicLong();
@@ -88,10 +89,15 @@ public class SyslogWalDispatcher {
scheduler.scheduleWithFixedDelay(this::logStatus,
properties.getSyslogStatusLogIntervalMs(), properties.getSyslogStatusLogIntervalMs(),
TimeUnit.MILLISECONDS);
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, maxInflightBytes={}, checkpointSequence={}",
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, maxInflightBytes={}, "
+ "terminalRecordEnabled={}, checkpointSequence={}",
properties.getSyslogDispatchWorkerThreads(), properties.getSyslogDispatchMaxInflight(),
properties.getSyslogDispatchMaxInflightBytes(),
properties.isSyslogDispatchTerminalRecordEnabled(),
walService.getCheckpointSequence());
if (!properties.isSyslogDispatchTerminalRecordEnabled()) {
log.warn("无效设备日志直接丢弃已启用: 新 Syslog 正文不进 WAL，历史记录不创建 Terminal 文件");
}
}
private void readLoop() {
@@ -147,7 +153,11 @@ public class SyslogWalDispatcher {
if (!resolution.isRegistered()) {
try {
walService.recordTerminal(record, resolution.getType().name(), resolution.getReason());
if (properties.isSyslogDispatchTerminalRecordEnabled()) {
walService.recordTerminal(record, resolution.getType().name(), resolution.getReason());
} else {
terminalDropped.increment();
}
terminalFiltered.increment();
markComplete(record);
} catch (Throwable failure) {
@@ -252,14 +262,16 @@ public class SyslogWalDispatcher {
private void logStatus() {
if (!running.get()) return;
log.info("Syslog WAL状态: networkReceived={}, appended={}, admissionOversized={}, walPendingApprox={}, walPendingBytes={}, walQueue={}/{}, durableSequence={}, "
+ "checkpointSequence={}, kafkaInflight={}, kafkaInflightBytes={}, kafkaAcked={}, largeDispatched={}, terminalFiltered={}, "
+ "checkpointSequence={}, kafkaInflight={}, kafkaInflightBytes={}, kafkaAcked={}, largeDispatched={}, "
+ "terminalRecordEnabled={}, terminalFiltered={}, terminalDropped={}, "
+ "retry={}, tcpPauseTotal={}, udpDropped={}, deviceCacheSize={}, deviceCacheHits={}, "
+ "deviceCacheMisses={}",
SyslogMessageHandler.getNetworkReceivedCount(), walService.getAppendedCount(),
walService.getOversizedCount(), walService.getPendingRecordsApprox(), walService.getPendingAppendBytes(),
walService.getQueueSize(), walService.getQueueCapacity(), walService.getDurableSequence(),
walService.getCheckpointSequence(), inflight.get(), byteBudget == null ? 0L : byteBudget.used(),
kafkaAcked.sum(), largeDispatched.sum(), terminalFiltered.sum(),
kafkaAcked.sum(), largeDispatched.sum(), properties.isSyslogDispatchTerminalRecordEnabled(),
terminalFiltered.sum(), terminalDropped.sum(),
retryCount.sum(), SyslogMessageHandler.getTcpPauseCount(),
SyslogMessageHandler.getUdpDroppedCount(), deviceResolver.getCacheSize(),
deviceResolver.getHitCount(), deviceResolver.getMissCount());
@@ -318,6 +330,8 @@ public class SyslogWalDispatcher {
public long getInflight() { return inflight.get(); }
public long getKafkaAcked() { return kafkaAcked.sum(); }
public long getTerminalFiltered() { return terminalFiltered.sum(); }
public long getTerminalDropped() { return terminalDropped.sum(); }
public boolean isTerminalRecordEnabled() { return properties.isSyslogDispatchTerminalRecordEnabled(); }
public long getRetryCount() { return retryCount.sum(); }
public long getLargeDispatched() { return largeDispatched.sum(); }
public long getInflightBytes() { return byteBudget == null ? 0L : byteBudget.used(); }
haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalStatusController.java
+7
View File
@@ -30,6 +30,8 @@ public class SyslogWalStatusController {
Map<String, Object> value = new LinkedHashMap<>();
value.put("network_received", SyslogMessageHandler.getNetworkReceivedCount());
value.put("network_oversized_dropped", SyslogMessageHandler.getOversizedDroppedCount());
value.put("network_invalid_device_dropped", SyslogMessageHandler.getInvalidDeviceDroppedCount());
value.put("network_device_lookup_failed", SyslogMessageHandler.getDeviceLookupFailedCount());
value.put("wal_appended", wal.getAppendedCount());
value.put("wal_queue_size", wal.getQueueSize());
value.put("wal_queue_capacity", wal.getQueueCapacity());
@@ -55,13 +57,18 @@ public class SyslogWalStatusController {
value.put("dispatch_inflight_bytes", dispatcher.getInflightBytes());
value.put("kafka_acked", dispatcher.getKafkaAcked());
value.put("kafka_large_dispatched", dispatcher.getLargeDispatched());
value.put("dispatch_terminal_record_enabled", dispatcher.isTerminalRecordEnabled());
value.put("dispatch_terminal_filtered", dispatcher.getTerminalFiltered());
value.put("dispatch_terminal_dropped", dispatcher.getTerminalDropped());
value.put("dispatch_retry", dispatcher.getRetryCount());
value.put("tcp_pause_total", SyslogMessageHandler.getTcpPauseCount());
value.put("udp_dropped_total", SyslogMessageHandler.getUdpDroppedCount());
value.put("device_cache_size", resolver.getCacheSize());
value.put("device_cache_hits", resolver.getHitCount());
value.put("device_cache_misses", resolver.getMissCount());
value.put("device_ingress_snapshot_ready", resolver.isIngressSnapshotReady());
value.put("device_ingress_snapshot_size", resolver.getIngressSnapshotSize());
value.put("device_ingress_snapshot_refresh_failed", resolver.getIngressSnapshotRefreshFailedCount());
return value;
}
}