nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 新增设备 API 拉取采集及配套处理能力

Browse Source 
- 接入 NDRServer 告警和流量日志，支持凭据与任务管理
- 支持分页拉取、窗口拆分、调度优先级及 WAL 持久化后推进断点
- 增加迟到告警规则重放，优化多条泛化规则选择和事件时间处理
- 补充回归测试、数据库升级脚本及部署文档
This commit is contained in:
王绍宏
2026-09-06 16:46:21 +08:00
parent b61ec2e37c
commit 00c0a1fb7f
141 changed files with 11318 additions and 467 deletions
Download Patch File Download Diff File Expand all files Collapse all files
haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalService.java
+234 -41
View File
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.haobang.config.AppProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
@@ -47,9 +48,13 @@ import java.util.concurrent.atomic.LongAdder;
/**
* Kafka 接管前的本地持久化 WAL。
*
* <p>网络线程只向有界队列提交事件；单写线程顺序追加并批量 force。Dispatcher 只读取已经
* force 的记录，Kafka ACK 后按连续 sequence 推进持久化游标。崩溃恢复可能重复投递游标后
* 的少量记录，但 event_id 不变，由下游幂等约束消除业务重复。</p>
* <p>网络/API 线程只向有界队列提交事件；单写线程顺序追加并批量 force。Dispatcher 只读取已经
* force 的记录，Kafka ACK 后按连续 sequence 推进持久化游标。崩溃恢复可能重复投递游标后的
* 少量记录，但 event_id 不变，由下游幂等约束消除业务重复。
*
* <p>API 拉取使用事件级 durable receipt：事件进入队列只表示 accepted，只有覆盖该事件的
* {@code FileChannel.force(true)} 成功后 future 才完成。force 失败时 WAL 进入 fail-closed，所有
* 已接收但未 durable 的 future 异常完成，调用方因此不会错误推进设备 checkpoint。
*/
@Component
public class SyslogWalService {
@@ -57,12 +62,14 @@ public class SyslogWalService {
private static final DateTimeFormatter RECORD_TIME = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS");
private static final DateTimeFormatter FILE_TIME = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmssSSS");
private static final String SEGMENT_GLOB = "syslog-wal-*.jsonl";
static final long COMPLETED_CLEANUP_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(30L);
public enum AdmissionResult { ACCEPTED, BACKPRESSURE, UNAVAILABLE }
private final AppProperties properties;
private final ObjectMapper objectMapper;
private final Charset syslogCharset;
private final ForceOperation forceOperation;
private final PointTimeAllocator pointTimeAllocator = new PointTimeAllocator();
private final AtomicBoolean initialized = new AtomicBoolean();
private final AtomicLong nextSequence = new AtomicLong();
@@ -82,8 +89,14 @@ public class SyslogWalService {
private volatile long lastCheckpointFlushNanos;
private volatile DiskState diskState = DiskState.NORMAL;
private volatile long lastDiskCheckNanos;
private volatile long diskTotalBytes;
private volatile long diskUsableBytes;
private volatile long diskStopThresholdBytes;
private volatile long lastCompletedCleanupNanos;
private ArrayBlockingQueue<SyslogIngressEvent> appendQueue;
private ArrayBlockingQueue<PendingWalAppend> appendQueue;
/** 已 write、尚未被一次成功 force 覆盖的事件；仅由写线程访问。 */
private final List<PendingWalAppend> pendingForce = new ArrayList<>();
private Thread writerThread;
private Path rootDirectory;
private Path activeDirectory;
@@ -100,7 +113,21 @@ public class SyslogWalService {
private long lastWrittenSequence;
private long lastForceNanos;
/**
* Spring 生产构造器。
*
* <p>另一个接收 {@link ForceOperation} 的包级构造器用于测试刷盘失败。存在多个构造器时必须
* 显式标记生产构造器，否则 Spring 会尝试寻找无参构造器并导致应用启动失败。</p>
*/
@Autowired
public SyslogWalService(AppProperties properties, ObjectMapper objectMapper) {
this(properties, objectMapper, FileChannel::force);
}
SyslogWalService(
AppProperties properties,
ObjectMapper objectMapper,
ForceOperation forceOperation) {
this.properties = properties;
/*
* WAL 是内部、固定 schema 的持久化协议，不能直接复用 Redis 缓存的多态 ObjectMapper。
@@ -115,6 +142,7 @@ public class SyslogWalService {
*/
this.objectMapper = objectMapper.copy().deactivateDefaultTyping();
this.syslogCharset = resolveSyslogCharset(properties.getSyslogCharset());
this.forceOperation = forceOperation;
}
@PostConstruct
@@ -147,7 +175,7 @@ public class SyslogWalService {
long maxSequence = Math.max(persistedCheckpoint.sequence, scanMaxSequence());
nextSequence.set(maxSequence);
durableSequence.set(maxSequence);
cleanupCompletedSegments();
cleanupCompletedSegments(true);
refreshDiskState(true);
} catch (Exception failure) {
fatalFailure = failure;
@@ -163,9 +191,15 @@ public class SyslogWalService {
writerThread.setDaemon(true);
writerThread.start();
log.info("Syslog WAL 已启用: directory={}, charset={}, queueCapacity={}, highWatermark={}%, lowWatermark={}%, "
+ "retentionDays={}, completedCleanupIntervalMinutes={}, "
+ "diskUsedWatermark={}%, diskMaxFreeBytes={}, diskStopThresholdBytes={}, "
+ "flushRecords={}, flushIntervalMs={}, checkpointSequence={}",
rootDirectory, syslogCharset.name(), properties.getSyslogWalQueueCapacity(),
properties.getSyslogWalHighWatermarkPercent(), properties.getSyslogWalLowWatermarkPercent(),
properties.getSyslogWalRetentionDays(),
TimeUnit.NANOSECONDS.toMinutes(COMPLETED_CLEANUP_INTERVAL_NANOS),
properties.getSyslogWalDiskHighWatermarkPercent(), properties.getSyslogWalMinFreeBytes(),
diskStopThresholdBytes,
properties.getSyslogWalFlushRecords(), properties.getSyslogWalFlushIntervalMs(),
persistedCheckpoint.sequence);
}
@@ -174,15 +208,35 @@ public class SyslogWalService {
if (properties.isSyslogPointTimeWriteEnabled() && event.getPointTimeNs() == null) {
throw new IllegalArgumentException("point_time 写入已开启，SyslogIngressEvent 必须先固化 point_time_ns");
}
AppendAttempt attempt = tryAppendUnified(toUnified(event));
if (attempt.getStatus() == AdmissionStatus.ACCEPTED) return AdmissionResult.ACCEPTED;
if (attempt.getStatus() == AdmissionStatus.BACKPRESSURE) return AdmissionResult.BACKPRESSURE;
return AdmissionResult.UNAVAILABLE;
}
/**
* 非阻塞接入统一事件。
*
* <p>返回 ACCEPTED 只代表进入有界队列；调用方必须等待 durableStage 成功后才能提交自己的
* 上游进度。BACKPRESSURE 可以使用同一个事件对象重试，UNAVAILABLE 表示 WAL 已 fail-closed。
*/
public AppendAttempt tryAppendUnified(IngressEvent event) {
if (event == null) throw new IllegalArgumentException("IngressEvent 不能为空");
if (event.getPointTimeNs() == null
&& (event.getIngressType() == IngressType.API_PULL
|| properties.isSyslogPointTimeWriteEnabled())) {
throw new IllegalArgumentException("IngressEvent 必须先通过 prepareForWal 固化 point_time_ns");
}
refreshDiskState(false);
if (!running || fatalFailure != null || diskState == DiskState.CRITICAL) {
return AdmissionResult.UNAVAILABLE;
return AppendAttempt.rejected(AdmissionStatus.UNAVAILABLE);
}
if (!appendQueue.offer(event)) {
PendingWalAppend pending = new PendingWalAppend(event);
if (!appendQueue.offer(pending)) {
admissionBackpressure.increment();
return AdmissionResult.BACKPRESSURE;
return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE);
}
return AdmissionResult.ACCEPTED;
return AppendAttempt.accepted(pending.getDurableFuture());
}
public boolean shouldPauseTcp() {
@@ -219,6 +273,10 @@ public class SyslogWalService {
}
public Throwable getFatalFailure() { return fatalFailure; }
public boolean isDiskCritical() { refreshDiskState(false); return diskState == DiskState.CRITICAL; }
public String getDiskState() { refreshDiskState(false); return diskState.name(); }
public long getDiskTotalBytes() { refreshDiskState(false); return diskTotalBytes; }
public long getDiskUsableBytes() { refreshDiskState(false); return diskUsableBytes; }
public long getDiskStopThresholdBytes() { refreshDiskState(false); return diskStopThresholdBytes; }
/** 每个事件只调用一次；TCP 背压重试必须复用返回对象，不能重新分配 Point 身份。 */
public SyslogIngressEvent prepareForWal(SyslogIngressEvent event) {
@@ -230,6 +288,35 @@ public class SyslogWalService {
return event.withPointTimeNs(pointTimeAllocator.allocate(receiveTimeMillis));
}
/**
* 为事件分配稳定 point_time_ns。
* 每个事件只调用一次，API 背压重试必须复用返回对象，避免同一事件身份在重试时变化。
*/
public IngressEvent prepareIngressEvent(IngressEvent event) {
if (event.getPointTimeNs() != null) return event;
return event.withPointTimeNs(pointTimeAllocator.allocate(event.getEventTime().toEpochMilli()));
}
private IngressEvent toUnified(SyslogIngressEvent event) {
Instant received = event.getReceivedAt().atZone(ZoneId.of("Asia/Shanghai")).toInstant();
return new IngressEvent(
event.getEventId(),
received,
received,
IngressType.SYSLOG_PUSH,
event.getProtocol(),
null,
null,
null,
null,
event.getSourceIp(),
event.getSourcePort(),
"TEXT",
syslogCharset,
event.getRawMessage(),
event.getPointTimeNs());
}
public WalCursor openCursor() throws IOException {
return new WalCursor(persistedCheckpoint.copy());
}
@@ -303,8 +390,9 @@ public class SyslogWalService {
private void writerLoop() {
try {
while (running || !appendQueue.isEmpty()) {
SyslogIngressEvent event = appendQueue.poll(properties.getSyslogWalFlushIntervalMs(), TimeUnit.MILLISECONDS);
if (event != null) appendEvent(event);
PendingWalAppend pending = appendQueue.poll(
properties.getSyslogWalFlushIntervalMs(), TimeUnit.MILLISECONDS);
if (pending != null) appendEvent(pending);
long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastForceNanos);
if (recordsSinceForce > 0 && (recordsSinceForce >= properties.getSyslogWalFlushRecords()
|| elapsedMs >= properties.getSyslogWalFlushIntervalMs())) {
@@ -313,7 +401,9 @@ public class SyslogWalService {
flushCheckpointIfDue();
}
forceWriter();
synchronized (checkpointMonitor) { flushCheckpointLocked(); }
synchronized (checkpointMonitor) {
flushCheckpointLocked();
}
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
failWal(interrupted);
@@ -324,18 +414,26 @@ public class SyslogWalService {
}
}
private void appendEvent(SyslogIngressEvent event) throws Exception {
private void appendEvent(PendingWalAppend pending) throws Exception {
IngressEvent event = pending.getEvent();
long sequence = nextSequence.incrementAndGet();
pending.setSequence(sequence);
// 当前元素已经离开队列；先登记，确保序列化/write 异常也会异常完成 receipt。
pendingForce.add(pending);
SyslogWalRecord record = toRecord(sequence, event);
pending.setSchemaVersion(record.getSchemaVersion());
byte[] json = objectMapper.writeValueAsBytes(record);
byte[] line = new byte[json.length + 1];
System.arraycopy(json, 0, line, 0, json.length);
line[line.length - 1] = (byte) '\n';
LocalDate receiveDate = event.getReceivedAt().toLocalDate();
LocalDate receiveDate = event.getReceivedAt()
.atZone(ZoneId.of("Asia/Shanghai"))
.toLocalDate();
if (writerChannel == null) openWriter(sequence, receiveDate);
if (writerBytes > 0 && (writerBytes + line.length > properties.getSyslogWalSegmentMaxBytes()
|| !receiveDate.equals(writerDate))) {
if (writerBytes > 0
&& (writerBytes + line.length > properties.getSyslogWalSegmentMaxBytes()
|| !receiveDate.equals(writerDate))) {
forceWriter();
closeWriter();
openWriter(sequence, receiveDate);
@@ -346,10 +444,21 @@ public class SyslogWalService {
lastWrittenSequence = sequence;
}
private SyslogWalRecord toRecord(long sequence, SyslogIngressEvent event) throws Exception {
byte[] raw = event.getRawMessage().getBytes(syslogCharset);
private SyslogWalRecord toRecord(long sequence, IngressEvent event) throws Exception {
byte[] raw = event.getRawMessage().getBytes(event.getCharset());
SyslogWalRecord record = new SyslogWalRecord();
if (properties.isSyslogPointTimeWriteEnabled()) {
if (event.getIngressType() == IngressType.API_PULL) {
record.setSchemaVersion(4);
record.setEventTime(formatWalTime(event.getEventTime()));
record.setPointTimeNs(event.getPointTimeNs());
record.setIngressType(event.getIngressType().name());
record.setTransport(event.getTransport());
record.setDeviceId(event.getDeviceId());
record.setCollectTaskId(event.getCollectTaskId());
record.setAdapterType(event.getAdapterType());
record.setExternalEventId(event.getExternalEventId());
record.setContentType(event.getContentType());
} else if (properties.isSyslogPointTimeWriteEnabled()) {
record.setSchemaVersion(3);
record.setPointTimeNs(event.getPointTimeNs());
} else {
@@ -357,11 +466,11 @@ public class SyslogWalService {
}
record.setSequence(sequence);
record.setEventId(event.getEventId());
record.setReceiveTime(event.getReceivedAt().format(RECORD_TIME));
record.setReceiveTime(formatWalTime(event.getReceivedAt()));
record.setSourceIp(event.getSourceIp());
record.setSourcePort(event.getSourcePort());
record.setProtocol(event.getProtocol());
record.setCharset(syslogCharset.name());
record.setSourcePort(event.getSourcePort() == null ? 0 : event.getSourcePort());
record.setProtocol(event.getTransport());
record.setCharset(event.getCharset().name());
record.setRawLength(raw.length);
record.setRawSha256(sha256(raw));
record.setRawMessage(event.getRawMessage());
@@ -369,22 +478,42 @@ public class SyslogWalService {
return record;
}
private String formatWalTime(Instant value) {
return LocalDateTime.ofInstant(value, ZoneId.of("Asia/Shanghai")).format(RECORD_TIME);
}
private void forceWriter() throws IOException {
if (writerChannel == null || recordsSinceForce == 0) return;
writerChannel.force(false);
// receipt 代表可恢复边界，文件内容与长度元数据都必须先持久化。
forceOperation.force(writerChannel, true);
Instant durableAt = Instant.now();
durableSequence.set(lastWrittenSequence);
appended.add(recordsSinceForce);
for (PendingWalAppend pending : pendingForce) {
DurableReceipt receipt = new DurableReceipt(
pending.getEvent().getEventId(),
pending.getSequence(),
pending.getSchemaVersion(),
durableAt);
pending.getDurableFuture().complete(receipt);
}
pendingForce.clear();
recordsSinceForce = 0;
lastForceNanos = System.nanoTime();
synchronized (durableMonitor) { durableMonitor.notifyAll(); }
synchronized (durableMonitor) {
durableMonitor.notifyAll();
}
}
private void openWriter(long startSequence, LocalDate date) throws IOException {
String name = String.format("syslog-wal-%020d-%s.jsonl", startSequence,
LocalDateTime.now().format(FILE_TIME));
writerPath = activeDirectory.resolve(name);
writerChannel = FileChannel.open(writerPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
writerChannel = FileChannel.open(
writerPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
applyFilePermissions(writerPath);
// 新目录项也需要持久化；否则断电后文件数据虽 force，分段文件名仍可能丢失。
forceDirectoryBestEffort(activeDirectory);
writerDate = date;
writerBytes = 0L;
}
@@ -399,10 +528,26 @@ public class SyslogWalService {
private void failWal(Throwable failure) {
fatalFailure = failure;
running = false;
synchronized (durableMonitor) { durableMonitor.notifyAll(); }
// 任何尚未 durable 的调用方都必须收到失败，不能留下永久等待或假成功的 receipt。
completePendingExceptionally(failure);
synchronized (durableMonitor) {
durableMonitor.notifyAll();
}
log.error("Syslog WAL 发生致命故障，接入已进入 fail-closed 状态", failure);
}
private void completePendingExceptionally(Throwable failure) {
for (PendingWalAppend pending : pendingForce) {
pending.getDurableFuture().completeExceptionally(failure);
}
pendingForce.clear();
if (appendQueue == null) return;
PendingWalAppend pending;
while ((pending = appendQueue.poll()) != null) {
pending.getDurableFuture().completeExceptionally(failure);
}
}
private int percentageCount(int percent) {
return Math.max(1, (int) Math.floor(properties.getSyslogWalQueueCapacity() * percent / 100.0));
}
@@ -428,10 +573,8 @@ public class SyslogWalService {
throw new IllegalArgumentException("WAL 水位必须满足 0 <= low < high < 100");
}
if (properties.getSyslogWalDiskHighWatermarkPercent() <= 0
|| properties.getSyslogWalDiskCriticalWatermarkPercent()
<= properties.getSyslogWalDiskHighWatermarkPercent()
|| properties.getSyslogWalDiskCriticalWatermarkPercent() >= 100) {
throw new IllegalArgumentException("WAL 磁盘水位必须满足 0 < high < critical < 100");
|| properties.getSyslogWalDiskHighWatermarkPercent() >= 100) {
throw new IllegalArgumentException("WAL 磁盘使用率保护线必须满足 0 < high < 100");
}
if (properties.getSyslogWalRetentionDays() < 0 || properties.getSyslogWalMinFreeBytes() < 0) {
throw new IllegalArgumentException("WAL 保留天数和最小剩余字节不能小于 0");
@@ -557,16 +700,32 @@ public class SyslogWalService {
if (name.compareTo(checkpointSegment) >= 0 || segment.equals(writerPath)) continue;
Files.move(segment, completedDirectory.resolve(name), StandardCopyOption.REPLACE_EXISTING);
}
cleanupCompletedSegments();
cleanupCompletedSegments(false);
}
private void cleanupCompletedSegments() throws IOException {
private void cleanupCompletedSegments(boolean force) throws IOException {
int retentionDays = properties.getSyslogWalRetentionDays();
if (retentionDays <= 0) return;
long nowNanos = System.nanoTime();
if (!force && !isCompletedCleanupDue(lastCompletedCleanupNanos, nowNanos)) return;
lastCompletedCleanupNanos = nowNanos;
Instant cutoff = LocalDate.now().minusDays(retentionDays).atStartOfDay(ZoneId.systemDefault()).toInstant();
int deletedSegments = 0;
for (Path path : listSegments(completedDirectory)) {
if (Files.getLastModifiedTime(path).toInstant().isBefore(cutoff)) Files.deleteIfExists(path);
if (Files.getLastModifiedTime(path).toInstant().isBefore(cutoff)
&& Files.deleteIfExists(path)) {
deletedSegments++;
}
}
if (deletedSegments > 0) {
log.info("WAL 已完成分段清理完成: retentionDays={}, cutoff={}, deletedSegments={}",
retentionDays, cutoff, deletedSegments);
}
}
static boolean isCompletedCleanupDue(long lastCleanupNanos, long nowNanos) {
return nowNanos - lastCleanupNanos >= COMPLETED_CLEANUP_INTERVAL_NANOS;
}
private List<Path> listSegments(Path directory) throws IOException {
@@ -589,13 +748,20 @@ public class SyslogWalService {
long total = store.getTotalSpace();
long usable = store.getUsableSpace();
long usedPercent = total <= 0 ? 0 : ((total - usable) * 100L / total);
DiskState next = usedPercent >= properties.getSyslogWalDiskCriticalWatermarkPercent()
|| usable < properties.getSyslogWalMinFreeBytes() ? DiskState.CRITICAL
: usedPercent >= properties.getSyslogWalDiskHighWatermarkPercent()
? DiskState.HIGH : DiskState.NORMAL;
long stopThresholdBytes = calculateDiskStopThresholdBytes(total,
properties.getSyslogWalDiskHighWatermarkPercent(),
properties.getSyslogWalMinFreeBytes());
diskTotalBytes = total;
diskUsableBytes = usable;
diskStopThresholdBytes = stopThresholdBytes;
DiskState next = isDiskStopRequired(usable, stopThresholdBytes)
? DiskState.CRITICAL : DiskState.NORMAL;
if (next != diskState) {
log.warn("WAL 磁盘状态变化: previous={}, current={}, usedPercent={}%, usableBytes={}",
diskState, next, usedPercent, usable);
log.warn("WAL 磁盘状态变化: previous={}, current={}, usedPercent={}%, usableBytes={}, "
+ "stopThresholdBytes={}, configuredUsedPercent={}%, configuredMaxFreeBytes={}",
diskState, next, usedPercent, usable, stopThresholdBytes,
properties.getSyslogWalDiskHighWatermarkPercent(),
properties.getSyslogWalMinFreeBytes());
}
diskState = next;
} catch (IOException failure) {
@@ -605,6 +771,26 @@ public class SyslogWalService {
}
}
/**
* 将使用率保护线换算为剩余字节数，再与固定容量上限取较小值。
* 例如 2TiB 磁盘在 90% 时约剩 205GiB，最终按 10GiB 触发；
* 50GiB 磁盘在 90% 时剩 5GiB，最终按 5GiB 触发。
*/
static long calculateDiskStopThresholdBytes(long totalBytes, int usedWatermarkPercent,
long maxFreeBytes) {
if (totalBytes <= 0 || maxFreeBytes <= 0) return 0L;
int remainingPercent = 100 - usedWatermarkPercent;
long quotient = totalBytes / 100L;
long remainder = totalBytes % 100L;
long percentageThreshold = quotient * remainingPercent
+ (remainder * remainingPercent + 99L) / 100L;
return Math.min(percentageThreshold, maxFreeBytes);
}
static boolean isDiskStopRequired(long usableBytes, long stopThresholdBytes) {
return stopThresholdBytes > 0L && usableBytes <= stopThresholdBytes;
}
private String sha256(byte[] value) throws Exception {
byte[] digest = MessageDigest.getInstance("SHA-256").digest(value);
StringBuilder hex = new StringBuilder(digest.length * 2);
@@ -658,6 +844,7 @@ public class SyslogWalService {
if (writerThread.isAlive()) {
log.error("WAL 写线程未在超时内退出: remaining={}", appendQueue.size());
writerThread.interrupt();
completePendingExceptionally(new IOException("WAL 优雅关闭超时"));
}
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
@@ -830,5 +1017,11 @@ public class SyslogWalService {
private Checkpoint copy() { return new Checkpoint(sequence, segment, offset); }
}
private enum DiskState { NORMAL, HIGH, CRITICAL }
private enum DiskState { NORMAL, CRITICAL }
/** 测试注入点，用于验证 force 失败时 receipt 和 fail-closed 语义。 */
@FunctionalInterface
interface ForceOperation {
void force(FileChannel channel, boolean metadata) throws IOException;
}
}