feat: 新增设备 API 拉取采集及配套处理能力

- 接入 NDRServer 告警和流量日志,支持凭据与任务管理
- 支持分页拉取、窗口拆分、调度优先级及 WAL 持久化后推进断点
- 增加迟到告警规则重放,优化多条泛化规则选择和事件时间处理
- 补充回归测试、数据库升级脚本及部署文档
This commit is contained in:
王绍宏
2026-09-06 16:46:21 +08:00
parent b61ec2e37c
commit 00c0a1fb7f
141 changed files with 11318 additions and 467 deletions
@@ -25,7 +25,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
/** 从 WAL 有界并发异步投递 Kafka,并只按连续 ACK 推进 WAL 游标。 */
/**
* 从 WAL 有界并发异步投递 Kafka,并只按连续 ACK 推进 WAL 游标。
*
* <p>v2/v3 保持原有按 source_ip 识别设备和单时间参数的生产路径;v4 改为按可信 device_id
* 识别设备,并分别传递 receive_time/event_time。兼容分支留在 Dispatcher,Kafka 消息外层结构
* 不因接入方式改变。
*/
@Component
public class SyslogWalDispatcher {
private static final Logger log = LoggerFactory.getLogger(SyslogWalDispatcher.class);
@@ -120,7 +126,12 @@ public class SyslogWalDispatcher {
}
final DeviceContextResolver.Resolution resolution;
try {
resolution = deviceResolver.resolve(record.getSourceIp(), record.getProtocol());
// API 的 HTTP 对端可能是代理或网关;v4 必须使用任务写入 WAL 的 device_id。
if (record.getSchemaVersion() >= 4 && record.getDeviceId() != null) {
resolution = deviceResolver.resolveDeviceId(record.getDeviceId());
} else {
resolution = deviceResolver.resolve(record.getSourceIp(), record.getProtocol());
}
} catch (Throwable failure) {
scheduleRetry(record, attempt, "DEVICE_LOOKUP", failure);
return;
@@ -138,10 +149,31 @@ public class SyslogWalDispatcher {
}
final String receiveTime;
final String eventTime;
try {
receiveTime = LocalDateTime.parse(record.getReceiveTime(), WAL_TIME).format(HEADER_TIME);
producer.messagePush(resolution.getDevice(), record.getRawMessage(), receiveTime,
record.getEventId(), record.getPointTimeNs())
String walEventTime = record.getEventTime() == null
? record.getReceiveTime()
: record.getEventTime();
eventTime = LocalDateTime.parse(walEventTime, WAL_TIME).format(HEADER_TIME);
// v2/v3 继续调用旧重载,确保升级前已落盘记录产生的 Kafka 消息完全不变。
java.util.concurrent.CompletableFuture<kafkaProducer.DeliveryResult> deliveryFuture =
record.getSchemaVersion() >= 4
? producer.messagePush(
resolution.getDevice(),
record.getRawMessage(),
receiveTime,
eventTime,
record.getEventId(),
record.getPointTimeNs())
: producer.messagePush(
resolution.getDevice(),
record.getRawMessage(),
receiveTime,
record.getEventId(),
record.getPointTimeNs());
deliveryFuture
.whenComplete((delivery, failure) -> {
if (failure == null) {
kafkaAcked.increment();