feat: 新增设备 API 拉取采集及配套处理能力
- 接入 NDRServer 告警和流量日志,支持凭据与任务管理 - 支持分页拉取、窗口拆分、调度优先级及 WAL 持久化后推进断点 - 增加迟到告警规则重放,优化多条泛化规则选择和事件时间处理 - 补充回归测试、数据库升级脚本及部署文档
This commit is contained in:
+36
-4
@@ -25,7 +25,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
|
||||
/** 从 WAL 有界并发异步投递 Kafka,并只按连续 ACK 推进 WAL 游标。 */
|
||||
/**
|
||||
* 从 WAL 有界并发异步投递 Kafka,并只按连续 ACK 推进 WAL 游标。
|
||||
*
|
||||
* <p>v2/v3 保持原有按 source_ip 识别设备和单时间参数的生产路径;v4 改为按可信 device_id
|
||||
* 识别设备,并分别传递 receive_time/event_time。兼容分支留在 Dispatcher,Kafka 消息外层结构
|
||||
* 不因接入方式改变。
|
||||
*/
|
||||
@Component
|
||||
public class SyslogWalDispatcher {
|
||||
private static final Logger log = LoggerFactory.getLogger(SyslogWalDispatcher.class);
|
||||
@@ -120,7 +126,12 @@ public class SyslogWalDispatcher {
|
||||
}
|
||||
final DeviceContextResolver.Resolution resolution;
|
||||
try {
|
||||
resolution = deviceResolver.resolve(record.getSourceIp(), record.getProtocol());
|
||||
// API 的 HTTP 对端可能是代理或网关;v4 必须使用任务写入 WAL 的 device_id。
|
||||
if (record.getSchemaVersion() >= 4 && record.getDeviceId() != null) {
|
||||
resolution = deviceResolver.resolveDeviceId(record.getDeviceId());
|
||||
} else {
|
||||
resolution = deviceResolver.resolve(record.getSourceIp(), record.getProtocol());
|
||||
}
|
||||
} catch (Throwable failure) {
|
||||
scheduleRetry(record, attempt, "DEVICE_LOOKUP", failure);
|
||||
return;
|
||||
@@ -138,10 +149,31 @@ public class SyslogWalDispatcher {
|
||||
}
|
||||
|
||||
final String receiveTime;
|
||||
final String eventTime;
|
||||
try {
|
||||
receiveTime = LocalDateTime.parse(record.getReceiveTime(), WAL_TIME).format(HEADER_TIME);
|
||||
producer.messagePush(resolution.getDevice(), record.getRawMessage(), receiveTime,
|
||||
record.getEventId(), record.getPointTimeNs())
|
||||
String walEventTime = record.getEventTime() == null
|
||||
? record.getReceiveTime()
|
||||
: record.getEventTime();
|
||||
eventTime = LocalDateTime.parse(walEventTime, WAL_TIME).format(HEADER_TIME);
|
||||
|
||||
// v2/v3 继续调用旧重载,确保升级前已落盘记录产生的 Kafka 消息完全不变。
|
||||
java.util.concurrent.CompletableFuture<kafkaProducer.DeliveryResult> deliveryFuture =
|
||||
record.getSchemaVersion() >= 4
|
||||
? producer.messagePush(
|
||||
resolution.getDevice(),
|
||||
record.getRawMessage(),
|
||||
receiveTime,
|
||||
eventTime,
|
||||
record.getEventId(),
|
||||
record.getPointTimeNs())
|
||||
: producer.messagePush(
|
||||
resolution.getDevice(),
|
||||
record.getRawMessage(),
|
||||
receiveTime,
|
||||
record.getEventId(),
|
||||
record.getPointTimeNs());
|
||||
deliveryFuture
|
||||
.whenComplete((delivery, failure) -> {
|
||||
if (failure == null) {
|
||||
kafkaAcked.increment();
|
||||
|
||||
Reference in New Issue
Block a user