feat: 新增设备 API 拉取采集及配套处理能力
- 接入 NDRServer 告警和流量日志，支持凭据与任务管理 - 支持分页拉取、窗口拆分、调度优先级及 WAL 持久化后推进断点 - 增加迟到告警规则重放，优化多条泛化规则选择和事件时间处理 - 补充回归测试、数据库升级脚本及部署文档
This commit is contained in:
王绍宏
+10
-1
@@ -32,6 +32,7 @@ public class DmStandardizedBatchExecutor {
|
||||
private final StatsChangeCaptureService statsChangeCaptureService;
|
||||
private final DirectStatsProjectionWriter directStatsProjectionWriter;
|
||||
private final DmDirectTransactionMetrics transactionMetrics;
|
||||
private final LateAlarmBucketWriter lateAlarmBucketWriter;
|
||||
|
||||
@org.springframework.beans.factory.annotation.Value("${stats.direct.transaction-log-enabled:false}")
|
||||
private boolean transactionLogEnabled;
|
||||
@@ -46,7 +47,8 @@ public class DmStandardizedBatchExecutor {
|
||||
PlatformTransactionManager transactionManager,
|
||||
StatsChangeCaptureService statsChangeCaptureService,
|
||||
DirectStatsProjectionWriter directStatsProjectionWriter,
|
||||
DmDirectTransactionMetrics transactionMetrics) {
|
||||
DmDirectTransactionMetrics transactionMetrics,
|
||||
LateAlarmBucketWriter lateAlarmBucketWriter) {
|
||||
// SqlSessionTemplate 本身线程安全。ExecutorType.BATCH 会把相同 SQL 的参数暂存在当前事务会话中，
|
||||
// 直到 flushStatements()/commit 时才真正交给 JDBC 驱动批量执行。
|
||||
this.batchSqlSession = new SqlSessionTemplate(sqlSessionFactory, ExecutorType.BATCH);
|
||||
@@ -54,6 +56,7 @@ public class DmStandardizedBatchExecutor {
|
||||
this.statsChangeCaptureService = statsChangeCaptureService;
|
||||
this.directStatsProjectionWriter = directStatsProjectionWriter;
|
||||
this.transactionMetrics = transactionMetrics;
|
||||
this.lateAlarmBucketWriter = lateAlarmBucketWriter;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -126,6 +129,12 @@ public class DmStandardizedBatchExecutor {
|
||||
batchSqlSession.flushStatements();
|
||||
timing.factFlushMs = elapsedMs(flushStarted);
|
||||
|
||||
// 与标准化告警事实共用同一事务写入事件时间桶。事实成功而通知失败时必须整体回滚，
|
||||
// 否则迟到数据可能永久失去重放机会。
|
||||
if (alarmTable) {
|
||||
lateAlarmBucketWriter.record(rows);
|
||||
}
|
||||
|
||||
// DIRECT/SHADOW 模式在当前事务内按设备、规则聚合写统计；LEGACY 模式会直接返回。
|
||||
// 任一统计 SQL 失败都会回滚本事务的事实行，禁止产生无法定位的统计缺口。
|
||||
timing.statsResult = directStatsProjectionWriter.writeStandardizedRows(rows);
|
||||
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
package com.common.service;
|
||||
|
||||
import java.sql.Timestamp;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.dao.CannotAcquireLockException;
|
||||
import org.springframework.dao.DeadlockLoserDataAccessException;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 在标准化告警事实事务内登记事件时间桶。
|
||||
*
|
||||
* <p>该通知不是“告警一定迟到”的结论。通知与事实在同一事务提交；下游能领取通知时，事实必然
|
||||
* 已经可见，再根据领取时已经完成的规则窗口决定是否重放。这里刻意不使用事实的
|
||||
* {@code created_at} 判断提交先后，因为 Kafka 积压时接收时间可能早于实际入库时间。
|
||||
*/
|
||||
@Component
|
||||
public class LateAlarmBucketWriter {
|
||||
|
||||
private static final String UPSERT_SQL =
|
||||
"MERGE INTO analysis_late_alarm_bucket t "
|
||||
+ "USING (SELECT ? device_id, ? bucket_start FROM dual) s "
|
||||
+ "ON (t.device_id=s.device_id AND t.bucket_start=s.bucket_start) "
|
||||
+ "WHEN MATCHED THEN UPDATE SET "
|
||||
+ "t.requested_generation=t.requested_generation+1, "
|
||||
+ "t.last_observed_at=SYSDATE, "
|
||||
+ "t.status=CASE WHEN t.status='RUNNING' THEN 'RUNNING' ELSE 'PENDING' END, "
|
||||
+ "t.updated_at=SYSDATE "
|
||||
+ "WHEN NOT MATCHED THEN INSERT "
|
||||
+ "(id,device_id,bucket_start,bucket_end,last_observed_at,requested_generation,"
|
||||
+ "claimed_generation,processed_generation,status,attempt_count,fencing_token,created_at,updated_at) "
|
||||
+ "VALUES (?,?,?,?,SYSDATE,1,0,0,'PENDING',0,0,SYSDATE,SYSDATE)";
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
|
||||
@Value("${analysis.late-replay.outbox-enabled:true}")
|
||||
private boolean enabled;
|
||||
|
||||
public LateAlarmBucketWriter(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
}
|
||||
|
||||
/**
|
||||
* 每个设备每分钟只写一条通知。调用方必须已经开启标准化事实事务。
|
||||
*
|
||||
* @param rows 已确认写入 {@code syslog_normal_alarm} 的同构小批次
|
||||
*/
|
||||
public void record(List<Map<String, Object>> rows) {
|
||||
if (!enabled || rows == null || rows.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Set<BucketKey> buckets = new LinkedHashSet<>();
|
||||
for (Map<String, Object> row : rows) {
|
||||
Integer deviceId = integerValue(row.get("device_id"));
|
||||
LocalDateTime eventTime = timeValue(row.get("log_time"));
|
||||
if (deviceId == null || eventTime == null) {
|
||||
throw new IllegalArgumentException("迟到告警通知缺少 device_id 或 log_time");
|
||||
}
|
||||
buckets.add(new BucketKey(deviceId, eventTime.truncatedTo(ChronoUnit.MINUTES)));
|
||||
}
|
||||
|
||||
for (BucketKey key : buckets) {
|
||||
LocalDateTime bucketEnd = key.bucketStart.plusMinutes(1);
|
||||
try {
|
||||
jdbcTemplate.update(
|
||||
UPSERT_SQL,
|
||||
key.deviceId,
|
||||
Timestamp.valueOf(key.bucketStart),
|
||||
UUID.randomUUID().toString(),
|
||||
key.deviceId,
|
||||
Timestamp.valueOf(key.bucketStart),
|
||||
Timestamp.valueOf(bucketEnd));
|
||||
} catch (RuntimeException failure) {
|
||||
// 多个 Kafka 分区可能同时首次写同一设备分钟，MERGE 的并发插入会竞争唯一键。
|
||||
// 交给外层事实事务整体回滚并有限重试，不能只吞掉 outbox 失败。
|
||||
if (isConcurrentWrite(failure)) {
|
||||
throw new DirectStatsProjectionWriter.ConcurrentStatsWriteException(
|
||||
"迟到告警分钟桶并发写入失败，可安全回滚后有限重试", failure);
|
||||
}
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Integer integerValue(Object value) {
|
||||
if (value instanceof Number) {
|
||||
return ((Number) value).intValue();
|
||||
}
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
return Integer.valueOf(value.toString());
|
||||
}
|
||||
|
||||
private LocalDateTime timeValue(Object value) {
|
||||
if (value instanceof LocalDateTime) {
|
||||
return (LocalDateTime) value;
|
||||
}
|
||||
if (value instanceof Timestamp) {
|
||||
return ((Timestamp) value).toLocalDateTime();
|
||||
}
|
||||
if (value instanceof java.util.Date) {
|
||||
return new Timestamp(((java.util.Date) value).getTime()).toLocalDateTime();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean isConcurrentWrite(Throwable failure) {
|
||||
Throwable current = failure;
|
||||
while (current != null) {
|
||||
if (current instanceof DuplicateKeyException
|
||||
|| current instanceof CannotAcquireLockException
|
||||
|| current instanceof DeadlockLoserDataAccessException) {
|
||||
return true;
|
||||
}
|
||||
String message = String.valueOf(current.getMessage()).toLowerCase(java.util.Locale.ROOT);
|
||||
if (message.contains("unique constraint") || message.contains("duplicate key")) {
|
||||
return true;
|
||||
}
|
||||
current = current.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static final class BucketKey {
|
||||
private final Integer deviceId;
|
||||
private final LocalDateTime bucketStart;
|
||||
|
||||
private BucketKey(Integer deviceId, LocalDateTime bucketStart) {
|
||||
this.deviceId = deviceId;
|
||||
this.bucketStart = bucketStart;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other) {
|
||||
if (this == other) {
|
||||
return true;
|
||||
}
|
||||
if (!(other instanceof BucketKey)) {
|
||||
return false;
|
||||
}
|
||||
BucketKey that = (BucketKey) other;
|
||||
return Objects.equals(deviceId, that.deviceId)
|
||||
&& Objects.equals(bucketStart, that.bucketStart);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(deviceId, bucketStart);
|
||||
}
|
||||
}
|
||||
}
|
||||
+5
@@ -30,6 +30,8 @@ public class SyslogNormalDataService {
|
||||
private StatsChangeCaptureService statsChangeCaptureService;
|
||||
@Autowired
|
||||
private DirectStatsProjectionWriter directStatsProjectionWriter;
|
||||
@Autowired
|
||||
private LateAlarmBucketWriter lateAlarmBucketWriter;
|
||||
|
||||
|
||||
/**
|
||||
@@ -49,6 +51,9 @@ public class SyslogNormalDataService {
|
||||
? syslogNormalAlarmMapper.insertDynamic(dataMap)
|
||||
: syslogNormalDataMapper.insertDynamic(dataMap);
|
||||
if (inserted > 0) {
|
||||
if (alarmData && lateAlarmBucketWriter != null) {
|
||||
lateAlarmBucketWriter.record(java.util.Collections.singletonList(dataMap));
|
||||
}
|
||||
statsChangeCaptureService.captureStandardizedInsert(dataMap, alarmData);
|
||||
directStatsProjectionWriter.writeStandardizedRow(dataMap);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user