feat: 新增设备 API 拉取采集及配套处理能力
- 接入 NDRServer 告警和流量日志，支持凭据与任务管理 - 支持分页拉取、窗口拆分、调度优先级及 WAL 持久化后推进断点 - 增加迟到告警规则重放，优化多条泛化规则选择和事件时间处理 - 补充回归测试、数据库升级脚本及部署文档
This commit is contained in:
王绍宏
+2
-2
@@ -78,9 +78,9 @@ services:
|
||||
- INFLUXDB_ORG=influxdb
|
||||
- INFLUXDB_BUCKET=yelangbucket
|
||||
- INFLUXDB_RAW_WRITE_SCHEMA=V2
|
||||
- INFLUXDB_RAW_READ_MODE=V2_ONLY
|
||||
- INFLUXDB_RAW_READ_MODE=V1_V2
|
||||
- INFLUXDB_RAW_V1_BUCKET=yelangbucket
|
||||
- INFLUXDB_RAW_V2_BUCKET=yelangbucket
|
||||
- INFLUXDB_RAW_V2_BUCKET=yelangbucket_v2
|
||||
- INFLUXDB_RAW_UUID_BUCKET_COUNT=4096
|
||||
- INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED=true
|
||||
- INFLUXDB_RAW_QUERY_MAX_ROWS=10000
|
||||
|
||||
+24
-6
@@ -75,6 +75,8 @@ public class LogNormalProcessor {
|
||||
Map<String, String> deviceInfoMap;
|
||||
String strSyslogUUID;
|
||||
String strSyslogTopic;
|
||||
LocalDateTime initialReceivedAt;
|
||||
LocalDateTime eventAt;
|
||||
boolean isSaveNonNormal = false;
|
||||
SyslogNonNormalMessage syslogNonNormalMessage = new SyslogNonNormalMessage();
|
||||
List<Map<String, Object>> dmNormalizeRuleList;
|
||||
@@ -156,6 +158,11 @@ public class LogNormalProcessor {
|
||||
// [流程节点6.2] 解析设备上下文，确定 device_id 和 data_type，作为选择标准化规则的依据。
|
||||
Map<String, String> mapdev = SyslogParser.parseKeyValuePairs(ctx.strDeviceInfo);
|
||||
ctx.deviceInfoMap = mapdev;
|
||||
ctx.initialReceivedAt = parseReceiveTime(mapdev.get("receive_time"));
|
||||
if (ctx.initialReceivedAt == null) {
|
||||
throw new IllegalArgumentException("日志缺少合法 receive_time");
|
||||
}
|
||||
ctx.eventAt = resolveEventAt(mapdev.get("event_time"), ctx.initialReceivedAt);
|
||||
long deviceID = Long.parseLong(mapdev.get("device_id"));
|
||||
logger.debug("标准化开始: eventId={}, deviceId={}", ctx.strSyslogUUID, deviceID);
|
||||
ctx.strDataType = mapdev.get("data_type");
|
||||
@@ -464,12 +471,9 @@ public class LogNormalProcessor {
|
||||
CompletionCollectTaskInfo(columnMap,ctx);
|
||||
//补全日志基础信息
|
||||
columnMap.put("device_id", deviceId);
|
||||
// [SYSLOG-ALARM][ETIME] 这里用当前标准化处理时间覆盖规则已经解析出的 log_time，混淆设备事件时间与处理时间。
|
||||
// 积压或迟到时，事件会被移动到错误分析窗口。建议分别保存 device_event_time、receive_time、normalized_at，
|
||||
// 仅在设备时间确实缺失且规则明确允许时回退到 receive_time，不应无条件覆盖。
|
||||
if (!columnMap.containsKey("log_time") || columnMap.get("log_time") == null) {
|
||||
columnMap.put("log_time", logtime);
|
||||
}
|
||||
// log_time 是两张标准化表的 RANGE 分区键，使用设备事件时间；created_at 保留首次接收时间。
|
||||
// TCP/UDP 的 event_time 等于 receive_time，因此原有实时接入的分区行为保持不变。
|
||||
applyIngressTimes(columnMap, ctx.eventAt, ctx.initialReceivedAt);
|
||||
// 同一 event_id + 规则在重试/回放时生成相同主键，数据库唯一约束可把重复执行收敛为幂等结果。
|
||||
columnMap.put("id", UUID.nameUUIDFromBytes((ctx.strSyslogUUID + ":" + normalizeRuleId + ":legacy")
|
||||
.getBytes(java.nio.charset.StandardCharsets.UTF_8)).toString());
|
||||
@@ -568,4 +572,18 @@ public class LogNormalProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
/** 新消息使用 event_time；历史 TCP/UDP 或旧 Kafka 消息缺失时回退到 receive_time。 */
|
||||
static LocalDateTime resolveEventAt(String eventTime, LocalDateTime initialReceivedAt) {
|
||||
if (eventTime == null || eventTime.trim().isEmpty()) {
|
||||
return initialReceivedAt;
|
||||
}
|
||||
return IngressReceiveTimeUtil.parse(eventTime);
|
||||
}
|
||||
|
||||
static void applyIngressTimes(Map<String, Object> columnMap, LocalDateTime eventAt,
|
||||
LocalDateTime initialReceivedAt) {
|
||||
columnMap.put("log_time", eventAt);
|
||||
columnMap.put("created_at", initialReceivedAt);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+22
-20
@@ -36,7 +36,6 @@ import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
@@ -323,32 +322,33 @@ public class SysLogProcessor {
|
||||
}
|
||||
|
||||
String receiveTime = deviceInfo.get("receive_time");
|
||||
ReceiveTimeParser.ParsedReceiveTime parsed = ReceiveTimeParser.parse(receiveTime);
|
||||
ReceiveTimeParser.ParsedReceiveTime parsedReceiveTime = ReceiveTimeParser.parse(receiveTime);
|
||||
String eventTime = effectiveEventTime(deviceInfo.get("event_time"), receiveTime);
|
||||
ReceiveTimeParser.ParsedReceiveTime parsedEventTime = ReceiveTimeParser.parse(eventTime);
|
||||
boolean headerHasPointTime = deviceInfo.containsKey("point_time_ns");
|
||||
Long envelopePointTime = messageContext.getPointTimeNs();
|
||||
Long headerPointTime = pointTimeResolver.resolve(headerHasPointTime,
|
||||
deviceInfo.get("point_time_ns"), parsed.getEpochMillis(), false);
|
||||
deviceInfo.get("point_time_ns"), parsedEventTime.getEpochMillis(), false);
|
||||
if (envelopePointTime != null) {
|
||||
PointTimeResolver.validate(envelopePointTime, parsed.getEpochMillis());
|
||||
PointTimeResolver.validate(envelopePointTime, parsedEventTime.getEpochMillis());
|
||||
if (headerPointTime != null && !headerPointTime.equals(envelopePointTime)) {
|
||||
throw new IllegalArgumentException("Retry/Replay point_time_ns 与原始 Kafka 载荷不一致");
|
||||
}
|
||||
}
|
||||
Long pointTimeNs = envelopePointTime != null ? envelopePointTime : headerPointTime;
|
||||
if (pointTimeNs == null && rawStorageProperties.getWriteSchema() == RawStorageProperties.WriteSchema.V2) {
|
||||
pointTimeNs = pointTimeResolver.resolve(false, null, parsed.getEpochMillis(), true);
|
||||
pointTimeNs = pointTimeResolver.resolve(false, null, parsedEventTime.getEpochMillis(), true);
|
||||
}
|
||||
messageContext.setPointTimeNs(pointTimeNs);
|
||||
String originalTopic = messageContext.getOriginalTopic();
|
||||
InfluxRawPointFactory.RawPoint rawPoint = rawPointFactory.create(eventId, decryptedMessage,
|
||||
receiveTime, deviceInfo.get("device_id"), deviceInfo.get("device_collect_id"),
|
||||
originalTopic, parsed, pointTimeNs);
|
||||
ResolvedInfluxTime influxTime = new ResolvedInfluxTime(parsed.getEpochMillis(),
|
||||
parsed.getMeasurementDate(), parsed.getLocalDateTime());
|
||||
originalTopic, parsedEventTime, pointTimeNs);
|
||||
Long deviceId = optionalLong(deviceInfo.get("device_id"));
|
||||
Long collectId = optionalLong(deviceInfo.get("device_collect_id"));
|
||||
return new PreparedMessage(record, messageContext, decryptedMessage, eventId, rawPoint.getPoint(),
|
||||
rawPoint.getBucket(), rawPoint.isV2(), influxTime.getReceivedAt(), deviceId, collectId, startTime);
|
||||
rawPoint.getBucket(), rawPoint.isV2(), parsedReceiveTime.getLocalDateTime(),
|
||||
deviceId, collectId, startTime);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -458,26 +458,28 @@ public class SysLogProcessor {
|
||||
catch (NumberFormatException invalid) { return null; }
|
||||
}
|
||||
|
||||
/**
|
||||
* 为 InfluxDB 选择跨 Kafka Retry/Replay 稳定的时间戳。
|
||||
* 优先使用接入端 receive_time；旧消息缺失或格式异常时，回退到首次 Kafka 接收时间。
|
||||
*/
|
||||
static ResolvedInfluxTime resolveInfluxTime(String receiveTime, Instant ignoredFallbackTime) {
|
||||
ReceiveTimeParser.ParsedReceiveTime parsed = ReceiveTimeParser.parse(receiveTime);
|
||||
/** 新消息使用 event_time；历史 Kafka 消息缺失该字段时回退到 receive_time。 */
|
||||
static ResolvedInfluxTime resolveInfluxTime(String eventTime, String receiveTime) {
|
||||
ReceiveTimeParser.ParsedReceiveTime parsed = ReceiveTimeParser.parse(
|
||||
effectiveEventTime(eventTime, receiveTime));
|
||||
return new ResolvedInfluxTime(
|
||||
parsed.getEpochMillis(), parsed.getMeasurementDate(), parsed.getLocalDateTime());
|
||||
}
|
||||
|
||||
/** InfluxDB 点的时间戳和 measurement 日期必须由同一个接收时间计算。 */
|
||||
static String effectiveEventTime(String eventTime, String receiveTime) {
|
||||
return eventTime == null || eventTime.trim().isEmpty() ? receiveTime : eventTime;
|
||||
}
|
||||
|
||||
/** InfluxDB 点的时间戳和 measurement 日期必须由同一个事件时间计算。 */
|
||||
static final class ResolvedInfluxTime {
|
||||
private final long timestampMs;
|
||||
private final String measurementDate;
|
||||
private final LocalDateTime receivedAt;
|
||||
private final LocalDateTime eventAt;
|
||||
|
||||
private ResolvedInfluxTime(long timestampMs, String measurementDate, LocalDateTime receivedAt) {
|
||||
private ResolvedInfluxTime(long timestampMs, String measurementDate, LocalDateTime eventAt) {
|
||||
this.timestampMs = timestampMs;
|
||||
this.measurementDate = measurementDate;
|
||||
this.receivedAt = receivedAt;
|
||||
this.eventAt = eventAt;
|
||||
}
|
||||
|
||||
long getTimestampMs() {
|
||||
@@ -488,7 +490,7 @@ public class SysLogProcessor {
|
||||
return measurementDate;
|
||||
}
|
||||
|
||||
LocalDateTime getReceivedAt() { return receivedAt; }
|
||||
LocalDateTime getEventAt() { return eventAt; }
|
||||
}
|
||||
|
||||
|
||||
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
package com.common.schedule;
|
||||
|
||||
import com.common.entity.AnalysisAnalysisRule;
|
||||
import com.common.mapper.AnalysisAnalysisRuleMapper;
|
||||
import com.common.service.AnalysisRuleService;
|
||||
import com.common.service.LateRuleReplayStore;
|
||||
import com.common.service.LateRuleReplayStore.ReplayWindow;
|
||||
import com.common.service.RuleExecutionWindow;
|
||||
import com.common.service.RuleWindowContext;
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 迟到标准化告警的独立规则重放调度器。
|
||||
*
|
||||
* <p>该调度器只消费 outbox 派生出的历史窗口，不修改正常实时规则 checkpoint 和下次执行时间。
|
||||
* 重放仍调用统一规则引擎，因告警 ID 由规则版本、窗口和聚合结果确定，已存在告警会覆盖更新
|
||||
* log_count、时间范围和来源日志集合，不会把同一窗口重复累加。
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class LateRuleReplayScheduler {
|
||||
|
||||
private final LateRuleReplayStore store;
|
||||
private final AnalysisRuleService analysisRuleService;
|
||||
private final AnalysisAnalysisRuleMapper ruleMapper;
|
||||
private final RuleWindowContext ruleWindowContext;
|
||||
private final String owner = ManagementFactory.getRuntimeMXBean().getName();
|
||||
|
||||
@Value("${analysis.late-replay.enabled:true}")
|
||||
private boolean enabled;
|
||||
|
||||
@Value("${analysis.late-replay.lease-seconds:1800}")
|
||||
private int leaseSeconds;
|
||||
|
||||
@Value("${analysis.late-replay.max-buckets-per-scan:100}")
|
||||
private int maxBucketsPerScan;
|
||||
|
||||
@Value("${analysis.late-replay.max-windows-per-scan:10}")
|
||||
private int maxWindowsPerScan;
|
||||
|
||||
@Value("${analysis.late-replay.retry-delay-seconds:30}")
|
||||
private int retryDelaySeconds;
|
||||
|
||||
public LateRuleReplayScheduler(
|
||||
LateRuleReplayStore store,
|
||||
AnalysisRuleService analysisRuleService,
|
||||
AnalysisAnalysisRuleMapper ruleMapper,
|
||||
RuleWindowContext ruleWindowContext) {
|
||||
this.store = store;
|
||||
this.analysisRuleService = analysisRuleService;
|
||||
this.ruleMapper = ruleMapper;
|
||||
this.ruleWindowContext = ruleWindowContext;
|
||||
}
|
||||
|
||||
/** 先展开到达通知，再有界执行重放，避免一次扫描长期占用调度线程。 */
|
||||
@Scheduled(fixedDelayString = "${analysis.late-replay.scan-interval-ms:5000}")
|
||||
public void scanAndReplay() {
|
||||
if (!enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
expandBuckets();
|
||||
} catch (Exception failure) {
|
||||
log.error("迟到告警通知展开失败，本轮暂停领取新的重放窗口", failure);
|
||||
return;
|
||||
}
|
||||
|
||||
for (int i = 0; i < Math.max(1, maxWindowsPerScan); i++) {
|
||||
ReplayWindow window = store.claimOneReplay(owner, leaseSeconds);
|
||||
if (window == null) {
|
||||
return;
|
||||
}
|
||||
replay(window);
|
||||
}
|
||||
}
|
||||
|
||||
private void expandBuckets() {
|
||||
for (int i = 0; i < Math.max(1, maxBucketsPerScan); i++) {
|
||||
if (store.expandOneBucket(owner, leaseSeconds) == 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void replay(ReplayWindow window) {
|
||||
try {
|
||||
AnalysisAnalysisRule rule = ruleMapper.selectByRuleId(window.getRuleId());
|
||||
if (rule == null
|
||||
|| !Integer.valueOf(1).equals(rule.getRuleStatus())
|
||||
|| !"realtime".equalsIgnoreCase(rule.getRunMode())) {
|
||||
store.blockReplay(window, "RULE_NOT_ACTIVE", "规则不存在、未启用或不是实时规则");
|
||||
return;
|
||||
}
|
||||
|
||||
String currentVersion = LateRuleReplayStore.checkpointVersion(rule.getVersion());
|
||||
if (!currentVersion.equals(window.getRuleVersion())) {
|
||||
store.blockReplay(
|
||||
window,
|
||||
"RULE_VERSION_CHANGED",
|
||||
"规则版本已变化，禁止用新规则配置覆盖旧窗口结果");
|
||||
return;
|
||||
}
|
||||
|
||||
RuleExecutionWindow executionWindow =
|
||||
new RuleExecutionWindow(
|
||||
window.getWindowStart(),
|
||||
window.getWindowEnd(),
|
||||
window.getWindowEnd(),
|
||||
"LATE_REPLAY");
|
||||
Map<String, Object> result;
|
||||
ruleWindowContext.set(executionWindow);
|
||||
try {
|
||||
result = analysisRuleService.executeRealtimeRule(window.getRuleId());
|
||||
} finally {
|
||||
ruleWindowContext.clear();
|
||||
}
|
||||
|
||||
if (!Boolean.TRUE.equals(result.get("success"))) {
|
||||
store.failReplay(
|
||||
window,
|
||||
"RULE_REPLAY_FAILED",
|
||||
String.valueOf(result.get("message")),
|
||||
retryDelaySeconds);
|
||||
return;
|
||||
}
|
||||
|
||||
long inputCount = sumResultCount(result, "processedCount");
|
||||
long outputCount = sumResultCount(result, "alarmCount");
|
||||
store.completeReplay(window, inputCount, outputCount);
|
||||
log.info(
|
||||
"迟到告警规则窗口重放完成: replayId={}, ruleId={}, windowStart={}, windowEnd={},"
|
||||
+ " inputCount={}, outputCount={}, generation={}",
|
||||
window.getId(),
|
||||
window.getRuleId(),
|
||||
window.getWindowStart(),
|
||||
window.getWindowEnd(),
|
||||
inputCount,
|
||||
outputCount,
|
||||
window.getClaimedGeneration());
|
||||
} catch (Exception failure) {
|
||||
log.warn(
|
||||
"迟到告警规则窗口重放失败: replayId={}, ruleId={}, windowStart={}, windowEnd={}",
|
||||
window.getId(),
|
||||
window.getRuleId(),
|
||||
window.getWindowStart(),
|
||||
window.getWindowEnd(),
|
||||
failure);
|
||||
try {
|
||||
store.failReplay(
|
||||
window, "RULE_REPLAY_EXCEPTION", safeMessage(failure), retryDelaySeconds);
|
||||
} catch (Exception stateFailure) {
|
||||
log.error("迟到告警重放失败状态写入失败: replayId={}", window.getId(), stateFailure);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private long sumResultCount(Map<String, Object> result, String key) {
|
||||
Object values = result.get("results");
|
||||
if (!(values instanceof List)) {
|
||||
return 0L;
|
||||
}
|
||||
long total = 0L;
|
||||
for (Map<String, Object> item : (List<Map<String, Object>>) values) {
|
||||
Object value = item.get(key);
|
||||
if (value instanceof Number) {
|
||||
total += ((Number) value).longValue();
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
private String safeMessage(Throwable failure) {
|
||||
String message = failure.getMessage();
|
||||
if (message == null || message.trim().isEmpty()) {
|
||||
return failure.getClass().getSimpleName();
|
||||
}
|
||||
return message.length() <= 2000 ? message : message.substring(0, 2000);
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -30,7 +30,7 @@ import java.util.Map;
|
||||
@Component
|
||||
public class RealtimeAnalysisScheduler {
|
||||
|
||||
/** 窗口语义版本，用于隔离应用管理的窗口与旧版未对齐检查点。 */
|
||||
/** Keeps application-managed windows isolated from legacy unaligned checkpoints. */
|
||||
private static final String WINDOW_SEMANTICS_VERSION = "window-v2";
|
||||
|
||||
@Autowired
|
||||
|
||||
+370
@@ -0,0 +1,370 @@
|
||||
package com.common.service;
|
||||
|
||||
import java.sql.Timestamp;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* 迟到告警通知和规则重放窗口的达梦状态机。
|
||||
*
|
||||
* <p>标准化服务只写事件分钟通知。通知能够被本服务领取，表示对应事实事务已经提交；此时凡是
|
||||
* 已经完成且与事件分钟相交的规则窗口都登记重放。该判断不比较事务内 SYSDATE 与 checkpoint
|
||||
* 完成时间，避免“事实先写、checkpoint 后完成、事实最后提交”时漏掉重放。重放使用独立状态表，
|
||||
* 不回退正常水位。
|
||||
*/
|
||||
@Service
|
||||
public class LateRuleReplayStore {
|
||||
|
||||
private static final String WINDOW_SEMANTICS_VERSION = "window-v2";
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
|
||||
public LateRuleReplayStore(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
}
|
||||
|
||||
/**
|
||||
* 领取一个事件分钟通知并登记受影响的规则窗口。
|
||||
*
|
||||
* @return 0 表示没有可领取通知；正数表示已处理一个通知
|
||||
*/
|
||||
@Transactional
|
||||
public int expandOneBucket(String owner, int leaseSeconds) {
|
||||
List<String> ids =
|
||||
jdbcTemplate.query(
|
||||
"SELECT id FROM (SELECT id FROM analysis_late_alarm_bucket "
|
||||
+ "WHERE (status IN ('PENDING','FAILED') "
|
||||
+ "OR (status='RUNNING' AND lease_expires_at<SYSDATE)) "
|
||||
+ "ORDER BY bucket_start,id) WHERE ROWNUM=1",
|
||||
(rs, rowNum) -> rs.getString(1));
|
||||
if (ids.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
String id = ids.get(0);
|
||||
int claimed =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE analysis_late_alarm_bucket SET status='RUNNING',lease_owner=?,"
|
||||
+ "lease_expires_at=SYSDATE+(CAST(? AS DECIMAL(18,6))/86400),"
|
||||
+ "fencing_token=fencing_token+1,claimed_generation=requested_generation,"
|
||||
+ "attempt_count=attempt_count+1,last_error_code=NULL,last_error_summary=NULL,"
|
||||
+ "updated_at=SYSDATE WHERE id=? AND (status IN ('PENDING','FAILED') "
|
||||
+ "OR (status='RUNNING' AND lease_expires_at<SYSDATE))",
|
||||
owner,
|
||||
leaseSeconds,
|
||||
id);
|
||||
if (claimed != 1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
LateBucket bucket = selectBucket(id);
|
||||
List<ReplayKey> windows = findLateCompletedWindows(bucket);
|
||||
for (ReplayKey window : windows) {
|
||||
enqueueReplay(window, bucket.id);
|
||||
}
|
||||
|
||||
int completed =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE analysis_late_alarm_bucket SET processed_generation=claimed_generation,"
|
||||
+ "status=CASE WHEN requested_generation=claimed_generation "
|
||||
+ "THEN 'COMPLETED' ELSE 'PENDING' END,lease_owner=NULL,lease_expires_at=NULL,"
|
||||
+ "last_error_code=NULL,last_error_summary=NULL,updated_at=SYSDATE "
|
||||
+ "WHERE id=? AND lease_owner=? AND fencing_token=? AND status='RUNNING'",
|
||||
id,
|
||||
owner,
|
||||
bucket.fencingToken);
|
||||
requireOwnedUpdate(completed, "完成通知");
|
||||
return windows.size() + 1;
|
||||
}
|
||||
|
||||
private LateBucket selectBucket(String id) {
|
||||
return jdbcTemplate.queryForObject(
|
||||
"SELECT id,device_id,bucket_start,bucket_end,fencing_token "
|
||||
+ "FROM analysis_late_alarm_bucket WHERE id=?",
|
||||
(rs, rowNum) ->
|
||||
new LateBucket(
|
||||
rs.getString("id"),
|
||||
rs.getLong("device_id"),
|
||||
requiredTime(rs.getTimestamp("bucket_start")),
|
||||
requiredTime(rs.getTimestamp("bucket_end")),
|
||||
rs.getLong("fencing_token")),
|
||||
id);
|
||||
}
|
||||
|
||||
private List<ReplayKey> findLateCompletedWindows(LateBucket bucket) {
|
||||
String sql =
|
||||
"SELECT DISTINCT c.task_id,c.task_version,c.window_start,c.window_end "
|
||||
+ "FROM analysis_checkpoint c WHERE c.task_type='RULE' AND c.status='COMPLETED' "
|
||||
+ "AND c.window_start<? AND c.window_end>? AND EXISTS ("
|
||||
+ "SELECT 1 FROM syslog_normal_alarm a WHERE a.device_id=? "
|
||||
+ "AND a.log_time>=c.window_start AND a.log_time<c.window_end "
|
||||
+ "AND a.log_time>=? AND a.log_time<?)";
|
||||
return jdbcTemplate.query(
|
||||
sql,
|
||||
(rs, rowNum) ->
|
||||
new ReplayKey(
|
||||
rs.getString("task_id"),
|
||||
rs.getString("task_version"),
|
||||
requiredTime(rs.getTimestamp("window_start")),
|
||||
requiredTime(rs.getTimestamp("window_end"))),
|
||||
Timestamp.valueOf(bucket.bucketEnd),
|
||||
Timestamp.valueOf(bucket.bucketStart),
|
||||
bucket.deviceId,
|
||||
Timestamp.valueOf(bucket.bucketStart),
|
||||
Timestamp.valueOf(bucket.bucketEnd));
|
||||
}
|
||||
|
||||
private void enqueueReplay(ReplayKey window, String sourceBucketId) {
|
||||
String sql =
|
||||
"MERGE INTO analysis_rule_replay_window t USING ("
|
||||
+ "SELECT ? rule_id,? rule_version,? window_start,? window_end FROM dual) s "
|
||||
+ "ON (t.rule_id=s.rule_id AND t.rule_version=s.rule_version "
|
||||
+ "AND t.window_start=s.window_start AND t.window_end=s.window_end) "
|
||||
+ "WHEN MATCHED THEN UPDATE SET t.requested_generation=t.requested_generation+1,"
|
||||
+ "t.source_bucket_id=?,t.status=CASE WHEN t.status='RUNNING' THEN 'RUNNING' "
|
||||
+ "ELSE 'PENDING' END,t.next_retry_at=NULL,t.updated_at=SYSDATE "
|
||||
+ "WHEN NOT MATCHED THEN INSERT "
|
||||
+ "(id,source_bucket_id,rule_id,rule_version,window_start,window_end,"
|
||||
+ "requested_generation,claimed_generation,processed_generation,status,attempt_count,"
|
||||
+ "input_count,output_count,fencing_token,created_at,updated_at) "
|
||||
+ "VALUES (?,?,?,?,?,?,1,0,0,'PENDING',0,0,0,0,SYSDATE,SYSDATE)";
|
||||
jdbcTemplate.update(
|
||||
sql,
|
||||
window.ruleId,
|
||||
window.ruleVersion,
|
||||
Timestamp.valueOf(window.windowStart),
|
||||
Timestamp.valueOf(window.windowEnd),
|
||||
sourceBucketId,
|
||||
UUID.randomUUID().toString(),
|
||||
sourceBucketId,
|
||||
window.ruleId,
|
||||
window.ruleVersion,
|
||||
Timestamp.valueOf(window.windowStart),
|
||||
Timestamp.valueOf(window.windowEnd));
|
||||
}
|
||||
|
||||
/** 领取一个到期的历史规则重放窗口。 */
|
||||
@Transactional
|
||||
public ReplayWindow claimOneReplay(String owner, int leaseSeconds) {
|
||||
List<String> ids =
|
||||
jdbcTemplate.query(
|
||||
"SELECT id FROM (SELECT id FROM analysis_rule_replay_window "
|
||||
+ "WHERE ((status IN ('PENDING','FAILED') "
|
||||
+ "AND (next_retry_at IS NULL OR next_retry_at<=SYSDATE)) "
|
||||
+ "OR (status='RUNNING' AND lease_expires_at<SYSDATE)) "
|
||||
+ "ORDER BY window_start,id) WHERE ROWNUM=1",
|
||||
(rs, rowNum) -> rs.getString(1));
|
||||
if (ids.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String id = ids.get(0);
|
||||
int claimed =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE analysis_rule_replay_window SET status='RUNNING',lease_owner=?,"
|
||||
+ "lease_expires_at=SYSDATE+(CAST(? AS DECIMAL(18,6))/86400),"
|
||||
+ "fencing_token=fencing_token+1,claimed_generation=requested_generation,"
|
||||
+ "attempt_count=attempt_count+1,started_at=SYSDATE,finished_at=NULL,"
|
||||
+ "last_error_code=NULL,last_error_summary=NULL,updated_at=SYSDATE WHERE id=? "
|
||||
+ "AND ((status IN ('PENDING','FAILED') "
|
||||
+ "AND (next_retry_at IS NULL OR next_retry_at<=SYSDATE)) "
|
||||
+ "OR (status='RUNNING' AND lease_expires_at<SYSDATE))",
|
||||
owner,
|
||||
leaseSeconds,
|
||||
id);
|
||||
if (claimed != 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return jdbcTemplate.queryForObject(
|
||||
"SELECT id,rule_id,rule_version,window_start,window_end,claimed_generation,"
|
||||
+ "fencing_token FROM analysis_rule_replay_window WHERE id=?",
|
||||
(rs, rowNum) ->
|
||||
new ReplayWindow(
|
||||
rs.getString("id"),
|
||||
rs.getString("rule_id"),
|
||||
rs.getString("rule_version"),
|
||||
requiredTime(rs.getTimestamp("window_start")),
|
||||
requiredTime(rs.getTimestamp("window_end")),
|
||||
rs.getLong("claimed_generation"),
|
||||
rs.getLong("fencing_token"),
|
||||
owner),
|
||||
id);
|
||||
}
|
||||
|
||||
/** 完成重放；若执行期间请求代数增加，则回到 PENDING 等待下一轮。 */
|
||||
@Transactional
|
||||
public void completeReplay(ReplayWindow window, long inputCount, long outputCount) {
|
||||
int updated =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE analysis_rule_replay_window SET processed_generation=claimed_generation,"
|
||||
+ "status=CASE WHEN requested_generation=claimed_generation "
|
||||
+ "THEN 'COMPLETED' ELSE 'PENDING' END,input_count=?,output_count=?,"
|
||||
+ "lease_owner=NULL,lease_expires_at=NULL,next_retry_at=NULL,finished_at=SYSDATE,"
|
||||
+ "last_error_code=NULL,last_error_summary=NULL,updated_at=SYSDATE "
|
||||
+ "WHERE id=? AND status='RUNNING' AND lease_owner=? AND fencing_token=?",
|
||||
inputCount,
|
||||
outputCount,
|
||||
window.id,
|
||||
window.owner,
|
||||
window.fencingToken);
|
||||
requireOwnedUpdate(updated, "完成重放");
|
||||
}
|
||||
|
||||
/** 记录可重试失败并释放租约。 */
|
||||
@Transactional
|
||||
public void failReplay(
|
||||
ReplayWindow window, String errorCode, String errorSummary, int retrySeconds) {
|
||||
int updated =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE analysis_rule_replay_window SET status='FAILED',lease_owner=NULL,"
|
||||
+ "lease_expires_at=NULL,next_retry_at=SYSDATE+(CAST(? AS DECIMAL(18,6))/86400),"
|
||||
+ "finished_at=SYSDATE,last_error_code=?,last_error_summary=?,updated_at=SYSDATE "
|
||||
+ "WHERE id=? AND status='RUNNING' AND lease_owner=? AND fencing_token=?",
|
||||
retrySeconds,
|
||||
errorCode,
|
||||
truncate(errorSummary),
|
||||
window.id,
|
||||
window.owner,
|
||||
window.fencingToken);
|
||||
requireOwnedUpdate(updated, "记录失败");
|
||||
}
|
||||
|
||||
/** 规则已删除、停用或版本变化时保留审计记录，不用新配置重算旧窗口。 */
|
||||
@Transactional
|
||||
public void blockReplay(ReplayWindow window, String errorCode, String errorSummary) {
|
||||
int updated =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE analysis_rule_replay_window SET status='BLOCKED',lease_owner=NULL,"
|
||||
+ "lease_expires_at=NULL,next_retry_at=NULL,finished_at=SYSDATE,last_error_code=?,"
|
||||
+ "last_error_summary=?,updated_at=SYSDATE WHERE id=? AND status='RUNNING' "
|
||||
+ "AND lease_owner=? AND fencing_token=?",
|
||||
errorCode,
|
||||
truncate(errorSummary),
|
||||
window.id,
|
||||
window.owner,
|
||||
window.fencingToken);
|
||||
requireOwnedUpdate(updated, "阻断重放");
|
||||
}
|
||||
|
||||
public static String checkpointVersion(Integer ruleVersion) {
|
||||
return String.valueOf(ruleVersion == null ? 0 : ruleVersion) + ":" + WINDOW_SEMANTICS_VERSION;
|
||||
}
|
||||
|
||||
private static void requireOwnedUpdate(int updated, String action) {
|
||||
if (updated != 1) {
|
||||
throw new IllegalStateException("迟到规则重放租约失效，拒绝" + action);
|
||||
}
|
||||
}
|
||||
|
||||
private static LocalDateTime requiredTime(Timestamp value) {
|
||||
if (value == null) {
|
||||
throw new IllegalStateException("迟到规则重放状态缺少时间字段");
|
||||
}
|
||||
return value.toLocalDateTime();
|
||||
}
|
||||
|
||||
private static String truncate(String value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
return value.length() <= 2000 ? value : value.substring(0, 2000);
|
||||
}
|
||||
|
||||
private static final class LateBucket {
|
||||
private final String id;
|
||||
private final long deviceId;
|
||||
private final LocalDateTime bucketStart;
|
||||
private final LocalDateTime bucketEnd;
|
||||
private final long fencingToken;
|
||||
|
||||
private LateBucket(
|
||||
String id,
|
||||
long deviceId,
|
||||
LocalDateTime bucketStart,
|
||||
LocalDateTime bucketEnd, long fencingToken) {
|
||||
this.id = id;
|
||||
this.deviceId = deviceId;
|
||||
this.bucketStart = bucketStart;
|
||||
this.bucketEnd = bucketEnd;
|
||||
this.fencingToken = fencingToken;
|
||||
}
|
||||
}
|
||||
|
||||
private static final class ReplayKey {
|
||||
private final String ruleId;
|
||||
private final String ruleVersion;
|
||||
private final LocalDateTime windowStart;
|
||||
private final LocalDateTime windowEnd;
|
||||
|
||||
private ReplayKey(
|
||||
String ruleId, String ruleVersion, LocalDateTime windowStart, LocalDateTime windowEnd) {
|
||||
this.ruleId = ruleId;
|
||||
this.ruleVersion = ruleVersion;
|
||||
this.windowStart = windowStart;
|
||||
this.windowEnd = windowEnd;
|
||||
}
|
||||
}
|
||||
|
||||
/** 调度器持有的不可变租约快照。 */
|
||||
public static final class ReplayWindow {
|
||||
private final String id;
|
||||
private final String ruleId;
|
||||
private final String ruleVersion;
|
||||
private final LocalDateTime windowStart;
|
||||
private final LocalDateTime windowEnd;
|
||||
private final long claimedGeneration;
|
||||
private final long fencingToken;
|
||||
private final String owner;
|
||||
|
||||
public ReplayWindow(
|
||||
String id,
|
||||
String ruleId,
|
||||
String ruleVersion,
|
||||
LocalDateTime windowStart,
|
||||
LocalDateTime windowEnd,
|
||||
long claimedGeneration,
|
||||
long fencingToken,
|
||||
String owner) {
|
||||
this.id = id;
|
||||
this.ruleId = ruleId;
|
||||
this.ruleVersion = ruleVersion;
|
||||
this.windowStart = windowStart;
|
||||
this.windowEnd = windowEnd;
|
||||
this.claimedGeneration = claimedGeneration;
|
||||
this.fencingToken = fencingToken;
|
||||
this.owner = owner;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public String getRuleId() {
|
||||
return ruleId;
|
||||
}
|
||||
|
||||
public String getRuleVersion() {
|
||||
return ruleVersion;
|
||||
}
|
||||
|
||||
public LocalDateTime getWindowStart() {
|
||||
return windowStart;
|
||||
}
|
||||
|
||||
public LocalDateTime getWindowEnd() {
|
||||
return windowEnd;
|
||||
}
|
||||
|
||||
public long getClaimedGeneration() {
|
||||
return claimedGeneration;
|
||||
}
|
||||
|
||||
public long getFencingToken() {
|
||||
return fencingToken;
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -13,17 +13,17 @@ public class InfluxRawPointFactory {
|
||||
|
||||
public RawPoint create(String eventId, String message, String receiveTime,
|
||||
String deviceId, String deviceCollectId, String originalTopic,
|
||||
ReceiveTimeParser.ParsedReceiveTime parsed, Long pointTimeNs) {
|
||||
ReceiveTimeParser.ParsedReceiveTime parsedEventTime, Long pointTimeNs) {
|
||||
require(eventId, "event_id"); require(receiveTime, "receive_time");
|
||||
require(deviceId, "device_id"); require(deviceCollectId, "device_collect_id");
|
||||
require(originalTopic, "originalTopic");
|
||||
String measurement = "syslog_security_" + parsed.getMeasurementDate();
|
||||
String measurement = "syslog_security_" + parsedEventTime.getMeasurementDate();
|
||||
if (properties.getWriteSchema() == RawStorageProperties.WriteSchema.V1) {
|
||||
Point point = Point.measurement(measurement)
|
||||
.addTag("device_id", deviceId).addTag("device_collect_id", deviceCollectId)
|
||||
.addTag("uuid", eventId).addTag("topic", originalTopic)
|
||||
.addField("message", message).addField("receive_time", receiveTime)
|
||||
.addField("uuid", eventId).time(parsed.getEpochMillis(), WritePrecision.MS);
|
||||
.addField("uuid", eventId).time(parsedEventTime.getEpochMillis(), WritePrecision.MS);
|
||||
return new RawPoint(point, properties.getV1Bucket(), false);
|
||||
}
|
||||
if (pointTimeNs == null) throw new IllegalArgumentException("V2 Point 缺少 point_time_ns");
|
||||
|
||||
+6
-6
@@ -14,7 +14,7 @@ public class PointTimeResolver {
|
||||
this.metrics = metrics;
|
||||
}
|
||||
|
||||
public Long resolve(boolean present, String rawValue, long receiveTimeMillis, boolean required) {
|
||||
public Long resolve(boolean present, String rawValue, long eventTimeMillis, boolean required) {
|
||||
if (!present) {
|
||||
if (!required) return null;
|
||||
metrics.legacyMissing();
|
||||
@@ -23,7 +23,7 @@ public class PointTimeResolver {
|
||||
throw new IllegalArgumentException("旧 Kafka 消息缺少 point_time_ns，且 legacy fallback 已关闭");
|
||||
}
|
||||
try {
|
||||
long assigned = allocator.allocate(receiveTimeMillis);
|
||||
long assigned = allocator.allocate(eventTimeMillis);
|
||||
metrics.legacyAssigned();
|
||||
return assigned;
|
||||
} catch (RuntimeException failure) {
|
||||
@@ -37,14 +37,14 @@ public class PointTimeResolver {
|
||||
final long value;
|
||||
try { value = Long.parseLong(rawValue); }
|
||||
catch (NumberFormatException invalid) { throw new IllegalArgumentException("point_time_ns 非数字或溢出", invalid); }
|
||||
validate(value, receiveTimeMillis);
|
||||
validate(value, eventTimeMillis);
|
||||
return value;
|
||||
}
|
||||
|
||||
public static void validate(long pointTimeNs, long receiveTimeMillis) {
|
||||
public static void validate(long pointTimeNs, long eventTimeMillis) {
|
||||
if (pointTimeNs <= 0L) throw new IllegalArgumentException("point_time_ns 必须为正整数");
|
||||
if (Math.floorDiv(pointTimeNs, PointTimeAllocator.NANOS_PER_MILLI) != receiveTimeMillis) {
|
||||
throw new IllegalArgumentException("point_time_ns 毫秒部分与 receive_time 不一致");
|
||||
if (Math.floorDiv(pointTimeNs, PointTimeAllocator.NANOS_PER_MILLI) != eventTimeMillis) {
|
||||
throw new IllegalArgumentException("point_time_ns 毫秒部分与 event_time 不一致");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -12,9 +12,9 @@ public class RawStorageProperties {
|
||||
public static final int UUID_BUCKET_COUNT = 4096;
|
||||
|
||||
@Value("${influxdb.raw.write-schema:V1}") private String writeSchemaValue;
|
||||
@Value("${influxdb.raw.read-mode:V2_ONLY}") private String readModeValue;
|
||||
@Value("${influxdb.raw.read-mode:V1_ONLY}") private String readModeValue;
|
||||
@Value("${influxdb.raw.v1-bucket:${influxdb.bucket:yelangbucket}}") private String v1Bucket;
|
||||
@Value("${influxdb.raw.v2-bucket:yelangbucket}") private String v2Bucket;
|
||||
@Value("${influxdb.raw.v2-bucket:yelangbucket_v2}") private String v2Bucket;
|
||||
@Value("${influxdb.raw.uuid-bucket-count:4096}") private int uuidBucketCount;
|
||||
@Value("${influxdb.raw.legacy-point-time-fallback-enabled:true}") private boolean legacyFallbackEnabled;
|
||||
@Value("${influxdb.raw.query-max-rows:10000}") private int queryMaxRows;
|
||||
|
||||
@@ -28,9 +28,9 @@ influxdb.retry.attempts=3
|
||||
influxdb.retry.delay=1000
|
||||
# 原始 Syslog V2 分阶段开关。兼容部署先保持 V1/V1_ONLY，切换时统一写 V2。
|
||||
influxdb.raw.write-schema=${INFLUXDB_RAW_WRITE_SCHEMA:V2}
|
||||
influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V2_ONLY}
|
||||
influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V1_V2}
|
||||
influxdb.raw.v1-bucket=${INFLUXDB_RAW_V1_BUCKET:${influxdb.bucket}}
|
||||
influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket}
|
||||
influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket_v2}
|
||||
influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
|
||||
influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
|
||||
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
|
||||
@@ -182,6 +182,14 @@ analysis.realtime.check-interval-seconds: 10
|
||||
# 完整窗口结束后默认等待 120 秒再聚合，无需额外配置；可用环境变量覆盖。
|
||||
analysis.realtime.allowed-lateness-seconds=${ANALYSIS_REALTIME_ALLOWED_LATENESS_SECONDS:120}
|
||||
|
||||
# 迟到标准化告警使用独立状态表重放已完成规则窗口，不回退正常实时分析 checkpoint。
|
||||
analysis.late-replay.enabled=${ANALYSIS_LATE_REPLAY_ENABLED:true}
|
||||
analysis.late-replay.scan-interval-ms=${ANALYSIS_LATE_REPLAY_SCAN_INTERVAL_MS:5000}
|
||||
analysis.late-replay.lease-seconds=${ANALYSIS_LATE_REPLAY_LEASE_SECONDS:1800}
|
||||
analysis.late-replay.max-buckets-per-scan=${ANALYSIS_LATE_REPLAY_MAX_BUCKETS_PER_SCAN:100}
|
||||
analysis.late-replay.max-windows-per-scan=${ANALYSIS_LATE_REPLAY_MAX_WINDOWS_PER_SCAN:10}
|
||||
analysis.late-replay.retry-delay-seconds=${ANALYSIS_LATE_REPLAY_RETRY_DELAY_SECONDS:30}
|
||||
|
||||
# 关联分析规则达梦安全版子查询模式开关（默认false，生产环境达梦安全版设为true）
|
||||
# true → 使用 RealtimeAnalysisEngineDm，子查询包装SQL，适配 GROUP BY 不支持自定义函数
|
||||
# false → 使用 RealtimeAnalysisEngine，平铺SQL（原有行为）
|
||||
|
||||
+22
@@ -3,6 +3,8 @@ package com.Modules.NormalData;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
@@ -20,4 +22,24 @@ class LogNormalProcessorReceiveTimeTest {
|
||||
assertNull(LogNormalProcessor.parseReceiveTime(null));
|
||||
assertNull(LogNormalProcessor.parseReceiveTime("invalid"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void eventTimeSetsPartitionAndReceiveTimeSetsCreationTime() {
|
||||
LocalDateTime eventTime = LocalDateTime.of(2026, 9, 1, 16, 17);
|
||||
LocalDateTime receiveTime = LocalDateTime.of(2026, 9, 2, 9, 18, 38);
|
||||
Map<String, Object> columns = new HashMap<>();
|
||||
|
||||
LogNormalProcessor.applyIngressTimes(columns, eventTime, receiveTime);
|
||||
|
||||
assertEquals(eventTime, columns.get("log_time"));
|
||||
assertEquals(receiveTime, columns.get("created_at"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingEventTimeFallsBackToReceiveTime() {
|
||||
LocalDateTime receiveTime = LocalDateTime.of(2026, 9, 2, 9, 18, 38);
|
||||
|
||||
assertEquals(receiveTime, LogNormalProcessor.resolveEventAt(null, receiveTime));
|
||||
assertEquals(receiveTime, LogNormalProcessor.resolveEventAt("", receiveTime));
|
||||
}
|
||||
}
|
||||
|
||||
+16
-9
@@ -2,27 +2,34 @@ package com.Modules.NormalData;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class SysLogProcessorTimeTest {
|
||||
|
||||
@Test
|
||||
void receiveTimeDeterminesBothPointTimestampAndMeasurementDate() {
|
||||
void eventTimeDeterminesBothPointTimestampAndMeasurementDate() {
|
||||
SysLogProcessor.ResolvedInfluxTime resolved = SysLogProcessor.resolveInfluxTime(
|
||||
"20260811035557109", Instant.parse("2026-08-09T00:00:00Z"));
|
||||
"20260810035557109", "20260811035557109");
|
||||
|
||||
assertEquals(Instant.parse("2026-08-10T19:55:57.109Z").toEpochMilli(),
|
||||
assertEquals(java.time.Instant.parse("2026-08-09T19:55:57.109Z").toEpochMilli(),
|
||||
resolved.getTimestampMs());
|
||||
assertEquals("20260810", resolved.getMeasurementDate());
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacyMessageWithoutEventTimeFallsBackToReceiveTime() {
|
||||
SysLogProcessor.ResolvedInfluxTime resolved = SysLogProcessor.resolveInfluxTime(
|
||||
null, "20260811035557109");
|
||||
|
||||
assertEquals(java.time.Instant.parse("2026-08-10T19:55:57.109Z").toEpochMilli(),
|
||||
resolved.getTimestampMs());
|
||||
assertEquals("20260811", resolved.getMeasurementDate());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidReceiveTimeIsRejectedInsteadOfUsingConsumerTime() {
|
||||
Instant firstReceivedAt = Instant.parse("2026-08-10T19:55:57.109Z");
|
||||
|
||||
void invalidEventTimeIsRejectedInsteadOfUsingReceiveOrConsumerTime() {
|
||||
org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
|
||||
() -> SysLogProcessor.resolveInfluxTime("invalid", firstReceivedAt));
|
||||
() -> SysLogProcessor.resolveInfluxTime(
|
||||
"invalid", "20260811035557109"));
|
||||
}
|
||||
}
|
||||
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
package com.common.schedule;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.common.entity.AnalysisAnalysisRule;
|
||||
import com.common.mapper.AnalysisAnalysisRuleMapper;
|
||||
import com.common.service.AnalysisRuleService;
|
||||
import com.common.service.LateRuleReplayStore;
|
||||
import com.common.service.LateRuleReplayStore.ReplayWindow;
|
||||
import com.common.service.RuleWindowContext;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
class LateRuleReplaySchedulerTest {
|
||||
|
||||
@Test
|
||||
void replaysExactWindowWithoutMovingNormalCheckpoint() {
|
||||
LateRuleReplayStore store = mock(LateRuleReplayStore.class);
|
||||
AnalysisRuleService ruleService = mock(AnalysisRuleService.class);
|
||||
AnalysisAnalysisRuleMapper ruleMapper = mock(AnalysisAnalysisRuleMapper.class);
|
||||
RuleWindowContext context = new RuleWindowContext();
|
||||
LateRuleReplayScheduler scheduler =
|
||||
new LateRuleReplayScheduler(store, ruleService, ruleMapper, context);
|
||||
ReflectionTestUtils.setField(scheduler, "enabled", true);
|
||||
ReflectionTestUtils.setField(scheduler, "leaseSeconds", 1800);
|
||||
ReflectionTestUtils.setField(scheduler, "maxBucketsPerScan", 10);
|
||||
ReflectionTestUtils.setField(scheduler, "maxWindowsPerScan", 10);
|
||||
ReflectionTestUtils.setField(scheduler, "retryDelaySeconds", 30);
|
||||
|
||||
LocalDateTime start = LocalDateTime.of(2026, 9, 4, 10, 0);
|
||||
LocalDateTime end = start.plusMinutes(5);
|
||||
ReplayWindow replay =
|
||||
new ReplayWindow("replay-1", "rule-1", "3:window-v2", start, end, 2, 9, "owner");
|
||||
when(store.expandOneBucket(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.anyInt()))
|
||||
.thenReturn(0);
|
||||
when(store.claimOneReplay(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.anyInt()))
|
||||
.thenReturn(replay)
|
||||
.thenReturn(null);
|
||||
when(ruleMapper.selectByRuleId("rule-1"))
|
||||
.thenReturn(
|
||||
AnalysisAnalysisRule.builder()
|
||||
.ruleId("rule-1")
|
||||
.version(3)
|
||||
.ruleStatus(1)
|
||||
.runMode("realtime")
|
||||
.build());
|
||||
when(ruleService.executeRealtimeRule("rule-1"))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
assertTrue(context.current().isPresent());
|
||||
Map<String, Object> item = new HashMap<>();
|
||||
item.put("processedCount", 20L);
|
||||
item.put("alarmCount", 2L);
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
result.put("success", true);
|
||||
result.put("results", Collections.singletonList(item));
|
||||
return result;
|
||||
});
|
||||
|
||||
scheduler.scanAndReplay();
|
||||
|
||||
verify(store).completeReplay(replay, 20L, 2L);
|
||||
assertFalse(context.current().isPresent());
|
||||
}
|
||||
}
|
||||
+17
-6
@@ -29,9 +29,9 @@ class InfluxRawV2ContractTest {
|
||||
@Test void configurationRejectsChangingPersistentBucketCount() {
|
||||
RawStorageProperties properties=new RawStorageProperties();
|
||||
ReflectionTestUtils.setField(properties,"writeSchemaValue","V2");
|
||||
ReflectionTestUtils.setField(properties,"readModeValue","V2_ONLY");
|
||||
ReflectionTestUtils.setField(properties,"readModeValue","V1_V2");
|
||||
ReflectionTestUtils.setField(properties,"v1Bucket","yelangbucket");
|
||||
ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket");
|
||||
ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket_v2");
|
||||
ReflectionTestUtils.setField(properties,"uuidBucketCount",1024);
|
||||
ReflectionTestUtils.setField(properties,"queryMaxRows",10000);
|
||||
ReflectionTestUtils.setField(properties,"queryMaxResponseBytes",10485760L);
|
||||
@@ -62,9 +62,20 @@ class InfluxRawV2ContractTest {
|
||||
}
|
||||
|
||||
@Test void v2HasUuidFieldButNeverUuidTagAndUsesNanoseconds() {
|
||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
|
||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
|
||||
InfluxRawPointFactory.RawPoint raw=new InfluxRawPointFactory(properties).create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5","message","20260814000000000","126","1","agent-01-syslog-topic",ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L);
|
||||
String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket",raw.getBucket());
|
||||
String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket_v2",raw.getBucket());
|
||||
}
|
||||
|
||||
@Test void v2UsesEventTimeForMeasurementButKeepsReceiveTimeAsField() {
|
||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
|
||||
String receiveTime="20260831000000000"; ReceiveTimeParser.ParsedReceiveTime eventTime=ReceiveTimeParser.parse("20260830000000000"); long pointTimeNs=eventTime.getEpochMillis()*PointTimeAllocator.NANOS_PER_MILLI+123L;
|
||||
|
||||
String line=new InfluxRawPointFactory(properties).create("20260830-event","message",receiveTime,"126","1","agent-01-syslog-topic",eventTime,pointTimeNs).getPoint().toLineProtocol();
|
||||
|
||||
assertTrue(line.startsWith("syslog_security_20260830,"));
|
||||
assertTrue(line.contains("receive_time=\"20260831000000000\""));
|
||||
assertTrue(line.endsWith(" "+pointTimeNs));
|
||||
}
|
||||
|
||||
@Test void onlyMissingPointTimeFallsBack() {
|
||||
@@ -83,7 +94,7 @@ class InfluxRawV2ContractTest {
|
||||
}
|
||||
|
||||
@Test void mixedLegacyAndNewMessagesAllProduceDistinctV2PointsAndRetryIsIdempotent() {
|
||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
|
||||
RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
|
||||
RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); InfluxRawPointFactory factory=new InfluxRawPointFactory(properties); ReceiveTimeParser.ParsedReceiveTime parsed=ReceiveTimeParser.parse("20260814000000000");
|
||||
long first=resolver.resolve(false,null,parsed.getEpochMillis(),true); long second=resolver.resolve(false,null,parsed.getEpochMillis(),true); assertNotEquals(first,second);
|
||||
String id="20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"; String originalTopic="agent-01-syslog-topic";
|
||||
@@ -100,7 +111,7 @@ class InfluxRawV2ContractTest {
|
||||
|
||||
@Test void uuidCandidateFluxComputesBucketEscapesUuidAndSelectsEarliestGlobally() {
|
||||
String uuid="20260814-a\" |> die() //";
|
||||
String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket",
|
||||
String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket_v2",
|
||||
"syslog_security_20260814",uuid,Instant.parse("2026-08-13T16:00:00Z"),
|
||||
Instant.parse("2026-08-14T16:00:00Z"));
|
||||
assertTrue(query.contains("r.event_bucket == \""+InfluxEventBucket.bucketOf(uuid)+"\""));
|
||||
|
||||
Reference in New Issue
Block a user