candidate, RuleMatchProfile profile) {
+ this.candidate = candidate;
+ this.profile = profile;
+ }
+ }
+
+ private static final class AutomaticScore {
+ private static final AutomaticScore ZERO = new AutomaticScore(0.0d, 0.0d, 0.0d, 0);
+ private final double ownershipScore;
+ private final double matchedWeight;
+ private final double coverage;
+ private final int matchedCount;
+
+ private AutomaticScore(double ownershipScore, double matchedWeight,
+ double coverage, int matchedCount) {
+ this.ownershipScore = ownershipScore;
+ this.matchedWeight = matchedWeight;
+ this.coverage = coverage;
+ this.matchedCount = matchedCount;
+ }
+ }
+}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/metrics/NormalizationRuleMetrics.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/metrics/NormalizationRuleMetrics.java
index a6084c7..09797b6 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/metrics/NormalizationRuleMetrics.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/metrics/NormalizationRuleMetrics.java
@@ -5,6 +5,7 @@ import org.springframework.stereotype.Component;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.DoubleAdder;
import java.util.concurrent.atomic.LongAdder;
/** 固定维度的进程内规则指标；不以 rule/device/event 作为长期标签。 */
@@ -18,7 +19,15 @@ public class NormalizationRuleMetrics {
"collect_task_hit", "collect_task_load", "collect_task_failure");
private final Counters invalidation = new Counters("local", "remote", "after_commit", "failure");
private final Counters messageParse = new Counters("hit", "miss");
+ private final Counters ruleSelection = new Counters("selected", "explicit_selected",
+ "automatic_selected", "ambiguous", "equivalent_tie", "conflicting_tie",
+ "no_match", "invalid_match_config", "shadow_selected", "shadow_no_match");
private final LongAdder ruleJsonParse = new LongAdder();
+ private final LongAdder selectionObservationCount = new LongAdder();
+ private final LongAdder selectionCandidateTotal = new LongAdder();
+ private final AtomicLong selectionCandidateMax = new AtomicLong();
+ private final DoubleAdder selectionMatchRateTotal = new DoubleAdder();
+ private final DoubleAdder selectionScoreMarginTotal = new DoubleAdder();
private final LongAdder compileDurationTotalNanos = new LongAdder();
private final LongAdder compileDurationCount = new LongAdder();
private final AtomicLong compileDurationMaxNanos = new AtomicLong();
@@ -39,6 +48,15 @@ public class NormalizationRuleMetrics {
}
public void invalidation(String outcome) { invalidation.increment(outcome); }
public void messageParse(String outcome) { messageParse.increment(outcome); }
+ public void ruleSelection(String outcome) { ruleSelection.increment(outcome); }
+ public void ruleSelectionObservation(int candidateCount, double matchRate, double scoreMargin) {
+ int candidates = Math.max(0, candidateCount);
+ selectionObservationCount.increment();
+ selectionCandidateTotal.add(candidates);
+ updateMax(selectionCandidateMax, candidates);
+ selectionMatchRateTotal.add(Math.max(0.0d, Math.min(1.0d, matchRate)));
+ selectionScoreMarginTotal.add(Math.max(0.0d, scoreMargin));
+ }
public void ruleJsonParse() { ruleJsonParse.increment(); }
public void compiledCacheGauge(long size, long evictions) {
compiledCacheSize.set(Math.max(0L, size));
@@ -61,6 +79,15 @@ public class NormalizationRuleMetrics {
result.put("enrichment_cache_get_total", enrichment.snapshot());
result.put("rule_invalidation_total", invalidation.snapshot());
result.put("message_parse_cache_get_total", messageParse.snapshot());
+ result.put("rule_selection_total", ruleSelection.snapshot());
+ result.put("rule_selection_observation_count", selectionObservationCount.sum());
+ result.put("rule_selection_candidate_count_total", selectionCandidateTotal.sum());
+ result.put("rule_selection_candidate_count_max", selectionCandidateMax.get());
+ long observations = selectionObservationCount.sum();
+ result.put("rule_selection_match_rate_avg", observations == 0L ? 0.0d
+ : selectionMatchRateTotal.sum() / observations);
+ result.put("rule_selection_score_margin_avg", observations == 0L ? 0.0d
+ : selectionScoreMarginTotal.sum() / observations);
result.put("rule_json_parse_total", ruleJsonParse.sum());
return result;
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java
index 6cc7a12..deacf5d 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java
@@ -32,6 +32,7 @@ public class DmStandardizedBatchExecutor {
private final StatsChangeCaptureService statsChangeCaptureService;
private final DirectStatsProjectionWriter directStatsProjectionWriter;
private final DmDirectTransactionMetrics transactionMetrics;
+ private final LateAlarmBucketWriter lateAlarmBucketWriter;
@org.springframework.beans.factory.annotation.Value("${stats.direct.transaction-log-enabled:false}")
private boolean transactionLogEnabled;
@@ -46,7 +47,8 @@ public class DmStandardizedBatchExecutor {
PlatformTransactionManager transactionManager,
StatsChangeCaptureService statsChangeCaptureService,
DirectStatsProjectionWriter directStatsProjectionWriter,
- DmDirectTransactionMetrics transactionMetrics) {
+ DmDirectTransactionMetrics transactionMetrics,
+ LateAlarmBucketWriter lateAlarmBucketWriter) {
// SqlSessionTemplate 本身线程安全。ExecutorType.BATCH 会把相同 SQL 的参数暂存在当前事务会话中，
// 直到 flushStatements()/commit 时才真正交给 JDBC 驱动批量执行。
this.batchSqlSession = new SqlSessionTemplate(sqlSessionFactory, ExecutorType.BATCH);
@@ -54,6 +56,7 @@ public class DmStandardizedBatchExecutor {
this.statsChangeCaptureService = statsChangeCaptureService;
this.directStatsProjectionWriter = directStatsProjectionWriter;
this.transactionMetrics = transactionMetrics;
+ this.lateAlarmBucketWriter = lateAlarmBucketWriter;
}
/**
@@ -126,6 +129,12 @@ public class DmStandardizedBatchExecutor {
batchSqlSession.flushStatements();
timing.factFlushMs = elapsedMs(flushStarted);
+ // 与标准化告警事实共用同一事务写入事件时间桶。事实成功而通知失败时必须整体回滚，
+ // 否则迟到数据可能永久失去重放机会。
+ if (alarmTable) {
+ lateAlarmBucketWriter.record(rows);
+ }
+
// DIRECT/SHADOW 模式在当前事务内按设备、规则聚合写统计；LEGACY 模式会直接返回。
// 任一统计 SQL 失败都会回滚本事务的事实行，禁止产生无法定位的统计缺口。
timing.statsResult = directStatsProjectionWriter.writeStandardizedRows(rows);
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/LateAlarmBucketWriter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/LateAlarmBucketWriter.java
new file mode 100644
index 0000000..8d7ddf6
--- /dev/null
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/LateAlarmBucketWriter.java
@@ -0,0 +1,162 @@
+package com.common.service;
+
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.dao.CannotAcquireLockException;
+import org.springframework.dao.DeadlockLoserDataAccessException;
+import org.springframework.dao.DuplicateKeyException;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Component;
+
+/**
+ * 在标准化告警事实事务内登记事件时间桶。
+ *
+ * 该通知不是“告警一定迟到”的结论。通知与事实在同一事务提交；下游能领取通知时，事实必然
+ * 已经可见，再根据领取时已经完成的规则窗口决定是否重放。这里刻意不使用事实的
+ * {@code created_at} 判断提交先后，因为 Kafka 积压时接收时间可能早于实际入库时间。
+ */
+@Component
+public class LateAlarmBucketWriter {
+
+ private static final String UPSERT_SQL =
+ "MERGE INTO analysis_late_alarm_bucket t "
+ + "USING (SELECT ? device_id, ? bucket_start FROM dual) s "
+ + "ON (t.device_id=s.device_id AND t.bucket_start=s.bucket_start) "
+ + "WHEN MATCHED THEN UPDATE SET "
+ + "t.requested_generation=t.requested_generation+1, "
+ + "t.last_observed_at=SYSDATE, "
+ + "t.status=CASE WHEN t.status='RUNNING' THEN 'RUNNING' ELSE 'PENDING' END, "
+ + "t.updated_at=SYSDATE "
+ + "WHEN NOT MATCHED THEN INSERT "
+ + "(id,device_id,bucket_start,bucket_end,last_observed_at,requested_generation,"
+ + "claimed_generation,processed_generation,status,attempt_count,fencing_token,created_at,updated_at) "
+ + "VALUES (?,?,?,?,SYSDATE,1,0,0,'PENDING',0,0,SYSDATE,SYSDATE)";
+
+ private final JdbcTemplate jdbcTemplate;
+
+ @Value("${analysis.late-replay.outbox-enabled:true}")
+ private boolean enabled;
+
+ public LateAlarmBucketWriter(JdbcTemplate jdbcTemplate) {
+ this.jdbcTemplate = jdbcTemplate;
+ }
+
+ /**
+ * 每个设备每分钟只写一条通知。调用方必须已经开启标准化事实事务。
+ *
+ * @param rows 已确认写入 {@code syslog_normal_alarm} 的同构小批次
+ */
+ public void record(List> rows) {
+ if (!enabled || rows == null || rows.isEmpty()) {
+ return;
+ }
+
+ Set buckets = new LinkedHashSet<>();
+ for (Map row : rows) {
+ Integer deviceId = integerValue(row.get("device_id"));
+ LocalDateTime eventTime = timeValue(row.get("log_time"));
+ if (deviceId == null || eventTime == null) {
+ throw new IllegalArgumentException("迟到告警通知缺少 device_id 或 log_time");
+ }
+ buckets.add(new BucketKey(deviceId, eventTime.truncatedTo(ChronoUnit.MINUTES)));
+ }
+
+ for (BucketKey key : buckets) {
+ LocalDateTime bucketEnd = key.bucketStart.plusMinutes(1);
+ try {
+ jdbcTemplate.update(
+ UPSERT_SQL,
+ key.deviceId,
+ Timestamp.valueOf(key.bucketStart),
+ UUID.randomUUID().toString(),
+ key.deviceId,
+ Timestamp.valueOf(key.bucketStart),
+ Timestamp.valueOf(bucketEnd));
+ } catch (RuntimeException failure) {
+ // 多个 Kafka 分区可能同时首次写同一设备分钟，MERGE 的并发插入会竞争唯一键。
+ // 交给外层事实事务整体回滚并有限重试，不能只吞掉 outbox 失败。
+ if (isConcurrentWrite(failure)) {
+ throw new DirectStatsProjectionWriter.ConcurrentStatsWriteException(
+ "迟到告警分钟桶并发写入失败，可安全回滚后有限重试", failure);
+ }
+ throw failure;
+ }
+ }
+ }
+
+ private Integer integerValue(Object value) {
+ if (value instanceof Number) {
+ return ((Number) value).intValue();
+ }
+ if (value == null) {
+ return null;
+ }
+ return Integer.valueOf(value.toString());
+ }
+
+ private LocalDateTime timeValue(Object value) {
+ if (value instanceof LocalDateTime) {
+ return (LocalDateTime) value;
+ }
+ if (value instanceof Timestamp) {
+ return ((Timestamp) value).toLocalDateTime();
+ }
+ if (value instanceof java.util.Date) {
+ return new Timestamp(((java.util.Date) value).getTime()).toLocalDateTime();
+ }
+ return null;
+ }
+
+ private boolean isConcurrentWrite(Throwable failure) {
+ Throwable current = failure;
+ while (current != null) {
+ if (current instanceof DuplicateKeyException
+ || current instanceof CannotAcquireLockException
+ || current instanceof DeadlockLoserDataAccessException) {
+ return true;
+ }
+ String message = String.valueOf(current.getMessage()).toLowerCase(java.util.Locale.ROOT);
+ if (message.contains("unique constraint") || message.contains("duplicate key")) {
+ return true;
+ }
+ current = current.getCause();
+ }
+ return false;
+ }
+
+ private static final class BucketKey {
+ private final Integer deviceId;
+ private final LocalDateTime bucketStart;
+
+ private BucketKey(Integer deviceId, LocalDateTime bucketStart) {
+ this.deviceId = deviceId;
+ this.bucketStart = bucketStart;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof BucketKey)) {
+ return false;
+ }
+ BucketKey that = (BucketKey) other;
+ return Objects.equals(deviceId, that.deviceId)
+ && Objects.equals(bucketStart, that.bucketStart);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(deviceId, bucketStart);
+ }
+ }
+}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java
index 9f091b3..91f9921 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java
@@ -30,6 +30,8 @@ public class SyslogNormalDataService {
private StatsChangeCaptureService statsChangeCaptureService;
@Autowired
private DirectStatsProjectionWriter directStatsProjectionWriter;
+ @Autowired
+ private LateAlarmBucketWriter lateAlarmBucketWriter;
/**
@@ -49,6 +51,9 @@ public class SyslogNormalDataService {
? syslogNormalAlarmMapper.insertDynamic(dataMap)
: syslogNormalDataMapper.insertDynamic(dataMap);
if (inserted > 0) {
+ if (alarmData && lateAlarmBucketWriter != null) {
+ lateAlarmBucketWriter.record(java.util.Collections.singletonList(dataMap));
+ }
statsChangeCaptureService.captureStandardizedInsert(dataMap, alarmData);
directStatsProjectionWriter.writeStandardizedRow(dataMap);
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/NormalizationOptimizationProperties.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/NormalizationOptimizationProperties.java
index d49b4ec..6c16037 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/NormalizationOptimizationProperties.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/NormalizationOptimizationProperties.java
@@ -5,17 +5,37 @@ import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Component;
/**
- * 灰度控制和本地缓存边界。默认值必须与 application*.properties 保持一致。
+ * 标准化热路径的执行策略、灰度开关和缓存边界。
+ *
+ * 配置前缀为 {@code normalization}。本类中的初始值是配置缺失时的最后保护值，必须与
+ * {@code application*.properties} 保持一致。修改开关时需要同时考虑下面三类影响：
+ *
+ *
+ * 结果语义：是否启用多规则唯一归属、是否仍可能产生多条标准化结果；
+ * 执行方式：使用编译规则还是旧解释执行器、是否进行影子计算；
+ * 缓存一致性：规则源、消息解析和设备补全数据是否允许复用。
+ *
+ *
+ * {@link #afterPropertiesSet()} 在 Spring 完成属性绑定后校验组合配置，防止应用带着
+ * 无效缓存依赖或未知歧义策略启动。
*/
@Component
@ConfigurationProperties(prefix = "normalization")
public class NormalizationOptimizationProperties implements InitializingBean {
+ /** JetCache 总开关；远端规则/补全缓存依赖该开关。 */
private final Toggle jetcache = new Toggle();
+ /** 规则 JSON 预编译和编译/旧执行器影子对比配置。 */
private final CompiledRule compiledRule = new CompiledRule();
+ /** 单个 Kafka 批次内复用设备和采集任务补全结果。 */
private final Toggle batchEnrichmentCache = new Toggle();
+ /** 同一条消息在解析签名相同时复用 crop + parse 结果。 */
private final Toggle messageParseCache = new Toggle();
+ /** 同设备多条泛化规则的唯一归属策略；默认启用。 */
+ private final RuleSelection ruleSelection = new RuleSelection();
+ /** 设备关联规则列表的本地/远端缓存参数。 */
private final RuleSourceCache ruleSourceCache = new RuleSourceCache();
+ /** 设备和采集任务基础信息的本地/远端缓存参数。 */
private final EnrichmentCache enrichmentCache = new EnrichmentCache();
public boolean isJetcacheEnabled() {
@@ -31,22 +51,32 @@ public class NormalizationOptimizationProperties implements InitializingBean {
return jetcache;
}
+ /** 对应 {@code normalization.compiled-rule.*}。 */
public CompiledRule getCompiledRule() {
return compiledRule;
}
+ /** 对应 {@code normalization.batch-enrichment-cache.enabled}。 */
public Toggle getBatchEnrichmentCache() {
return batchEnrichmentCache;
}
+ /** 对应 {@code normalization.message-parse-cache.enabled}。 */
public Toggle getMessageParseCache() {
return messageParseCache;
}
+ /** 对应 {@code normalization.rule-selection.*}。 */
+ public RuleSelection getRuleSelection() {
+ return ruleSelection;
+ }
+
+ /** 对应 {@code normalization.rule-source-cache.*}。 */
public RuleSourceCache getRuleSourceCache() {
return ruleSourceCache;
}
+ /** 对应 {@code normalization.enrichment-cache.*}。 */
public EnrichmentCache getEnrichmentCache() {
return enrichmentCache;
}
@@ -62,10 +92,12 @@ public class NormalizationOptimizationProperties implements InitializingBean {
"normalization.batch-enrichment-cache.enabled=true 要求 normalization.jetcache.enabled=true");
}
compiledRule.validate();
+ ruleSelection.validate();
ruleSourceCache.validate();
enrichmentCache.validate();
}
+ /** 只包含 {@code enabled} 的通用配置段。未显式赋值时默认关闭。 */
public static class Toggle {
private boolean enabled;
@@ -78,14 +110,28 @@ public class NormalizationOptimizationProperties implements InitializingBean {
}
}
+ /**
+ * 编译规则执行器配置。
+ *
+ * {@code enabled=false} 时使用旧解释执行器；编译失败的单条规则仍会自动回退旧执行器，
+ * 不会因为缓存中存在不兼容历史规则而阻塞整条 Kafka 消息。
+ */
public static class CompiledRule {
+ /** 是否让编译规则执行器成为主路径。 */
private boolean enabled;
+ /** 旧执行器为主路径时，是否抽样运行编译执行器并比较业务结果。 */
private boolean shadowCompareEnabled = true;
+ /** 编译执行器影子对比抽样率，范围 0～1。 */
private double shadowSampleRate = 0.001d;
+ /** JVM 内编译规则缓存最大条目数。 */
private int localLimit = 10_000;
+ /** 编译规则缓存访问后过期时间，单位分钟。 */
private int expireAfterAccessMinutes = 30;
+ /** 编译失败短缓存时间，避免每条日志反复编译同一条无效规则。 */
private int failureCacheSeconds = 10;
+ /** 单次影子对比最大等待时间，单位毫秒。 */
private int shadowTimeoutMs = 1_000;
+ /** 是否记录脱敏后的规则编译详情；不包含原始日志。 */
private boolean ruleDetailLogEnabled = true;
public boolean isEnabled() {
@@ -173,6 +219,77 @@ public class NormalizationOptimizationProperties implements InitializingBean {
}
}
+ /**
+ * 同设备多条泛化规则的唯一归属配置。
+ *
+ * 主流程是：对所有规则做无副作用试解析；先检查规则正文中的显式
+ * {@code match_params}；没有显式命中时，再对未启用显式配置的历史规则执行自动字段
+ * 指纹评分；最后只允许唯一胜出规则执行映射、过滤、补全和入库。
+ *
+ * {@code enabled} 继承自 {@link Toggle}，本配置段默认设为 {@code true}。关闭时恢复
+ * 历史“逐条完整执行所有设备规则”的行为，因此同一日志可能重新产生多条结果。
+ */
+ public static class RuleSelection extends Toggle {
+ /**
+ * 主开关关闭时是否仍在后台计算新选择结果。影子计算只记录指标，不改变旧路径输出；
+ * 主开关开启时直接执行唯一选择，本值不再决定业务输出。
+ */
+ private boolean shadowEnabled = true;
+ /**
+ * 候选排名完全相同时的策略：{@code LOWEST_RULE_ID} 稳定选择最小规则 ID；
+ * {@code STRICT} 会复核并列规则的业务结果，冲突时只保存一条非标记录。
+ */
+ private String ambiguousFallback = "LOWEST_RULE_ID";
+ /**
+ * 是否输出低频歧义详情日志。日志只包含 event hash、候选规则 ID 和分数，不输出原文。
+ */
+ private boolean detailLogEnabled;
+ /** 是否启用 {@code rule_content.match_params} 显式格式指纹。 */
+ private boolean explicitEnabled = true;
+ /** 没有显式命中时，是否允许未配置显式条件的历史规则使用自动字段指纹评分。 */
+ private boolean automaticFallbackEnabled = true;
+
+ /** 默认启用唯一归属，确保配置文件缺失时仍不会退回多规则重复产出。 */
+ public RuleSelection() {
+ setEnabled(true);
+ }
+
+ /** 返回是否启用关闭主开关后的影子计算。 */
+ public boolean isShadowEnabled() { return shadowEnabled; }
+ public void setShadowEnabled(boolean shadowEnabled) { this.shadowEnabled = shadowEnabled; }
+ /** 返回并列候选兜底策略名称。 */
+ public String getAmbiguousFallback() { return ambiguousFallback; }
+ public void setAmbiguousFallback(String ambiguousFallback) {
+ this.ambiguousFallback = ambiguousFallback;
+ }
+ /** 返回是否记录脱敏后的规则选择详情。 */
+ public boolean isDetailLogEnabled() { return detailLogEnabled; }
+ public void setDetailLogEnabled(boolean detailLogEnabled) {
+ this.detailLogEnabled = detailLogEnabled;
+ }
+ /** 返回是否启用规则正文中的显式 {@code match_params}。 */
+ public boolean isExplicitEnabled() { return explicitEnabled; }
+ public void setExplicitEnabled(boolean explicitEnabled) { this.explicitEnabled = explicitEnabled; }
+ /** 返回是否启用历史规则自动字段指纹兜底。 */
+ public boolean isAutomaticFallbackEnabled() { return automaticFallbackEnabled; }
+ public void setAutomaticFallbackEnabled(boolean automaticFallbackEnabled) {
+ this.automaticFallbackEnabled = automaticFallbackEnabled;
+ }
+ /** 是否使用“业务结果冲突即转非标”的严格歧义策略。 */
+ public boolean isStrictAmbiguity() {
+ return "STRICT".equalsIgnoreCase(ambiguousFallback);
+ }
+
+ private void validate() {
+ if (!("LOWEST_RULE_ID".equalsIgnoreCase(ambiguousFallback)
+ || "STRICT".equalsIgnoreCase(ambiguousFallback))) {
+ throw new IllegalArgumentException(
+ "ruleSelection.ambiguousFallback 仅支持 LOWEST_RULE_ID 或 STRICT");
+ }
+ }
+ }
+
+ /** 设备关联规则快照的两级缓存 TTL 配置，单位均为秒。 */
public static class RuleSourceCache extends Toggle {
private int localTtlSeconds = 30;
private int remoteTtlSeconds = 300;
@@ -191,6 +308,7 @@ public class NormalizationOptimizationProperties implements InitializingBean {
}
}
+ /** 设备和采集任务补全事实的两级缓存 TTL 配置，单位均为秒。 */
public static class EnrichmentCache {
private int localTtlSeconds = 60;
private int remoteTtlSeconds = 3600;
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/PointTimeResolver.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/PointTimeResolver.java
index d919d07..c5d6f38 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/PointTimeResolver.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/PointTimeResolver.java
@@ -14,7 +14,7 @@ public class PointTimeResolver {
this.metrics = metrics;
}
- public Long resolve(boolean present, String rawValue, long receiveTimeMillis, boolean required) {
+ public Long resolve(boolean present, String rawValue, long eventTimeMillis, boolean required) {
if (!present) {
if (!required) return null;
metrics.legacyMissing();
@@ -23,7 +23,7 @@ public class PointTimeResolver {
throw new IllegalArgumentException("旧 Kafka 消息缺少 point_time_ns，且 legacy fallback 已关闭");
}
try {
- long assigned = allocator.allocate(receiveTimeMillis);
+ long assigned = allocator.allocate(eventTimeMillis);
metrics.legacyAssigned();
return assigned;
} catch (RuntimeException failure) {
@@ -37,14 +37,14 @@ public class PointTimeResolver {
final long value;
try { value = Long.parseLong(rawValue); }
catch (NumberFormatException invalid) { throw new IllegalArgumentException("point_time_ns 非数字或溢出", invalid); }
- validate(value, receiveTimeMillis);
+ validate(value, eventTimeMillis);
return value;
}
- public static void validate(long pointTimeNs, long receiveTimeMillis) {
+ public static void validate(long pointTimeNs, long eventTimeMillis) {
if (pointTimeNs <= 0L) throw new IllegalArgumentException("point_time_ns 必须为正整数");
- if (Math.floorDiv(pointTimeNs, PointTimeAllocator.NANOS_PER_MILLI) != receiveTimeMillis) {
- throw new IllegalArgumentException("point_time_ns 毫秒部分与 receive_time 不一致");
+ if (Math.floorDiv(pointTimeNs, PointTimeAllocator.NANOS_PER_MILLI) != eventTimeMillis) {
+ throw new IllegalArgumentException("point_time_ns 毫秒部分与 event_time 不一致");
}
}
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawStorageProperties.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawStorageProperties.java
index 9ce5774..c2f1b8e 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawStorageProperties.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawStorageProperties.java
@@ -12,9 +12,9 @@ public class RawStorageProperties {
public static final int UUID_BUCKET_COUNT = 4096;
@Value("${influxdb.raw.write-schema:V1}") private String writeSchemaValue;
- @Value("${influxdb.raw.read-mode:V2_ONLY}") private String readModeValue;
+ @Value("${influxdb.raw.read-mode:V1_ONLY}") private String readModeValue;
@Value("${influxdb.raw.v1-bucket:${influxdb.bucket:yelangbucket}}") private String v1Bucket;
- @Value("${influxdb.raw.v2-bucket:yelangbucket}") private String v2Bucket;
+ @Value("${influxdb.raw.v2-bucket:yelangbucket_v2}") private String v2Bucket;
@Value("${influxdb.raw.uuid-bucket-count:4096}") private int uuidBucketCount;
@Value("${influxdb.raw.legacy-point-time-fallback-enabled:true}") private boolean legacyFallbackEnabled;
@Value("${influxdb.raw.query-max-rows:10000}") private int queryMaxRows;
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-dev.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-dev.properties
index 814dc2d..08ec9a0 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-dev.properties
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-dev.properties
@@ -139,6 +139,12 @@ normalization.compiled-rule.shadow-timeout-ms=1000
normalization.compiled-rule.rule-detail-log-enabled=true
normalization.batch-enrichment-cache.enabled=true
normalization.message-parse-cache.enabled=true
+normalization.rule-selection.enabled=true
+normalization.rule-selection.shadow-enabled=true
+normalization.rule-selection.ambiguous-fallback=LOWEST_RULE_ID
+normalization.rule-selection.detail-log-enabled=false
+normalization.rule-selection.explicit-enabled=true
+normalization.rule-selection.automatic-fallback-enabled=true
normalization.rule-source-cache.local-ttl-seconds=30
normalization.rule-source-cache.remote-ttl-seconds=300
normalization.rule-source-cache.empty-ttl-seconds=10
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-pre-nw.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-pre-nw.properties
index 39e9406..7630e7f 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-pre-nw.properties
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-pre-nw.properties
@@ -200,6 +200,12 @@ normalization.compiled-rule.shadow-timeout-ms=1000
normalization.compiled-rule.rule-detail-log-enabled=true
normalization.batch-enrichment-cache.enabled=true
normalization.message-parse-cache.enabled=true
+normalization.rule-selection.enabled=true
+normalization.rule-selection.shadow-enabled=true
+normalization.rule-selection.ambiguous-fallback=LOWEST_RULE_ID
+normalization.rule-selection.detail-log-enabled=false
+normalization.rule-selection.explicit-enabled=true
+normalization.rule-selection.automatic-fallback-enabled=true
normalization.rule-source-cache.local-ttl-seconds=30
normalization.rule-source-cache.remote-ttl-seconds=300
normalization.rule-source-cache.empty-ttl-seconds=10
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod-zc.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod-zc.properties
index c60df32..060b868 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod-zc.properties
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod-zc.properties
@@ -163,6 +163,12 @@ normalization.compiled-rule.shadow-timeout-ms=1000
normalization.compiled-rule.rule-detail-log-enabled=true
normalization.batch-enrichment-cache.enabled=true
normalization.message-parse-cache.enabled=true
+normalization.rule-selection.enabled=true
+normalization.rule-selection.shadow-enabled=true
+normalization.rule-selection.ambiguous-fallback=LOWEST_RULE_ID
+normalization.rule-selection.detail-log-enabled=false
+normalization.rule-selection.explicit-enabled=true
+normalization.rule-selection.automatic-fallback-enabled=true
normalization.rule-source-cache.local-ttl-seconds=30
normalization.rule-source-cache.remote-ttl-seconds=300
normalization.rule-source-cache.empty-ttl-seconds=10
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties
index 111c531..80567d6 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties
@@ -142,6 +142,12 @@ normalization.compiled-rule.shadow-timeout-ms=1000
normalization.compiled-rule.rule-detail-log-enabled=true
normalization.batch-enrichment-cache.enabled=true
normalization.message-parse-cache.enabled=true
+normalization.rule-selection.enabled=true
+normalization.rule-selection.shadow-enabled=true
+normalization.rule-selection.ambiguous-fallback=LOWEST_RULE_ID
+normalization.rule-selection.detail-log-enabled=false
+normalization.rule-selection.explicit-enabled=true
+normalization.rule-selection.automatic-fallback-enabled=true
normalization.rule-source-cache.local-ttl-seconds=30
normalization.rule-source-cache.remote-ttl-seconds=300
normalization.rule-source-cache.empty-ttl-seconds=10
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-test.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-test.properties
index 62e1961..797a1b8 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-test.properties
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-test.properties
@@ -101,6 +101,12 @@ normalization.compiled-rule.shadow-timeout-ms=1000
normalization.compiled-rule.rule-detail-log-enabled=true
normalization.batch-enrichment-cache.enabled=true
normalization.message-parse-cache.enabled=true
+normalization.rule-selection.enabled=true
+normalization.rule-selection.shadow-enabled=true
+normalization.rule-selection.ambiguous-fallback=LOWEST_RULE_ID
+normalization.rule-selection.detail-log-enabled=false
+normalization.rule-selection.explicit-enabled=true
+normalization.rule-selection.automatic-fallback-enabled=true
normalization.rule-source-cache.local-ttl-seconds=30
normalization.rule-source-cache.remote-ttl-seconds=300
normalization.rule-source-cache.empty-ttl-seconds=10
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties
index e6ef45c..efd1c5a 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties
@@ -27,9 +27,9 @@ influxdb.retry.attempts=3
influxdb.retry.delay=1000
# 原始 Syslog V2 分阶段开关。兼容部署先保持 V1/V1_ONLY，切换时统一写 V2。
influxdb.raw.write-schema=${INFLUXDB_RAW_WRITE_SCHEMA:V2}
-influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V2_ONLY}
+influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V1_V2}
influxdb.raw.v1-bucket=${INFLUXDB_RAW_V1_BUCKET:${influxdb.bucket}}
-influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket}
+influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket_v2}
influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
@@ -103,6 +103,9 @@ spring.kafka.listener.type=batch
# 定时任务配置
spring.task.scheduling.pool.size=10
+# 标准化告警事实与迟到重放通知使用同一个达梦事务。
+analysis.late-replay.outbox-enabled=${ANALYSIS_LATE_REPLAY_OUTBOX_ENABLED:true}
+
# 日志配置
logging.level.com.common.schedule=INFO
logging.level.com.common.service=INFO
@@ -204,6 +207,18 @@ normalization.compiled-rule.shadow-timeout-ms=1000
normalization.compiled-rule.rule-detail-log-enabled=true
normalization.batch-enrichment-cache.enabled=true
normalization.message-parse-cache.enabled=true
+# 同设备多规则唯一归属：默认开启，最终只允许一条胜出规则进入映射和入库。
+normalization.rule-selection.enabled=true
+# 仅在 enabled=false 时用于后台观察新选择结果，不改变旧路径的实际输出。
+normalization.rule-selection.shadow-enabled=true
+# 并列策略：LOWEST_RULE_ID=稳定选择最小规则ID；STRICT=业务结果冲突时转为一条非标记录。
+normalization.rule-selection.ambiguous-fallback=LOWEST_RULE_ID
+# 只记录 event hash、候选规则ID和分数等脱敏详情，不记录原始日志。
+normalization.rule-selection.detail-log-enabled=false
+# 优先使用 rule_content.match_params 中的显式格式条件。
+normalization.rule-selection.explicit-enabled=true
+# 没有显式命中时，允许未配置 match_params 的历史规则按自动字段指纹选优。
+normalization.rule-selection.automatic-fallback-enabled=true
normalization.rule-source-cache.local-ttl-seconds=30
normalization.rule-source-cache.remote-ttl-seconds=300
normalization.rule-source-cache.empty-ttl-seconds=10
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorCompiledEquivalenceTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorCompiledEquivalenceTest.java
index b5ae70f..f9fef7d 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorCompiledEquivalenceTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorCompiledEquivalenceTest.java
@@ -59,6 +59,8 @@ class LogNormalProcessorCompiledEquivalenceTest {
assertEquals(legacyResult.getOutcome(), compiledResult.getOutcome());
assertEquals(legacyResult.getRows(), compiledResult.getRows());
+ assertEquals(1, compiledResult.getRows().size());
+ assertEquals(31L, compiledResult.getRows().get(0).get("normalize_rule_id"));
assertEquals(1, metrics.getParseMisses());
assertEquals(1, metrics.getParseHits());
assertEquals(2, metrics.getCompiledMisses());
@@ -118,6 +120,8 @@ class LogNormalProcessorCompiledEquivalenceTest {
properties.getCompiledRule().setShadowCompareEnabled(false);
properties.getMessageParseCache().setEnabled(parseCache);
properties.getBatchEnrichmentCache().setEnabled(compiled);
+ properties.getRuleSelection().setEnabled(true);
+ properties.getRuleSelection().setShadowEnabled(false);
return properties;
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorReceiveTimeTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorReceiveTimeTest.java
index 619f594..6273c51 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorReceiveTimeTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorReceiveTimeTest.java
@@ -33,29 +33,46 @@ class LogNormalProcessorReceiveTimeTest {
}
@Test
- void initialReceiveTimeSetsBothPartitionAndCreationTime() {
+ void eventTimeSetsPartitionAndReceiveTimeSetsCreationTime() {
+ LocalDateTime eventTime = LocalDateTime.of(2026, 8, 10, 20, 11, 24, 736_717_000);
LocalDateTime receiveTime = LocalDateTime.of(2026, 8, 11, 3, 55, 57, 109_000_000);
Map columns = new HashMap<>();
- LogNormalProcessor.applyInitialReceiveTime(columns, receiveTime);
+ LogNormalProcessor.applyIngressTimes(columns, eventTime, receiveTime);
- assertEquals(receiveTime, columns.get("log_time"));
+ assertEquals(eventTime, columns.get("log_time"));
assertEquals(receiveTime, columns.get("created_at"));
}
@Test
- void ruleProducedLogTimeCannotChangeReceiveDayPartition() {
+ void eventTimeOverridesRuleValueAndSelectsEventDayPartition() {
LocalDateTime eventTime = LocalDateTime.of(2026, 8, 10, 20, 11, 24, 736_717_000);
LocalDateTime receiveTime = LocalDateTime.of(2026, 8, 11, 12, 30, 44, 199_000_000);
Map columns = new HashMap<>();
- columns.put("log_time", eventTime);
+ columns.put("log_time", receiveTime);
- LogNormalProcessor.applyInitialReceiveTime(columns, receiveTime);
+ LogNormalProcessor.applyIngressTimes(columns, eventTime, receiveTime);
- assertEquals(receiveTime, columns.get("log_time"));
+ assertEquals(eventTime, columns.get("log_time"));
assertEquals(receiveTime, columns.get("created_at"));
}
+ @Test
+ void missingEventTimeFallsBackToReceiveTimeForLegacyMessages() {
+ LocalDateTime receiveTime = LocalDateTime.of(2026, 8, 11, 12, 30, 44, 199_000_000);
+
+ assertEquals(receiveTime, LogNormalProcessor.resolveEventAt(null, receiveTime));
+ assertEquals(receiveTime, LogNormalProcessor.resolveEventAt("", receiveTime));
+ }
+
+ @Test
+ void explicitEventTimeIsParsedForHistoricalApiPull() {
+ LocalDateTime receiveTime = LocalDateTime.of(2026, 9, 2, 9, 18, 38);
+
+ assertEquals(LocalDateTime.of(2026, 9, 1, 16, 17),
+ LogNormalProcessor.resolveEventAt("20260901161700000", receiveTime));
+ }
+
@Test
void deterministicResultIdIsStableAndKeepsUuidLength() {
String first = LogNormalProcessor.deterministicResultId("20260811-event", 7L, "legacy");
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java
new file mode 100644
index 0000000..7a7ba61
--- /dev/null
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java
@@ -0,0 +1,236 @@
+package com.Modules.NormalData;
+
+import com.Modules.NormalData.compiled.BatchEnrichmentContext;
+import com.Modules.NormalData.compiled.BatchRuleSnapshot;
+import com.Modules.NormalData.compiled.CompiledRuleCache;
+import com.Modules.NormalData.compiled.CompiledRuleCompiler;
+import com.Modules.NormalData.compiled.RuleExecutionMetrics;
+import com.Modules.NormalData.compiled.RuleSource;
+import com.common.entity.SyslogNonNormalMessage;
+import com.common.mapper.DmColumnMapper;
+import com.common.mapper.DmNormalizeRuleMapper;
+import com.common.service.DeviceCollectTaskService;
+import com.common.service.DeviceDeviceService;
+import com.common.service.DmColumnService;
+import com.common.service.DmNormalizeRuleService;
+import com.common.service.LogDataCompleteService;
+import com.common.service.LogDataFilterService;
+import com.common.service.SyslogNonNormalMessageService;
+import com.common.service.SyslogNormalDataService;
+import com.config.AppProperties;
+import com.config.KafkaConsumerProperties;
+import com.config.NormalizationOptimizationProperties;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class LogNormalProcessorRuleSelectionTest {
+
+ @Test
+ void explicitRuleProducesTheOnlyRowAndOverridesHistoricalRule() {
+ RuleSource historical = jsonRule(10L, "historical", "", mapping("allow"), "");
+ RuleSource explicit = jsonRule(20L, "explicit",
+ "\"match_params\":{\"enabled\":true,\"priority\":100,"
+ + "\"all\":[{\"field\":\"type\",\"op\":\"eq\","
+ + "\"value\":\"firewall\"}]},",
+ mapping("deny"), "");
+ Fixture fixture = fixture(Arrays.asList(historical, explicit), config(false));
+
+ LogNormalProcessor.NormalizationResult result = fixture.prepare(
+ "{\"type\":\"firewall\",\"status\":\"x\"}");
+
+ assertEquals(LogNormalProcessor.ProcessingOutcome.SUCCESS, result.getOutcome());
+ assertEquals(1, result.getRows().size());
+ assertEquals(20L, result.getRows().get(0).get("normalize_rule_id"));
+ assertEquals("deny", result.getRows().get(0).get("action"));
+ verify(fixture.nonNormalService, never()).saveMessage(any());
+ }
+
+ @Test
+ void failedCandidateDoesNotCreateNonNormalWhenAnotherRuleWins() {
+ RuleSource failedRegex = new RuleSource(10L, "regex", "regex", "regex", "{"
+ + "\"regexp\":\"never=(\\\\S+)\","
+ + "\"mappers\":[{\"origin_field\":\"1\",\"dest_field\":\"value\","
+ + "\"action\":{\"type\":\"equal\",\"param\":{}}}]}", true, "0");
+ RuleSource valid = jsonRule(20L, "json", "", equalMapping(), "");
+ Fixture fixture = fixture(Arrays.asList(failedRegex, valid), config(true));
+
+ LogNormalProcessor.NormalizationResult result = fixture.prepare(
+ "{\"type\":\"firewall\",\"status\":\"ok\"}");
+
+ assertEquals(1, result.getRows().size());
+ assertEquals(20L, result.getRows().get(0).get("normalize_rule_id"));
+ verify(fixture.nonNormalService, never()).saveMessage(any());
+ }
+
+ @Test
+ void allCandidatesFailAndOnlyOneNonNormalRecordIsSaved() {
+ RuleSource first = regexRule(10L, "first", "first=(\\S+)");
+ RuleSource second = regexRule(20L, "second", "second=(\\S+)");
+ Fixture fixture = fixture(Arrays.asList(first, second), config(true));
+
+ LogNormalProcessor.NormalizationResult result = fixture.prepare("nothing matches");
+
+ assertEquals(LogNormalProcessor.ProcessingOutcome.FILTERED, result.getOutcome());
+ verify(fixture.nonNormalService).saveMessage(any(SyslogNonNormalMessage.class));
+ }
+
+ @Test
+ void nonNormalSaveFailureRemainsRetryable() {
+ Fixture fixture = fixture(Arrays.asList(
+ regexRule(10L, "first", "first=(\\S+)"),
+ regexRule(20L, "second", "second=(\\S+)")), config(true));
+ when(fixture.nonNormalService.saveMessage(any())).thenReturn(false);
+
+ assertThrows(RuntimeException.class, () -> fixture.prepare("nothing matches"));
+ }
+
+ @Test
+ void winnerFilterIsTerminalAndDoesNotTryTheRunnerUp() {
+ String filter = "\"filters_params\":{\"expressions\":[{"
+ + "\"filters_relation\":\"and\",\"expressions\":[{"
+ + "\"column_name\":\"action\",\"op\":\"=\",\"value\":\"deny\"}]}]},";
+ RuleSource historical = jsonRule(10L, "historical", "", equalMapping(), "");
+ RuleSource explicit = jsonRule(20L, "explicit",
+ "\"match_params\":{\"enabled\":true,\"all\":[{"
+ + "\"field\":\"type\",\"op\":\"eq\","
+ + "\"value\":\"firewall\"}]},",
+ equalMapping(), filter);
+ Fixture fixture = fixture(Arrays.asList(historical, explicit), config(true));
+
+ LogNormalProcessor.NormalizationResult result = fixture.prepare(
+ "{\"type\":\"firewall\",\"status\":\"deny\"}");
+
+ assertEquals(LogNormalProcessor.ProcessingOutcome.FILTERED, result.getOutcome());
+ assertTrue(result.getRows().isEmpty());
+ verify(fixture.nonNormalService, never()).saveMessage(any());
+ }
+
+ @Test
+ void strictAmbiguitySavesNonNormalWhenEqualCandidatesProduceDifferentBusinessResults() {
+ RuleSource allow = jsonRule(10L, "allow", "", mapping("allow"), "");
+ RuleSource deny = jsonRule(20L, "deny", "", mapping("deny"), "");
+ NormalizationOptimizationProperties properties = config(true);
+ properties.getRuleSelection().setAmbiguousFallback("STRICT");
+ Fixture fixture = fixture(Arrays.asList(allow, deny), properties);
+
+ LogNormalProcessor.NormalizationResult result = fixture.prepare(
+ "{\"type\":\"firewall\",\"status\":\"x\"}");
+
+ assertEquals(LogNormalProcessor.ProcessingOutcome.FILTERED, result.getOutcome());
+ assertTrue(result.getRows().isEmpty());
+ ArgumentCaptor captor =
+ ArgumentCaptor.forClass(SyslogNonNormalMessage.class);
+ verify(fixture.nonNormalService).saveMessage(captor.capture());
+ assertEquals("多规则归属歧义", captor.getValue().getReason());
+ assertTrue(captor.getValue().getReasonDetail().contains("10"));
+ assertTrue(captor.getValue().getReasonDetail().contains("20"));
+ }
+
+ @Test
+ void strictAmbiguityStillUsesLowestIdWhenBusinessResultsAreEquivalent() {
+ RuleSource highId = jsonRule(20L, "same-high", "", mapping("allow"), "");
+ RuleSource lowId = jsonRule(10L, "same-low", "", mapping("allow"), "");
+ NormalizationOptimizationProperties properties = config(true);
+ properties.getRuleSelection().setAmbiguousFallback("STRICT");
+ Fixture fixture = fixture(Arrays.asList(highId, lowId), properties);
+
+ LogNormalProcessor.NormalizationResult result = fixture.prepare(
+ "{\"type\":\"firewall\",\"status\":\"x\"}");
+
+ assertEquals(LogNormalProcessor.ProcessingOutcome.SUCCESS, result.getOutcome());
+ assertEquals(1, result.getRows().size());
+ assertEquals(10L, result.getRows().get(0).get("normalize_rule_id"));
+ verify(fixture.nonNormalService, never()).saveMessage(any());
+ }
+
+ private static NormalizationOptimizationProperties config(boolean compiled) {
+ NormalizationOptimizationProperties properties = new NormalizationOptimizationProperties();
+ properties.getCompiledRule().setEnabled(compiled);
+ properties.getCompiledRule().setShadowCompareEnabled(false);
+ properties.getMessageParseCache().setEnabled(true);
+ properties.getRuleSelection().setEnabled(true);
+ properties.getRuleSelection().setShadowEnabled(false);
+ return properties;
+ }
+
+ private static RuleSource jsonRule(long id, String name, String prefix,
+ String mapper, String extra) {
+ String content = "{" + prefix
+ + "\"cropper_params\":{\"head_key\":\"{\",\"head_offset\":0,"
+ + "\"tail_key\":\"\",\"tail_offset\":0},"
+ + "\"mappers\":[" + mapper + "]," + extra
+ + "\"complete_columns\":[]}";
+ return new RuleSource(id, name, name, "json", content, true, "0");
+ }
+
+ private static RuleSource regexRule(long id, String name, String regexp) {
+ String content = "{\"regexp\":\"" + regexp.replace("\\", "\\\\") + "\","
+ + "\"mappers\":[{\"origin_field\":\"1\",\"dest_field\":\"value\","
+ + "\"action\":{\"type\":\"equal\",\"param\":{}}}]}";
+ return new RuleSource(id, name, name, "regex", content, true, "0");
+ }
+
+ private static String equalMapping() {
+ return "{\"origin_field\":\"status\",\"dest_field\":\"action\","
+ + "\"action\":{\"type\":\"equal\",\"param\":{}}}";
+ }
+
+ private static String mapping(String value) {
+ return "{\"origin_field\":\"status\",\"dest_field\":\"action\","
+ + "\"action\":{\"type\":\"mapping\",\"param\":{\"mapping\":{"
+ + "\"x\":\"" + value + "\"}}}}";
+ }
+
+ private static Fixture fixture(List rules,
+ NormalizationOptimizationProperties properties) {
+ Fixture fixture = new Fixture();
+ fixture.taskService = mock(DeviceCollectTaskService.class);
+ fixture.deviceService = mock(DeviceDeviceService.class);
+ fixture.nonNormalService = mock(SyslogNonNormalMessageService.class);
+ when(fixture.nonNormalService.saveMessage(any())).thenReturn(true);
+ CompiledRuleCompiler compiler = new CompiledRuleCompiler(new ObjectMapper());
+ fixture.processor = new LogNormalProcessor(mock(DmNormalizeRuleService.class),
+ mock(DmColumnService.class), mock(SyslogNormalDataService.class),
+ fixture.nonNormalService, new LogDataFilterService(), new LogDataCompleteService(),
+ fixture.taskService, fixture.deviceService, mock(DmColumnMapper.class),
+ mock(DmNormalizeRuleMapper.class), mock(KafkaConsumerProperties.class),
+ mock(AppProperties.class), new CompiledRuleCache(compiler, properties), properties);
+ Map> byDevice = new LinkedHashMap<>();
+ byDevice.put(7L, rules);
+ fixture.snapshot = new BatchRuleSnapshot(byDevice, System.currentTimeMillis());
+ return fixture;
+ }
+
+ private static final class Fixture {
+ private LogNormalProcessor processor;
+ private SyslogNonNormalMessageService nonNormalService;
+ private DeviceCollectTaskService taskService;
+ private DeviceDeviceService deviceService;
+ private BatchRuleSnapshot snapshot;
+
+ private LogNormalProcessor.NormalizationResult prepare(String body) {
+ String message = "[receive_time=20260821150000000 event_id=event-selection device_id=7 "
+ + "device_name=fw vendor=vendor data_type=json device_collect_id=3]" + body;
+ return processor.prepare(message, "event-selection", "raw",
+ LocalDateTime.of(2026, 8, 21, 15, 0), snapshot,
+ new BatchEnrichmentContext(false, taskService, deviceService),
+ new RuleExecutionMetrics());
+ }
+ }
+}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorTimeTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorTimeTest.java
index 8abe4a5..8a16375 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorTimeTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorTimeTest.java
@@ -1,31 +1,53 @@
package com.Modules.NormalData;
+import com.influx.raw.PointTimeAllocator;
+import com.influx.raw.PointTimeResolver;
import org.junit.jupiter.api.Test;
-import java.time.Instant;
-import java.time.LocalDateTime;
-
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
class SysLogProcessorTimeTest {
@Test
- void receiveTimeDeterminesBothPointTimestampAndMeasurementDate() {
+ void eventTimeDeterminesBothPointTimestampAndMeasurementDate() {
SysLogProcessor.ResolvedInfluxTime resolved = SysLogProcessor.resolveInfluxTime(
- "20260811035557109", Instant.parse("2026-08-09T00:00:00Z"));
+ "20260810035557109", "20260811035557109");
- assertEquals(Instant.parse("2026-08-10T19:55:57.109Z").toEpochMilli(),
+ assertEquals(java.time.Instant.parse("2026-08-09T19:55:57.109Z").toEpochMilli(),
resolved.getTimestampMs());
- assertEquals("20260811", resolved.getMeasurementDate());
- assertEquals(LocalDateTime.of(2026, 8, 11, 3, 55, 57, 109_000_000),
- resolved.getReceivedAt());
+ assertEquals("20260810", resolved.getMeasurementDate());
+ assertEquals(java.time.LocalDateTime.of(2026, 8, 10, 3, 55, 57, 109_000_000),
+ resolved.getEventAt());
}
@Test
- void invalidReceiveTimeIsRejectedInsteadOfUsingConsumerTime() {
- Instant firstReceivedAt = Instant.parse("2026-08-10T19:55:57.109Z");
+ void legacyMessageWithoutEventTimeFallsBackToReceiveTime() {
+ SysLogProcessor.ResolvedInfluxTime resolved = SysLogProcessor.resolveInfluxTime(
+ null, "20260811035557109");
- org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
- () -> SysLogProcessor.resolveInfluxTime("invalid", firstReceivedAt));
+ assertEquals(java.time.Instant.parse("2026-08-10T19:55:57.109Z").toEpochMilli(),
+ resolved.getTimestampMs());
+ assertEquals("20260811", resolved.getMeasurementDate());
+ }
+
+ @Test
+ void invalidEventTimeIsRejectedInsteadOfUsingReceiveTime() {
+ assertThrows(IllegalArgumentException.class,
+ () -> SysLogProcessor.resolveInfluxTime("invalid", "20260811035557109"));
+ }
+
+ @Test
+ void historicalPointTimeIsValidatedAgainstEventTimeRatherThanNextDayReceiveTime() {
+ SysLogProcessor.ResolvedInfluxTime event = SysLogProcessor.resolveInfluxTime(
+ "20260901161700123", "20260902091800456");
+ long pointTimeNs = event.getTimestampMs() * PointTimeAllocator.NANOS_PER_MILLI + 789L;
+
+ assertDoesNotThrow(() -> PointTimeResolver.validate(pointTimeNs, event.getTimestampMs()));
+
+ long receiveTimeMs = java.time.Instant.parse("2026-09-02T01:18:00.456Z").toEpochMilli();
+ assertThrows(IllegalArgumentException.class,
+ () -> PointTimeResolver.validate(pointTimeNs, receiveTimeMs));
}
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/compiled/RuleSelectionEngineTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/compiled/RuleSelectionEngineTest.java
new file mode 100644
index 0000000..76ccd15
--- /dev/null
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/compiled/RuleSelectionEngineTest.java
@@ -0,0 +1,200 @@
+package com.Modules.NormalData.compiled;
+
+import com.Modules.NormalData.metrics.NormalizationRuleMetrics;
+import com.config.NormalizationOptimizationProperties;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class RuleSelectionEngineTest {
+ private NormalizationOptimizationProperties properties;
+ private NormalizationRuleMetrics metrics;
+ private CompiledRuleCache cache;
+ private RuleSelectionEngine engine;
+
+ @BeforeEach
+ void setUp() {
+ properties = new NormalizationOptimizationProperties();
+ properties.getRuleSelection().setEnabled(true);
+ properties.getRuleSelection().setShadowEnabled(false);
+ metrics = new NormalizationRuleMetrics();
+ cache = new CompiledRuleCache(new CompiledRuleCompiler(new ObjectMapper()), properties, metrics);
+ engine = new RuleSelectionEngine(cache, properties, metrics);
+ }
+
+ @Test
+ void explicitMatchWinsBeforeHistoricalAutomaticRule() {
+ RuleSource explicit = rule(20L, "title,client", "\"match_params\":{"
+ + "\"enabled\":true,\"priority\":10,"
+ + "\"all\":[{\"field\":\"title\",\"op\":\"eq\",\"value\":\"HFish\"}],"
+ + "\"any\":[{\"source\":\"raw\",\"op\":\"contains\",\"value\":\"threat\"}],"
+ + "\"none\":[{\"field\":\"honeypot_id\",\"op\":\"exists\"}]},");
+ RuleSource historical = rule(10L, "title,client", "");
+ Map fields = fields("title", "HFish", "client", "agent");
+
+ RuleSelectionEngine.Decision result = engine.select(Arrays.asList(
+ candidate(historical, fields), candidate(explicit, fields)), "HFish threat", false);
+
+ assertTrue(result.isSelected());
+ assertEquals(20L, result.getWinner().getSource().getId());
+ assertEquals(RuleSelectionEngine.SelectionMode.EXPLICIT, result.getMode());
+ }
+
+ @Test
+ void failedExplicitRuleCannotFallBackToAutomaticScoring() {
+ RuleSource explicit = rule(1L, "title,special", "\"match_params\":{"
+ + "\"enabled\":true,\"priority\":999,"
+ + "\"all\":[{\"field\":\"title\",\"op\":\"eq\",\"value\":\"other\"}]},");
+ RuleSource historical = rule(2L, "title", "");
+ Map fields = fields("title", "actual", "special", "yes");
+
+ RuleSelectionEngine.Decision result = engine.select(Arrays.asList(
+ candidate(explicit, fields), candidate(historical, fields)), "actual", false);
+
+ assertEquals(2L, result.getWinner().getSource().getId());
+ assertEquals(RuleSelectionEngine.SelectionMode.AUTOMATIC, result.getMode());
+ }
+
+ @Test
+ void explicitPriorityOnlyAppliesAfterConditionsMatch() {
+ RuleSource low = rule(10L, "kind", explicitEq("kind", "event", 10));
+ RuleSource high = rule(20L, "kind", explicitEq("kind", "event", 100));
+
+ RuleSelectionEngine.Decision result = engine.select(Arrays.asList(
+ candidate(low, fields("kind", "event")),
+ candidate(high, fields("kind", "event"))), "event", false);
+
+ assertEquals(20L, result.getWinner().getSource().getId());
+ assertFalse(result.isAmbiguous());
+ }
+
+ @Test
+ void weightedOwnershipPreventsTinyCommonRuleFromBeatingSpecificRule() {
+ RuleSource small = rule(10L, "src_ip,time", "");
+ RuleSource large = rule(20L, "src_ip,time,u1,u2,u3,u4,u5,u6,u7,u8", "");
+ Map parsed = fields("src_ip", "10.0.0.1", "time", "now",
+ "u1", 1, "u2", 2, "u3", 3, "u4", 4, "u5", 5, "u6", 6, "u7", 7);
+
+ RuleSelectionEngine.Decision result = engine.select(Arrays.asList(
+ candidate(small, parsed), candidate(large, parsed)), "raw", false);
+
+ assertEquals(20L, result.getWinner().getSource().getId());
+ assertEquals(RuleSelectionEngine.SelectionMode.AUTOMATIC, result.getMode());
+ }
+
+ @Test
+ void completelyEqualCandidatesUseLowestRuleIdAndExposeAmbiguity() {
+ RuleSource highId = rule(40L, "src_ip,type", "");
+ RuleSource lowId = rule(30L, "src_ip,type", "");
+ Map parsed = fields("src_ip", "10.0.0.1", "type", "deny");
+
+ RuleSelectionEngine.Decision result = engine.select(Arrays.asList(
+ candidate(highId, parsed), candidate(lowId, parsed)), "raw", false);
+
+ assertEquals(30L, result.getWinner().getSource().getId());
+ assertTrue(result.isAmbiguous());
+ assertEquals(2, result.getTiedCandidates().size());
+ }
+
+ @Test
+ void allExplicitRulesFailingProducesNoCandidate() {
+ RuleSource first = rule(1L, "kind", explicitEq("kind", "a", 0));
+ RuleSource second = rule(2L, "kind", explicitEq("kind", "b", 0));
+
+ RuleSelectionEngine.Decision result = engine.select(Arrays.asList(
+ candidate(first, fields("kind", "c")),
+ candidate(second, fields("kind", "c"))), "c", false);
+
+ assertFalse(result.isSelected());
+ }
+
+ @Test
+ void invalidMatchConfigurationIsRejectedDuringCompilation() {
+ RuleSource invalid = rule(1L, "kind", "\"match_params\":{"
+ + "\"enabled\":true,\"all\":[{\"field\":\"kind\","
+ + "\"op\":\"in\",\"value\":\"not-an-array\"}]},");
+
+ assertThrows(CompiledRuleCompiler.RuleCompilationException.class,
+ () -> cache.getMatchProfile(invalid));
+ }
+
+ @Test
+ void explicitProfileSupportsAllDocumentedOperatorsAndGroupSemantics() {
+ RuleSource source = rule(7L, "a", "\"match_params\":{\"enabled\":true,"
+ + "\"all\":["
+ + "{\"field\":\"a\",\"op\":\"exists\"},"
+ + "{\"field\":\"a\",\"op\":\"not_blank\"},"
+ + "{\"field\":\"a\",\"op\":\"eq\",\"value\":\"abc\"},"
+ + "{\"field\":\"a\",\"op\":\"ne\",\"value\":\"zzz\"},"
+ + "{\"field\":\"a\",\"op\":\"in\",\"value\":[\"abc\",\"def\"]},"
+ + "{\"field\":\"a\",\"op\":\"contains\",\"value\":\"b\"},"
+ + "{\"field\":\"a\",\"op\":\"regexp\",\"value\":\"^a.*\"},"
+ + "{\"field\":\"missing\",\"op\":\"not_exists\"},"
+ + "{\"source\":\"raw\",\"op\":\"not_blank\"},"
+ + "{\"source\":\"raw\",\"op\":\"contains\",\"value\":\"CEF:\"},"
+ + "{\"source\":\"raw\",\"op\":\"regexp\",\"value\":\"CEF:.*abc\"}],"
+ + "\"any\":[{\"field\":\"a\",\"op\":\"eq\",\"value\":\"no\"},"
+ + "{\"field\":\"a\",\"op\":\"eq\",\"value\":\"abc\"}],"
+ + "\"none\":[{\"field\":\"blocked\",\"op\":\"exists\"}]},");
+
+ RuleMatchProfile profile = cache.getMatchProfile(source);
+
+ assertTrue(profile.evaluate(fields("a", "abc"), "CEF: vendor abc").isMatched());
+ assertFalse(profile.evaluate(fields("a", "abc", "blocked", true),
+ "CEF: vendor abc").isMatched());
+ }
+
+ @Test
+ void rawSourceRejectsFieldOnlyOperations() {
+ RuleSource invalid = rule(8L, "a", "\"match_params\":{\"enabled\":true,"
+ + "\"all\":[{\"source\":\"raw\",\"op\":\"exists\"}]},");
+
+ assertThrows(CompiledRuleCompiler.RuleCompilationException.class,
+ () -> cache.getMatchProfile(invalid));
+ }
+
+ private static String explicitEq(String field, String value, int priority) {
+ return "\"match_params\":{\"enabled\":true,\"priority\":" + priority
+ + ",\"all\":[{\"field\":\"" + field + "\",\"op\":\"eq\","
+ + "\"value\":\"" + value + "\"}]},";
+ }
+
+ private static RuleSource rule(long id, String origins, String prefix) {
+ StringBuilder mappers = new StringBuilder();
+ if (!origins.isEmpty()) {
+ for (String field : origins.split(",")) {
+ if (mappers.length() > 0) mappers.append(',');
+ mappers.append("{\"origin_field\":\"").append(field)
+ .append("\",\"dest_field\":\"").append(field)
+ .append("\",\"action\":{\"type\":\"equal\",\"param\":{}}}");
+ }
+ }
+ String content = "{" + prefix + "\"mappers\":[" + mappers + "]}";
+ return new RuleSource(id, "rule-" + id, "rule-" + id,
+ "json", content, true, "0");
+ }
+
+ private static RuleSelectionEngine.Candidate candidate(
+ RuleSource source, Map fields) {
+ return new RuleSelectionEngine.Candidate<>(source, fields, "payload-" + source.getId());
+ }
+
+ private static Map fields(Object... values) {
+ Map result = new LinkedHashMap<>();
+ for (int i = 0; i < values.length; i += 2) {
+ result.put(values[i].toString(), values[i + 1]);
+ }
+ return result;
+ }
+}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/metrics/NormalizationRuleMetricsTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/metrics/NormalizationRuleMetricsTest.java
index 1bd9b86..c370ac3 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/metrics/NormalizationRuleMetricsTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/metrics/NormalizationRuleMetricsTest.java
@@ -19,6 +19,9 @@ class NormalizationRuleMetricsTest {
metrics.enrichment("device", "hit");
metrics.invalidation("after_commit");
metrics.messageParse("miss");
+ metrics.ruleSelection("selected");
+ metrics.ruleSelection("explicit_selected");
+ metrics.ruleSelectionObservation(3, 2.0d / 3.0d, 0.5d);
metrics.ruleJsonParse();
metrics.compiledCacheGauge(3, 1);
@@ -28,6 +31,9 @@ class NormalizationRuleMetricsTest {
assertTrue(snapshot.containsKey("rule_source_cache_get_total"));
assertTrue(snapshot.containsKey("enrichment_cache_get_total"));
assertTrue(snapshot.containsKey("rule_invalidation_total"));
+ assertEquals(1L, ((Map) snapshot.get("rule_selection_total")).get("selected"));
+ assertEquals(3L, snapshot.get("rule_selection_candidate_count_total"));
+ assertEquals(2.0d / 3.0d, (Double) snapshot.get("rule_selection_match_rate_avg"), 0.000001d);
assertEquals(1L, snapshot.get("rule_json_parse_total"));
}
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmNormalizeRuleServiceUpdateTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmNormalizeRuleServiceUpdateTest.java
index 6670ce7..e6a11b6 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmNormalizeRuleServiceUpdateTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmNormalizeRuleServiceUpdateTest.java
@@ -13,6 +13,7 @@ import org.springframework.test.util.ReflectionTestUtils;
import java.util.Arrays;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
@@ -62,6 +63,21 @@ class DmNormalizeRuleServiceUpdateTest {
verify(publisher, never()).publishEvent(any());
}
+ @Test
+ void invalidMatchParamsAreRejectedBeforeRuleUpdate() {
+ when(mapper.selectById(11L)).thenReturn(validRule());
+ DmNormalizeRule patch = new DmNormalizeRule();
+ patch.setId(11L);
+ patch.setRuleContent("{\"match_params\":{\"enabled\":true,\"all\":[{"
+ + "\"field\":\"type\",\"op\":\"in\",\"value\":\"bad\"}]},"
+ + "\"mappers\":[]}");
+
+ assertThrows(CompiledRuleCompiler.RuleCompilationException.class,
+ () -> service.updateById(patch));
+ verify(mapper, never()).updateById(any());
+ verify(publisher, never()).publishEvent(any());
+ }
+
private static DmNormalizeRule validRule() {
DmNormalizeRule rule = new DmNormalizeRule();
rule.setId(11L);
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/LateAlarmBucketWriterTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/LateAlarmBucketWriterTest.java
new file mode 100644
index 0000000..e7923d1
--- /dev/null
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/LateAlarmBucketWriterTest.java
@@ -0,0 +1,60 @@
+package com.common.service;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.util.ReflectionTestUtils;
+
+class LateAlarmBucketWriterTest {
+
+ @Test
+ void groupsRowsByDeviceAndEventMinute() {
+ JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class);
+ LateAlarmBucketWriter writer = new LateAlarmBucketWriter(jdbcTemplate);
+ ReflectionTestUtils.setField(writer, "enabled", true);
+
+ writer.record(
+ Arrays.asList(
+ row(126, LocalDateTime.of(2026, 9, 4, 10, 1, 2), 3),
+ row(126, LocalDateTime.of(2026, 9, 4, 10, 1, 59), 4),
+ row(126, LocalDateTime.of(2026, 9, 4, 10, 2, 1), 5)));
+
+ verify(jdbcTemplate, times(2))
+ .update(
+ anyString(),
+ any(),
+ any(),
+ any(),
+ any(),
+ any(),
+ any());
+ }
+
+ @Test
+ void rejectsAlarmRowsWithoutStableTimes() {
+ LateAlarmBucketWriter writer = new LateAlarmBucketWriter(mock(JdbcTemplate.class));
+ ReflectionTestUtils.setField(writer, "enabled", true);
+ Map invalid = new HashMap<>();
+ invalid.put("device_id", 126);
+
+ assertThrows(IllegalArgumentException.class, () -> writer.record(Arrays.asList(invalid)));
+ }
+
+ private Map row(int deviceId, LocalDateTime eventTime, int receiveSecond) {
+ Map row = new HashMap<>();
+ row.put("device_id", deviceId);
+ row.put("log_time", eventTime);
+ row.put("created_at", LocalDateTime.of(2026, 9, 5, 1, 0, receiveSecond));
+ return row;
+ }
+}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/NormalizationCachePropertiesTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/NormalizationCachePropertiesTest.java
index 9f77903..10cc574 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/NormalizationCachePropertiesTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/NormalizationCachePropertiesTest.java
@@ -22,6 +22,7 @@ class NormalizationCachePropertiesTest {
assertFalse(properties.isJetcacheEnabled());
assertFalse(properties.getRuleSourceCache().isEnabled());
assertFalse(properties.getCompiledRule().isEnabled());
+ assertEquals(true, properties.getRuleSelection().isEnabled());
assertEquals(1000, properties.getCompiledRule().getShadowTimeoutMs());
assertEquals(30, properties.getRuleSourceCache().getLocalTtlSeconds());
assertEquals(300, properties.getRuleSourceCache().getRemoteTtlSeconds());
@@ -58,6 +59,8 @@ class NormalizationCachePropertiesTest {
"normalization.batch-enrichment-cache.enabled"), resource);
assertEquals("true", values.getProperty(
"normalization.message-parse-cache.enabled"), resource);
+ assertEquals("true", values.getProperty(
+ "normalization.rule-selection.enabled"), resource);
NormalizationOptimizationProperties bound = new Binder(
new MapConfigurationPropertySource(values))
@@ -72,6 +75,7 @@ class NormalizationCachePropertiesTest {
assertEquals(true, bound.getCompiledRule().isRuleDetailLogEnabled(), resource);
assertEquals(true, bound.getBatchEnrichmentCache().isEnabled(), resource);
assertEquals(true, bound.getMessageParseCache().isEnabled(), resource);
+ assertEquals(true, bound.getRuleSelection().isEnabled(), resource);
}
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java
index 674e4c0..4bdce65 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java
@@ -29,9 +29,9 @@ class InfluxRawV2ContractTest {
@Test void configurationRejectsChangingPersistentBucketCount() {
RawStorageProperties properties=new RawStorageProperties();
ReflectionTestUtils.setField(properties,"writeSchemaValue","V2");
- ReflectionTestUtils.setField(properties,"readModeValue","V2_ONLY");
+ ReflectionTestUtils.setField(properties,"readModeValue","V1_V2");
ReflectionTestUtils.setField(properties,"v1Bucket","yelangbucket");
- ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket");
+ ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket_v2");
ReflectionTestUtils.setField(properties,"uuidBucketCount",1024);
ReflectionTestUtils.setField(properties,"queryMaxRows",10000);
ReflectionTestUtils.setField(properties,"queryMaxResponseBytes",10485760L);
@@ -62,9 +62,9 @@ class InfluxRawV2ContractTest {
}
@Test void v2HasUuidFieldButNeverUuidTagAndUsesNanoseconds() {
- RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
+ RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
InfluxRawPointFactory.RawPoint raw=new InfluxRawPointFactory(properties).create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5","message","20260814000000000","126","1","agent-01-syslog-topic",ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L);
- String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket",raw.getBucket());
+ String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket_v2",raw.getBucket());
}
@Test void onlyMissingPointTimeFallsBack() {
@@ -83,7 +83,7 @@ class InfluxRawV2ContractTest {
}
@Test void mixedLegacyAndNewMessagesAllProduceDistinctV2PointsAndRetryIsIdempotent() {
- RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
+ RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); InfluxRawPointFactory factory=new InfluxRawPointFactory(properties); ReceiveTimeParser.ParsedReceiveTime parsed=ReceiveTimeParser.parse("20260814000000000");
long first=resolver.resolve(false,null,parsed.getEpochMillis(),true); long second=resolver.resolve(false,null,parsed.getEpochMillis(),true); assertNotEquals(first,second);
String id="20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"; String originalTopic="agent-01-syslog-topic";
@@ -100,7 +100,7 @@ class InfluxRawV2ContractTest {
@Test void uuidCandidateFluxComputesBucketEscapesUuidAndSelectsEarliestGlobally() {
String uuid="20260814-a\" |> die() //";
- String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket",
+ String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket_v2",
"syslog_security_20260814",uuid,Instant.parse("2026-08-13T16:00:00Z"),
Instant.parse("2026-08-14T16:00:00Z"));
assertTrue(query.contains("r.event_bucket == \""+InfluxEventBucket.bucketOf(uuid)+"\""));
diff --git a/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-prod.yaml b/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-prod.yaml
index 3dfba92..7f5565f 100644
--- a/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-prod.yaml
+++ b/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-prod.yaml
@@ -78,9 +78,9 @@ services:
- INFLUXDB_ORG=influxdb
- INFLUXDB_BUCKET=yelangbucket
- INFLUXDB_RAW_WRITE_SCHEMA=V2
- - INFLUXDB_RAW_READ_MODE=V2_ONLY
+ - INFLUXDB_RAW_READ_MODE=V1_V2
- INFLUXDB_RAW_V1_BUCKET=yelangbucket
- - INFLUXDB_RAW_V2_BUCKET=yelangbucket
+ - INFLUXDB_RAW_V2_BUCKET=yelangbucket_v2
- INFLUXDB_RAW_UUID_BUCKET_COUNT=4096
- INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED=true
- INFLUXDB_RAW_QUERY_MAX_ROWS=10000
diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java
index 851aa4c..7a213ba 100644
--- a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java
+++ b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java
@@ -75,6 +75,8 @@ public class LogNormalProcessor {
Map deviceInfoMap;
String strSyslogUUID;
String strSyslogTopic;
+ LocalDateTime initialReceivedAt;
+ LocalDateTime eventAt;
boolean isSaveNonNormal = false;
SyslogNonNormalMessage syslogNonNormalMessage = new SyslogNonNormalMessage();
List> dmNormalizeRuleList;
@@ -156,6 +158,11 @@ public class LogNormalProcessor {
// [流程节点6.2] 解析设备上下文，确定 device_id 和 data_type，作为选择标准化规则的依据。
Map mapdev = SyslogParser.parseKeyValuePairs(ctx.strDeviceInfo);
ctx.deviceInfoMap = mapdev;
+ ctx.initialReceivedAt = parseReceiveTime(mapdev.get("receive_time"));
+ if (ctx.initialReceivedAt == null) {
+ throw new IllegalArgumentException("日志缺少合法 receive_time");
+ }
+ ctx.eventAt = resolveEventAt(mapdev.get("event_time"), ctx.initialReceivedAt);
long deviceID = Long.parseLong(mapdev.get("device_id"));
logger.debug("标准化开始: eventId={}, deviceId={}", ctx.strSyslogUUID, deviceID);
ctx.strDataType = mapdev.get("data_type");
@@ -464,12 +471,9 @@ public class LogNormalProcessor {
CompletionCollectTaskInfo(columnMap,ctx);
//补全日志基础信息
columnMap.put("device_id", deviceId);
- // [SYSLOG-ALARM][ETIME] 这里用当前标准化处理时间覆盖规则已经解析出的 log_time，混淆设备事件时间与处理时间。
- // 积压或迟到时，事件会被移动到错误分析窗口。建议分别保存 device_event_time、receive_time、normalized_at，
- // 仅在设备时间确实缺失且规则明确允许时回退到 receive_time，不应无条件覆盖。
- if (!columnMap.containsKey("log_time") || columnMap.get("log_time") == null) {
- columnMap.put("log_time", logtime);
- }
+ // log_time 是两张标准化表的 RANGE 分区键，使用设备事件时间；created_at 保留首次接收时间。
+ // TCP/UDP 的 event_time 等于 receive_time，因此原有实时接入的分区行为保持不变。
+ applyIngressTimes(columnMap, ctx.eventAt, ctx.initialReceivedAt);
// 同一 event_id + 规则在重试/回放时生成相同主键，数据库唯一约束可把重复执行收敛为幂等结果。
columnMap.put("id", UUID.nameUUIDFromBytes((ctx.strSyslogUUID + ":" + normalizeRuleId + ":legacy")
.getBytes(java.nio.charset.StandardCharsets.UTF_8)).toString());
@@ -568,4 +572,18 @@ public class LogNormalProcessor {
}
}
+ /** 新消息使用 event_time；历史 TCP/UDP 或旧 Kafka 消息缺失时回退到 receive_time。 */
+ static LocalDateTime resolveEventAt(String eventTime, LocalDateTime initialReceivedAt) {
+ if (eventTime == null || eventTime.trim().isEmpty()) {
+ return initialReceivedAt;
+ }
+ return IngressReceiveTimeUtil.parse(eventTime);
+ }
+
+ static void applyIngressTimes(Map columnMap, LocalDateTime eventAt,
+ LocalDateTime initialReceivedAt) {
+ columnMap.put("log_time", eventAt);
+ columnMap.put("created_at", initialReceivedAt);
+ }
+
}
diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java
index 387e214..1320295 100644
--- a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java
+++ b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java
@@ -36,7 +36,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.time.LocalDate;
import java.time.LocalDateTime;
-import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
@@ -323,32 +322,33 @@ public class SysLogProcessor {
}
String receiveTime = deviceInfo.get("receive_time");
- ReceiveTimeParser.ParsedReceiveTime parsed = ReceiveTimeParser.parse(receiveTime);
+ ReceiveTimeParser.ParsedReceiveTime parsedReceiveTime = ReceiveTimeParser.parse(receiveTime);
+ String eventTime = effectiveEventTime(deviceInfo.get("event_time"), receiveTime);
+ ReceiveTimeParser.ParsedReceiveTime parsedEventTime = ReceiveTimeParser.parse(eventTime);
boolean headerHasPointTime = deviceInfo.containsKey("point_time_ns");
Long envelopePointTime = messageContext.getPointTimeNs();
Long headerPointTime = pointTimeResolver.resolve(headerHasPointTime,
- deviceInfo.get("point_time_ns"), parsed.getEpochMillis(), false);
+ deviceInfo.get("point_time_ns"), parsedEventTime.getEpochMillis(), false);
if (envelopePointTime != null) {
- PointTimeResolver.validate(envelopePointTime, parsed.getEpochMillis());
+ PointTimeResolver.validate(envelopePointTime, parsedEventTime.getEpochMillis());
if (headerPointTime != null && !headerPointTime.equals(envelopePointTime)) {
throw new IllegalArgumentException("Retry/Replay point_time_ns 与原始 Kafka 载荷不一致");
}
}
Long pointTimeNs = envelopePointTime != null ? envelopePointTime : headerPointTime;
if (pointTimeNs == null && rawStorageProperties.getWriteSchema() == RawStorageProperties.WriteSchema.V2) {
- pointTimeNs = pointTimeResolver.resolve(false, null, parsed.getEpochMillis(), true);
+ pointTimeNs = pointTimeResolver.resolve(false, null, parsedEventTime.getEpochMillis(), true);
}
messageContext.setPointTimeNs(pointTimeNs);
String originalTopic = messageContext.getOriginalTopic();
InfluxRawPointFactory.RawPoint rawPoint = rawPointFactory.create(eventId, decryptedMessage,
receiveTime, deviceInfo.get("device_id"), deviceInfo.get("device_collect_id"),
- originalTopic, parsed, pointTimeNs);
- ResolvedInfluxTime influxTime = new ResolvedInfluxTime(parsed.getEpochMillis(),
- parsed.getMeasurementDate(), parsed.getLocalDateTime());
+ originalTopic, parsedEventTime, pointTimeNs);
Long deviceId = optionalLong(deviceInfo.get("device_id"));
Long collectId = optionalLong(deviceInfo.get("device_collect_id"));
return new PreparedMessage(record, messageContext, decryptedMessage, eventId, rawPoint.getPoint(),
- rawPoint.getBucket(), rawPoint.isV2(), influxTime.getReceivedAt(), deviceId, collectId, startTime);
+ rawPoint.getBucket(), rawPoint.isV2(), parsedReceiveTime.getLocalDateTime(),
+ deviceId, collectId, startTime);
}
/**
@@ -458,26 +458,28 @@ public class SysLogProcessor {
catch (NumberFormatException invalid) { return null; }
}
- /**
- * 为 InfluxDB 选择跨 Kafka Retry/Replay 稳定的时间戳。
- * 优先使用接入端 receive_time；旧消息缺失或格式异常时，回退到首次 Kafka 接收时间。
- */
- static ResolvedInfluxTime resolveInfluxTime(String receiveTime, Instant ignoredFallbackTime) {
- ReceiveTimeParser.ParsedReceiveTime parsed = ReceiveTimeParser.parse(receiveTime);
+ /** 新消息使用 event_time；历史 Kafka 消息缺失该字段时回退到 receive_time。 */
+ static ResolvedInfluxTime resolveInfluxTime(String eventTime, String receiveTime) {
+ ReceiveTimeParser.ParsedReceiveTime parsed = ReceiveTimeParser.parse(
+ effectiveEventTime(eventTime, receiveTime));
return new ResolvedInfluxTime(
parsed.getEpochMillis(), parsed.getMeasurementDate(), parsed.getLocalDateTime());
}
- /** InfluxDB 点的时间戳和 measurement 日期必须由同一个接收时间计算。 */
+ static String effectiveEventTime(String eventTime, String receiveTime) {
+ return eventTime == null || eventTime.trim().isEmpty() ? receiveTime : eventTime;
+ }
+
+ /** InfluxDB 点的时间戳和 measurement 日期必须由同一个事件时间计算。 */
static final class ResolvedInfluxTime {
private final long timestampMs;
private final String measurementDate;
- private final LocalDateTime receivedAt;
+ private final LocalDateTime eventAt;
- private ResolvedInfluxTime(long timestampMs, String measurementDate, LocalDateTime receivedAt) {
+ private ResolvedInfluxTime(long timestampMs, String measurementDate, LocalDateTime eventAt) {
this.timestampMs = timestampMs;
this.measurementDate = measurementDate;
- this.receivedAt = receivedAt;
+ this.eventAt = eventAt;
}
long getTimestampMs() {
@@ -488,7 +490,7 @@ public class SysLogProcessor {
return measurementDate;
}
- LocalDateTime getReceivedAt() { return receivedAt; }
+ LocalDateTime getEventAt() { return eventAt; }
}
diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/LateRuleReplayScheduler.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/LateRuleReplayScheduler.java
new file mode 100644
index 0000000..3f9b0f2
--- /dev/null
+++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/LateRuleReplayScheduler.java
@@ -0,0 +1,187 @@
+package com.common.schedule;
+
+import com.common.entity.AnalysisAnalysisRule;
+import com.common.mapper.AnalysisAnalysisRuleMapper;
+import com.common.service.AnalysisRuleService;
+import com.common.service.LateRuleReplayStore;
+import com.common.service.LateRuleReplayStore.ReplayWindow;
+import com.common.service.RuleExecutionWindow;
+import com.common.service.RuleWindowContext;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+import java.util.Map;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+/**
+ * 迟到标准化告警的独立规则重放调度器。
+ *
+ * 该调度器只消费 outbox 派生出的历史窗口，不修改正常实时规则 checkpoint 和下次执行时间。
+ * 重放仍调用统一规则引擎，因告警 ID 由规则版本、窗口和聚合结果确定，已存在告警会覆盖更新
+ * log_count、时间范围和来源日志集合，不会把同一窗口重复累加。
+ */
+@Slf4j
+@Component
+public class LateRuleReplayScheduler {
+
+ private final LateRuleReplayStore store;
+ private final AnalysisRuleService analysisRuleService;
+ private final AnalysisAnalysisRuleMapper ruleMapper;
+ private final RuleWindowContext ruleWindowContext;
+ private final String owner = ManagementFactory.getRuntimeMXBean().getName();
+
+ @Value("${analysis.late-replay.enabled:true}")
+ private boolean enabled;
+
+ @Value("${analysis.late-replay.lease-seconds:1800}")
+ private int leaseSeconds;
+
+ @Value("${analysis.late-replay.max-buckets-per-scan:100}")
+ private int maxBucketsPerScan;
+
+ @Value("${analysis.late-replay.max-windows-per-scan:10}")
+ private int maxWindowsPerScan;
+
+ @Value("${analysis.late-replay.retry-delay-seconds:30}")
+ private int retryDelaySeconds;
+
+ public LateRuleReplayScheduler(
+ LateRuleReplayStore store,
+ AnalysisRuleService analysisRuleService,
+ AnalysisAnalysisRuleMapper ruleMapper,
+ RuleWindowContext ruleWindowContext) {
+ this.store = store;
+ this.analysisRuleService = analysisRuleService;
+ this.ruleMapper = ruleMapper;
+ this.ruleWindowContext = ruleWindowContext;
+ }
+
+ /** 先展开到达通知，再有界执行重放，避免一次扫描长期占用调度线程。 */
+ @Scheduled(fixedDelayString = "${analysis.late-replay.scan-interval-ms:5000}")
+ public void scanAndReplay() {
+ if (!enabled) {
+ return;
+ }
+
+ try {
+ expandBuckets();
+ } catch (Exception failure) {
+ log.error("迟到告警通知展开失败，本轮暂停领取新的重放窗口", failure);
+ return;
+ }
+
+ for (int i = 0; i < Math.max(1, maxWindowsPerScan); i++) {
+ ReplayWindow window = store.claimOneReplay(owner, leaseSeconds);
+ if (window == null) {
+ return;
+ }
+ replay(window);
+ }
+ }
+
+ private void expandBuckets() {
+ for (int i = 0; i < Math.max(1, maxBucketsPerScan); i++) {
+ if (store.expandOneBucket(owner, leaseSeconds) == 0) {
+ return;
+ }
+ }
+ }
+
+ private void replay(ReplayWindow window) {
+ try {
+ AnalysisAnalysisRule rule = ruleMapper.selectByRuleId(window.getRuleId());
+ if (rule == null
+ || !Integer.valueOf(1).equals(rule.getRuleStatus())
+ || !"realtime".equalsIgnoreCase(rule.getRunMode())) {
+ store.blockReplay(window, "RULE_NOT_ACTIVE", "规则不存在、未启用或不是实时规则");
+ return;
+ }
+
+ String currentVersion = LateRuleReplayStore.checkpointVersion(rule.getVersion());
+ if (!currentVersion.equals(window.getRuleVersion())) {
+ store.blockReplay(
+ window,
+ "RULE_VERSION_CHANGED",
+ "规则版本已变化，禁止用新规则配置覆盖旧窗口结果");
+ return;
+ }
+
+ RuleExecutionWindow executionWindow =
+ new RuleExecutionWindow(
+ window.getWindowStart(),
+ window.getWindowEnd(),
+ window.getWindowEnd(),
+ "LATE_REPLAY");
+ Map result;
+ ruleWindowContext.set(executionWindow);
+ try {
+ result = analysisRuleService.executeRealtimeRule(window.getRuleId());
+ } finally {
+ ruleWindowContext.clear();
+ }
+
+ if (!Boolean.TRUE.equals(result.get("success"))) {
+ store.failReplay(
+ window,
+ "RULE_REPLAY_FAILED",
+ String.valueOf(result.get("message")),
+ retryDelaySeconds);
+ return;
+ }
+
+ long inputCount = sumResultCount(result, "processedCount");
+ long outputCount = sumResultCount(result, "alarmCount");
+ store.completeReplay(window, inputCount, outputCount);
+ log.info(
+ "迟到告警规则窗口重放完成: replayId={}, ruleId={}, windowStart={}, windowEnd={},"
+ + " inputCount={}, outputCount={}, generation={}",
+ window.getId(),
+ window.getRuleId(),
+ window.getWindowStart(),
+ window.getWindowEnd(),
+ inputCount,
+ outputCount,
+ window.getClaimedGeneration());
+ } catch (Exception failure) {
+ log.warn(
+ "迟到告警规则窗口重放失败: replayId={}, ruleId={}, windowStart={}, windowEnd={}",
+ window.getId(),
+ window.getRuleId(),
+ window.getWindowStart(),
+ window.getWindowEnd(),
+ failure);
+ try {
+ store.failReplay(
+ window, "RULE_REPLAY_EXCEPTION", safeMessage(failure), retryDelaySeconds);
+ } catch (Exception stateFailure) {
+ log.error("迟到告警重放失败状态写入失败: replayId={}", window.getId(), stateFailure);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private long sumResultCount(Map result, String key) {
+ Object values = result.get("results");
+ if (!(values instanceof List)) {
+ return 0L;
+ }
+ long total = 0L;
+ for (Map item : (List>) values) {
+ Object value = item.get(key);
+ if (value instanceof Number) {
+ total += ((Number) value).longValue();
+ }
+ }
+ return total;
+ }
+
+ private String safeMessage(Throwable failure) {
+ String message = failure.getMessage();
+ if (message == null || message.trim().isEmpty()) {
+ return failure.getClass().getSimpleName();
+ }
+ return message.length() <= 2000 ? message : message.substring(0, 2000);
+ }
+}
diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/RealtimeAnalysisScheduler.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/RealtimeAnalysisScheduler.java
index 3dc8487..00a7a20 100644
--- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/RealtimeAnalysisScheduler.java
+++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/RealtimeAnalysisScheduler.java
@@ -30,7 +30,7 @@ import java.util.Map;
@Component
public class RealtimeAnalysisScheduler {
- /** 窗口语义版本，用于隔离应用管理的窗口与旧版未对齐检查点。 */
+ /** Keeps application-managed windows isolated from legacy unaligned checkpoints. */
private static final String WINDOW_SEMANTICS_VERSION = "window-v2";
@Autowired
diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/LateRuleReplayStore.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/LateRuleReplayStore.java
new file mode 100644
index 0000000..6dee0a4
--- /dev/null
+++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/LateRuleReplayStore.java
@@ -0,0 +1,370 @@
+package com.common.service;
+
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.UUID;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+/**
+ * 迟到告警通知和规则重放窗口的达梦状态机。
+ *
+ * 标准化服务只写事件分钟通知。通知能够被本服务领取，表示对应事实事务已经提交；此时凡是
+ * 已经完成且与事件分钟相交的规则窗口都登记重放。该判断不比较事务内 SYSDATE 与 checkpoint
+ * 完成时间，避免“事实先写、checkpoint 后完成、事实最后提交”时漏掉重放。重放使用独立状态表，
+ * 不回退正常水位。
+ */
+@Service
+public class LateRuleReplayStore {
+
+ private static final String WINDOW_SEMANTICS_VERSION = "window-v2";
+ private final JdbcTemplate jdbcTemplate;
+
+ public LateRuleReplayStore(JdbcTemplate jdbcTemplate) {
+ this.jdbcTemplate = jdbcTemplate;
+ }
+
+ /**
+ * 领取一个事件分钟通知并登记受影响的规则窗口。
+ *
+ * @return 0 表示没有可领取通知；正数表示已处理一个通知
+ */
+ @Transactional
+ public int expandOneBucket(String owner, int leaseSeconds) {
+ List ids =
+ jdbcTemplate.query(
+ "SELECT id FROM (SELECT id FROM analysis_late_alarm_bucket "
+ + "WHERE (status IN ('PENDING','FAILED') "
+ + "OR (status='RUNNING' AND lease_expires_at rs.getString(1));
+ if (ids.isEmpty()) {
+ return 0;
+ }
+
+ String id = ids.get(0);
+ int claimed =
+ jdbcTemplate.update(
+ "UPDATE analysis_late_alarm_bucket SET status='RUNNING',lease_owner=?,"
+ + "lease_expires_at=SYSDATE+(CAST(? AS DECIMAL(18,6))/86400),"
+ + "fencing_token=fencing_token+1,claimed_generation=requested_generation,"
+ + "attempt_count=attempt_count+1,last_error_code=NULL,last_error_summary=NULL,"
+ + "updated_at=SYSDATE WHERE id=? AND (status IN ('PENDING','FAILED') "
+ + "OR (status='RUNNING' AND lease_expires_at windows = findLateCompletedWindows(bucket);
+ for (ReplayKey window : windows) {
+ enqueueReplay(window, bucket.id);
+ }
+
+ int completed =
+ jdbcTemplate.update(
+ "UPDATE analysis_late_alarm_bucket SET processed_generation=claimed_generation,"
+ + "status=CASE WHEN requested_generation=claimed_generation "
+ + "THEN 'COMPLETED' ELSE 'PENDING' END,lease_owner=NULL,lease_expires_at=NULL,"
+ + "last_error_code=NULL,last_error_summary=NULL,updated_at=SYSDATE "
+ + "WHERE id=? AND lease_owner=? AND fencing_token=? AND status='RUNNING'",
+ id,
+ owner,
+ bucket.fencingToken);
+ requireOwnedUpdate(completed, "完成通知");
+ return windows.size() + 1;
+ }
+
+ private LateBucket selectBucket(String id) {
+ return jdbcTemplate.queryForObject(
+ "SELECT id,device_id,bucket_start,bucket_end,fencing_token "
+ + "FROM analysis_late_alarm_bucket WHERE id=?",
+ (rs, rowNum) ->
+ new LateBucket(
+ rs.getString("id"),
+ rs.getLong("device_id"),
+ requiredTime(rs.getTimestamp("bucket_start")),
+ requiredTime(rs.getTimestamp("bucket_end")),
+ rs.getLong("fencing_token")),
+ id);
+ }
+
+ private List findLateCompletedWindows(LateBucket bucket) {
+ String sql =
+ "SELECT DISTINCT c.task_id,c.task_version,c.window_start,c.window_end "
+ + "FROM analysis_checkpoint c WHERE c.task_type='RULE' AND c.status='COMPLETED' "
+ + "AND c.window_start? AND EXISTS ("
+ + "SELECT 1 FROM syslog_normal_alarm a WHERE a.device_id=? "
+ + "AND a.log_time>=c.window_start AND a.log_time=? AND a.log_time
+ new ReplayKey(
+ rs.getString("task_id"),
+ rs.getString("task_version"),
+ requiredTime(rs.getTimestamp("window_start")),
+ requiredTime(rs.getTimestamp("window_end"))),
+ Timestamp.valueOf(bucket.bucketEnd),
+ Timestamp.valueOf(bucket.bucketStart),
+ bucket.deviceId,
+ Timestamp.valueOf(bucket.bucketStart),
+ Timestamp.valueOf(bucket.bucketEnd));
+ }
+
+ private void enqueueReplay(ReplayKey window, String sourceBucketId) {
+ String sql =
+ "MERGE INTO analysis_rule_replay_window t USING ("
+ + "SELECT ? rule_id,? rule_version,? window_start,? window_end FROM dual) s "
+ + "ON (t.rule_id=s.rule_id AND t.rule_version=s.rule_version "
+ + "AND t.window_start=s.window_start AND t.window_end=s.window_end) "
+ + "WHEN MATCHED THEN UPDATE SET t.requested_generation=t.requested_generation+1,"
+ + "t.source_bucket_id=?,t.status=CASE WHEN t.status='RUNNING' THEN 'RUNNING' "
+ + "ELSE 'PENDING' END,t.next_retry_at=NULL,t.updated_at=SYSDATE "
+ + "WHEN NOT MATCHED THEN INSERT "
+ + "(id,source_bucket_id,rule_id,rule_version,window_start,window_end,"
+ + "requested_generation,claimed_generation,processed_generation,status,attempt_count,"
+ + "input_count,output_count,fencing_token,created_at,updated_at) "
+ + "VALUES (?,?,?,?,?,?,1,0,0,'PENDING',0,0,0,0,SYSDATE,SYSDATE)";
+ jdbcTemplate.update(
+ sql,
+ window.ruleId,
+ window.ruleVersion,
+ Timestamp.valueOf(window.windowStart),
+ Timestamp.valueOf(window.windowEnd),
+ sourceBucketId,
+ UUID.randomUUID().toString(),
+ sourceBucketId,
+ window.ruleId,
+ window.ruleVersion,
+ Timestamp.valueOf(window.windowStart),
+ Timestamp.valueOf(window.windowEnd));
+ }
+
+ /** 领取一个到期的历史规则重放窗口。 */
+ @Transactional
+ public ReplayWindow claimOneReplay(String owner, int leaseSeconds) {
+ List ids =
+ jdbcTemplate.query(
+ "SELECT id FROM (SELECT id FROM analysis_rule_replay_window "
+ + "WHERE ((status IN ('PENDING','FAILED') "
+ + "AND (next_retry_at IS NULL OR next_retry_at<=SYSDATE)) "
+ + "OR (status='RUNNING' AND lease_expires_at rs.getString(1));
+ if (ids.isEmpty()) {
+ return null;
+ }
+
+ String id = ids.get(0);
+ int claimed =
+ jdbcTemplate.update(
+ "UPDATE analysis_rule_replay_window SET status='RUNNING',lease_owner=?,"
+ + "lease_expires_at=SYSDATE+(CAST(? AS DECIMAL(18,6))/86400),"
+ + "fencing_token=fencing_token+1,claimed_generation=requested_generation,"
+ + "attempt_count=attempt_count+1,started_at=SYSDATE,finished_at=NULL,"
+ + "last_error_code=NULL,last_error_summary=NULL,updated_at=SYSDATE WHERE id=? "
+ + "AND ((status IN ('PENDING','FAILED') "
+ + "AND (next_retry_at IS NULL OR next_retry_at<=SYSDATE)) "
+ + "OR (status='RUNNING' AND lease_expires_at
+ new ReplayWindow(
+ rs.getString("id"),
+ rs.getString("rule_id"),
+ rs.getString("rule_version"),
+ requiredTime(rs.getTimestamp("window_start")),
+ requiredTime(rs.getTimestamp("window_end")),
+ rs.getLong("claimed_generation"),
+ rs.getLong("fencing_token"),
+ owner),
+ id);
+ }
+
+ /** 完成重放；若执行期间请求代数增加，则回到 PENDING 等待下一轮。 */
+ @Transactional
+ public void completeReplay(ReplayWindow window, long inputCount, long outputCount) {
+ int updated =
+ jdbcTemplate.update(
+ "UPDATE analysis_rule_replay_window SET processed_generation=claimed_generation,"
+ + "status=CASE WHEN requested_generation=claimed_generation "
+ + "THEN 'COMPLETED' ELSE 'PENDING' END,input_count=?,output_count=?,"
+ + "lease_owner=NULL,lease_expires_at=NULL,next_retry_at=NULL,finished_at=SYSDATE,"
+ + "last_error_code=NULL,last_error_summary=NULL,updated_at=SYSDATE "
+ + "WHERE id=? AND status='RUNNING' AND lease_owner=? AND fencing_token=?",
+ inputCount,
+ outputCount,
+ window.id,
+ window.owner,
+ window.fencingToken);
+ requireOwnedUpdate(updated, "完成重放");
+ }
+
+ /** 记录可重试失败并释放租约。 */
+ @Transactional
+ public void failReplay(
+ ReplayWindow window, String errorCode, String errorSummary, int retrySeconds) {
+ int updated =
+ jdbcTemplate.update(
+ "UPDATE analysis_rule_replay_window SET status='FAILED',lease_owner=NULL,"
+ + "lease_expires_at=NULL,next_retry_at=SYSDATE+(CAST(? AS DECIMAL(18,6))/86400),"
+ + "finished_at=SYSDATE,last_error_code=?,last_error_summary=?,updated_at=SYSDATE "
+ + "WHERE id=? AND status='RUNNING' AND lease_owner=? AND fencing_token=?",
+ retrySeconds,
+ errorCode,
+ truncate(errorSummary),
+ window.id,
+ window.owner,
+ window.fencingToken);
+ requireOwnedUpdate(updated, "记录失败");
+ }
+
+ /** 规则已删除、停用或版本变化时保留审计记录，不用新配置重算旧窗口。 */
+ @Transactional
+ public void blockReplay(ReplayWindow window, String errorCode, String errorSummary) {
+ int updated =
+ jdbcTemplate.update(
+ "UPDATE analysis_rule_replay_window SET status='BLOCKED',lease_owner=NULL,"
+ + "lease_expires_at=NULL,next_retry_at=NULL,finished_at=SYSDATE,last_error_code=?,"
+ + "last_error_summary=?,updated_at=SYSDATE WHERE id=? AND status='RUNNING' "
+ + "AND lease_owner=? AND fencing_token=?",
+ errorCode,
+ truncate(errorSummary),
+ window.id,
+ window.owner,
+ window.fencingToken);
+ requireOwnedUpdate(updated, "阻断重放");
+ }
+
+ public static String checkpointVersion(Integer ruleVersion) {
+ return String.valueOf(ruleVersion == null ? 0 : ruleVersion) + ":" + WINDOW_SEMANTICS_VERSION;
+ }
+
+ private static void requireOwnedUpdate(int updated, String action) {
+ if (updated != 1) {
+ throw new IllegalStateException("迟到规则重放租约失效，拒绝" + action);
+ }
+ }
+
+ private static LocalDateTime requiredTime(Timestamp value) {
+ if (value == null) {
+ throw new IllegalStateException("迟到规则重放状态缺少时间字段");
+ }
+ return value.toLocalDateTime();
+ }
+
+ private static String truncate(String value) {
+ if (value == null) {
+ return null;
+ }
+ return value.length() <= 2000 ? value : value.substring(0, 2000);
+ }
+
+ private static final class LateBucket {
+ private final String id;
+ private final long deviceId;
+ private final LocalDateTime bucketStart;
+ private final LocalDateTime bucketEnd;
+ private final long fencingToken;
+
+ private LateBucket(
+ String id,
+ long deviceId,
+ LocalDateTime bucketStart,
+ LocalDateTime bucketEnd, long fencingToken) {
+ this.id = id;
+ this.deviceId = deviceId;
+ this.bucketStart = bucketStart;
+ this.bucketEnd = bucketEnd;
+ this.fencingToken = fencingToken;
+ }
+ }
+
+ private static final class ReplayKey {
+ private final String ruleId;
+ private final String ruleVersion;
+ private final LocalDateTime windowStart;
+ private final LocalDateTime windowEnd;
+
+ private ReplayKey(
+ String ruleId, String ruleVersion, LocalDateTime windowStart, LocalDateTime windowEnd) {
+ this.ruleId = ruleId;
+ this.ruleVersion = ruleVersion;
+ this.windowStart = windowStart;
+ this.windowEnd = windowEnd;
+ }
+ }
+
+ /** 调度器持有的不可变租约快照。 */
+ public static final class ReplayWindow {
+ private final String id;
+ private final String ruleId;
+ private final String ruleVersion;
+ private final LocalDateTime windowStart;
+ private final LocalDateTime windowEnd;
+ private final long claimedGeneration;
+ private final long fencingToken;
+ private final String owner;
+
+ public ReplayWindow(
+ String id,
+ String ruleId,
+ String ruleVersion,
+ LocalDateTime windowStart,
+ LocalDateTime windowEnd,
+ long claimedGeneration,
+ long fencingToken,
+ String owner) {
+ this.id = id;
+ this.ruleId = ruleId;
+ this.ruleVersion = ruleVersion;
+ this.windowStart = windowStart;
+ this.windowEnd = windowEnd;
+ this.claimedGeneration = claimedGeneration;
+ this.fencingToken = fencingToken;
+ this.owner = owner;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public String getRuleId() {
+ return ruleId;
+ }
+
+ public String getRuleVersion() {
+ return ruleVersion;
+ }
+
+ public LocalDateTime getWindowStart() {
+ return windowStart;
+ }
+
+ public LocalDateTime getWindowEnd() {
+ return windowEnd;
+ }
+
+ public long getClaimedGeneration() {
+ return claimedGeneration;
+ }
+
+ public long getFencingToken() {
+ return fencingToken;
+ }
+ }
+}
diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/InfluxRawPointFactory.java b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/InfluxRawPointFactory.java
index 7c1ce76..d056bba 100644
--- a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/InfluxRawPointFactory.java
+++ b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/InfluxRawPointFactory.java
@@ -13,17 +13,17 @@ public class InfluxRawPointFactory {
public RawPoint create(String eventId, String message, String receiveTime,
String deviceId, String deviceCollectId, String originalTopic,
- ReceiveTimeParser.ParsedReceiveTime parsed, Long pointTimeNs) {
+ ReceiveTimeParser.ParsedReceiveTime parsedEventTime, Long pointTimeNs) {
require(eventId, "event_id"); require(receiveTime, "receive_time");
require(deviceId, "device_id"); require(deviceCollectId, "device_collect_id");
require(originalTopic, "originalTopic");
- String measurement = "syslog_security_" + parsed.getMeasurementDate();
+ String measurement = "syslog_security_" + parsedEventTime.getMeasurementDate();
if (properties.getWriteSchema() == RawStorageProperties.WriteSchema.V1) {
Point point = Point.measurement(measurement)
.addTag("device_id", deviceId).addTag("device_collect_id", deviceCollectId)
.addTag("uuid", eventId).addTag("topic", originalTopic)
.addField("message", message).addField("receive_time", receiveTime)
- .addField("uuid", eventId).time(parsed.getEpochMillis(), WritePrecision.MS);
+ .addField("uuid", eventId).time(parsedEventTime.getEpochMillis(), WritePrecision.MS);
return new RawPoint(point, properties.getV1Bucket(), false);
}
if (pointTimeNs == null) throw new IllegalArgumentException("V2 Point 缺少 point_time_ns");
diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/PointTimeResolver.java b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/PointTimeResolver.java
index d919d07..c5d6f38 100644
--- a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/PointTimeResolver.java
+++ b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/PointTimeResolver.java
@@ -14,7 +14,7 @@ public class PointTimeResolver {
this.metrics = metrics;
}
- public Long resolve(boolean present, String rawValue, long receiveTimeMillis, boolean required) {
+ public Long resolve(boolean present, String rawValue, long eventTimeMillis, boolean required) {
if (!present) {
if (!required) return null;
metrics.legacyMissing();
@@ -23,7 +23,7 @@ public class PointTimeResolver {
throw new IllegalArgumentException("旧 Kafka 消息缺少 point_time_ns，且 legacy fallback 已关闭");
}
try {
- long assigned = allocator.allocate(receiveTimeMillis);
+ long assigned = allocator.allocate(eventTimeMillis);
metrics.legacyAssigned();
return assigned;
} catch (RuntimeException failure) {
@@ -37,14 +37,14 @@ public class PointTimeResolver {
final long value;
try { value = Long.parseLong(rawValue); }
catch (NumberFormatException invalid) { throw new IllegalArgumentException("point_time_ns 非数字或溢出", invalid); }
- validate(value, receiveTimeMillis);
+ validate(value, eventTimeMillis);
return value;
}
- public static void validate(long pointTimeNs, long receiveTimeMillis) {
+ public static void validate(long pointTimeNs, long eventTimeMillis) {
if (pointTimeNs <= 0L) throw new IllegalArgumentException("point_time_ns 必须为正整数");
- if (Math.floorDiv(pointTimeNs, PointTimeAllocator.NANOS_PER_MILLI) != receiveTimeMillis) {
- throw new IllegalArgumentException("point_time_ns 毫秒部分与 receive_time 不一致");
+ if (Math.floorDiv(pointTimeNs, PointTimeAllocator.NANOS_PER_MILLI) != eventTimeMillis) {
+ throw new IllegalArgumentException("point_time_ns 毫秒部分与 event_time 不一致");
}
}
}
diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/RawStorageProperties.java b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/RawStorageProperties.java
index 9ce5774..c2f1b8e 100644
--- a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/RawStorageProperties.java
+++ b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/raw/RawStorageProperties.java
@@ -12,9 +12,9 @@ public class RawStorageProperties {
public static final int UUID_BUCKET_COUNT = 4096;
@Value("${influxdb.raw.write-schema:V1}") private String writeSchemaValue;
- @Value("${influxdb.raw.read-mode:V2_ONLY}") private String readModeValue;
+ @Value("${influxdb.raw.read-mode:V1_ONLY}") private String readModeValue;
@Value("${influxdb.raw.v1-bucket:${influxdb.bucket:yelangbucket}}") private String v1Bucket;
- @Value("${influxdb.raw.v2-bucket:yelangbucket}") private String v2Bucket;
+ @Value("${influxdb.raw.v2-bucket:yelangbucket_v2}") private String v2Bucket;
@Value("${influxdb.raw.uuid-bucket-count:4096}") private int uuidBucketCount;
@Value("${influxdb.raw.legacy-point-time-fallback-enabled:true}") private boolean legacyFallbackEnabled;
@Value("${influxdb.raw.query-max-rows:10000}") private int queryMaxRows;
diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application.properties
index a0febf5..aea0a48 100644
--- a/haobang-security-dm/syslog-consumer/src/main/resources/application.properties
+++ b/haobang-security-dm/syslog-consumer/src/main/resources/application.properties
@@ -28,9 +28,9 @@ influxdb.retry.attempts=3
influxdb.retry.delay=1000
# 原始 Syslog V2 分阶段开关。兼容部署先保持 V1/V1_ONLY，切换时统一写 V2。
influxdb.raw.write-schema=${INFLUXDB_RAW_WRITE_SCHEMA:V2}
-influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V2_ONLY}
+influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V1_V2}
influxdb.raw.v1-bucket=${INFLUXDB_RAW_V1_BUCKET:${influxdb.bucket}}
-influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket}
+influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket_v2}
influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
@@ -182,6 +182,14 @@ analysis.realtime.check-interval-seconds: 10
# 完整窗口结束后默认等待 120 秒再聚合，无需额外配置；可用环境变量覆盖。
analysis.realtime.allowed-lateness-seconds=${ANALYSIS_REALTIME_ALLOWED_LATENESS_SECONDS:120}
+# 迟到标准化告警使用独立状态表重放已完成规则窗口，不回退正常实时分析 checkpoint。
+analysis.late-replay.enabled=${ANALYSIS_LATE_REPLAY_ENABLED:true}
+analysis.late-replay.scan-interval-ms=${ANALYSIS_LATE_REPLAY_SCAN_INTERVAL_MS:5000}
+analysis.late-replay.lease-seconds=${ANALYSIS_LATE_REPLAY_LEASE_SECONDS:1800}
+analysis.late-replay.max-buckets-per-scan=${ANALYSIS_LATE_REPLAY_MAX_BUCKETS_PER_SCAN:100}
+analysis.late-replay.max-windows-per-scan=${ANALYSIS_LATE_REPLAY_MAX_WINDOWS_PER_SCAN:10}
+analysis.late-replay.retry-delay-seconds=${ANALYSIS_LATE_REPLAY_RETRY_DELAY_SECONDS:30}
+
# 关联分析规则达梦安全版子查询模式开关（默认false，生产环境达梦安全版设为true）
# true → 使用 RealtimeAnalysisEngineDm，子查询包装SQL，适配 GROUP BY 不支持自定义函数
# false → 使用 RealtimeAnalysisEngine，平铺SQL（原有行为）
diff --git a/haobang-security-dm/syslog-consumer/src/test/java/com/Modules/NormalData/LogNormalProcessorReceiveTimeTest.java b/haobang-security-dm/syslog-consumer/src/test/java/com/Modules/NormalData/LogNormalProcessorReceiveTimeTest.java
index da9936b..275b84f 100644
--- a/haobang-security-dm/syslog-consumer/src/test/java/com/Modules/NormalData/LogNormalProcessorReceiveTimeTest.java
+++ b/haobang-security-dm/syslog-consumer/src/test/java/com/Modules/NormalData/LogNormalProcessorReceiveTimeTest.java
@@ -3,6 +3,8 @@ package com.Modules.NormalData;
import org.junit.jupiter.api.Test;
import java.time.LocalDateTime;
+import java.util.HashMap;
+import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -20,4 +22,24 @@ class LogNormalProcessorReceiveTimeTest {
assertNull(LogNormalProcessor.parseReceiveTime(null));
assertNull(LogNormalProcessor.parseReceiveTime("invalid"));
}
+
+ @Test
+ void eventTimeSetsPartitionAndReceiveTimeSetsCreationTime() {
+ LocalDateTime eventTime = LocalDateTime.of(2026, 9, 1, 16, 17);
+ LocalDateTime receiveTime = LocalDateTime.of(2026, 9, 2, 9, 18, 38);
+ Map columns = new HashMap<>();
+
+ LogNormalProcessor.applyIngressTimes(columns, eventTime, receiveTime);
+
+ assertEquals(eventTime, columns.get("log_time"));
+ assertEquals(receiveTime, columns.get("created_at"));
+ }
+
+ @Test
+ void missingEventTimeFallsBackToReceiveTime() {
+ LocalDateTime receiveTime = LocalDateTime.of(2026, 9, 2, 9, 18, 38);
+
+ assertEquals(receiveTime, LogNormalProcessor.resolveEventAt(null, receiveTime));
+ assertEquals(receiveTime, LogNormalProcessor.resolveEventAt("", receiveTime));
+ }
}
diff --git a/haobang-security-dm/syslog-consumer/src/test/java/com/Modules/NormalData/SysLogProcessorTimeTest.java b/haobang-security-dm/syslog-consumer/src/test/java/com/Modules/NormalData/SysLogProcessorTimeTest.java
index baf33b3..99fbbb5 100644
--- a/haobang-security-dm/syslog-consumer/src/test/java/com/Modules/NormalData/SysLogProcessorTimeTest.java
+++ b/haobang-security-dm/syslog-consumer/src/test/java/com/Modules/NormalData/SysLogProcessorTimeTest.java
@@ -2,27 +2,34 @@ package com.Modules.NormalData;
import org.junit.jupiter.api.Test;
-import java.time.Instant;
-
import static org.junit.jupiter.api.Assertions.assertEquals;
class SysLogProcessorTimeTest {
@Test
- void receiveTimeDeterminesBothPointTimestampAndMeasurementDate() {
+ void eventTimeDeterminesBothPointTimestampAndMeasurementDate() {
SysLogProcessor.ResolvedInfluxTime resolved = SysLogProcessor.resolveInfluxTime(
- "20260811035557109", Instant.parse("2026-08-09T00:00:00Z"));
+ "20260810035557109", "20260811035557109");
- assertEquals(Instant.parse("2026-08-10T19:55:57.109Z").toEpochMilli(),
+ assertEquals(java.time.Instant.parse("2026-08-09T19:55:57.109Z").toEpochMilli(),
+ resolved.getTimestampMs());
+ assertEquals("20260810", resolved.getMeasurementDate());
+ }
+
+ @Test
+ void legacyMessageWithoutEventTimeFallsBackToReceiveTime() {
+ SysLogProcessor.ResolvedInfluxTime resolved = SysLogProcessor.resolveInfluxTime(
+ null, "20260811035557109");
+
+ assertEquals(java.time.Instant.parse("2026-08-10T19:55:57.109Z").toEpochMilli(),
resolved.getTimestampMs());
assertEquals("20260811", resolved.getMeasurementDate());
}
@Test
- void invalidReceiveTimeIsRejectedInsteadOfUsingConsumerTime() {
- Instant firstReceivedAt = Instant.parse("2026-08-10T19:55:57.109Z");
-
+ void invalidEventTimeIsRejectedInsteadOfUsingReceiveOrConsumerTime() {
org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
- () -> SysLogProcessor.resolveInfluxTime("invalid", firstReceivedAt));
+ () -> SysLogProcessor.resolveInfluxTime(
+ "invalid", "20260811035557109"));
}
}
diff --git a/haobang-security-dm/syslog-consumer/src/test/java/com/common/schedule/LateRuleReplaySchedulerTest.java b/haobang-security-dm/syslog-consumer/src/test/java/com/common/schedule/LateRuleReplaySchedulerTest.java
new file mode 100644
index 0000000..00bb5ad
--- /dev/null
+++ b/haobang-security-dm/syslog-consumer/src/test/java/com/common/schedule/LateRuleReplaySchedulerTest.java
@@ -0,0 +1,73 @@
+package com.common.schedule;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.common.entity.AnalysisAnalysisRule;
+import com.common.mapper.AnalysisAnalysisRuleMapper;
+import com.common.service.AnalysisRuleService;
+import com.common.service.LateRuleReplayStore;
+import com.common.service.LateRuleReplayStore.ReplayWindow;
+import com.common.service.RuleWindowContext;
+import java.time.LocalDateTime;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+import org.springframework.test.util.ReflectionTestUtils;
+
+class LateRuleReplaySchedulerTest {
+
+ @Test
+ void replaysExactWindowWithoutMovingNormalCheckpoint() {
+ LateRuleReplayStore store = mock(LateRuleReplayStore.class);
+ AnalysisRuleService ruleService = mock(AnalysisRuleService.class);
+ AnalysisAnalysisRuleMapper ruleMapper = mock(AnalysisAnalysisRuleMapper.class);
+ RuleWindowContext context = new RuleWindowContext();
+ LateRuleReplayScheduler scheduler =
+ new LateRuleReplayScheduler(store, ruleService, ruleMapper, context);
+ ReflectionTestUtils.setField(scheduler, "enabled", true);
+ ReflectionTestUtils.setField(scheduler, "leaseSeconds", 1800);
+ ReflectionTestUtils.setField(scheduler, "maxBucketsPerScan", 10);
+ ReflectionTestUtils.setField(scheduler, "maxWindowsPerScan", 10);
+ ReflectionTestUtils.setField(scheduler, "retryDelaySeconds", 30);
+
+ LocalDateTime start = LocalDateTime.of(2026, 9, 4, 10, 0);
+ LocalDateTime end = start.plusMinutes(5);
+ ReplayWindow replay =
+ new ReplayWindow("replay-1", "rule-1", "3:window-v2", start, end, 2, 9, "owner");
+ when(store.expandOneBucket(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.anyInt()))
+ .thenReturn(0);
+ when(store.claimOneReplay(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.anyInt()))
+ .thenReturn(replay)
+ .thenReturn(null);
+ when(ruleMapper.selectByRuleId("rule-1"))
+ .thenReturn(
+ AnalysisAnalysisRule.builder()
+ .ruleId("rule-1")
+ .version(3)
+ .ruleStatus(1)
+ .runMode("realtime")
+ .build());
+ when(ruleService.executeRealtimeRule("rule-1"))
+ .thenAnswer(
+ invocation -> {
+ assertTrue(context.current().isPresent());
+ Map item = new HashMap<>();
+ item.put("processedCount", 20L);
+ item.put("alarmCount", 2L);
+ Map result = new HashMap<>();
+ result.put("success", true);
+ result.put("results", Collections.singletonList(item));
+ return result;
+ });
+
+ scheduler.scanAndReplay();
+
+ verify(store).completeReplay(replay, 20L, 2L);
+ assertFalse(context.current().isPresent());
+ }
+}
diff --git a/haobang-security-dm/syslog-consumer/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java b/haobang-security-dm/syslog-consumer/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java
index 674e4c0..0a6bd0c 100644
--- a/haobang-security-dm/syslog-consumer/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java
+++ b/haobang-security-dm/syslog-consumer/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java
@@ -29,9 +29,9 @@ class InfluxRawV2ContractTest {
@Test void configurationRejectsChangingPersistentBucketCount() {
RawStorageProperties properties=new RawStorageProperties();
ReflectionTestUtils.setField(properties,"writeSchemaValue","V2");
- ReflectionTestUtils.setField(properties,"readModeValue","V2_ONLY");
+ ReflectionTestUtils.setField(properties,"readModeValue","V1_V2");
ReflectionTestUtils.setField(properties,"v1Bucket","yelangbucket");
- ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket");
+ ReflectionTestUtils.setField(properties,"v2Bucket","yelangbucket_v2");
ReflectionTestUtils.setField(properties,"uuidBucketCount",1024);
ReflectionTestUtils.setField(properties,"queryMaxRows",10000);
ReflectionTestUtils.setField(properties,"queryMaxResponseBytes",10485760L);
@@ -62,9 +62,20 @@ class InfluxRawV2ContractTest {
}
@Test void v2HasUuidFieldButNeverUuidTagAndUsesNanoseconds() {
- RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
+ RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
InfluxRawPointFactory.RawPoint raw=new InfluxRawPointFactory(properties).create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5","message","20260814000000000","126","1","agent-01-syslog-topic",ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L);
- String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket",raw.getBucket());
+ String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket_v2",raw.getBucket());
+ }
+
+ @Test void v2UsesEventTimeForMeasurementButKeepsReceiveTimeAsField() {
+ RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
+ String receiveTime="20260831000000000"; ReceiveTimeParser.ParsedReceiveTime eventTime=ReceiveTimeParser.parse("20260830000000000"); long pointTimeNs=eventTime.getEpochMillis()*PointTimeAllocator.NANOS_PER_MILLI+123L;
+
+ String line=new InfluxRawPointFactory(properties).create("20260830-event","message",receiveTime,"126","1","agent-01-syslog-topic",eventTime,pointTimeNs).getPoint().toLineProtocol();
+
+ assertTrue(line.startsWith("syslog_security_20260830,"));
+ assertTrue(line.contains("receive_time=\"20260831000000000\""));
+ assertTrue(line.endsWith(" "+pointTimeNs));
}
@Test void onlyMissingPointTimeFallsBack() {
@@ -83,7 +94,7 @@ class InfluxRawV2ContractTest {
}
@Test void mixedLegacyAndNewMessagesAllProduceDistinctV2PointsAndRetryIsIdempotent() {
- RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket");
+ RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); InfluxRawPointFactory factory=new InfluxRawPointFactory(properties); ReceiveTimeParser.ParsedReceiveTime parsed=ReceiveTimeParser.parse("20260814000000000");
long first=resolver.resolve(false,null,parsed.getEpochMillis(),true); long second=resolver.resolve(false,null,parsed.getEpochMillis(),true); assertNotEquals(first,second);
String id="20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"; String originalTopic="agent-01-syslog-topic";
@@ -100,7 +111,7 @@ class InfluxRawV2ContractTest {
@Test void uuidCandidateFluxComputesBucketEscapesUuidAndSelectsEarliestGlobally() {
String uuid="20260814-a\" |> die() //";
- String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket",
+ String query=InfluxDBClient.buildUuidV2CandidateQuery("yelangbucket_v2",
"syslog_security_20260814",uuid,Instant.parse("2026-08-13T16:00:00Z"),
Instant.parse("2026-08-14T16:00:00Z"));
assertTrue(query.contains("r.event_bucket == \""+InfluxEventBucket.bucketOf(uuid)+"\""));
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceApiCredentialMapper.java b/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceApiCredentialMapper.java
new file mode 100644
index 0000000..f2576b4
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceApiCredentialMapper.java
@@ -0,0 +1,21 @@
+package com.common.mapper;
+
+import com.haobang.apipull.persistence.DeviceApiCredential;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+/** 设备 API 凭据的达梦访问接口；任何调用方都不得记录查询结果对象。 */
+@Mapper
+public interface DeviceApiCredentialMapper {
+ int insert(DeviceApiCredential credential);
+
+ DeviceApiCredential selectByRef(@Param("credentialRef") String credentialRef);
+
+ DeviceApiCredential selectForLoad(
+ @Param("credentialRef") String credentialRef,
+ @Param("deviceId") Integer deviceId,
+ @Param("adapterType") String adapterType);
+
+ /** 按版本更新完整凭据并递增版本，返回 0 表示并发冲突。 */
+ int updateCas(DeviceApiCredential credential);
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceCollectApiMapper.java b/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceCollectApiMapper.java
new file mode 100644
index 0000000..f419770
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceCollectApiMapper.java
@@ -0,0 +1,92 @@
+package com.common.mapper;
+
+import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
+import com.haobang.apipull.persistence.DeviceCollectApiConfig;
+import com.haobang.apipull.persistence.DeviceCollectApiRun;
+import java.time.LocalDateTime;
+import java.util.List;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+/**
+ * API 拉取配置、租约、checkpoint 和运行审计的达梦原子操作。
+ *
+ * 带 {@code Cas}、lease 或 checkpoint 的更新都必须检查返回行数；0 表示版本冲突、失租或 fencing token 已过期，调用方不得继续提交进度。
+ */
+@Mapper
+public interface DeviceCollectApiMapper {
+ int insertConfig(DeviceCollectApiConfig config);
+
+ int insertCheckpoint(DeviceCollectApiCheckpoint checkpoint);
+
+ DeviceCollectApiConfig selectConfig(@Param("taskId") Integer taskId);
+
+ DeviceCollectApiCheckpoint selectCheckpoint(@Param("taskId") Integer taskId);
+
+ List selectDueConfigs();
+
+ /** 仅在任务暂停且 configVersion 匹配时更新，并原子递增版本。 */
+ int updateConfigCas(DeviceCollectApiConfig config);
+
+ int pause(@Param("taskId") Integer taskId, @Param("configVersion") long configVersion);
+
+ int resume(@Param("taskId") Integer taskId, @Param("configVersion") long configVersion);
+
+ /** 抢占已过期/空闲租约，并原子递增 fencing_token；返回 1 才代表成功持有。 */
+ int acquireLease(
+ @Param("taskId") Integer taskId,
+ @Param("configVersion") long configVersion,
+ @Param("owner") String owner,
+ @Param("leaseSeconds") int leaseSeconds);
+
+ /** 只有当前 owner 与 fencing token 同时匹配时才续租。 */
+ int renewLease(
+ @Param("taskId") Integer taskId,
+ @Param("owner") String owner,
+ @Param("fencingToken") long fencingToken,
+ @Param("leaseSeconds") int leaseSeconds);
+
+ int releaseLease(
+ @Param("taskId") Integer taskId,
+ @Param("owner") String owner,
+ @Param("fencingToken") long fencingToken);
+
+ /** 在一个 UPDATE 中校验 checkpoint 版本、租约、fencing 和配置版本并推进窗口。 返回 0 时调用方必须放弃提交，不能重试为无条件 UPDATE。 */
+ int commitCheckpoint(
+ @Param("taskId") Integer taskId,
+ @Param("checkpointVersion") long checkpointVersion,
+ @Param("owner") String owner,
+ @Param("fencingToken") long fencingToken,
+ @Param("configVersion") long configVersion,
+ @Param("checkpointValue") String checkpointValue,
+ @Param("committedWindowEnd") LocalDateTime committedWindowEnd,
+ @Param("lastEventTime") LocalDateTime lastEventTime,
+ @Param("lastExternalEventId") String lastExternalEventId);
+
+ /** 仅用于人工运维重置；依靠 checkpointVersion 防止覆盖正在变化的进度。 */
+ int resetCheckpoint(
+ @Param("taskId") Integer taskId,
+ @Param("checkpointVersion") long checkpointVersion,
+ @Param("checkpointValue") String checkpointValue,
+ @Param("committedWindowEnd") LocalDateTime committedWindowEnd);
+
+ int scheduleNext(
+ @Param("taskId") Integer taskId,
+ @Param("configVersion") long configVersion,
+ @Param("delaySeconds") int delaySeconds);
+
+ int insertRun(DeviceCollectApiRun run);
+
+ int updateRun(DeviceCollectApiRun run);
+
+ DeviceCollectApiRun selectRun(@Param("runId") String runId);
+
+ DeviceCollectApiRun selectLatestRun(@Param("taskId") Integer taskId);
+
+ List selectRuns(
+ @Param("taskId") Integer taskId,
+ @Param("offset") int offset,
+ @Param("pageSize") int pageSize);
+
+ long countRuns(@Param("taskId") Integer taskId);
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java b/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java
index 332dfc8..7e9ffe0 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java
@@ -69,6 +69,25 @@ public interface DeviceCollectTaskMapper {
@Options(useGeneratedKeys = true, keyProperty = "id")
int insert(DeviceCollectTask task);
+ /**
+ * API 拉取专用插入。
+ *
+ * 达梦 JDBC 在“列默认值为序列 NEXTVAL”的表上通过 getGeneratedKeys 返回了错误主键，
+ * 会导致后续 API 配置引用不存在的主任务。这里先显式取得同一序列值，再携带该 ID 插入，
+ * 使事务内三个表始终使用完全一致的 collect_task_id。
+ */
+ @SelectKey(statement = "SELECT device_collect_task_id_seq.NEXTVAL FROM DUAL",
+ keyProperty = "id", before = true, resultType = Integer.class)
+ @Insert("INSERT INTO device_collect_task (id, created_at, updated_at, device_id, method, task_name, " +
+ "first_time, last_success_time, last_failed_time, detail_id, epm, epm_peak, " +
+ "process_architecture, task_count, recent_discover_time, epm_upper_limit, " +
+ "expire_time, remark, device_ip, organization_id) " +
+ "VALUES (#{id}, NOW(), NOW(), #{deviceId}, #{method}, #{taskName}, #{firstTime}, " +
+ "#{lastSuccessTime}, #{lastFailedTime}, #{detailId}, #{epm}, #{epmPeak}, " +
+ "#{processArchitecture}, #{taskCount}, #{recentDiscoverTime}, #{epmUpperLimit}, " +
+ "#{expireTime}, #{remark}, #{deviceIp}, #{organizationId})")
+ int insertApiPullTask(DeviceCollectTask task);
+
/**
* 更新采集任务
*/
@@ -94,6 +113,10 @@ public interface DeviceCollectTaskMapper {
"WHERE id = #{id}")
int update(DeviceCollectTask task);
+ /** API 拉取专用：避免用全量 update 把主任务其他字段覆盖为 null。 */
+ @Update("UPDATE device_collect_task SET task_name=#{taskName}, updated_at=NOW() WHERE id=#{id} AND method=4")
+ int updateTaskName(@Param("id") Integer id, @Param("taskName") String taskName);
+
/**
* 删除采集任务
*/
@@ -146,4 +169,4 @@ public interface DeviceCollectTaskMapper {
*/
@Select("SELECT * FROM device_collect_task WHERE device_id = #{deviceId} ORDER BY updated_at DESC LIMIT 1")
DeviceCollectTask selectLatestByDeviceId(Integer deviceId);
-}
\ No newline at end of file
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/common/service/DeviceCollectTaskService.java b/haobang-security-dm/syslog-serve/src/main/java/com/common/service/DeviceCollectTaskService.java
index b77e8f4..60a7929 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/common/service/DeviceCollectTaskService.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/common/service/DeviceCollectTaskService.java
@@ -2,6 +2,7 @@ package com.common.service;
import com.common.entity.DeviceCollectTask;
import com.common.mapper.DeviceCollectTaskMapper;
+import com.common.mapper.DeviceCollectApiMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.CacheEvict;
@@ -9,6 +10,8 @@ import org.springframework.cache.annotation.Cacheable;
import org.springframework.cache.annotation.CachePut;
import org.springframework.stereotype.Service;
import java.util.List;
+import org.springframework.web.server.ResponseStatusException;
+import org.springframework.http.HttpStatus;
@Service
@CacheConfig(cacheNames = "collectTask")
@@ -17,6 +20,9 @@ public class DeviceCollectTaskService {
@Autowired
private DeviceCollectTaskMapper deviceCollectTaskMapper;
+ @Autowired
+ private DeviceCollectApiMapper deviceCollectApiMapper;
+
/**
* 根据ID查询采集任务 - 缓存
*/
@@ -60,6 +66,10 @@ public class DeviceCollectTaskService {
*/
@CacheEvict(allEntries = true)
public int create(DeviceCollectTask task) {
+ if (task != null && Integer.valueOf(4).equals(task.getMethod())) {
+ throw new ResponseStatusException(HttpStatus.CONFLICT,
+ "API_PULL 任务必须通过 /api/api-pull-tasks 创建");
+ }
return deviceCollectTaskMapper.insert(task);
}
@@ -69,6 +79,7 @@ public class DeviceCollectTaskService {
@CachePut(key = "'id:' + #task.id")
@CacheEvict(key = "'device:' + #task.deviceId")
public DeviceCollectTask update(DeviceCollectTask task) {
+ rejectApiPullMutation(task == null ? null : task.getId());
deviceCollectTaskMapper.update(task);
return deviceCollectTaskMapper.selectById(task.getId());
}
@@ -78,9 +89,17 @@ public class DeviceCollectTaskService {
*/
@CacheEvict(key = "'id:' + #id")
public int delete(Integer id) {
+ rejectApiPullMutation(id);
return deviceCollectTaskMapper.deleteById(id);
}
+ private void rejectApiPullMutation(Integer id) {
+ if (id != null && deviceCollectApiMapper.selectConfig(id) != null) {
+ throw new ResponseStatusException(HttpStatus.CONFLICT,
+ "API_PULL 任务必须通过 /api/api-pull-tasks 管理，禁止通用更新或删除");
+ }
+ }
+
/**
* 标记任务成功 - 更新缓存
*/
@@ -133,4 +152,4 @@ public class DeviceCollectTaskService {
public int countByDeviceId(Integer deviceId) {
return deviceCollectTaskMapper.countByDeviceId(deviceId);
}
-}
\ No newline at end of file
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/config/CacheConfig.java b/haobang-security-dm/syslog-serve/src/main/java/com/config/CacheConfig.java
index e2408cd..b5405d9 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/config/CacheConfig.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/config/CacheConfig.java
@@ -8,6 +8,7 @@ import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSeriali
import org.springframework.data.redis.serializer.RedisSerializationContext;
import org.springframework.data.redis.serializer.StringRedisSerializer;
import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Primary;
import org.springframework.context.annotation.Configuration;
import java.time.Duration;
@@ -32,6 +33,19 @@ import java.io.IOException;
@EnableCaching // 启用缓存
public class CacheConfig {
+ /**
+ * MVC、WAL 和外部协议使用不带 Java 类型元数据的标准 JSON Mapper。
+ * Redis 的多态 Mapper 仍由 objectMapper() 专用于缓存序列化，不能再污染 HTTP/WAL 协议。
+ */
+ @Bean("webObjectMapper")
+ @Primary
+ public ObjectMapper webObjectMapper() {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.registerModule(new JavaTimeModule());
+ mapper.disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+ return mapper;
+ }
+
/**
* 配置支持类型信息的 ObjectMapper
*/
@@ -139,4 +153,4 @@ public class CacheConfig {
.transactionAware()
.build();
}
-}
\ No newline at end of file
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiCredentialService.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiCredentialService.java
new file mode 100644
index 0000000..f3a6130
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiCredentialService.java
@@ -0,0 +1,123 @@
+package com.haobang.apipull;
+
+import com.common.entity.DeviceDevice;
+import com.common.mapper.DeviceApiCredentialMapper;
+import com.common.service.DeviceDeviceService;
+import com.haobang.apipull.adapter.AdapterRegistry;
+import com.haobang.apipull.api.ApiPullDtos.ApiCredentialResponse;
+import com.haobang.apipull.api.ApiPullDtos.CreateApiCredentialRequest;
+import com.haobang.apipull.api.ApiPullDtos.UpdateApiCredentialRequest;
+import com.haobang.apipull.persistence.DeviceApiCredential;
+import com.haobang.config.AppProperties;
+import org.springframework.http.HttpStatus;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+/**
+ * 设备 API 凭据管理服务。
+ *
+ * 第一阶段按实施决定将 AppKey/AppSecret 明文写入独立表。服务从不记录请求对象，响应对象中也没有 AppKey/AppSecret；任务只通过 credentialRef
+ * 使用凭据。
+ */
+@Service
+public class ApiCredentialService {
+ private static final String NDRSERVER_V1 = "NDRSERVER_V1";
+
+ private final DeviceApiCredentialMapper mapper;
+ private final DeviceDeviceService devices;
+ private final AdapterRegistry adapters;
+ private final AppProperties app;
+
+ public ApiCredentialService(
+ DeviceApiCredentialMapper mapper,
+ DeviceDeviceService devices,
+ AdapterRegistry adapters,
+ AppProperties app) {
+ this.mapper = mapper;
+ this.devices = devices;
+ this.adapters = adapters;
+ this.app = app;
+ }
+
+ /** 新建凭据；引用全局唯一，同一引用可由同一设备的多个日志流共享。 */
+ @Transactional
+ public ApiCredentialResponse create(CreateApiCredentialRequest request) {
+ requireOwnedDevice(request.getDeviceId());
+ adapters.require(request.getAdapterType());
+ if (!NDRSERVER_V1.equals(request.getAdapterType())) {
+ throw error("CREDENTIAL_SCHEMA", "当前凭据字段只支持 NDRSERVER_V1", HttpStatus.BAD_REQUEST);
+ }
+
+ DeviceApiCredential credential = new DeviceApiCredential();
+ credential.setCredentialRef(request.getCredentialRef());
+ credential.setDeviceId(request.getDeviceId());
+ credential.setAdapterType(request.getAdapterType());
+ credential.setCredentialVersion(request.getCredentialVersion());
+ credential.setAppKey(request.getAppKey());
+ credential.setAppSecret(request.getAppSecret());
+ credential.setConfigVersion(1L);
+ if (mapper.insert(credential) != 1) {
+ throw error("INTERNAL_ERROR", "创建设备凭据失败", HttpStatus.INTERNAL_SERVER_ERROR);
+ }
+ return response(require(request.getCredentialRef()));
+ }
+
+ public ApiCredentialResponse get(String credentialRef) {
+ DeviceApiCredential credential = require(credentialRef);
+ requireOwnedDevice(credential.getDeviceId());
+ return response(credential);
+ }
+
+ /** 使用乐观锁完整轮换 AppKey/AppSecret，避免并发管理请求互相覆盖。 */
+ @Transactional
+ public ApiCredentialResponse update(String credentialRef, UpdateApiCredentialRequest request) {
+ DeviceApiCredential current = require(credentialRef);
+ requireOwnedDevice(current.getDeviceId());
+
+ DeviceApiCredential update = new DeviceApiCredential();
+ update.setCredentialRef(credentialRef);
+ update.setCredentialVersion(request.getCredentialVersion());
+ update.setAppKey(request.getAppKey());
+ update.setAppSecret(request.getAppSecret());
+ update.setConfigVersion(request.getConfigVersion());
+ if (mapper.updateCas(update) != 1) {
+ throw error("CONFIG_VERSION_CONFLICT", "凭据版本冲突", HttpStatus.CONFLICT);
+ }
+ return response(require(credentialRef));
+ }
+
+ private DeviceApiCredential require(String credentialRef) {
+ DeviceApiCredential credential = mapper.selectByRef(credentialRef);
+ if (credential == null) {
+ throw error("CREDENTIAL_NOT_FOUND", "设备凭据不存在", HttpStatus.NOT_FOUND);
+ }
+ return credential;
+ }
+
+ private void requireOwnedDevice(Integer deviceId) {
+ DeviceDevice device = devices.getByIdSafely(deviceId);
+ if (device == null) {
+ throw error("INVALID_REQUEST", "设备不存在", HttpStatus.BAD_REQUEST);
+ }
+ if (device.getDeviceCollectId() == null
+ || device.getDeviceCollectId() != app.getAppServiceDeviceCollectId()) {
+ throw error("FORBIDDEN", "设备不属于当前采集探针", HttpStatus.FORBIDDEN);
+ }
+ }
+
+ private ApiCredentialResponse response(DeviceApiCredential credential) {
+ ApiCredentialResponse response = new ApiCredentialResponse();
+ response.setCredentialRef(credential.getCredentialRef());
+ response.setDeviceId(credential.getDeviceId());
+ response.setAdapterType(credential.getAdapterType());
+ response.setCredentialVersion(credential.getCredentialVersion());
+ response.setConfigVersion(credential.getConfigVersion());
+ response.setCreatedAt(ApiPullTime.toOffset(credential.getCreatedAt()));
+ response.setUpdatedAt(ApiPullTime.toOffset(credential.getUpdatedAt()));
+ return response;
+ }
+
+ private ApiPullException error(String code, String message, HttpStatus status) {
+ return new ApiPullException(code, message, status);
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullException.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullException.java
new file mode 100644
index 0000000..ae2e455
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullException.java
@@ -0,0 +1,33 @@
+package com.haobang.apipull;
+
+import org.springframework.http.HttpStatus;
+
+/**
+ * API 拉取管理面的业务异常。
+ *
+ *
{@code code} 是稳定的机器可读错误码，供前端和运维脚本判断； {@code status} 只负责映射 HTTP 语义。异常消息不得包含凭据、令牌或响应原文。
+ */
+public class ApiPullException extends RuntimeException {
+ private final String code;
+ private final HttpStatus status;
+
+ public ApiPullException(String code, String message, HttpStatus status) {
+ super(message);
+ this.code = code;
+ this.status = status;
+ }
+
+ public ApiPullException(String code, String message, HttpStatus status, Throwable cause) {
+ super(message, cause);
+ this.code = code;
+ this.status = status;
+ }
+
+ public String getCode() {
+ return code;
+ }
+
+ public HttpStatus getStatus() {
+ return status;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullProperties.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullProperties.java
new file mode 100644
index 0000000..2ec8692
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullProperties.java
@@ -0,0 +1,65 @@
+package com.haobang.apipull;
+
+import lombok.Data;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+/**
+ * 进程级 API 拉取开关和资源上限。
+ *
+ *
这里只保存所有任务共享的运行保护参数；页大小、时间窗口、厂商适配器等任务级参数 固化在达梦数据库中。总开关默认关闭，避免系统升级后未经验证就访问真实设备。
+ */
+@Data
+@Component
+public class ApiPullProperties {
+ @Value("${device.api.enabled:false}")
+ private boolean enabled;
+
+ @Value("${device.api.scheduler.instance-id:${HOSTNAME:local}-${random.uuid}}")
+ private String instanceId;
+
+ @Value("${device.api.scheduler.scan-interval-ms:5000}")
+ private long scanIntervalMs;
+
+ @Value("${device.api.scheduler.lease-seconds:120}")
+ private int leaseSeconds;
+
+ @Value("${device.api.scheduler.global-concurrency:4}")
+ private int globalConcurrency;
+
+ @Value("${device.api.scheduler.per-host-concurrency:1}")
+ private int perHostConcurrency;
+
+ /**
+ * 单次主机许可期望处理的最大页数。
+ *
+ *
Runner 会在写入 WAL 前根据第一页 total 缩小时间窗口，使一个高流量任务完成较小的
+ * durable 子窗口后尽快释放主机许可。该值不替代任务级 maxPagesPerRun 的绝对安全上限。
+ */
+ @Value("${device.api.scheduler.max-pages-per-slice:20}")
+ private int maxPagesPerSlice = 20;
+
+ @Value("${device.api.page.slow-threshold-ms:1000}")
+ private long slowPageThresholdMs;
+
+ @Value("${device.api.page.backpressure-timeout-ms:120000}")
+ private long backpressureTimeoutMs;
+
+ @Value("${device.api.allowed-hosts:}")
+ private String allowedHosts;
+
+ /** 在调度器创建线程池前快速失败，防止以危险的并发或租约参数启动。 */
+ public void validate() {
+ if (scanIntervalMs < 100
+ || leaseSeconds < 30
+ || leaseSeconds > 3600
+ || globalConcurrency < 1
+ || perHostConcurrency < 1
+ || maxPagesPerSlice < 1
+ || maxPagesPerSlice > 100000
+ || slowPageThresholdMs < 1
+ || backpressureTimeoutMs < 1) {
+ throw new IllegalArgumentException("设备 API 调度、租约、并发和超时参数非法");
+ }
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullRunner.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullRunner.java
new file mode 100644
index 0000000..3450f01
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullRunner.java
@@ -0,0 +1,506 @@
+package com.haobang.apipull;
+
+import com.common.mapper.DeviceCollectApiMapper;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.haobang.apipull.adapter.AdapterException;
+import com.haobang.apipull.adapter.AdapterRegistry;
+import com.haobang.apipull.adapter.ApiPullAdapter;
+import com.haobang.apipull.adapter.PullPage;
+import com.haobang.apipull.adapter.PullRequest;
+import com.haobang.apipull.adapter.PulledEvent;
+import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
+import com.haobang.apipull.persistence.DeviceCollectApiConfig;
+import com.haobang.apipull.persistence.DeviceCollectApiRun;
+import com.haobang.wal.AdmissionStatus;
+import com.haobang.wal.AppendAttempt;
+import com.haobang.wal.IngressEvent;
+import com.haobang.wal.IngressType;
+import com.haobang.wal.SyslogWalService;
+import com.haobang.wal.UnifiedIngressService;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletionStage;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+/**
+ * 单个 API 拉取任务的一次运行执行器。
+ *
+ *
执行顺序固定为：计算已闭合时间窗口、根据第一页 total 自适应拆分超容量窗口、串行分页、
+ * 逐事件写入 WAL、等待整页 durable，最后以 CAS 推进 checkpoint。只要任一事件尚未通过
+ * {@code FileChannel.force(true)}，本次实际子窗口就不会提交，因此进程崩溃后最多重复拉取，
+ * 不会因为提前移动游标而丢日志。
+ *
+ *
同一任务的互斥和数据库租约由 {@link ApiPullScheduler} 建立；本类在耗时 HTTP、WAL 背压
+ * 和 durable 等待期间持续续租，并在提交前再次验证 fencing token 与配置版本。
+ */
+@Component
+public class ApiPullRunner {
+ private static final Logger log = LoggerFactory.getLogger(ApiPullRunner.class);
+ private final DeviceCollectApiMapper mapper;
+ private final AdapterRegistry adapters;
+ private final UnifiedIngressService ingress;
+ private final SyslogWalService wal;
+ private final ApiPullProperties properties;
+ private final ObjectMapper json;
+
+ public ApiPullRunner(
+ DeviceCollectApiMapper mapper,
+ AdapterRegistry adapters,
+ UnifiedIngressService ingress,
+ SyslogWalService wal,
+ ApiPullProperties properties,
+ ObjectMapper json) {
+ this.mapper = mapper;
+ this.adapters = adapters;
+ this.ingress = ingress;
+ this.wal = wal;
+ this.properties = properties;
+ this.json = json.copy().deactivateDefaultTyping();
+ }
+
+ /**
+ * 执行调度器已经取得租约的任务快照。
+ *
+ * @param config 本次运行不可变的任务配置快照
+ * @param checkpoint 取得租约后的 checkpoint 与 fencing token
+ * @param run 已写入数据库的运行审计记录
+ */
+ public void execute(
+ DeviceCollectApiConfig config,
+ DeviceCollectApiCheckpoint checkpoint,
+ DeviceCollectApiRun run) {
+ ApiPullAdapter adapter = adapters.require(config.getAdapterType());
+ long fencing = checkpoint.getFencingToken();
+ long checkpointVersion = checkpoint.getVersion();
+ Instant lastEvent = null;
+ String lastExternal = null;
+ long lastRenewNanos = System.nanoTime();
+ long lastConfigCheckNanos = 0L;
+ AtomicBoolean leaseLost = new AtomicBoolean(false);
+ // 独立心跳避免业务线程阻塞在设备 HTTP 或 durable future 时导致租约自然过期。
+ ScheduledExecutorService heartbeat =
+ Executors.newSingleThreadScheduledExecutor(
+ r -> {
+ Thread t = new Thread(r, "device-api-lease-heartbeat");
+ t.setDaemon(true);
+ return t;
+ });
+ long heartbeatSeconds = Math.max(5L, Math.min(40L, properties.getLeaseSeconds() / 3L));
+ heartbeat.scheduleWithFixedDelay(
+ () -> {
+ try {
+ if (mapper.renewLease(
+ config.getCollectTaskId(),
+ properties.getInstanceId(),
+ fencing,
+ properties.getLeaseSeconds())
+ != 1) leaseLost.set(true);
+ } catch (Throwable failure) {
+ leaseLost.set(true);
+ }
+ },
+ heartbeatSeconds,
+ heartbeatSeconds,
+ TimeUnit.SECONDS);
+ try {
+ // safetyDelay 之后的数据仍可能在设备端持续变化，只处理已经闭合的逻辑窗口。
+ Instant start = ApiPullTime.toInstant(checkpoint.getCommittedWindowEnd());
+ Instant now = Instant.now();
+ Instant closedAt = now.minusSeconds(config.getSafetyDelaySeconds());
+ Instant plannedEnd = min(start.plusSeconds(config.getWindowSizeSeconds()), closedAt);
+ if (!plannedEnd.isAfter(start)) {
+ finish(run, "SKIPPED", null, null);
+ mapper.scheduleNext(
+ config.getCollectTaskId(), config.getConfigVersion(), config.getPollIntervalSeconds());
+ return;
+ }
+
+ Instant initial = ApiPullTime.toInstant(config.getInitialStartTime());
+ // lookback 有意重叠查询，用稳定 eventId 将迟到数据转化为下游可幂等的重复数据。
+ Instant queryStart = max(initial, start.minusSeconds(config.getLookbackSeconds()));
+
+ URI base = new URI(config.getBaseUrl());
+ long hardPageCapacity = calculatePageCapacity(config.getPageSize(), config.getMaxPagesPerRun());
+ int targetSlicePages = Math.min(config.getMaxPagesPerRun(), properties.getMaxPagesPerSlice());
+ long slicePageCapacity = calculatePageCapacity(config.getPageSize(), targetSlicePages);
+ int minimumWindowSeconds = adapter.minimumWindowSeconds();
+ if (minimumWindowSeconds < 1) {
+ throw new AdapterException("ADAPTER_CONTRACT_ERROR", "适配器最小窗口必须大于等于 1 秒");
+ }
+ Instant end = plannedEnd;
+ int splitDepth = 0;
+ PullPage firstPage;
+ long firstPageStarted;
+
+ /*
+ * 先读取第一页的 total 做容量预判。父窗口超限时不接纳探测页中的事件，
+ * 因而可以安全缩小窗口，不会先写入一批 WAL 后又从 offset=0 重放。
+ */
+ while (true) {
+ if (leaseLost.get()) throw new AdapterException("LEASE_LOST", "设备 API 租约心跳失败");
+ lastConfigCheckNanos = verifyConfigIfDue(config, lastConfigCheckNanos);
+ lastRenewNanos = renewIfDue(config, fencing, lastRenewNanos);
+
+ updateRunWindow(run, start, end, queryStart);
+ mapper.updateRun(run);
+
+ firstPageStarted = System.nanoTime();
+ firstPage =
+ adapter.pull(new PullRequest(config, queryStart, end, config.getPageSize(), 0L));
+ run.setLastHttpStatus(firstPage.getHttpStatus());
+
+ if (firstPage.getTotal() <= slicePageCapacity) {
+ break;
+ }
+
+ // 超限探测页不会进入正常分页循环，因此在此单独保留既有的慢页观测。
+ logSlowPage(config, run, firstPage, 0L, 0L, firstPageStarted);
+
+ if (!canSplitWindow(start, end, minimumWindowSeconds)) {
+ // 协议已经到达最小时间粒度时不能再安全拆分。如果结果仍在任务的绝对分页上限内，
+ // 为了不制造永久卡点，本次允许超过公平调度目标并完整提交该最小窗口。
+ if (firstPage.getTotal() <= hardPageCapacity) {
+ log.warn(
+ "设备 API 最小窗口超过分页时间片: taskId={}, runId={}, stream={}, total={},"
+ + " sliceCapacity={}, hardCapacity={}, minimumWindowSeconds={}",
+ config.getCollectTaskId(),
+ run.getRunId(),
+ config.getStreamKey(),
+ firstPage.getTotal(),
+ slicePageCapacity,
+ hardPageCapacity,
+ minimumWindowSeconds);
+ break;
+ }
+ throw new AdapterException(
+ "MIN_WINDOW_CAPACITY_EXCEEDED",
+ "最小 "
+ + minimumWindowSeconds
+ + " 秒窗口仍超过分页容量: total="
+ + firstPage.getTotal()
+ + ", capacity="
+ + hardPageCapacity
+ + ", windowStart="
+ + start
+ + ", windowEnd="
+ + end);
+ }
+
+ Instant previousEnd = end;
+ end = splitWindowEnd(start, end, minimumWindowSeconds);
+ splitDepth++;
+ log.info(
+ "设备 API 高密度窗口自动拆分: taskId={}, runId={}, adapter={}, stream={},"
+ + " originalWindowEnd={}, previousWindowEnd={}, splitWindowEnd={}, total={},"
+ + " capacity={}, splitDepth={}",
+ config.getCollectTaskId(),
+ run.getRunId(),
+ config.getAdapterType(),
+ config.getStreamKey(),
+ plannedEnd,
+ previousEnd,
+ end,
+ firstPage.getTotal(),
+ slicePageCapacity,
+ splitDepth);
+ }
+
+ long offset = 0;
+ Long firstTotal = null;
+ boolean complete = false;
+
+ // 同一任务严格串行分页，offset 只在整页进入 WAL 后递增。
+ for (int pageNo = 1; pageNo <= config.getMaxPagesPerRun(); pageNo++) {
+ PullPage page;
+ long pageStarted;
+ long pageOffset = offset;
+
+ if (pageNo == 1) {
+ // 容量可接受时直接复用探测页，正常窗口不会额外增加一次 HTTP 请求。
+ page = firstPage;
+ pageStarted = firstPageStarted;
+ } else {
+ if (leaseLost.get())
+ throw new AdapterException("LEASE_LOST", "设备 API 租约心跳失败");
+ lastConfigCheckNanos = verifyConfigIfDue(config, lastConfigCheckNanos);
+ lastRenewNanos = renewIfDue(config, fencing, lastRenewNanos);
+ pageStarted = System.nanoTime();
+ page =
+ adapter.pull(
+ new PullRequest(config, queryStart, end, config.getPageSize(), pageOffset));
+ }
+
+ run.setLastHttpStatus(page.getHttpStatus());
+ run.setPagesFetched(pageNo);
+ // offset 分页依赖稳定数据集；total 漂移时继续翻页可能静默漏数或重复。
+ if (firstTotal == null) firstTotal = page.getTotal();
+ else if (firstTotal.longValue() != page.getTotal())
+ throw new AdapterException("PAGINATION_DATASET_CHANGED", "固定窗口内 NDR total 发生变化");
+ if (page.getEvents().isEmpty()) {
+ if (offset >= firstTotal) {
+ complete = true;
+ logSlowPage(config, run, page, pageOffset, 0, pageStarted);
+ break;
+ }
+ throw new AdapterException("PAGINATION_INCONSISTENT", "NDR 在 total 之前返回空页");
+ }
+ run.setRecordsFetched(run.getRecordsFetched() + page.getEvents().size());
+ List> receipts = new ArrayList<>();
+ long walStarted = System.nanoTime();
+ for (PulledEvent e : page.getEvents()) {
+ if (e.getEventTime().isBefore(start))
+ run.setDuplicateCandidates(run.getDuplicateCandidates() + 1L);
+ IngressEvent event =
+ new IngressEvent(
+ e.getEventId(),
+ Instant.now(),
+ e.getEventTime(),
+ IngressType.API_PULL,
+ "HTTPS",
+ config.getDeviceId(),
+ config.getCollectTaskId(),
+ config.getAdapterType(),
+ e.getExternalEventId(),
+ base.getHost(),
+ base.getPort() < 0 ? 443 : base.getPort(),
+ "JSON",
+ StandardCharsets.UTF_8,
+ e.getRawMessage(),
+ null);
+ event = wal.prepareIngressEvent(event);
+ receipts.add(appendWithBackpressure(event, config, fencing));
+ if (lastEvent == null || e.getEventTime().isAfter(lastEvent)) {
+ lastEvent = e.getEventTime();
+ lastExternal = e.getExternalEventId();
+ }
+ }
+ long durableTimeout =
+ Math.min(600000L, Math.max(120000L, config.getRequestTimeoutMs().longValue()));
+ // 使用一个页级截止时间，避免 N 个 future 各自拥有完整超时而把总等待放大 N 倍。
+ long durableDeadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durableTimeout);
+ for (CompletionStage receipt : receipts) {
+ long remaining = durableDeadline - System.nanoTime();
+ if (remaining <= 0)
+ throw new java.util.concurrent.TimeoutException("本页 WAL durable 等待超时");
+ receipt.toCompletableFuture().get(remaining, TimeUnit.NANOSECONDS);
+ run.setRecordsDurable(run.getRecordsDurable() + 1L);
+ }
+ long walMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - walStarted);
+ offset += page.getEvents().size();
+ if (offset > firstTotal)
+ throw new AdapterException("PAGINATION_INCONSISTENT", "NDR 返回条数超过首次 total");
+ mapper.updateRun(run);
+ logSlowPage(config, run, page, pageOffset, walMs, pageStarted);
+ if (offset >= firstTotal) {
+ complete = true;
+ break;
+ }
+ }
+ if (!complete)
+ throw new AdapterException("MAX_PAGES_EXCEEDED", "达到 maxPagesPerRun，实际子窗口尚未完成");
+ ObjectNode cp = json.createObjectNode();
+ cp.put("schemaVersion", 1);
+ cp.put(
+ "committedWindowEnd",
+ end.atZone(ApiPullTime.DATABASE_ZONE).toOffsetDateTime().toString());
+ // CAS 同时检查旧版本、持有者、fencing token 和配置版本，防止失租实例覆盖新进度。
+ if (leaseLost.get()) throw new AdapterException("LEASE_LOST", "提交 checkpoint 前租约心跳失败");
+ int updated =
+ mapper.commitCheckpoint(
+ config.getCollectTaskId(),
+ checkpointVersion,
+ properties.getInstanceId(),
+ fencing,
+ config.getConfigVersion(),
+ json.writeValueAsString(cp),
+ ApiPullTime.toDatabase(end),
+ ApiPullTime.toDatabase(lastEvent),
+ lastExternal);
+ if (updated != 1)
+ throw new AdapterException("CHECKPOINT_CAS_FAILED", "checkpoint CAS 失败，禁止覆盖提交");
+ mapper.scheduleNext(
+ config.getCollectTaskId(), config.getConfigVersion(), config.getPollIntervalSeconds());
+ finish(run, "SUCCESS", null, null);
+ } catch (Throwable failure) {
+ String code = code(failure);
+ String summary = sanitize(failure);
+ if (failure instanceof AdapterException
+ && ((AdapterException) failure).getHttpStatus() != null)
+ run.setLastHttpStatus(((AdapterException) failure).getHttpStatus());
+ finish(run, "FAILED", code, summary);
+ mapper.scheduleNext(
+ config.getCollectTaskId(), config.getConfigVersion(), config.getPollIntervalSeconds());
+ log.warn(
+ "设备 API 拉取失败: taskId={}, runId={}, adapter={}, stream={}, code={}, error={}",
+ config.getCollectTaskId(),
+ run.getRunId(),
+ config.getAdapterType(),
+ config.getStreamKey(),
+ code,
+ summary);
+ } finally {
+ heartbeat.shutdownNow();
+ if (mapper.releaseLease(config.getCollectTaskId(), properties.getInstanceId(), fencing) != 1)
+ log.warn(
+ "设备 API 租约释放失败: taskId={}, runId={}, fencingToken={}",
+ config.getCollectTaskId(),
+ run.getRunId(),
+ fencing);
+ }
+ }
+
+ /** 使用 long 计算容量，避免两个 Integer 配置相乘时溢出。 */
+ private long calculatePageCapacity(int pageSize, int pageCount) {
+ return Math.multiplyExact((long) pageSize, (long) pageCount);
+ }
+
+ /** 到达厂商协议能表达的最小粒度后停止拆分，避免生成等价查询。 */
+ private boolean canSplitWindow(Instant start, Instant end, int minimumWindowSeconds) {
+ return Duration.between(start, end).compareTo(Duration.ofSeconds(minimumWindowSeconds)) > 0;
+ }
+
+ /**
+ * 只保留原窗口左半部分，并将拆分粒度向下取整到整秒。这样下一个 checkpoint
+ * 仍从原起点连续推进，不会为了降低单次容量而跳过任何时间区间。
+ */
+ private Instant splitWindowEnd(Instant start, Instant end, int minimumWindowSeconds) {
+ long durationSeconds = Duration.between(start, end).getSeconds();
+ long halfSeconds = Math.max(minimumWindowSeconds, durationSeconds / 2L);
+ return start.plusSeconds(halfSeconds);
+ }
+
+ /** 运行审计始终记录最终实际处理的子窗口，而不是拆分前的计划窗口。 */
+ private void updateRunWindow(
+ DeviceCollectApiRun run, Instant start, Instant end, Instant queryStart) {
+ run.setLogicalWindowStart(ApiPullTime.toDatabase(start));
+ run.setLogicalWindowEnd(ApiPullTime.toDatabase(end));
+ run.setQueryStart(ApiPullTime.toDatabase(queryStart));
+ run.setQueryEnd(ApiPullTime.toDatabase(end));
+ }
+
+ private CompletionStage appendWithBackpressure(
+ IngressEvent event, DeviceCollectApiConfig config, long fencing) throws Exception {
+ long deadline =
+ System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(properties.getBackpressureTimeoutMs());
+ while (true) {
+ AppendAttempt attempt = ingress.tryAppend(event);
+ if (attempt.getStatus() == AdmissionStatus.ACCEPTED) return attempt.getDurableStage();
+ if (attempt.getStatus() == AdmissionStatus.UNAVAILABLE)
+ throw new AdapterException("WAL_UNAVAILABLE", "WAL 当前不可用");
+ if (System.nanoTime() >= deadline)
+ throw new AdapterException("WAL_BACKPRESSURE_TIMEOUT", "等待 WAL 背压超时");
+ // 背压重试不能成为不可取消的死循环：同时检查暂停/改配并主动维持数据库租约。
+ DeviceCollectApiConfig current = mapper.selectConfig(config.getCollectTaskId());
+ if (current == null
+ || !current.isEnabled()
+ || !current.getConfigVersion().equals(config.getConfigVersion()))
+ throw new AdapterException("TASK_CONFIGURATION_CHANGED", "任务已暂停或配置版本变化");
+ if (mapper.renewLease(
+ config.getCollectTaskId(),
+ properties.getInstanceId(),
+ fencing,
+ properties.getLeaseSeconds())
+ != 1) throw new AdapterException("LEASE_LOST", "等待 WAL 时租约已失效");
+ Thread.sleep(Math.max(1L, wal.getRetryIntervalMs()));
+ }
+ }
+
+ private long renewIfDue(DeviceCollectApiConfig config, long fencing, long last) {
+ if (TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - last) < 40) return last;
+ if (mapper.renewLease(
+ config.getCollectTaskId(),
+ properties.getInstanceId(),
+ fencing,
+ properties.getLeaseSeconds())
+ != 1) throw new AdapterException("LEASE_LOST", "设备 API 租约续租失败");
+ return System.nanoTime();
+ }
+
+ private long verifyConfigIfDue(DeviceCollectApiConfig snapshot, long last) {
+ long now = System.nanoTime();
+ if (last != 0L && TimeUnit.NANOSECONDS.toMillis(now - last) < 1000L) return last;
+ DeviceCollectApiConfig current = mapper.selectConfig(snapshot.getCollectTaskId());
+ if (current == null
+ || !current.isEnabled()
+ || !snapshot.getConfigVersion().equals(current.getConfigVersion()))
+ throw new AdapterException("TASK_CONFIGURATION_CHANGED", "任务已暂停或配置版本变化");
+ return now;
+ }
+
+ private void logSlowPage(
+ DeviceCollectApiConfig c,
+ DeviceCollectApiRun r,
+ PullPage p,
+ long offset,
+ long walMs,
+ long started) {
+ long total = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started);
+ if (total < properties.getSlowPageThresholdMs()) return;
+ // 一条结构化日志覆盖网络、解析和 durable 三段耗时，便于定位千万级吞吐瓶颈。
+ double rate = p.getEvents().isEmpty() ? 0d : p.getEvents().size() * 1000d / Math.max(1, total);
+ log.warn(
+ "设备 API 慢页: taskId={}, runId={}, adapter={}, stream={}, windowStart={}, windowEnd={},"
+ + " offset={}, limit={}, count={}, responseBytes={}, httpMs={}, parseMs={},"
+ + " walDurableMs={}, totalMs={}, recordsPerSecond={}",
+ c.getCollectTaskId(),
+ r.getRunId(),
+ c.getAdapterType(),
+ c.getStreamKey(),
+ r.getLogicalWindowStart(),
+ r.getLogicalWindowEnd(),
+ offset,
+ c.getPageSize(),
+ p.getEvents().size(),
+ p.getResponseBytes(),
+ p.getHttpDurationMs(),
+ p.getParseDurationMs(),
+ walMs,
+ total,
+ String.format("%.2f", rate));
+ }
+
+ private void finish(DeviceCollectApiRun run, String status, String code, String summary) {
+ run.setStatus(status);
+ run.setErrorCode(code);
+ run.setErrorSummary(summary);
+ run.setFinishedAt(LocalDateTime.now(ApiPullTime.DATABASE_ZONE));
+ mapper.updateRun(run);
+ }
+
+ private String code(Throwable e) {
+ return e instanceof AdapterException
+ ? ((AdapterException) e).getCode()
+ : e instanceof java.util.concurrent.TimeoutException ? "WAL_DURABLE_TIMEOUT" : "RUN_FAILED";
+ }
+
+ private String sanitize(Throwable e) {
+ String s = e.getMessage();
+ if (s == null) s = e.getClass().getSimpleName();
+ s =
+ s.replaceAll(
+ "(?i)(authorization|token|answer|app_secret|app_key)\\s*[:=]\\s*[^,\\s]+",
+ "$1=[REDACTED]");
+ s = s.replaceAll("[A-Za-z0-9+/=_-]{32,}", "[REDACTED]");
+ return s.length() > 2000 ? s.substring(0, 2000) : s;
+ }
+
+ private Instant min(Instant a, Instant b) {
+ return a.isBefore(b) ? a : b;
+ }
+
+ private Instant max(Instant a, Instant b) {
+ return a.isAfter(b) ? a : b;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullScheduler.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullScheduler.java
new file mode 100644
index 0000000..fbb39ba
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullScheduler.java
@@ -0,0 +1,209 @@
+package com.haobang.apipull;
+
+import com.common.mapper.DeviceCollectApiMapper;
+import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
+import com.haobang.apipull.persistence.DeviceCollectApiConfig;
+import com.haobang.apipull.persistence.DeviceCollectApiRun;
+import com.haobang.wal.SyslogWalService;
+import java.net.URI;
+import java.time.LocalDateTime;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.*;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+/**
+ * API 拉取任务调度器和并发闸门。
+ *
+ * 并发限制分三层：同一 JVM 内同一任务只能运行一次、全局并发不超过线程池大小、同一设备 主机不超过 per-host 上限。跨实例互斥由达梦租约和单调递增的 fencing token
+ * 保证。
+ */
+@Component
+public class ApiPullScheduler {
+ private static final Logger log = LoggerFactory.getLogger(ApiPullScheduler.class);
+ private final ApiPullProperties properties;
+ private final DeviceCollectApiMapper mapper;
+ private final ApiPullRunner runner;
+ private final SyslogWalService wal;
+ private ExecutorService executor;
+ private Semaphore global;
+ private final ConcurrentHashMap hosts = new ConcurrentHashMap<>();
+ private final Set localRunning = ConcurrentHashMap.newKeySet();
+
+ public ApiPullScheduler(
+ ApiPullProperties properties,
+ DeviceCollectApiMapper mapper,
+ ApiPullRunner runner,
+ SyslogWalService wal) {
+ this.properties = properties;
+ this.mapper = mapper;
+ this.runner = runner;
+ this.wal = wal;
+ }
+
+ @PostConstruct
+ public void init() {
+ properties.validate();
+ global = new Semaphore(properties.getGlobalConcurrency());
+ executor =
+ Executors.newFixedThreadPool(
+ properties.getGlobalConcurrency(),
+ r -> {
+ Thread t = new Thread(r, "device-api-pull");
+ t.setDaemon(true);
+ return t;
+ });
+ log.info(
+ "设备 API 拉取调度器配置: enabled={}, globalConcurrency={}, perHostConcurrency={}, leaseSeconds={},"
+ + " maxPagesPerSlice={}, slowPageThresholdMs={}",
+ properties.isEnabled(),
+ properties.getGlobalConcurrency(),
+ properties.getPerHostConcurrency(),
+ properties.getLeaseSeconds(),
+ properties.getMaxPagesPerSlice(),
+ properties.getSlowPageThresholdMs());
+ }
+
+ /** 扫描到期任务；总开关关闭时不查询数据库，也不创建任何设备请求。 */
+ @Scheduled(fixedDelayString = "${device.api.scheduler.scan-interval-ms:5000}")
+ public void scan() {
+ if (!properties.isEnabled()) return;
+ for (DeviceCollectApiConfig c : mapper.selectDueConfigs())
+ try {
+ trySubmit(c, "SCHEDULED");
+ } catch (Throwable failure) {
+ log.warn("设备 API 调度提交失败: taskId={}, error={}", c.getCollectTaskId(), failure.toString());
+ }
+ }
+
+ /** 提交一次人工运行；手工操作不会绕过任务启用状态或资源保护。 */
+ public DeviceCollectApiRun trigger(Integer taskId) {
+ if (!properties.isEnabled())
+ throw new ApiPullException(
+ "API_PULL_DISABLED",
+ "设备 API 拉取总开关未开启",
+ org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE);
+ if (wal.getFatalFailure() != null || wal.isDiskCritical() || wal.shouldPauseTcp())
+ throw new ApiPullException(
+ "WAL_UNAVAILABLE",
+ "WAL 不可用或处于接入保护水位",
+ org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE);
+ DeviceCollectApiConfig c = mapper.selectConfig(taskId);
+ if (c == null)
+ throw new ApiPullException(
+ "TASK_NOT_FOUND", "API 拉取任务不存在", org.springframework.http.HttpStatus.NOT_FOUND);
+ if (!c.isEnabled())
+ throw new ApiPullException(
+ "TASK_NOT_PAUSED", "任务处于暂停状态，不能手工触发", org.springframework.http.HttpStatus.CONFLICT);
+ DeviceCollectApiRun run = trySubmit(c, "MANUAL");
+ if (run == null)
+ throw new ApiPullException(
+ "LEASE_CONFLICT", "任务正在运行或无可用并发槽", org.springframework.http.HttpStatus.CONFLICT);
+ return run;
+ }
+
+ private DeviceCollectApiRun trySubmit(DeviceCollectApiConfig c, String trigger) {
+ if (!c.isEnabled()) return null;
+ if (wal.getFatalFailure() != null || wal.isDiskCritical() || wal.shouldPauseTcp()) {
+ mapper.scheduleNext(c.getCollectTaskId(), c.getConfigVersion(), c.getPollIntervalSeconds());
+ return null;
+ }
+ // 获取顺序固定为任务互斥 -> 全局许可 -> 主机许可 -> DB 租约，失败时按持有情况释放。
+ if (!localRunning.add(c.getCollectTaskId())) return null;
+ if (!global.tryAcquire()) {
+ localRunning.remove(c.getCollectTaskId());
+ return null;
+ }
+ String host;
+ try {
+ host = new URI(c.getBaseUrl()).getHost();
+ } catch (Exception e) {
+ releaseLocal(c.getCollectTaskId(), null);
+ return null;
+ }
+ Semaphore hostPermit =
+ hosts.computeIfAbsent(host, h -> new Semaphore(properties.getPerHostConcurrency()));
+ if (!hostPermit.tryAcquire()) {
+ releaseLocal(c.getCollectTaskId(), null);
+ return null;
+ }
+ DeviceCollectApiCheckpoint cp = null;
+ DeviceCollectApiRun submittedRun = null;
+ try {
+ // DB 租约是跨实例最终互斥；本地 semaphore 只用于资源控制，不能代替 fencing。
+ if (mapper.acquireLease(
+ c.getCollectTaskId(),
+ c.getConfigVersion(),
+ properties.getInstanceId(),
+ properties.getLeaseSeconds())
+ != 1) {
+ releaseLocal(c.getCollectTaskId(), hostPermit);
+ return null;
+ }
+ cp = mapper.selectCheckpoint(c.getCollectTaskId());
+ DeviceCollectApiRun run = new DeviceCollectApiRun();
+ run.setRunId(UUID.randomUUID().toString());
+ run.setCollectTaskId(c.getCollectTaskId());
+ run.setDeviceId(c.getDeviceId());
+ run.setAdapterType(c.getAdapterType());
+ run.setAdapterVersion(c.getAdapterVersion());
+ run.setStreamKey(c.getStreamKey());
+ run.setInstanceId(properties.getInstanceId());
+ run.setConfigVersion(c.getConfigVersion());
+ run.setFencingToken(cp.getFencingToken());
+ run.setTriggerType(trigger);
+ run.setStatus("RUNNING");
+ run.setScheduledAt(LocalDateTime.now(ApiPullTime.DATABASE_ZONE));
+ mapper.insertRun(run);
+ submittedRun = run;
+ final DeviceCollectApiCheckpoint leasedCheckpoint = cp;
+ // 成功提交后，许可的所有权交给工作线程并由其 finally 统一释放。
+ executor.execute(
+ () -> {
+ try {
+ runner.execute(c, leasedCheckpoint, run);
+ } finally {
+ releaseLocal(c.getCollectTaskId(), hostPermit);
+ }
+ });
+ return run;
+ } catch (Throwable e) {
+ if (submittedRun != null) {
+ submittedRun.setStatus("FAILED");
+ submittedRun.setErrorCode("SCHEDULER_REJECTED");
+ submittedRun.setErrorSummary("任务未能提交到本地执行器");
+ submittedRun.setFinishedAt(LocalDateTime.now(ApiPullTime.DATABASE_ZONE));
+ mapper.updateRun(submittedRun);
+ }
+ if (cp != null && cp.getFencingToken() != null)
+ mapper.releaseLease(c.getCollectTaskId(), properties.getInstanceId(), cp.getFencingToken());
+ releaseLocal(c.getCollectTaskId(), hostPermit);
+ throw e;
+ }
+ }
+
+ private void releaseLocal(Integer taskId, Semaphore host) {
+ if (host != null) host.release();
+ global.release();
+ localRunning.remove(taskId);
+ }
+
+ @PreDestroy
+ public void shutdown() {
+ // 等待在途窗口正常收尾；超时中断时，未提交的 checkpoint 会使该窗口下次重新拉取。
+ if (executor == null) return;
+ executor.shutdown();
+ try {
+ if (!executor.awaitTermination(30, TimeUnit.SECONDS)) executor.shutdownNow();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ executor.shutdownNow();
+ }
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullTaskService.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullTaskService.java
new file mode 100644
index 0000000..c8d5a35
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullTaskService.java
@@ -0,0 +1,611 @@
+package com.haobang.apipull;
+
+import com.common.entity.DeviceCollectTask;
+import com.common.entity.DeviceDevice;
+import com.common.mapper.DeviceCollectApiMapper;
+import com.common.mapper.DeviceCollectTaskMapper;
+import com.common.service.DeviceDeviceService;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.haobang.apipull.adapter.AdapterException;
+import com.haobang.apipull.adapter.AdapterRegistry;
+import com.haobang.apipull.adapter.ApiPullAdapter;
+import com.haobang.apipull.adapter.PullRequest;
+import com.haobang.apipull.api.ApiPullDtos.*;
+import com.haobang.apipull.credential.CredentialProvider;
+import com.haobang.apipull.credential.CredentialSnapshot;
+import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
+import com.haobang.apipull.persistence.DeviceCollectApiConfig;
+import com.haobang.apipull.persistence.DeviceCollectApiRun;
+import com.haobang.apipull.security.SafeBaseUrlValidator;
+import com.haobang.config.AppProperties;
+import java.sql.SQLException;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.util.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataIntegrityViolationException;
+import org.springframework.http.HttpStatus;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+/**
+ * API 拉取任务的管理面服务。
+ *
+ * 主任务仍复用 {@code device_collect_task}，API 专属配置、checkpoint 和运行审计存放在独立表。
+ * 创建、改配、暂停、恢复和重置均通过本服务维护事务与乐观锁，通用 TCP/UDP 接口不能绕过这些 约束。任务创建后默认暂停，只有完整验证通过后才能恢复调度。
+ */
+@Service
+public class ApiPullTaskService {
+ public static final int API_PULL_METHOD = CollectMethod.API_PULL.getCode();
+ private static final Logger log = LoggerFactory.getLogger(ApiPullTaskService.class);
+ private final DeviceCollectTaskMapper tasks;
+ private final DeviceCollectApiMapper mapper;
+ private final DeviceDeviceService devices;
+ private final AdapterRegistry adapters;
+ private final CredentialProvider credentials;
+ private final SafeBaseUrlValidator urls;
+ private final ApiPullScheduler scheduler;
+ private final AppProperties app;
+ private final ApiPullProperties apiProperties;
+ private final ObjectMapper json;
+
+ public ApiPullTaskService(
+ DeviceCollectTaskMapper tasks,
+ DeviceCollectApiMapper mapper,
+ DeviceDeviceService devices,
+ AdapterRegistry adapters,
+ CredentialProvider credentials,
+ SafeBaseUrlValidator urls,
+ ApiPullScheduler scheduler,
+ AppProperties app,
+ ApiPullProperties apiProperties,
+ ObjectMapper json) {
+ this.tasks = tasks;
+ this.mapper = mapper;
+ this.devices = devices;
+ this.adapters = adapters;
+ this.credentials = credentials;
+ this.urls = urls;
+ this.scheduler = scheduler;
+ this.app = app;
+ this.apiProperties = apiProperties;
+ this.json = json.copy().deactivateDefaultTyping();
+ }
+
+ /** 创建主任务、API 配置和初始 checkpoint；新任务始终保持暂停。 */
+ @Transactional
+ public ApiPullTaskResponse create(CreateApiPullTaskRequest request) {
+ // 设备必须归属当前探针，避免一个采集实例替其他探针访问设备内网地址。
+ DeviceDevice device = devices.getByIdSafely(request.getDeviceId());
+ if (device == null) throw error("INVALID_REQUEST", "设备不存在", HttpStatus.BAD_REQUEST);
+ if (device.getDeviceCollectId() == null
+ || device.getDeviceCollectId() != app.getAppServiceDeviceCollectId())
+ throw error("FORBIDDEN", "设备不属于当前采集探针", HttpStatus.FORBIDDEN);
+ DeviceCollectApiConfig config = fromCreate(request);
+ ApiPullAdapter adapter = adapters.require(config.getAdapterType());
+ adapter.validateConfig(config);
+ validateCredentialBinding(config);
+ if (request
+ .getInitialStartTime()
+ .toInstant()
+ .isAfter(Instant.now().minusSeconds(config.getSafetyDelaySeconds())))
+ throw error(
+ "INVALID_REQUEST", "initialStartTime 必须早于当前时间减 safetyDelay", HttpStatus.BAD_REQUEST);
+ DeviceCollectTask task = new DeviceCollectTask();
+ task.setDeviceId(request.getDeviceId());
+ task.setMethod(API_PULL_METHOD);
+ task.setTaskName(request.getTaskName());
+ task.setFirstTime(request.getInitialStartTime());
+ task.setDeviceIp(device.getIp());
+ task.setOrganizationId(
+ device.getOrganizationId() == null ? null : device.getOrganizationId().longValue());
+ try {
+ if (tasks.insertApiPullTask(task) != 1 || task.getId() == null)
+ throw error("INTERNAL_ERROR", "创建采集主任务失败", HttpStatus.INTERNAL_SERVER_ERROR);
+ } catch (DataIntegrityViolationException e) {
+ logCreateIntegrityFailure("MAIN_TASK", task.getId(), e);
+ throw e;
+ }
+ config.setCollectTaskId(task.getId());
+ try {
+ mapper.insertConfig(config);
+ } catch (DataIntegrityViolationException e) {
+ logCreateIntegrityFailure("API_CONFIG", task.getId(), e);
+ throw e;
+ }
+ // 初始 checkpoint 与配置在同一事务写入，任一步失败都会连同主任务一起回滚。
+ DeviceCollectApiCheckpoint cp = new DeviceCollectApiCheckpoint();
+ cp.setCollectTaskId(task.getId());
+ cp.setCheckpointType("TIME_WINDOW");
+ cp.setCommittedWindowEnd(config.getInitialStartTime());
+ cp.setAdapterType(config.getAdapterType());
+ cp.setAdapterVersion(config.getAdapterVersion());
+ cp.setCheckpointValue(checkpointJson(request.getInitialStartTime().toInstant()));
+ try {
+ mapper.insertCheckpoint(cp);
+ } catch (DataIntegrityViolationException e) {
+ logCreateIntegrityFailure("CHECKPOINT", task.getId(), e);
+ throw e;
+ }
+ return response(task, mapper.selectConfig(task.getId()), mapper.selectCheckpoint(task.getId()));
+ }
+
+ /**
+ * 记录任务创建失败的数据库阶段和达梦错误标识，但不记录 SQL 参数、请求正文或凭据内容。
+ *
+ *
管理接口仍只向调用方返回稳定的 409 错误；详细信息仅进入服务端日志，便于区分主任务、API 配置和 checkpoint 三个事务写入阶段。
+ */
+ private void logCreateIntegrityFailure(
+ String stage, Integer taskId, DataIntegrityViolationException failure) {
+ SQLException sql = findSqlException(failure);
+ String sqlState = sql == null ? null : sql.getSQLState();
+ Integer vendorCode = sql == null ? null : sql.getErrorCode();
+ Throwable specific = sql == null ? failure.getMostSpecificCause() : sql;
+ log.error(
+ "API 拉取任务创建数据库约束失败: stage={}, taskId={}, sqlState={}, vendorCode={}, detail={}",
+ stage,
+ taskId,
+ sqlState,
+ vendorCode,
+ safeDatabaseDetail(specific == null ? null : specific.getMessage()));
+ }
+
+ private SQLException findSqlException(Throwable failure) {
+ Throwable current = failure;
+ while (current != null) {
+ if (current instanceof SQLException) return (SQLException) current;
+ current = current.getCause();
+ }
+ return null;
+ }
+
+ private String safeDatabaseDetail(String value) {
+ if (value == null || value.trim().isEmpty()) return "未提供数据库错误摘要";
+ String safe = value.replaceAll("[\\r\\n\\t]+", " ");
+ safe =
+ safe.replaceAll(
+ "(?i)(authorization|token|answer|app_secret|app_key)\\s*[:=]\\s*[^,\\s]+",
+ "$1=[REDACTED]");
+ return safe.length() > 500 ? safe.substring(0, 500) : safe;
+ }
+
+ public ApiPullTaskResponse get(Integer taskId) {
+ DeviceCollectApiConfig c = requireConfig(taskId);
+ return response(requireTask(taskId), c, mapper.selectCheckpoint(taskId));
+ }
+
+ /** 按 configVersion 修改暂停任务，适配器类型和日志流在 v1 中不可原地变更。 */
+ @Transactional
+ public ApiPullTaskResponse update(Integer taskId, UpdateApiPullTaskRequest request) {
+ DeviceCollectApiConfig current = requireConfig(taskId);
+ DeviceCollectApiConfig c = copy(current);
+ c.setConfigVersion(request.getConfigVersion());
+ c.setBaseUrl(urls.validate(request.getBaseUrl()).toString());
+ c.setCredentialRef(request.getCredentialRef());
+ c.setPollIntervalSeconds(request.getPollIntervalSeconds());
+ c.setPageSize(request.getPageSize());
+ c.setMaxPagesPerRun(request.getMaxPagesPerRun());
+ // schedulerPriority 是本次升级新增字段。旧管理端未传该字段时保留数据库现值，
+ // 避免一次普通改配把 NOT NULL 列更新为 NULL。
+ if (request.getSchedulerPriority() != null) {
+ c.setSchedulerPriority(request.getSchedulerPriority());
+ }
+ c.setConnectTimeoutMs(request.getConnectTimeoutMs());
+ c.setRequestTimeoutMs(request.getRequestTimeoutMs());
+ c.setWindowSizeSeconds(request.getWindowSizeSeconds());
+ c.setSafetyDelaySeconds(request.getSafetyDelaySeconds());
+ c.setLookbackSeconds(request.getLookbackSeconds());
+ c.setMaxResponseBytes(request.getMaxResponseBytes());
+ c.setExtraConfigJson(writeJson(request.getExtraConfig()));
+ adapters.require(c.getAdapterType()).validateConfig(c);
+ validateCredentialBinding(c);
+ // 只允许暂停状态下按 configVersion CAS 改配，运行中的配置快照不会被原地替换。
+ if (mapper.updateConfigCas(c) != 1)
+ throw error("CONFIG_VERSION_CONFLICT", "配置版本冲突或任务未暂停", HttpStatus.CONFLICT);
+ if (tasks.updateTaskName(taskId, request.getTaskName()) != 1)
+ throw error("TASK_NOT_FOUND", "主任务不存在", HttpStatus.NOT_FOUND);
+ return get(taskId);
+ }
+
+ /** 以配置版本为条件暂停任务；正在运行的快照会在周期检查中自行终止。 */
+ @Transactional
+ public ApiPullTaskResponse pause(Integer taskId, long version) {
+ requireConfig(taskId);
+ if (mapper.pause(taskId, version) != 1)
+ throw error("CONFIG_VERSION_CONFLICT", "配置版本冲突或任务未运行", HttpStatus.CONFLICT);
+ return get(taskId);
+ }
+
+ /** 完成配置、凭据、TLS、认证和最小查询验证后才允许恢复调度。 */
+ @Transactional
+ public ApiPullTaskResponse resume(Integer taskId, long version) {
+ ValidateApiPullTaskResponse validation = validate(taskId, version);
+ if (!validation.isValid())
+ throw error("CONFIG_SCHEMA_ERROR", "验证未通过，任务不能恢复", HttpStatus.UNPROCESSABLE_ENTITY);
+ if (mapper.resume(taskId, version) != 1)
+ throw error("CONFIG_VERSION_CONFLICT", "配置版本冲突或任务未暂停", HttpStatus.CONFLICT);
+ return get(taskId);
+ }
+
+ /** 手工触发一次异步运行，但仍受 WAL 水位、并发槽和数据库租约约束。 */
+ public DeviceCollectApiRun trigger(Integer taskId, long version) {
+ DeviceCollectApiConfig c = requireConfig(taskId);
+ if (c.getConfigVersion() != version)
+ throw error("CONFIG_VERSION_CONFLICT", "配置版本冲突", HttpStatus.CONFLICT);
+ return scheduler.trigger(taskId);
+ }
+
+ /** 执行只读连通性验证并保存逐项审计，不写 WAL，也不推进 checkpoint。 */
+ public ValidateApiPullTaskResponse validate(Integer taskId, long version) {
+ DeviceCollectApiConfig c = requireConfig(taskId);
+ if (c.getConfigVersion() != version)
+ throw error("CONFIG_VERSION_CONFLICT", "配置版本冲突", HttpStatus.CONFLICT);
+ DeviceCollectApiRun audit = new DeviceCollectApiRun();
+ audit.setRunId(UUID.randomUUID().toString());
+ audit.setCollectTaskId(taskId);
+ audit.setDeviceId(c.getDeviceId());
+ audit.setAdapterType(c.getAdapterType());
+ audit.setAdapterVersion(c.getAdapterVersion());
+ audit.setStreamKey(c.getStreamKey());
+ audit.setInstanceId(apiProperties.getInstanceId());
+ audit.setConfigVersion(c.getConfigVersion());
+ audit.setFencingToken(0L);
+ audit.setTriggerType("VALIDATE");
+ audit.setStatus("RUNNING");
+ mapper.insertRun(audit);
+ // 每个检查项独立记录，便于管理端定位；异常摘要统一脱敏后才允许进入数据库。
+ List checks = new ArrayList<>();
+ String credentialVersion = null;
+ boolean valid = true;
+ ApiPullAdapter adapter = null;
+ try {
+ long s = System.nanoTime();
+ adapter = adapters.require(c.getAdapterType());
+ adapter.validateConfig(c);
+ checks.add(check("CONFIG_SCHEMA", "PASS", null, "配置符合适配器 Schema", s));
+ } catch (Throwable e) {
+ valid = false;
+ checks.add(check("CONFIG_SCHEMA", "FAIL", code(e), safe(e), System.nanoTime()));
+ }
+ try {
+ long s = System.nanoTime();
+ try (CredentialSnapshot snapshot =
+ credentials.load(c.getCredentialRef(), c.getAdapterType(), c.getDeviceId())) {
+ credentialVersion = snapshot.getCredentialVersion();
+ }
+ checks.add(check("CREDENTIAL_FILE", "PASS", null, "凭据文件有效", s));
+ } catch (Throwable e) {
+ valid = false;
+ checks.add(check("CREDENTIAL_FILE", "FAIL", code(e), safe(e), System.nanoTime()));
+ }
+ try {
+ long s = System.nanoTime();
+ urls.validate(c.getBaseUrl());
+ checks.add(check("SSRF_ALLOWLIST", "PASS", null, "主机在允许列表中", s));
+ checks.add(check("DNS_RESOLUTION", "PASS", null, "主机解析成功", s));
+ } catch (Throwable e) {
+ valid = false;
+ checks.add(check("SSRF_ALLOWLIST", "FAIL", code(e), safe(e), System.nanoTime()));
+ checks.add(skipped("DNS_RESOLUTION"));
+ }
+ if (valid && adapter != null) {
+ long pullStarted = System.nanoTime();
+ try {
+ Instant end = Instant.now().minusSeconds(c.getSafetyDelaySeconds());
+ Instant start = end.minusSeconds(1);
+ // 验证仅做 limit=1 的闭合窗口查询，不推进 checkpoint，也不写入 WAL/Kafka。
+ adapter.pull(new PullRequest(c, start, end, 1, 0));
+ checks.add(check("TLS_HANDSHAKE", "PASS", null, "TLS 和主机名校验通过", pullStarted));
+ checks.add(check("AUTHENTICATION", "PASS", null, "Challenge/Verify 认证通过", pullStarted));
+ checks.add(check("API_PERMISSION", "PASS", null, "业务接口可访问", pullStarted));
+ checks.add(
+ check(
+ "NDR_ALERT".equals(c.getStreamKey()) ? "MINIMAL_ALERT_QUERY" : "MINIMAL_FLOW_QUERY",
+ "PASS",
+ null,
+ "最小查询通过",
+ pullStarted));
+ } catch (Throwable e) {
+ valid = false;
+ String failureCode = code(e);
+ String queryCheck =
+ "NDR_ALERT".equals(c.getStreamKey()) ? "MINIMAL_ALERT_QUERY" : "MINIMAL_FLOW_QUERY";
+ if (failureCode.startsWith("QUERY_")) {
+ // QUERY_ 表示已完成 TLS、Challenge/Verify 并收到业务接口的 2xx 响应，不能误报为认证失败。
+ checks.add(check("TLS_HANDSHAKE", "PASS", null, "TLS 和主机名校验通过", pullStarted));
+ checks.add(check("AUTHENTICATION", "PASS", null, "Challenge/Verify 认证通过", pullStarted));
+ checks.add(check("API_PERMISSION", "PASS", null, "业务接口可访问", pullStarted));
+ checks.add(check(queryCheck, "FAIL", failureCode, safe(e), pullStarted));
+ } else {
+ checks.add(check("AUTHENTICATION", "FAIL", failureCode, safe(e), pullStarted));
+ checks.add(skipped("API_PERMISSION"));
+ checks.add(skipped(queryCheck));
+ }
+ }
+ }
+ ensureCheck(checks, "TLS_HANDSHAKE");
+ ensureCheck(checks, "AUTHENTICATION");
+ ensureCheck(checks, "API_PERMISSION");
+ ensureCheck(
+ checks,
+ "NDR_ALERT".equals(c.getStreamKey()) ? "MINIMAL_ALERT_QUERY" : "MINIMAL_FLOW_QUERY");
+ ValidateApiPullTaskResponse r = new ValidateApiPullTaskResponse();
+ r.setTaskId(taskId);
+ r.setConfigVersion(version);
+ r.setValid(valid);
+ r.setAdapterType(c.getAdapterType());
+ r.setCredentialVersion(credentialVersion);
+ r.setChecks(checks);
+ r.setValidatedAt(OffsetDateTime.now(ApiPullTime.DATABASE_ZONE));
+ audit.setStatus(valid ? "SUCCESS" : "FAILED");
+ audit.setFinishedAt(LocalDateTime.now(ApiPullTime.DATABASE_ZONE));
+ if (!valid) {
+ for (ValidationCheckResponse check : checks)
+ if ("FAIL".equals(check.getStatus())) {
+ audit.setErrorCode(check.getErrorCode());
+ audit.setErrorSummary(check.getSummary());
+ break;
+ }
+ }
+ mapper.updateRun(audit);
+ return r;
+ }
+
+ public ApiPullRuntimeResponse runtime(Integer taskId) {
+ DeviceCollectApiConfig c = requireConfig(taskId);
+ DeviceCollectApiCheckpoint cp = mapper.selectCheckpoint(taskId);
+ DeviceCollectApiRun latest = mapper.selectLatestRun(taskId);
+ ApiPullRuntimeResponse r = new ApiPullRuntimeResponse();
+ r.setTaskId(taskId);
+ r.setEnabled(c.isEnabled());
+ r.setHealth(
+ latest == null
+ ? "UNKNOWN"
+ : "SUCCESS".equals(latest.getStatus())
+ ? "HEALTHY"
+ : "FAILED".equals(latest.getStatus()) ? "ERROR" : "DEGRADED");
+ r.setNextRunAt(ApiPullTime.toOffset(c.getNextRunAt()));
+ r.setCommittedWindowEnd(ApiPullTime.toOffset(cp.getCommittedWindowEnd()));
+ r.setCheckpointVersion(cp.getVersion());
+ r.setLeaseOwner(cp.getLeaseOwner());
+ r.setLeaseExpiresAt(ApiPullTime.toOffset(cp.getLeaseExpiresAt()));
+ r.setCheckpointLagSeconds(
+ Math.max(
+ 0,
+ Duration.between(ApiPullTime.toInstant(cp.getCommittedWindowEnd()), Instant.now())
+ .getSeconds()));
+ r.setLatestRun(runResponse(latest));
+ return r;
+ }
+
+ public PagedResponse runs(Integer taskId, int page, int pageSize) {
+ requireConfig(taskId);
+ int offset = (page - 1) * pageSize;
+ List items = new ArrayList<>();
+ for (DeviceCollectApiRun run : mapper.selectRuns(taskId, offset, pageSize))
+ items.add(runResponse(run));
+ PagedResponse r = new PagedResponse<>();
+ r.setItems(items);
+ r.setPage(page);
+ r.setPageSize(pageSize);
+ r.setTotal(mapper.countRuns(taskId));
+ return r;
+ }
+
+ /** 人工重置已提交窗口。 仅允许暂停且无有效租约的任务，并要求说明原因和 checkpoint 乐观锁版本。 */
+ @Transactional
+ public ApiPullRuntimeResponse reset(Integer taskId, ResetApiCheckpointRequest request) {
+ DeviceCollectApiConfig c = requireConfig(taskId);
+ if (c.isEnabled())
+ throw error("TASK_NOT_PAUSED", "checkpoint reset 要求任务暂停", HttpStatus.CONFLICT);
+ if (mapper.resetCheckpoint(
+ taskId,
+ request.getExpectedCheckpointVersion(),
+ checkpointJson(request.getCommittedWindowEnd().toInstant()),
+ ApiPullTime.toDatabase(request.getCommittedWindowEnd()))
+ != 1) throw error("CONFIG_VERSION_CONFLICT", "checkpoint 版本冲突或租约仍有效", HttpStatus.CONFLICT);
+ org.slf4j.LoggerFactory.getLogger(ApiPullTaskService.class)
+ .warn(
+ "API checkpoint 人工重置: taskId={}, newWindowEnd={}, reason={}",
+ taskId,
+ request.getCommittedWindowEnd(),
+ request.getReason());
+ return runtime(taskId);
+ }
+
+ private DeviceCollectApiConfig fromCreate(CreateApiPullTaskRequest r) {
+ DeviceCollectApiConfig c = new DeviceCollectApiConfig();
+ c.setDeviceId(r.getDeviceId());
+ c.setAdapterType(r.getAdapterType());
+ c.setAdapterVersion(1);
+ c.setBaseUrl(urls.validate(r.getBaseUrl()).toString());
+ c.setStreamKey(r.getStreamKey());
+ c.setCredentialRef(r.getCredentialRef());
+ c.setEnabled(0);
+ c.setPollIntervalSeconds(r.getPollIntervalSeconds());
+ c.setPageSize(r.getPageSize());
+ c.setMaxPagesPerRun(r.getMaxPagesPerRun());
+ c.setSchedulerPriority(r.getSchedulerPriority());
+ c.setConnectTimeoutMs(r.getConnectTimeoutMs());
+ c.setRequestTimeoutMs(r.getRequestTimeoutMs());
+ c.setWindowSizeSeconds(r.getWindowSizeSeconds());
+ c.setSafetyDelaySeconds(r.getSafetyDelaySeconds());
+ c.setLookbackSeconds(r.getLookbackSeconds());
+ c.setMaxResponseBytes(r.getMaxResponseBytes());
+ c.setInitialStartTime(ApiPullTime.toDatabase(r.getInitialStartTime()));
+ c.setExtraConfigJson(writeJson(r.getExtraConfig()));
+ c.setConfigVersion(1L);
+ return c;
+ }
+
+ /** 在写任务前验证凭据确实属于同一设备和适配器，避免只依赖数据库外键返回晦涩错误。 */
+ private void validateCredentialBinding(DeviceCollectApiConfig config) {
+ try (CredentialSnapshot ignored =
+ credentials.load(
+ config.getCredentialRef(), config.getAdapterType(), config.getDeviceId())) {
+ // 成功构造并关闭快照即可；认证连通性由 validate 接口负责。
+ }
+ }
+
+ private DeviceCollectApiConfig copy(DeviceCollectApiConfig a) {
+ DeviceCollectApiConfig c = new DeviceCollectApiConfig();
+ c.setCollectTaskId(a.getCollectTaskId());
+ c.setDeviceId(a.getDeviceId());
+ c.setAdapterType(a.getAdapterType());
+ c.setAdapterVersion(a.getAdapterVersion());
+ c.setStreamKey(a.getStreamKey());
+ c.setInitialStartTime(a.getInitialStartTime());
+ c.setSchedulerPriority(a.getSchedulerPriority());
+ return c;
+ }
+
+ private ApiPullTaskResponse response(
+ DeviceCollectTask t, DeviceCollectApiConfig c, DeviceCollectApiCheckpoint cp) {
+ ApiPullTaskResponse r = new ApiPullTaskResponse();
+ r.setTaskId(t.getId());
+ r.setDeviceId(c.getDeviceId());
+ r.setTaskName(t.getTaskName());
+ r.setAdapterType(c.getAdapterType());
+ r.setAdapterVersion(c.getAdapterVersion());
+ r.setStreamKey(c.getStreamKey());
+ r.setBaseUrl(c.getBaseUrl());
+ r.setCredentialRef(c.getCredentialRef());
+ r.setEnabled(c.isEnabled());
+ r.setConfigVersion(c.getConfigVersion());
+ r.setInitialStartTime(ApiPullTime.toOffset(c.getInitialStartTime()));
+ r.setCommittedWindowEnd(ApiPullTime.toOffset(cp.getCommittedWindowEnd()));
+ ApiPullTuningResponse x = new ApiPullTuningResponse();
+ x.setPollIntervalSeconds(c.getPollIntervalSeconds());
+ x.setPageSize(c.getPageSize());
+ x.setMaxPagesPerRun(c.getMaxPagesPerRun());
+ x.setSchedulerPriority(c.getSchedulerPriority());
+ x.setConnectTimeoutMs(c.getConnectTimeoutMs());
+ x.setRequestTimeoutMs(c.getRequestTimeoutMs());
+ x.setWindowSizeSeconds(c.getWindowSizeSeconds());
+ x.setSafetyDelaySeconds(c.getSafetyDelaySeconds());
+ x.setLookbackSeconds(c.getLookbackSeconds());
+ x.setMaxResponseBytes(c.getMaxResponseBytes());
+ r.setTuning(x);
+ r.setExtraConfig(readJson(c.getExtraConfigJson()));
+ r.setCreatedAt(ApiPullTime.toOffset(c.getCreatedAt()));
+ r.setUpdatedAt(ApiPullTime.toOffset(c.getUpdatedAt()));
+ return r;
+ }
+
+ private ApiPullRunResponse runResponse(DeviceCollectApiRun x) {
+ if (x == null) return null;
+ ApiPullRunResponse r = new ApiPullRunResponse();
+ r.setRunId(x.getRunId());
+ r.setTriggerType(x.getTriggerType());
+ r.setStatus(x.getStatus());
+ r.setLogicalWindowStart(ApiPullTime.toOffset(x.getLogicalWindowStart()));
+ r.setLogicalWindowEnd(ApiPullTime.toOffset(x.getLogicalWindowEnd()));
+ r.setPagesFetched(n(x.getPagesFetched()));
+ r.setRecordsFetched(n(x.getRecordsFetched()));
+ r.setRecordsDurable(n(x.getRecordsDurable()));
+ r.setRecordsQuarantined(n(x.getRecordsQuarantined()));
+ r.setLastHttpStatus(x.getLastHttpStatus());
+ r.setErrorCode(x.getErrorCode());
+ r.setErrorSummary(x.getErrorSummary());
+ r.setStartedAt(ApiPullTime.toOffset(x.getStartedAt()));
+ r.setFinishedAt(ApiPullTime.toOffset(x.getFinishedAt()));
+ return r;
+ }
+
+ private DeviceCollectApiConfig requireConfig(Integer id) {
+ DeviceCollectApiConfig c = mapper.selectConfig(id);
+ if (c == null) throw error("TASK_NOT_FOUND", "API 拉取任务不存在", HttpStatus.NOT_FOUND);
+ return c;
+ }
+
+ private DeviceCollectTask requireTask(Integer id) {
+ DeviceCollectTask t = tasks.selectById(id);
+ if (t == null || t.getMethod() == null || t.getMethod() != API_PULL_METHOD)
+ throw error("TASK_NOT_FOUND", "API 拉取主任务不存在", HttpStatus.NOT_FOUND);
+ return t;
+ }
+
+ private String checkpointJson(Instant instant) {
+ try {
+ ObjectNode n = json.createObjectNode();
+ n.put("schemaVersion", 1);
+ n.put(
+ "committedWindowEnd",
+ instant.atZone(ApiPullTime.DATABASE_ZONE).toOffsetDateTime().toString());
+ return json.writeValueAsString(n);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private String writeJson(Map value) {
+ try {
+ return json.writeValueAsString(value == null ? Collections.emptyMap() : value);
+ } catch (Exception e) {
+ throw error("INVALID_REQUEST", "extraConfig 无法序列化", HttpStatus.BAD_REQUEST);
+ }
+ }
+
+ private Map readJson(String value) {
+ try {
+ return json.readValue(value, new TypeReference>() {});
+ } catch (Exception e) {
+ return Collections.emptyMap();
+ }
+ }
+
+ private ValidationCheckResponse check(
+ String name, String status, String code, String summary, long started) {
+ ValidationCheckResponse r = new ValidationCheckResponse();
+ r.setName(name);
+ r.setStatus(status);
+ r.setErrorCode(code);
+ r.setSummary(summary);
+ r.setDurationMs(
+ java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started));
+ return r;
+ }
+
+ private ValidationCheckResponse skipped(String name) {
+ return check(name, "SKIPPED", null, "前置检查未通过", System.nanoTime());
+ }
+
+ private void ensureCheck(List checks, String name) {
+ for (ValidationCheckResponse c : checks) if (name.equals(c.getName())) return;
+ checks.add(skipped(name));
+ }
+
+ private String code(Throwable e) {
+ if (e instanceof AdapterException) return ((AdapterException) e).getCode();
+ if (e instanceof com.haobang.apipull.credential.CredentialException)
+ return ((com.haobang.apipull.credential.CredentialException) e).getCode();
+ return "VALIDATION_FAILED";
+ }
+
+ private String safe(Throwable e) {
+ String s = e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage();
+ s =
+ s.replaceAll(
+ "(?i)(authorization|token|answer|app_secret|app_key)\\s*[:=]\\s*[^,\\s]+",
+ "$1=[REDACTED]");
+ s = s.replaceAll("[A-Za-z0-9+/=_-]{32,}", "[REDACTED]");
+ return s.length() > 500 ? s.substring(0, 500) : s;
+ }
+
+ private ApiPullException error(String c, String m, HttpStatus s) {
+ return new ApiPullException(c, m, s);
+ }
+
+ private int n(Integer v) {
+ return v == null ? 0 : v;
+ }
+
+ private long n(Long v) {
+ return v == null ? 0 : v;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullTime.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullTime.java
new file mode 100644
index 0000000..86eed46
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullTime.java
@@ -0,0 +1,36 @@
+package com.haobang.apipull;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+
+/**
+ * API 拉取模块统一的数据库时间转换器。
+ *
+ * 达梦字段使用不带时区的 {@link LocalDateTime}，实施契约约定其业务时区固定为 Asia/Shanghai；设备事件在内存和 WAL 中始终使用 {@link
+ * Instant}，避免把 JVM 默认时区 带入 checkpoint 计算。
+ */
+public final class ApiPullTime {
+ public static final ZoneId DATABASE_ZONE = ZoneId.of("Asia/Shanghai");
+
+ private ApiPullTime() {}
+
+ /** 将绝对时间转换为达梦业务时区下的无时区字段值。 */
+ public static LocalDateTime toDatabase(Instant v) {
+ return v == null ? null : LocalDateTime.ofInstant(v, DATABASE_ZONE);
+ }
+
+ public static LocalDateTime toDatabase(OffsetDateTime v) {
+ return v == null ? null : toDatabase(v.toInstant());
+ }
+
+ /** 按约定的数据库业务时区解释无时区字段，而不是使用 JVM 默认时区。 */
+ public static Instant toInstant(LocalDateTime v) {
+ return v == null ? null : v.atZone(DATABASE_ZONE).toInstant();
+ }
+
+ public static OffsetDateTime toOffset(LocalDateTime v) {
+ return v == null ? null : v.atZone(DATABASE_ZONE).toOffsetDateTime();
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/CollectMethod.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/CollectMethod.java
new file mode 100644
index 0000000..e90ea35
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/CollectMethod.java
@@ -0,0 +1,18 @@
+package com.haobang.apipull;
+
+/** {@code device_collect_task.collect_method} 的稳定编码。 API 拉取固定使用 4，不能通过通用 TCP/UDP 管理接口创建或修改。 */
+public enum CollectMethod {
+ SYSLOG_TCP(1),
+ SYSLOG_UDP(2),
+ SYSLOG_TCP_UDP(3),
+ API_PULL(4);
+ private final int code;
+
+ CollectMethod(int code) {
+ this.code = code;
+ }
+
+ public int getCode() {
+ return code;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/AdapterException.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/AdapterException.java
new file mode 100644
index 0000000..976e29d
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/AdapterException.java
@@ -0,0 +1,33 @@
+package com.haobang.apipull.adapter;
+
+/**
+ * 厂商适配器执行异常。
+ *
+ *
错误码会写入运行审计记录；当异常来自设备 HTTP 响应时可携带状态码。 调用方只记录经过脱敏的摘要，禁止向日志传播请求头、令牌和完整响应体。
+ */
+public class AdapterException extends RuntimeException {
+ private final String code;
+ private final Integer httpStatus;
+
+ public AdapterException(String code, String message) {
+ this(code, message, null, null);
+ }
+
+ public AdapterException(String code, String message, Integer httpStatus) {
+ this(code, message, httpStatus, null);
+ }
+
+ public AdapterException(String code, String message, Integer httpStatus, Throwable cause) {
+ super(message, cause);
+ this.code = code;
+ this.httpStatus = httpStatus;
+ }
+
+ public String getCode() {
+ return code;
+ }
+
+ public Integer getHttpStatus() {
+ return httpStatus;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/AdapterRegistry.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/AdapterRegistry.java
new file mode 100644
index 0000000..c705e46
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/AdapterRegistry.java
@@ -0,0 +1,30 @@
+package com.haobang.apipull.adapter;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.springframework.stereotype.Component;
+
+/**
+ * Spring 注入的厂商适配器注册表。
+ *
+ *
适配器类型是数据库中的持久化契约，因此启动时拒绝重复类型，运行时也不做模糊匹配。
+ */
+@Component
+public class AdapterRegistry {
+ private final Map adapters = new HashMap<>();
+
+ public AdapterRegistry(List list) {
+ for (ApiPullAdapter a : list) {
+ if (adapters.put(a.type(), a) != null)
+ throw new IllegalStateException("重复 API 适配器: " + a.type());
+ }
+ }
+
+ /** 精确查找任务声明的适配器；缺失时以可审计错误结束本次运行。 */
+ public ApiPullAdapter require(String type) {
+ ApiPullAdapter a = adapters.get(type);
+ if (a == null) throw new AdapterException("ADAPTER_NOT_FOUND", "未安装适配器: " + type);
+ return a;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/ApiPullAdapter.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/ApiPullAdapter.java
new file mode 100644
index 0000000..f61f1de
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/ApiPullAdapter.java
@@ -0,0 +1,34 @@
+package com.haobang.apipull.adapter;
+
+import com.haobang.apipull.persistence.DeviceCollectApiConfig;
+import java.util.Set;
+
+/**
+ * 一个设备厂商协议版本的完整适配边界。
+ *
+ * 通用调度层只负责窗口、分页、租约和 WAL；认证方式、请求字段、响应解析及厂商时间格式 必须全部封装在适配器内，不能退化成可任意配置的认证脚本。
+ */
+public interface ApiPullAdapter {
+ /** 写入任务配置的稳定适配器标识。 */
+ String type();
+
+ /** 协议实现版本，用于审计和升级兼容判断。 */
+ int version();
+
+ /** 此协议版本明确支持的独立日志流。 */
+ Set streams();
+
+ /**
+ * 厂商时间查询能够可靠表达的最小窗口，单位为秒。
+ *
+ * Runner 的自适应拆分不得小于此值。当前管理 DTO 的下限为 1 秒，每个厂商适配器仍须
+ * 根据自身查询协议明确声明，不能由通用配置猜测。
+ */
+ int minimumWindowSeconds();
+
+ /** 在保存任务和执行请求前校验本厂商的全部约束。 */
+ void validateConfig(DeviceCollectApiConfig config);
+
+ /** 拉取固定时间窗口内的一页数据；不得在适配器内部推进 checkpoint。 */
+ PullPage pull(PullRequest request);
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/PullPage.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/PullPage.java
new file mode 100644
index 0000000..f6caf43
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/PullPage.java
@@ -0,0 +1,57 @@
+package com.haobang.apipull.adapter;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * 厂商适配器返回的单页标准结果。
+ *
+ *
{@code total} 是当前固定查询窗口的数据集总数；执行器要求后续各页 total 保持一致， 防止 offset 分页过程中数据集漂移而静默漏数。耗时和字节数用于慢页日志。
+ */
+public final class PullPage {
+ private final List events;
+ private final long total;
+ private final int httpStatus;
+ private final long responseBytes;
+ private final long httpDurationMs;
+ private final long parseDurationMs;
+
+ public PullPage(
+ List events,
+ long total,
+ int httpStatus,
+ long responseBytes,
+ long httpDurationMs,
+ long parseDurationMs) {
+ this.events = Collections.unmodifiableList(events);
+ this.total = total;
+ this.httpStatus = httpStatus;
+ this.responseBytes = responseBytes;
+ this.httpDurationMs = httpDurationMs;
+ this.parseDurationMs = parseDurationMs;
+ }
+
+ public List getEvents() {
+ return events;
+ }
+
+ public long getTotal() {
+ return total;
+ }
+
+ public int getHttpStatus() {
+ return httpStatus;
+ }
+
+ public long getResponseBytes() {
+ return responseBytes;
+ }
+
+ public long getHttpDurationMs() {
+ return httpDurationMs;
+ }
+
+ public long getParseDurationMs() {
+ return parseDurationMs;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/PullRequest.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/PullRequest.java
new file mode 100644
index 0000000..27a41a4
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/PullRequest.java
@@ -0,0 +1,42 @@
+package com.haobang.apipull.adapter;
+
+import com.haobang.apipull.persistence.DeviceCollectApiConfig;
+import java.time.Instant;
+
+/** 通用执行器传给厂商适配器的不可变分页请求。 queryStart/queryEnd 在同一次运行中保持不变，只有 offset 单调递增。 */
+public final class PullRequest {
+ private final DeviceCollectApiConfig config;
+ private final Instant queryStart;
+ private final Instant queryEnd;
+ private final int limit;
+ private final long offset;
+
+ public PullRequest(
+ DeviceCollectApiConfig config, Instant queryStart, Instant queryEnd, int limit, long offset) {
+ this.config = config;
+ this.queryStart = queryStart;
+ this.queryEnd = queryEnd;
+ this.limit = limit;
+ this.offset = offset;
+ }
+
+ public DeviceCollectApiConfig getConfig() {
+ return config;
+ }
+
+ public Instant getQueryStart() {
+ return queryStart;
+ }
+
+ public Instant getQueryEnd() {
+ return queryEnd;
+ }
+
+ public int getLimit() {
+ return limit;
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/PulledEvent.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/PulledEvent.java
new file mode 100644
index 0000000..22c47f2
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/PulledEvent.java
@@ -0,0 +1,39 @@
+package com.haobang.apipull.adapter;
+
+import java.time.Instant;
+
+/**
+ * 厂商事件转换后的最小通用模型。
+ *
+ * {@code externalEventId} 是设备侧稳定主键；{@code eventId} 是据此生成的确定性 UUID， 重叠窗口重复拉到同一事件时仍得到相同
+ * ID，供下游幂等处理。
+ */
+public final class PulledEvent {
+ private final String eventId;
+ private final String externalEventId;
+ private final Instant eventTime;
+ private final String rawMessage;
+
+ public PulledEvent(String eventId, String externalEventId, Instant eventTime, String rawMessage) {
+ this.eventId = eventId;
+ this.externalEventId = externalEventId;
+ this.eventTime = eventTime;
+ this.rawMessage = rawMessage;
+ }
+
+ public String getEventId() {
+ return eventId;
+ }
+
+ public String getExternalEventId() {
+ return externalEventId;
+ }
+
+ public Instant getEventTime() {
+ return eventTime;
+ }
+
+ public String getRawMessage() {
+ return rawMessage;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/ndr/NdrServerApiAdapter.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/ndr/NdrServerApiAdapter.java
new file mode 100644
index 0000000..01bf638
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/ndr/NdrServerApiAdapter.java
@@ -0,0 +1,633 @@
+package com.haobang.apipull.adapter.ndr;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.haobang.apipull.adapter.AdapterException;
+import com.haobang.apipull.adapter.ApiPullAdapter;
+import com.haobang.apipull.adapter.PullPage;
+import com.haobang.apipull.adapter.PullRequest;
+import com.haobang.apipull.adapter.PulledEvent;
+import com.haobang.apipull.credential.CredentialProvider;
+import com.haobang.apipull.credential.CredentialSnapshot;
+import com.haobang.apipull.persistence.DeviceCollectApiConfig;
+import com.haobang.apipull.security.SafeBaseUrlValidator;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoField;
+import java.time.temporal.TemporalAccessor;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.annotation.PreDestroy;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+/**
+ * {@code ndrserver接口文档.docx} 对应的固定厂商协议实现。
+ *
+ *
认证流程固定为 Challenge -> {@code MD5(challenge + app_secret)} -> Verify；业务请求的 Authorization
+ * 值按厂商协议直接使用 token，不增加 Bearer 前缀。告警流和流量流共享认证缓存， 但保持独立 streamKey、接口路径和事件 ID 命名空间。
+ *
+ *
HTTP 客户端禁止自动重试和重定向；401 只允许清理缓存、重载凭据、重新认证并重试业务请求 一次，避免认证错误形成无限请求风暴。
+ */
+@Component
+public class NdrServerApiAdapter implements ApiPullAdapter {
+ public static final String TYPE = "NDRSERVER_V1";
+ public static final String ALERT = "NDR_ALERT";
+ public static final String FLOW = "NDR_FLOW_LOG";
+ private static final int MIN_PAGE_SIZE = 1;
+ private static final int MAX_PAGE_SIZE = 1000;
+ /*
+ * NDR 的 TCP/UDP 日志外发负载配置上限是 10,000 bytes。API 正文可能明显更大；为了兼容
+ * 设备 126 的既有推送规则，正文仅在该边界内增加旧字段别名，避免序列化后的单条事件因正文
+ * 同时出现在厂商字段和兼容字段中而无界膨胀。
+ */
+ private static final int MAX_BODY_ALIAS_UTF8_BYTES = 10000;
+ private static final ZoneId DEVICE_ZONE = ZoneId.of("Asia/Shanghai");
+ private static final DateTimeFormatter QUERY_TIME =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(DEVICE_ZONE);
+ private static final DateTimeFormatter LOCAL_EVENT_TIME =
+ new DateTimeFormatterBuilder()
+ .appendPattern("uuuu-MM-dd HH:mm:ss")
+ .optionalStart()
+ .appendFraction(ChronoField.NANO_OF_SECOND, 1, 9, true)
+ .optionalEnd()
+ .toFormatter();
+ private static final DateTimeFormatter EVENT_TIME =
+ new DateTimeFormatterBuilder()
+ .append(LOCAL_EVENT_TIME)
+ .optionalStart()
+ .appendOffset("+HH:MM", "Z")
+ .optionalEnd()
+ .toFormatter();
+ private static final DateTimeFormatter EVENT_ID_DATE =
+ DateTimeFormatter.ofPattern("yyyyMMdd").withZone(DEVICE_ZONE);
+ private static final UUID EVENT_NAMESPACE =
+ UUID.fromString("a8cc91a6-f674-5f2f-a33f-bbd3ccae422a");
+ private final CredentialProvider credentials;
+ private final SafeBaseUrlValidator urlValidator;
+ private final ObjectMapper mapper;
+ private final CloseableHttpClient http;
+ private final Map tokens = new ConcurrentHashMap<>();
+ private final Map tokenLocks = new ConcurrentHashMap<>();
+
+ /**
+ * Spring 生产构造器。
+ *
+ * 本类还保留了一个接收 {@link CloseableHttpClient} 的包级构造器，供单元测试注入可控的 HTTP 客户端。存在多个构造器时必须显式标记生产构造器，否则
+ * Spring 会尝试寻找无参构造器，导致应用启动失败。
+ */
+ @Autowired
+ public NdrServerApiAdapter(
+ CredentialProvider credentials, SafeBaseUrlValidator urlValidator, ObjectMapper mapper) {
+ this(credentials, urlValidator, mapper, HttpClients.custom().disableAutomaticRetries().build());
+ }
+
+ NdrServerApiAdapter(
+ CredentialProvider credentials,
+ SafeBaseUrlValidator urlValidator,
+ ObjectMapper mapper,
+ CloseableHttpClient http) {
+ this.credentials = credentials;
+ this.urlValidator = urlValidator;
+ this.mapper = mapper.copy().deactivateDefaultTyping();
+ this.http = http;
+ }
+
+ @Override
+ public String type() {
+ return TYPE;
+ }
+
+ @Override
+ public int version() {
+ return 1;
+ }
+
+ @Override
+ public Set streams() {
+ return Collections.unmodifiableSet(new java.util.LinkedHashSet<>(Arrays.asList(ALERT, FLOW)));
+ }
+
+ /** NDR 查询时间格式只精确到秒，自适应窗口不能继续拆到亚秒级。 */
+ @Override
+ public int minimumWindowSeconds() {
+ return 1;
+ }
+
+ @Override
+ public void validateConfig(DeviceCollectApiConfig c) {
+ if (c == null || !TYPE.equals(c.getAdapterType()))
+ throw new AdapterException("CONFIG_SCHEMA_ERROR", "适配器类型不匹配");
+ if (!streams().contains(c.getStreamKey()))
+ throw new AdapterException("STREAM_NOT_SUPPORTED", "NDR 不支持日志流: " + c.getStreamKey());
+ // 厂商文档中的 limit=10/100 只是请求样例，并未声明最大值。这里的 1000 是平台保护上限，
+ // 用来阻止无界响应；具体设备能否稳定支持 500/1000 条，仍需通过真实环境压测确认。
+ requireRange(c.getPageSize(), MIN_PAGE_SIZE, MAX_PAGE_SIZE, "pageSize");
+ requireRange(c.getPollIntervalSeconds(), 1, 3600, "pollIntervalSeconds");
+ requireRange(c.getMaxPagesPerRun(), 1, 1000, "maxPagesPerRun");
+ requireRange(c.getConnectTimeoutMs(), 100, 30000, "connectTimeoutMs");
+ requireRange(c.getRequestTimeoutMs(), 1000, 120000, "requestTimeoutMs");
+ requireRange(c.getWindowSizeSeconds(), 1, 3600, "windowSizeSeconds");
+ requireRange(c.getSafetyDelaySeconds(), 0, 86400, "safetyDelaySeconds");
+ requireRange(c.getLookbackSeconds(), 0, 86400, "lookbackSeconds");
+ if (c.getMaxResponseBytes() == null
+ || c.getMaxResponseBytes() < 1024
+ || c.getMaxResponseBytes() > 67108864)
+ throw new AdapterException("CONFIG_SCHEMA_ERROR", "maxResponseBytes 超出 NDRSERVER_V1 范围");
+ URI normalized = urlValidator.validate(c.getBaseUrl());
+ if (!"/ndr/api".equals(normalized.getPath()))
+ throw new AdapterException("CONFIG_SCHEMA_ERROR", "NDRSERVER_V1 baseUrl 路径必须是 /ndr/api");
+ if (normalized.toString().endsWith("/"))
+ throw new AdapterException("CONFIG_SCHEMA_ERROR", "baseUrl 不能以 / 结尾");
+ if (c.getExtraConfigJson() != null && !"{}".equals(c.getExtraConfigJson().trim()))
+ throw new AdapterException("CONFIG_SCHEMA_ERROR", "NDRSERVER_V1 第一版 extraConfig 只能是空对象");
+ }
+
+ @Override
+ public PullPage pull(PullRequest request) {
+ DeviceCollectApiConfig config = request.getConfig();
+ validateConfig(config);
+ URI base = urlValidator.validate(config.getBaseUrl());
+ String path =
+ ALERT.equals(config.getStreamKey()) ? "/threatalert/investigate" : "/netlog/investigate";
+ ObjectNode body = mapper.createObjectNode();
+ body.put("start_time", QUERY_TIME.format(request.getQueryStart()));
+ body.put("end_time", QUERY_TIME.format(request.getQueryEnd()));
+ if (FLOW.equals(config.getStreamKey())) {
+ body.putArray("src_ip");
+ body.putArray("dest_ip");
+ body.putArray("flow_id");
+ body.putArray("pcap_file");
+ body.putArray("src_port");
+ body.putArray("dest_port");
+ }
+ body.put("limit", request.getLimit());
+ body.put("offset", request.getOffset());
+ TokenLease lease = token(config, base, false);
+ long httpStarted = System.nanoTime();
+ HttpResult result = post(base.resolve(base.getPath() + path), body, lease.token, config);
+ // 401 可能表示 token 过期或凭据已轮换；仅强制刷新一次并重放本次只读查询。
+ if (result.status == 401) {
+ tokens.remove(lease.cacheKey);
+ lease = token(config, base, true);
+ result = post(base.resolve(base.getPath() + path), body, lease.token, config);
+ }
+ long httpMs = elapsed(httpStarted);
+ if (result.status == 401) throw new AdapterException("AUTH_ERROR", "NDR 重新认证后仍返回 401", 401);
+ if (result.status == 403) throw new AdapterException("PERMISSION_DENIED", "NDR API 权限不足", 403);
+ if (result.status < 200 || result.status >= 300)
+ throw new AdapterException(
+ "DEVICE_HTTP_ERROR", "NDR API HTTP " + result.status, result.status);
+ long parseStarted = System.nanoTime();
+ try {
+ JsonNode root = mapper.readTree(result.body);
+ ensureSuccess(root, "NDR 业务查询失败");
+ JsonNode data = root.get("data");
+ if (data == null || !data.isObject())
+ throw new AdapterException("RESPONSE_SCHEMA_ERROR", "NDR 响应缺少 data");
+ JsonNode totalNode = data.get("total");
+ if (totalNode == null || !totalNode.canConvertToLong() || totalNode.longValue() < 0)
+ throw new AdapterException("RESPONSE_SCHEMA_ERROR", "NDR 响应 total 非法");
+ String arrayName = ALERT.equals(config.getStreamKey()) ? "alerts" : "logs";
+ JsonNode array = data.get(arrayName);
+ /*
+ * 厂商文档示例固定返回 alerts/logs，但现场设备在 total=0 时会省略空数组。
+ * 仅在总数明确为 0 时兼容为空页；total>0 仍严格要求数组存在，防止响应漂移造成静默丢日志。
+ */
+ if ((array == null || array.isNull()) && totalNode.longValue() == 0)
+ array = mapper.createArrayNode();
+ if (array == null || !array.isArray())
+ throw new AdapterException("RESPONSE_SCHEMA_ERROR", "NDR 响应缺少 data." + arrayName);
+ ArrayList events = new ArrayList<>();
+ for (JsonNode item : array) {
+ if (!item.isObject()) throw new AdapterException("RESPONSE_SCHEMA_ERROR", "NDR 日志元素不是对象");
+ String external = externalId(item, config.getStreamKey());
+ Instant eventTime = parseEventTime(text(item, "timestamp"));
+ if (ALERT.equals(config.getStreamKey())) addAlertCompatibilityAliases((ObjectNode) item);
+ else addFlowCompatibilityAliases((ObjectNode) item);
+ // 两级 UUIDv5 隔离任务/流：同一设备事件重拉得到相同 ID，不同任务或流不会碰撞。
+ UUID taskNamespace =
+ uuidV5(
+ EVENT_NAMESPACE,
+ config.getCollectTaskId() + "|" + TYPE + "|" + config.getStreamKey());
+ String eventId = EVENT_ID_DATE.format(eventTime) + "-" + uuidV5(taskNamespace, external);
+ events.add(new PulledEvent(eventId, external, eventTime, mapper.writeValueAsString(item)));
+ }
+ return new PullPage(
+ events,
+ totalNode.longValue(),
+ result.status,
+ result.body.length,
+ httpMs,
+ elapsed(parseStarted));
+ } catch (AdapterException e) {
+ // 此代码块只处理已通过认证且 HTTP 2xx 的业务查询响应，增加 QUERY_ 前缀供验证端准确归类阶段。
+ throw new AdapterException("QUERY_" + e.getCode(), e.getMessage(), e.getHttpStatus(), e);
+ } catch (Exception e) {
+ throw new AdapterException("QUERY_RESPONSE_PARSE_ERROR", "无法解析 NDR 响应", result.status, e);
+ } finally {
+ Arrays.fill(result.body, (byte) 0);
+ }
+ }
+
+ private TokenLease token(DeviceCollectApiConfig config, URI base, boolean force) {
+ try (CredentialSnapshot snapshot =
+ credentials.load(config.getCredentialRef(), TYPE, config.getDeviceId())) {
+ // credentialVersion 进入缓存键，数据库凭据轮换后会自然淘汰旧 token。
+ String key =
+ TYPE
+ + "|"
+ + base
+ + "|"
+ + config.getCredentialRef()
+ + "|"
+ + snapshot.getCredentialVersion();
+ String prefix = TYPE + "|" + base + "|" + config.getCredentialRef() + "|";
+ tokens.keySet().removeIf(existing -> existing.startsWith(prefix) && !existing.equals(key));
+ tokenLocks
+ .keySet()
+ .removeIf(existing -> existing.startsWith(prefix) && !existing.equals(key));
+ if (!force) {
+ String cached = tokens.get(key);
+ if (cached != null) return new TokenLease(key, cached);
+ }
+ // 同一设备和凭据版本只允许一个线程刷新，避免告警流与流量流同时发起认证风暴。
+ Object lock = tokenLocks.computeIfAbsent(key, k -> new Object());
+ synchronized (lock) {
+ String cached = tokens.get(key);
+ if (cached != null) return new TokenLease(key, cached);
+ char[] appKeyChars = snapshot.require("app_key").copy();
+ char[] secretChars = snapshot.require("app_secret").copy();
+ try {
+ String appKey = new String(appKeyChars);
+ String encoded = URLEncoder.encode(appKey, "UTF-8").replace("+", "%20");
+ HttpResult challenge =
+ get(base.resolve(base.getPath() + "/openapp/challenge/" + encoded), config);
+ try {
+ if (challenge.status < 200 || challenge.status >= 300)
+ throw new AdapterException(
+ "AUTH_CHALLENGE_HTTP",
+ "NDR challenge HTTP " + challenge.status,
+ challenge.status);
+ JsonNode challengeRoot = mapper.readTree(challenge.body);
+ ensureSuccess(challengeRoot, "NDR challenge 失败");
+ String challengeText = text(challengeRoot.path("data"), "challenge");
+ String answer = answer(challengeText, secretChars);
+ ObjectNode verify = mapper.createObjectNode();
+ verify.put("app_key", appKey);
+ verify.put("answer", answer);
+ HttpResult verified =
+ post(base.resolve(base.getPath() + "/openapp/verify"), verify, null, config);
+ try {
+ if (verified.status < 200 || verified.status >= 300)
+ throw new AdapterException(
+ "AUTH_VERIFY_HTTP", "NDR verify HTTP " + verified.status, verified.status);
+ JsonNode verifyRoot = mapper.readTree(verified.body);
+ ensureSuccess(verifyRoot, "NDR verify 失败");
+ String token = text(verifyRoot.path("data"), "token");
+ tokens.put(key, token);
+ return new TokenLease(key, token);
+ } finally {
+ Arrays.fill(verified.body, (byte) 0);
+ }
+ } finally {
+ Arrays.fill(challenge.body, (byte) 0);
+ }
+ } catch (AdapterException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AdapterException("AUTH_ERROR", "NDR Challenge/Verify 认证失败", null, e);
+ } finally {
+ Arrays.fill(appKeyChars, '\0');
+ Arrays.fill(secretChars, '\0');
+ }
+ }
+ }
+ }
+
+ private HttpResult get(URI uri, DeviceCollectApiConfig config) {
+ HttpGet get = new HttpGet(uri);
+ get.setConfig(requestConfig(config));
+ return execute(get, config.getMaxResponseBytes());
+ }
+
+ private HttpResult post(URI uri, ObjectNode body, String token, DeviceCollectApiConfig config) {
+ HttpPost post = new HttpPost(uri);
+ post.setConfig(requestConfig(config));
+ post.setHeader("Accept", "application/json");
+ if (token != null) post.setHeader("Authorization", token);
+ try {
+ post.setEntity(
+ new StringEntity(mapper.writeValueAsString(body), ContentType.APPLICATION_JSON));
+ } catch (Exception e) {
+ throw new AdapterException("REQUEST_SERIALIZE_ERROR", "无法构造 NDR 请求", null, e);
+ }
+ return execute(post, config.getMaxResponseBytes());
+ }
+
+ private RequestConfig requestConfig(DeviceCollectApiConfig c) {
+ return RequestConfig.custom()
+ .setConnectTimeout(c.getConnectTimeoutMs())
+ .setConnectionRequestTimeout(c.getConnectTimeoutMs())
+ .setSocketTimeout(c.getRequestTimeoutMs())
+ // 禁止跟随设备返回的 Location，避免已通过白名单的 URL 被重定向到内网其他地址。
+ .setRedirectsEnabled(false)
+ .build();
+ }
+
+ private HttpResult execute(org.apache.http.client.methods.HttpUriRequest request, long maxBytes) {
+ try (CloseableHttpResponse response = http.execute(request)) {
+ HttpEntity entity = response.getEntity();
+ byte[] body = readBounded(entity, maxBytes);
+ return new HttpResult(response.getStatusLine().getStatusCode(), body);
+ } catch (AdapterException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AdapterException("DEVICE_API_UNAVAILABLE", "NDR API 请求失败", null, e);
+ }
+ }
+
+ private byte[] readBounded(HttpEntity entity, long maxBytes) throws Exception {
+ if (entity == null) return new byte[0];
+ // 同时检查 Content-Length 和实际流量，防止缺失/伪造长度导致堆内存被超大响应耗尽。
+ if (entity.getContentLength() > maxBytes)
+ throw new AdapterException("RESPONSE_TOO_LARGE", "NDR 响应超过 maxResponseBytes");
+ try (InputStream in = entity.getContent();
+ ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+ byte[] buffer = new byte[8192];
+ long count = 0;
+ int n;
+ while ((n = in.read(buffer)) != -1) {
+ count += n;
+ if (count > maxBytes)
+ throw new AdapterException("RESPONSE_TOO_LARGE", "NDR 响应超过 maxResponseBytes");
+ out.write(buffer, 0, n);
+ }
+ return out.toByteArray();
+ }
+ }
+
+ private void ensureSuccess(JsonNode root, String message) {
+ validateJsonLimits(root, 0);
+ if (root == null || !root.isObject() || root.path("result").asInt(0) != 1) {
+ String reason =
+ root == null ? "unknown" : sanitizeReason(root.path("reason").asText("unknown"));
+ throw new AdapterException("DEVICE_RESULT_ERROR", message + ": " + reason);
+ }
+ }
+
+ private String sanitizeReason(String reason) {
+ String value = reason == null ? "unknown" : reason.replace('\r', ' ').replace('\n', ' ');
+ value =
+ value.replaceAll(
+ "(?i)(authorization|token|answer|app_secret|app_key)\\s*[:=]\\s*[^,\\s]+",
+ "$1=[REDACTED]");
+ value = value.replaceAll("[A-Za-z0-9+/=_-]{32,}", "[REDACTED]");
+ return value.length() > 200 ? value.substring(0, 200) : value;
+ }
+
+ private void validateJsonLimits(JsonNode node, int depth) {
+ // 限制深度、字段名和字符串长度，避免小字节响应构造病态 JSON 消耗过多 CPU/内存。
+ if (node == null) return;
+ if (depth > 64) throw new AdapterException("RESPONSE_SCHEMA_ERROR", "NDR JSON 嵌套深度超过 64");
+ if (node.isTextual() && node.textValue().length() > 1048576)
+ throw new AdapterException("RESPONSE_SCHEMA_ERROR", "NDR JSON 字符串超过 1MiB");
+ if (node.isContainerNode()) {
+ java.util.Iterator> fields = node.fields();
+ while (fields.hasNext()) {
+ Map.Entry field = fields.next();
+ if (field.getKey().length() > 512)
+ throw new AdapterException("RESPONSE_SCHEMA_ERROR", "NDR JSON 字段名过长");
+ validateJsonLimits(field.getValue(), depth + 1);
+ }
+ if (node.isArray()) for (JsonNode child : node) validateJsonLimits(child, depth + 1);
+ }
+ }
+
+ private String externalId(JsonNode item, String stream) {
+ String value =
+ ALERT.equals(stream) ? text(item, "index") + ":" + text(item, "id") : text(item, "_id");
+ if (value.length() > 512)
+ throw new AdapterException("DATA_CONTRACT_ERROR", "NDR 外部事件 ID 超过 512 字符");
+ return value;
+ }
+
+ /**
+ * 将 API 流量日志的 HTTP 字段补成设备 126 既有 TCP 推送规则使用的字段名。
+ *
+ * 厂商字段始终保留；兼容字段已存在时不覆盖。Jackson 的 {@link JsonNode} 引用被直接复用，
+ * 不复制字符串或子树。正文还受 10,000 UTF-8 bytes 门禁限制，使最终 JSON 序列化的额外开销有界。
+ */
+ private void addFlowCompatibilityAliases(ObjectNode item) {
+ addCompatibilityAlias(item, "request_headers", "http_req_header", false);
+ addCompatibilityAlias(item, "request_body", "http_req_body", true);
+ addCompatibilityAlias(item, "response_headers", "http_resp_header", false);
+ addCompatibilityAlias(item, "response_body", "http_resp_body", true);
+ }
+
+ /**
+ * 将 API 告警 payload 中的 HTTP 字段补成标准字段和旧 TCP 规则兼容字段。
+ *
+ *
现场设备使用 {@code http_req_*}/{@code http_rsp_*}，厂商文档使用
+ * {@code request_*}/{@code response_*}。平台标准字段是 {@code http_req_header} 等，而既有 NDR TCP
+ * 规则的输入字段是 {@code request_headers}/{@code data}/{@code response_headers}/
+ * {@code http_response_body}。这里同时提供两套别名，让不同环境的规则按自己的 {@code origin_field}
+ * 匹配；适配器不查询或写死设备 ID、规则 ID。
+ *
+ *
原始 payload 始终保留，目标字段已存在时不覆盖，正文沿用与流量日志相同的大小门禁。
+ */
+ private void addAlertCompatibilityAliases(ObjectNode item) {
+ JsonNode payload = item.get("payload");
+ if (payload == null || !payload.isObject()) return;
+
+ // 平台标准别名，供使用标准 origin_field 的新规则直接消费。
+ addCompatibilityAlias(item, payload, "http_req_head", "http_req_header", false);
+ addCompatibilityAlias(item, payload, "request_headers", "http_req_header", false);
+ addCompatibilityAlias(item, payload, "http_req_body", "http_req_body", true);
+ addCompatibilityAlias(item, payload, "request_body", "http_req_body", true);
+ addCompatibilityAlias(item, payload, "http_rsp_head", "http_resp_header", false);
+ addCompatibilityAlias(item, payload, "response_headers", "http_resp_header", false);
+ addCompatibilityAlias(item, payload, "http_rsp_body", "http_resp_body", true);
+ addCompatibilityAlias(item, payload, "response_body", "http_resp_body", true);
+
+ // 旧 NDR TCP 推送规则兼容别名。规则选择仍由标准化服务完成，适配器不绑定规则主键。
+ addCompatibilityAlias(item, payload, "http_req_head", "request_headers", false);
+ addCompatibilityAlias(item, payload, "request_headers", "request_headers", false);
+ addCompatibilityAlias(item, payload, "http_req_body", "data", true);
+ addCompatibilityAlias(item, payload, "request_body", "data", true);
+ addCompatibilityAlias(item, payload, "http_rsp_head", "response_headers", false);
+ addCompatibilityAlias(item, payload, "response_headers", "response_headers", false);
+ addCompatibilityAlias(item, payload, "http_rsp_body", "http_response_body", true);
+ addCompatibilityAlias(item, payload, "response_body", "http_response_body", true);
+ }
+
+ private void addCompatibilityAlias(
+ ObjectNode item, String vendorField, String compatibilityField, boolean body) {
+ addCompatibilityAlias(item, item, vendorField, compatibilityField, body);
+ }
+
+ private void addCompatibilityAlias(
+ ObjectNode item,
+ JsonNode source,
+ String vendorField,
+ String compatibilityField,
+ boolean body) {
+ if (item.has(compatibilityField)) return;
+ JsonNode value = source.get(vendorField);
+ if (value == null || value.isNull()) return;
+ if (body
+ && (!value.isTextual()
+ || !utf8LengthAtMost(value.textValue(), MAX_BODY_ALIAS_UTF8_BYTES))) return;
+ item.set(compatibilityField, value);
+ }
+
+ /** 计算 UTF-8 长度是否在边界内，不创建临时 byte[]。 */
+ private boolean utf8LengthAtMost(String value, int maxBytes) {
+ int bytes = 0;
+ for (int i = 0; i < value.length(); i++) {
+ char c = value.charAt(i);
+ if (c <= 0x7f) {
+ bytes++;
+ } else if (c <= 0x7ff) {
+ bytes += 2;
+ } else if (Character.isHighSurrogate(c)
+ && i + 1 < value.length()
+ && Character.isLowSurrogate(value.charAt(i + 1))) {
+ bytes += 4;
+ i++;
+ } else {
+ bytes += 3;
+ }
+ if (bytes > maxBytes) return false;
+ }
+ return true;
+ }
+
+ private String text(JsonNode node, String field) {
+ JsonNode value = node.get(field);
+ if (value == null || !value.isValueNode() || value.asText().trim().isEmpty())
+ throw new AdapterException("RESPONSE_SCHEMA_ERROR", "NDR 响应缺少字段: " + field);
+ return value.asText();
+ }
+
+ static Instant parseEventTime(String value) {
+ try {
+ // 厂商不同版本会返回空格/T 分隔及 +0800/+08:00，也可能省略时区。
+ // NDR 查询参数本身采用北京时间，因此无偏移的 timestamp 明确按 Asia/Shanghai 解释。
+ String normalized = value.trim();
+ if (normalized.length() > 10 && normalized.charAt(10) == 'T')
+ normalized = normalized.substring(0, 10) + " " + normalized.substring(11);
+ int n = normalized.length();
+ if (n >= 5
+ && (normalized.charAt(n - 5) == '+' || normalized.charAt(n - 5) == '-')
+ && normalized.charAt(n - 3) != ':')
+ normalized = normalized.substring(0, n - 2) + ":" + normalized.substring(n - 2);
+ TemporalAccessor parsed =
+ EVENT_TIME.parseBest(normalized, OffsetDateTime::from, LocalDateTime::from);
+ if (parsed instanceof OffsetDateTime) return ((OffsetDateTime) parsed).toInstant();
+ return ((LocalDateTime) parsed).atZone(DEVICE_ZONE).toInstant();
+ } catch (Exception e) {
+ throw new AdapterException("EVENT_TIME_INVALID", "NDR timestamp 格式非法");
+ }
+ }
+
+ static String answer(String challenge, char[] secret) throws Exception {
+ byte[] a = challenge.getBytes(StandardCharsets.UTF_8);
+ byte[] b = new String(secret).getBytes(StandardCharsets.UTF_8);
+ byte[] joined = new byte[a.length + b.length];
+ try {
+ System.arraycopy(a, 0, joined, 0, a.length);
+ System.arraycopy(b, 0, joined, a.length, b.length);
+ byte[] digest = MessageDigest.getInstance("MD5").digest(joined);
+ StringBuilder hex = new StringBuilder(32);
+ for (byte x : digest) hex.append(String.format("%02x", x & 0xff));
+ Arrays.fill(digest, (byte) 0);
+ return hex.toString();
+ } finally {
+ // 尽力缩短 challenge、secret 拼接字节和摘要在堆中的存活时间。
+ Arrays.fill(a, (byte) 0);
+ Arrays.fill(b, (byte) 0);
+ Arrays.fill(joined, (byte) 0);
+ }
+ }
+
+ static UUID uuidV5(UUID namespace, String name) throws Exception {
+ java.nio.ByteBuffer ns =
+ java.nio.ByteBuffer.allocate(16)
+ .putLong(namespace.getMostSignificantBits())
+ .putLong(namespace.getLeastSignificantBits());
+ MessageDigest sha1 = MessageDigest.getInstance("SHA-1");
+ sha1.update(ns.array());
+ byte[] hash = sha1.digest(name.getBytes(StandardCharsets.UTF_8));
+ hash[6] = (byte) ((hash[6] & 0x0f) | 0x50);
+ hash[8] = (byte) ((hash[8] & 0x3f) | 0x80);
+ java.nio.ByteBuffer b = java.nio.ByteBuffer.wrap(hash);
+ UUID result = new UUID(b.getLong(), b.getLong());
+ Arrays.fill(hash, (byte) 0);
+ return result;
+ }
+
+ private void requireRange(Integer v, int min, int max, String name) {
+ if (v == null || v < min || v > max)
+ throw new AdapterException("CONFIG_SCHEMA_ERROR", name + " 超出 NDRSERVER_V1 范围");
+ }
+
+ private long elapsed(long started) {
+ return java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started);
+ }
+
+ @PreDestroy
+ public void close() {
+ tokens.clear();
+ tokenLocks.clear();
+ try {
+ http.close();
+ } catch (Exception ignored) {
+ }
+ }
+
+ private static final class HttpResult {
+ final int status;
+ final byte[] body;
+
+ HttpResult(int s, byte[] b) {
+ status = s;
+ body = b;
+ }
+ }
+
+ private static final class TokenLease {
+ final String cacheKey;
+ final String token;
+
+ TokenLease(String k, String t) {
+ cacheKey = k;
+ token = t;
+ }
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullDtos.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullDtos.java
new file mode 100644
index 0000000..bdb62ff
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullDtos.java
@@ -0,0 +1,337 @@
+package com.haobang.apipull.api;
+
+import com.fasterxml.jackson.annotation.JsonAnySetter;
+import java.time.OffsetDateTime;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.validation.constraints.*;
+import lombok.Data;
+
+/** API 拉取管理接口的请求与响应契约集合；任何响应 DTO 都不包含凭据内容或认证 token。 */
+public final class ApiPullDtos {
+ private ApiPullDtos() {}
+
+ /** 所有写请求的严格基类。 未知 JSON 字段立即失败，防止客户端拼错参数后误以为配置已经生效。 */
+ public abstract static class StrictRequest {
+ @JsonAnySetter
+ public void unknown(String name, Object value) {
+ throw new IllegalArgumentException("未知字段: " + name);
+ }
+ }
+
+ /** 新建任务请求；服务端无论请求内容如何都将任务初始状态设为暂停。 */
+ @Data
+ public static class CreateApiPullTaskRequest extends StrictRequest {
+ @NotNull private Integer deviceId;
+
+ @NotBlank
+ @Size(max = 255)
+ private String taskName;
+
+ @NotBlank
+ @Size(max = 64)
+ private String adapterType;
+
+ @NotBlank
+ @Size(max = 64)
+ private String streamKey;
+
+ @NotBlank
+ @Size(max = 1024)
+ private String baseUrl;
+
+ @NotBlank
+ @Pattern(regexp = "^(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
+ private String credentialRef;
+
+ @Min(1)
+ @Max(86400)
+ private Integer pollIntervalSeconds = 10;
+
+ @Min(1)
+ @Max(10000)
+ private Integer pageSize = 100;
+
+ @Min(1)
+ @Max(100000)
+ private Integer maxPagesPerRun = 1000;
+
+ @Min(-1000)
+ @Max(1000)
+ private Integer schedulerPriority = 0;
+
+ @Min(100)
+ @Max(120000)
+ private Integer connectTimeoutMs = 5000;
+
+ @Min(100)
+ @Max(600000)
+ private Integer requestTimeoutMs = 30000;
+
+ @Min(1)
+ @Max(1209600)
+ private Integer windowSizeSeconds = 60;
+
+ @Min(0)
+ @Max(604800)
+ private Integer safetyDelaySeconds = 120;
+
+ @Min(0)
+ @Max(1209600)
+ private Integer lookbackSeconds = 300;
+
+ @Min(1024)
+ @Max(268435456)
+ private Long maxResponseBytes = 16777216L;
+
+ @NotNull private OffsetDateTime initialStartTime;
+ @NotNull private Map extraConfig = Collections.emptyMap();
+ }
+
+ /** 修改请求必须携带当前 configVersion，并且仅允许修改暂停中的任务。 */
+ @Data
+ public static class UpdateApiPullTaskRequest extends StrictRequest {
+ @Min(1)
+ private long configVersion;
+
+ @NotBlank
+ @Size(max = 255)
+ private String taskName;
+
+ @NotBlank
+ @Size(max = 1024)
+ private String baseUrl;
+
+ @NotBlank
+ @Pattern(regexp = "^(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
+ private String credentialRef;
+
+ @Min(1)
+ @Max(86400)
+ private Integer pollIntervalSeconds;
+
+ @Min(1)
+ @Max(10000)
+ private Integer pageSize;
+
+ @Min(1)
+ @Max(100000)
+ private Integer maxPagesPerRun;
+
+ @Min(-1000)
+ @Max(1000)
+ private Integer schedulerPriority;
+
+ @Min(100)
+ @Max(120000)
+ private Integer connectTimeoutMs;
+
+ @Min(100)
+ @Max(600000)
+ private Integer requestTimeoutMs;
+
+ @Min(1)
+ @Max(1209600)
+ private Integer windowSizeSeconds;
+
+ @Min(0)
+ @Max(604800)
+ private Integer safetyDelaySeconds;
+
+ @Min(0)
+ @Max(1209600)
+ private Integer lookbackSeconds;
+
+ @Min(1024)
+ @Max(268435456)
+ private Long maxResponseBytes;
+
+ @NotNull private Map extraConfig;
+ }
+
+ @Data
+ public static class TaskVersionRequest extends StrictRequest {
+ @Min(1)
+ private long configVersion;
+ }
+
+ @Data
+ public static class TriggerApiPullTaskRequest extends StrictRequest {
+ @Min(1)
+ private long configVersion;
+ }
+
+ /** checkpoint 人工回退/前移请求；expectedCheckpointVersion 用于防止并发覆盖。 */
+ @Data
+ public static class ResetApiCheckpointRequest extends StrictRequest {
+ @Min(1)
+ private long expectedCheckpointVersion;
+
+ @NotNull private OffsetDateTime committedWindowEnd;
+
+ @NotBlank
+ @Size(max = 500)
+ private String reason;
+ }
+
+ @Data
+ public static class ApiPullTaskResponse {
+ private Integer taskId;
+ private Integer deviceId;
+ private String taskName;
+ private String adapterType;
+ private Integer adapterVersion;
+ private String streamKey;
+ private String baseUrl;
+ private String credentialRef;
+ private boolean enabled;
+ private long configVersion;
+ private OffsetDateTime initialStartTime;
+ private OffsetDateTime committedWindowEnd;
+ private ApiPullTuningResponse tuning;
+ private Map extraConfig;
+ private OffsetDateTime createdAt;
+ private OffsetDateTime updatedAt;
+ }
+
+ /** 新建设备凭据；AppSecret 仅允许出现在请求中，不会进入任何响应 DTO。 */
+ @Data
+ public static class CreateApiCredentialRequest extends StrictRequest {
+ @NotBlank
+ @Pattern(regexp = "^(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
+ private String credentialRef;
+
+ @NotNull private Integer deviceId;
+
+ @NotBlank
+ @Size(max = 64)
+ private String adapterType;
+
+ @NotBlank
+ @Pattern(regexp = "^[A-Za-z0-9._-]{1,64}$")
+ private String credentialVersion;
+
+ @NotBlank
+ @Size(max = 512)
+ private String appKey;
+
+ @NotBlank
+ @Size(max = 2048)
+ private String appSecret;
+ }
+
+ /** 轮换凭据时必须携带当前 configVersion，设备、适配器和引用不可原地修改。 */
+ @Data
+ public static class UpdateApiCredentialRequest extends StrictRequest {
+ @Min(1)
+ private long configVersion;
+
+ @NotBlank
+ @Pattern(regexp = "^[A-Za-z0-9._-]{1,64}$")
+ private String credentialVersion;
+
+ @NotBlank
+ @Size(max = 512)
+ private String appKey;
+
+ @NotBlank
+ @Size(max = 2048)
+ private String appSecret;
+ }
+
+ /** 凭据管理响应；刻意不定义 appKey/appSecret 字段，避免序列化或日志误回显。 */
+ @Data
+ public static class ApiCredentialResponse {
+ private String credentialRef;
+ private Integer deviceId;
+ private String adapterType;
+ private String credentialVersion;
+ private long configVersion;
+ private OffsetDateTime createdAt;
+ private OffsetDateTime updatedAt;
+ }
+
+ @Data
+ public static class ApiPullTuningResponse {
+ private Integer pollIntervalSeconds;
+ private Integer pageSize;
+ private Integer maxPagesPerRun;
+ private Integer schedulerPriority;
+ private Integer connectTimeoutMs;
+ private Integer requestTimeoutMs;
+ private Integer windowSizeSeconds;
+ private Integer safetyDelaySeconds;
+ private Integer lookbackSeconds;
+ private Long maxResponseBytes;
+ }
+
+ @Data
+ public static class ValidationCheckResponse {
+ private String name;
+ private String status;
+ private String errorCode;
+ private String summary;
+ private Long durationMs;
+ }
+
+ @Data
+ public static class ValidateApiPullTaskResponse {
+ private Integer taskId;
+ private long configVersion;
+ private boolean valid;
+ private String adapterType;
+ private String credentialVersion;
+ private List checks;
+ private OffsetDateTime validatedAt;
+ }
+
+ @Data
+ public static class ApiPullRuntimeResponse {
+ private Integer taskId;
+ private boolean enabled;
+ private String health;
+ private OffsetDateTime nextRunAt;
+ private OffsetDateTime committedWindowEnd;
+ private long checkpointVersion;
+ private String leaseOwner;
+ private OffsetDateTime leaseExpiresAt;
+ private Long checkpointLagSeconds;
+ private ApiPullRunResponse latestRun;
+ }
+
+ @Data
+ public static class ApiPullRunResponse {
+ private String runId;
+ private String triggerType;
+ private String status;
+ private OffsetDateTime logicalWindowStart;
+ private OffsetDateTime logicalWindowEnd;
+ private int pagesFetched;
+ private long recordsFetched;
+ private long recordsDurable;
+ private long recordsQuarantined;
+ private Integer lastHttpStatus;
+ private String errorCode;
+ private String errorSummary;
+ private OffsetDateTime startedAt;
+ private OffsetDateTime finishedAt;
+ }
+
+ @Data
+ public static class PagedResponse {
+ private List items;
+ private int page;
+ private int pageSize;
+ private long total;
+ }
+
+ @Data
+ public static class ApiErrorResponse {
+ private String code;
+ private String message;
+ private String requestId;
+ private Map fieldErrors;
+ private OffsetDateTime timestamp;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullExceptionHandler.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullExceptionHandler.java
new file mode 100644
index 0000000..acbf034
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullExceptionHandler.java
@@ -0,0 +1,94 @@
+package com.haobang.apipull.api;
+
+import com.haobang.apipull.ApiPullException;
+import com.haobang.apipull.adapter.AdapterException;
+import com.haobang.apipull.api.ApiPullDtos.ApiErrorResponse;
+import com.haobang.apipull.credential.CredentialException;
+import java.time.OffsetDateTime;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+import javax.servlet.http.HttpServletRequest;
+import javax.validation.ConstraintViolationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataIntegrityViolationException;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.http.converter.HttpMessageNotReadableException;
+import org.springframework.web.bind.MethodArgumentNotValidException;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.RestControllerAdvice;
+
+/**
+ * API 拉取管理面的统一错误边界。
+ *
+ * 已知异常转换为稳定错误码；未知异常只在服务端记录类型，不将堆栈、SQL、设备响应或凭据 回显给调用方。requestId 可用于关联服务端日志。
+ */
+@RestControllerAdvice(basePackages = "com.haobang.apipull.api")
+public class ApiPullExceptionHandler {
+ private static final Logger log = LoggerFactory.getLogger(ApiPullExceptionHandler.class);
+
+ @ExceptionHandler(ApiPullException.class)
+ public ResponseEntity api(ApiPullException e, HttpServletRequest r) {
+ return response(e.getStatus(), e.getCode(), e.getMessage(), null, r);
+ }
+
+ @ExceptionHandler(AdapterException.class)
+ public ResponseEntity adapter(AdapterException e, HttpServletRequest r) {
+ return response(HttpStatus.UNPROCESSABLE_ENTITY, e.getCode(), e.getMessage(), null, r);
+ }
+
+ @ExceptionHandler(CredentialException.class)
+ public ResponseEntity credential(CredentialException e, HttpServletRequest r) {
+ return response(HttpStatus.UNPROCESSABLE_ENTITY, e.getCode(), e.getMessage(), null, r);
+ }
+
+ @ExceptionHandler(MethodArgumentNotValidException.class)
+ public ResponseEntity validation(
+ MethodArgumentNotValidException e, HttpServletRequest r) {
+ Map fields = new LinkedHashMap<>();
+ e.getBindingResult()
+ .getFieldErrors()
+ .forEach(x -> fields.put(x.getField(), x.getDefaultMessage()));
+ return response(HttpStatus.BAD_REQUEST, "INVALID_REQUEST", "请求字段校验失败", fields, r);
+ }
+
+ @ExceptionHandler({
+ HttpMessageNotReadableException.class,
+ IllegalArgumentException.class,
+ ConstraintViolationException.class
+ })
+ public ResponseEntity bad(Exception e, HttpServletRequest r) {
+ return response(HttpStatus.BAD_REQUEST, "INVALID_REQUEST", "请求格式错误", null, r);
+ }
+
+ @ExceptionHandler(DataIntegrityViolationException.class)
+ public ResponseEntity conflict(
+ DataIntegrityViolationException e, HttpServletRequest r) {
+ return response(HttpStatus.CONFLICT, "CONFIG_VERSION_CONFLICT", "配置与现有记录冲突", null, r);
+ }
+
+ @ExceptionHandler(Exception.class)
+ public ResponseEntity unexpected(Exception e, HttpServletRequest r) {
+ // 不记录异常 message，避免第三方 SDK 把含 Authorization 的请求细节拼入消息。
+ log.error("设备 API 管理请求失败: path={}, errorType={}", r.getRequestURI(), e.getClass().getName());
+ return response(HttpStatus.INTERNAL_SERVER_ERROR, "INTERNAL_ERROR", "内部处理失败", null, r);
+ }
+
+ private ResponseEntity response(
+ HttpStatus status,
+ String code,
+ String message,
+ Map fields,
+ HttpServletRequest request) {
+ ApiErrorResponse body = new ApiErrorResponse();
+ body.setCode(code);
+ body.setMessage(message);
+ body.setFieldErrors(fields);
+ String id = request.getHeader("X-Request-Id");
+ body.setRequestId(id == null || id.trim().isEmpty() ? UUID.randomUUID().toString() : id);
+ body.setTimestamp(OffsetDateTime.now());
+ return ResponseEntity.status(status).body(body);
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/DeviceApiCredentialController.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/DeviceApiCredentialController.java
new file mode 100644
index 0000000..8b0f024
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/DeviceApiCredentialController.java
@@ -0,0 +1,51 @@
+package com.haobang.apipull.api;
+
+import com.haobang.apipull.ApiCredentialService;
+import com.haobang.apipull.api.ApiPullDtos.ApiCredentialResponse;
+import com.haobang.apipull.api.ApiPullDtos.CreateApiCredentialRequest;
+import com.haobang.apipull.api.ApiPullDtos.UpdateApiCredentialRequest;
+import javax.validation.Valid;
+import javax.validation.constraints.Pattern;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+/** 明文数据库凭据的管理端点；所有响应均不包含 AppKey/AppSecret。 */
+@Validated
+@RestController
+@RequestMapping("/api/api-pull-credentials")
+public class DeviceApiCredentialController {
+ private final ApiCredentialService service;
+
+ public DeviceApiCredentialController(ApiCredentialService service) {
+ this.service = service;
+ }
+
+ @PostMapping
+ public ResponseEntity create(
+ @Valid @RequestBody CreateApiCredentialRequest request) {
+ return ResponseEntity.status(HttpStatus.CREATED).body(service.create(request));
+ }
+
+ @GetMapping("/{credentialRef}")
+ public ApiCredentialResponse get(
+ @PathVariable @Pattern(regexp = "^(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
+ String credentialRef) {
+ return service.get(credentialRef);
+ }
+
+ @PutMapping("/{credentialRef}")
+ public ApiCredentialResponse update(
+ @PathVariable @Pattern(regexp = "^(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
+ String credentialRef,
+ @Valid @RequestBody UpdateApiCredentialRequest request) {
+ return service.update(credentialRef, request);
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/DeviceApiPullTaskController.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/DeviceApiPullTaskController.java
new file mode 100644
index 0000000..42843b0
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/DeviceApiPullTaskController.java
@@ -0,0 +1,99 @@
+package com.haobang.apipull.api;
+
+import com.haobang.apipull.ApiPullTaskService;
+import com.haobang.apipull.api.ApiPullDtos.*;
+import com.haobang.apipull.persistence.DeviceCollectApiRun;
+import javax.validation.Valid;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.*;
+
+/**
+ * API 拉取任务的专用管理端点。
+ *
+ * 使用独立路由而不是复用原有 TCP/UDP 任务接口，以确保适配器校验、乐观锁、凭据引用和 checkpoint 操作都经过 {@link ApiPullTaskService}
+ * 的实施契约。
+ */
+@Validated
+@RestController
+@RequestMapping("/api/api-pull-tasks")
+public class DeviceApiPullTaskController {
+ private final ApiPullTaskService service;
+
+ public DeviceApiPullTaskController(ApiPullTaskService service) {
+ this.service = service;
+ }
+
+ @PostMapping
+ public ResponseEntity create(
+ @Valid @RequestBody CreateApiPullTaskRequest r) {
+ return ResponseEntity.status(HttpStatus.CREATED).body(service.create(r));
+ }
+
+ @GetMapping("/{taskId}")
+ public ApiPullTaskResponse get(@PathVariable Integer taskId) {
+ return service.get(taskId);
+ }
+
+ @PutMapping("/{taskId}")
+ public ApiPullTaskResponse update(
+ @PathVariable Integer taskId, @Valid @RequestBody UpdateApiPullTaskRequest r) {
+ return service.update(taskId, r);
+ }
+
+ @PostMapping("/{taskId}/validate")
+ public ValidateApiPullTaskResponse validate(
+ @PathVariable Integer taskId, @Valid @RequestBody TaskVersionRequest r) {
+ return service.validate(taskId, r.getConfigVersion());
+ }
+
+ /** 手工触发是异步操作，返回 202 和已创建的 runId，不等待设备请求完成。 */
+ @PostMapping("/{taskId}/trigger")
+ public ResponseEntity trigger(
+ @PathVariable Integer taskId, @Valid @RequestBody TriggerApiPullTaskRequest r) {
+ DeviceCollectApiRun run = service.trigger(taskId, r.getConfigVersion());
+ return ResponseEntity.accepted().body(toRun(run));
+ }
+
+ @PostMapping("/{taskId}/pause")
+ public ApiPullTaskResponse pause(
+ @PathVariable Integer taskId, @Valid @RequestBody TaskVersionRequest r) {
+ return service.pause(taskId, r.getConfigVersion());
+ }
+
+ @PostMapping("/{taskId}/resume")
+ public ApiPullTaskResponse resume(
+ @PathVariable Integer taskId, @Valid @RequestBody TaskVersionRequest r) {
+ return service.resume(taskId, r.getConfigVersion());
+ }
+
+ @GetMapping("/{taskId}/runtime")
+ public ApiPullRuntimeResponse runtime(@PathVariable Integer taskId) {
+ return service.runtime(taskId);
+ }
+
+ @GetMapping("/{taskId}/runs")
+ public PagedResponse runs(
+ @PathVariable Integer taskId,
+ @RequestParam(defaultValue = "1") int page,
+ @RequestParam(defaultValue = "20") int pageSize) {
+ if (page < 1 || pageSize < 1 || pageSize > 100)
+ throw new IllegalArgumentException("page 必须 >=1，pageSize 必须在 1..100");
+ return service.runs(taskId, page, pageSize);
+ }
+
+ @PostMapping("/{taskId}/checkpoint/reset")
+ public ApiPullRuntimeResponse reset(
+ @PathVariable Integer taskId, @Valid @RequestBody ResetApiCheckpointRequest r) {
+ return service.reset(taskId, r);
+ }
+
+ private ApiPullRunResponse toRun(DeviceCollectApiRun x) {
+ ApiPullRunResponse r = new ApiPullRunResponse();
+ r.setRunId(x.getRunId());
+ r.setTriggerType(x.getTriggerType());
+ r.setStatus(x.getStatus());
+ return r;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/CredentialException.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/CredentialException.java
new file mode 100644
index 0000000..be5ffe7
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/CredentialException.java
@@ -0,0 +1,20 @@
+package com.haobang.apipull.credential;
+
+/** 凭据定位、设备/适配器绑定或字段校验失败时使用的稳定业务异常。 */
+public class CredentialException extends RuntimeException {
+ private final String code;
+
+ public CredentialException(String code, String message) {
+ super(message);
+ this.code = code;
+ }
+
+ public CredentialException(String code, String message, Throwable cause) {
+ super(message, cause);
+ this.code = code;
+ }
+
+ public String getCode() {
+ return code;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/CredentialProvider.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/CredentialProvider.java
new file mode 100644
index 0000000..81f491f
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/CredentialProvider.java
@@ -0,0 +1,12 @@
+package com.haobang.apipull.credential;
+
+/**
+ * 厂商适配器读取凭据的唯一入口。
+ *
+ * 任务配置只保存 {@code credentialRef}，真实凭据由独立凭据存储管理。返回值必须由调用方及时关闭， 以清零内存中的字符数组。
+ */
+public interface CredentialProvider {
+ /** 加载一次不可变凭据快照，并验证其适配器类型与调用方完全一致。 */
+ CredentialSnapshot load(
+ String credentialRef, String expectedAdapterType, Integer expectedDeviceId);
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/CredentialSnapshot.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/CredentialSnapshot.java
new file mode 100644
index 0000000..6443b66
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/CredentialSnapshot.java
@@ -0,0 +1,64 @@
+package com.haobang.apipull.credential;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * 单次加载得到的凭据版本快照。
+ *
+ *
快照实现 {@link AutoCloseable}，关闭时会级联清零所有敏感字段。 {@link #toString()} 永远不输出字段内容，避免被调试日志或异常对象意外泄漏。
+ */
+public final class CredentialSnapshot implements AutoCloseable {
+ private final int schemaVersion;
+ private final String credentialVersion;
+ private final String adapterType;
+ private final Map values;
+
+ public CredentialSnapshot(
+ int schemaVersion,
+ String credentialVersion,
+ String adapterType,
+ Map values) {
+ this.schemaVersion = schemaVersion;
+ this.credentialVersion = credentialVersion;
+ this.adapterType = adapterType;
+ this.values = Collections.unmodifiableMap(new LinkedHashMap<>(values));
+ }
+
+ public int getSchemaVersion() {
+ return schemaVersion;
+ }
+
+ public String getCredentialVersion() {
+ return credentialVersion;
+ }
+
+ public String getAdapterType() {
+ return adapterType;
+ }
+
+ /** 返回指定敏感字段；缺失字段以稳定错误码失败，避免厂商适配器继续发出畸形请求。 */
+ public SecretValue require(String name) {
+ SecretValue value = values.get(name);
+ if (value == null) throw new CredentialException("CREDENTIAL_FIELD_MISSING", "凭据缺少字段: " + name);
+ return value;
+ }
+
+ /** 尽力覆盖内存中的所有敏感字符；调用方应使用 try-with-resources。 */
+ @Override
+ public void close() {
+ for (SecretValue value : values.values()) value.close();
+ }
+
+ @Override
+ public String toString() {
+ return "CredentialSnapshot{schemaVersion="
+ + schemaVersion
+ + ", credentialVersion='"
+ + credentialVersion
+ + "', adapterType='"
+ + adapterType
+ + "', values=[REDACTED]}";
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/DatabaseCredentialProvider.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/DatabaseCredentialProvider.java
new file mode 100644
index 0000000..89e2358
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/DatabaseCredentialProvider.java
@@ -0,0 +1,51 @@
+package com.haobang.apipull.credential;
+
+import com.common.mapper.DeviceApiCredentialMapper;
+import com.haobang.apipull.persistence.DeviceApiCredential;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import org.springframework.stereotype.Component;
+
+/**
+ * 从达梦独立凭据表读取设备 API 凭据。
+ *
+ * 查询条件同时包含引用、设备和适配器，防止任务误用另一台设备或另一厂商的凭据。第一阶段数据库字段为明文， 但返回给适配器的仍是可主动清零的 {@link
+ * CredentialSnapshot}，并且异常信息不包含凭据正文。
+ */
+@Component
+public class DatabaseCredentialProvider implements CredentialProvider {
+ private final DeviceApiCredentialMapper mapper;
+
+ public DatabaseCredentialProvider(DeviceApiCredentialMapper mapper) {
+ this.mapper = mapper;
+ }
+
+ @Override
+ public CredentialSnapshot load(
+ String credentialRef, String expectedAdapterType, Integer deviceId) {
+ if (isBlank(credentialRef) || isBlank(expectedAdapterType) || deviceId == null) {
+ throw new CredentialException("CREDENTIAL_REF_INVALID", "凭据引用、设备或适配器非法");
+ }
+
+ DeviceApiCredential credential =
+ mapper.selectForLoad(credentialRef, deviceId, expectedAdapterType);
+ if (credential == null) {
+ throw new CredentialException("CREDENTIAL_NOT_FOUND", "设备凭据不存在或绑定关系不匹配");
+ }
+ if (isBlank(credential.getCredentialVersion())
+ || isBlank(credential.getAppKey())
+ || isBlank(credential.getAppSecret())) {
+ throw new CredentialException("CREDENTIAL_SCHEMA", "设备凭据字段不完整");
+ }
+
+ Map values = new LinkedHashMap<>();
+ values.put("app_key", new SecretValue(credential.getAppKey().toCharArray()));
+ values.put("app_secret", new SecretValue(credential.getAppSecret().toCharArray()));
+ return new CredentialSnapshot(
+ 1, credential.getCredentialVersion(), credential.getAdapterType(), values);
+ }
+
+ private boolean isBlank(String value) {
+ return value == null || value.trim().isEmpty();
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/SecretValue.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/SecretValue.java
new file mode 100644
index 0000000..32ddf0f
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/credential/SecretValue.java
@@ -0,0 +1,31 @@
+package com.haobang.apipull.credential;
+
+import java.util.Arrays;
+
+/**
+ * 可主动清零的敏感字符容器。
+ *
+ * 使用 char[] 而不是长期驻留的 String；{@link #copy()} 的返回数组同样必须由调用方清零。
+ */
+public final class SecretValue implements AutoCloseable {
+ private final char[] value;
+
+ public SecretValue(char[] value) {
+ this.value = value == null ? new char[0] : value.clone();
+ }
+
+ /** 返回防御性副本，避免厂商适配器修改快照内部状态。 */
+ public char[] copy() {
+ return value.clone();
+ }
+
+ @Override
+ public void close() {
+ Arrays.fill(value, '\0');
+ }
+
+ @Override
+ public String toString() {
+ return "[REDACTED]";
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceApiCredential.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceApiCredential.java
new file mode 100644
index 0000000..0b03ab4
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceApiCredential.java
@@ -0,0 +1,93 @@
+package com.haobang.apipull.persistence;
+
+import java.time.LocalDateTime;
+
+/**
+ * 设备 API 凭据数据库记录。
+ *
+ *
第一阶段按照当前实施决定明文保存 AppKey/AppSecret。该对象禁止实现会展开字段的 {@code toString()}，调用方也不得将实例直接写入日志。后续加密改造保持
+ * {@code credentialRef} 不变，只替换持久化字段和凭据提供器的解密步骤。
+ */
+public class DeviceApiCredential {
+ private String credentialRef;
+ private Integer deviceId;
+ private String adapterType;
+ private String credentialVersion;
+ private String appKey;
+ private String appSecret;
+ private long configVersion;
+ private LocalDateTime createdAt;
+ private LocalDateTime updatedAt;
+
+ public String getCredentialRef() {
+ return credentialRef;
+ }
+
+ public void setCredentialRef(String credentialRef) {
+ this.credentialRef = credentialRef;
+ }
+
+ public Integer getDeviceId() {
+ return deviceId;
+ }
+
+ public void setDeviceId(Integer deviceId) {
+ this.deviceId = deviceId;
+ }
+
+ public String getAdapterType() {
+ return adapterType;
+ }
+
+ public void setAdapterType(String adapterType) {
+ this.adapterType = adapterType;
+ }
+
+ public String getCredentialVersion() {
+ return credentialVersion;
+ }
+
+ public void setCredentialVersion(String credentialVersion) {
+ this.credentialVersion = credentialVersion;
+ }
+
+ public String getAppKey() {
+ return appKey;
+ }
+
+ public void setAppKey(String appKey) {
+ this.appKey = appKey;
+ }
+
+ public String getAppSecret() {
+ return appSecret;
+ }
+
+ public void setAppSecret(String appSecret) {
+ this.appSecret = appSecret;
+ }
+
+ public long getConfigVersion() {
+ return configVersion;
+ }
+
+ public void setConfigVersion(long configVersion) {
+ this.configVersion = configVersion;
+ }
+
+ public LocalDateTime getCreatedAt() {
+ return createdAt;
+ }
+
+ public void setCreatedAt(LocalDateTime createdAt) {
+ this.createdAt = createdAt;
+ }
+
+ public LocalDateTime getUpdatedAt() {
+ return updatedAt;
+ }
+
+ public void setUpdatedAt(LocalDateTime updatedAt) {
+ this.updatedAt = updatedAt;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceCollectApiCheckpoint.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceCollectApiCheckpoint.java
new file mode 100644
index 0000000..679908b
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceCollectApiCheckpoint.java
@@ -0,0 +1,138 @@
+package com.haobang.apipull.persistence;
+
+import java.time.LocalDateTime;
+
+/**
+ * {@code device_collect_api_checkpoint} 行模型。
+ *
+ *
{@code version} 用于 checkpoint CAS；{@code fencingToken} 每次成功取得租约时递增，阻止 已失租的旧实例提交进度。{@code
+ * committedWindowEnd} 只表示已经完整 durable 的窗口末端。
+ */
+public class DeviceCollectApiCheckpoint {
+ private Integer collectTaskId;
+ private String checkpointType;
+ private String checkpointValue;
+ private LocalDateTime committedWindowEnd;
+ private String adapterType;
+ private Integer adapterVersion;
+ private Long version;
+ private String leaseOwner;
+ private LocalDateTime leaseExpiresAt;
+ private Long fencingToken;
+ private LocalDateTime lastEventTime;
+ private String lastExternalEventId;
+ private LocalDateTime createdAt;
+ private LocalDateTime updatedAt;
+
+ public Integer getCollectTaskId() {
+ return collectTaskId;
+ }
+
+ public void setCollectTaskId(Integer v) {
+ collectTaskId = v;
+ }
+
+ public String getCheckpointType() {
+ return checkpointType;
+ }
+
+ public void setCheckpointType(String v) {
+ checkpointType = v;
+ }
+
+ public String getCheckpointValue() {
+ return checkpointValue;
+ }
+
+ public void setCheckpointValue(String v) {
+ checkpointValue = v;
+ }
+
+ public LocalDateTime getCommittedWindowEnd() {
+ return committedWindowEnd;
+ }
+
+ public void setCommittedWindowEnd(LocalDateTime v) {
+ committedWindowEnd = v;
+ }
+
+ public String getAdapterType() {
+ return adapterType;
+ }
+
+ public void setAdapterType(String v) {
+ adapterType = v;
+ }
+
+ public Integer getAdapterVersion() {
+ return adapterVersion;
+ }
+
+ public void setAdapterVersion(Integer v) {
+ adapterVersion = v;
+ }
+
+ public Long getVersion() {
+ return version;
+ }
+
+ public void setVersion(Long v) {
+ version = v;
+ }
+
+ public String getLeaseOwner() {
+ return leaseOwner;
+ }
+
+ public void setLeaseOwner(String v) {
+ leaseOwner = v;
+ }
+
+ public LocalDateTime getLeaseExpiresAt() {
+ return leaseExpiresAt;
+ }
+
+ public void setLeaseExpiresAt(LocalDateTime v) {
+ leaseExpiresAt = v;
+ }
+
+ public Long getFencingToken() {
+ return fencingToken;
+ }
+
+ public void setFencingToken(Long v) {
+ fencingToken = v;
+ }
+
+ public LocalDateTime getLastEventTime() {
+ return lastEventTime;
+ }
+
+ public void setLastEventTime(LocalDateTime v) {
+ lastEventTime = v;
+ }
+
+ public String getLastExternalEventId() {
+ return lastExternalEventId;
+ }
+
+ public void setLastExternalEventId(String v) {
+ lastExternalEventId = v;
+ }
+
+ public LocalDateTime getCreatedAt() {
+ return createdAt;
+ }
+
+ public void setCreatedAt(LocalDateTime v) {
+ createdAt = v;
+ }
+
+ public LocalDateTime getUpdatedAt() {
+ return updatedAt;
+ }
+
+ public void setUpdatedAt(LocalDateTime v) {
+ updatedAt = v;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceCollectApiConfig.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceCollectApiConfig.java
new file mode 100644
index 0000000..4acadf4
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceCollectApiConfig.java
@@ -0,0 +1,232 @@
+package com.haobang.apipull.persistence;
+
+import java.time.LocalDateTime;
+
+/**
+ * {@code device_collect_api_config} 行模型。
+ *
+ *
只保存非敏感任务参数和 {@code credentialRef}；真实凭据不得进入数据库。 {@code configVersion} 是管理修改、调度快照与 checkpoint
+ * 提交共同使用的乐观锁版本。
+ */
+public class DeviceCollectApiConfig {
+ private Integer collectTaskId;
+ private Integer deviceId;
+ private String adapterType;
+ private Integer adapterVersion;
+ private String baseUrl;
+ private String streamKey;
+ private String credentialRef;
+ private Integer enabled;
+ private LocalDateTime nextRunAt;
+ private Integer pollIntervalSeconds;
+ private Integer pageSize;
+ private Integer maxPagesPerRun;
+ private Integer schedulerPriority;
+ private Integer connectTimeoutMs;
+ private Integer requestTimeoutMs;
+ private Integer windowSizeSeconds;
+ private Integer safetyDelaySeconds;
+ private Integer lookbackSeconds;
+ private Long maxResponseBytes;
+ private LocalDateTime initialStartTime;
+ private String extraConfigJson;
+ private Long configVersion;
+ private LocalDateTime createdAt;
+ private LocalDateTime updatedAt;
+
+ public Integer getCollectTaskId() {
+ return collectTaskId;
+ }
+
+ public void setCollectTaskId(Integer v) {
+ collectTaskId = v;
+ }
+
+ public Integer getDeviceId() {
+ return deviceId;
+ }
+
+ public void setDeviceId(Integer v) {
+ deviceId = v;
+ }
+
+ public String getAdapterType() {
+ return adapterType;
+ }
+
+ public void setAdapterType(String v) {
+ adapterType = v;
+ }
+
+ public Integer getAdapterVersion() {
+ return adapterVersion;
+ }
+
+ public void setAdapterVersion(Integer v) {
+ adapterVersion = v;
+ }
+
+ public String getBaseUrl() {
+ return baseUrl;
+ }
+
+ public void setBaseUrl(String v) {
+ baseUrl = v;
+ }
+
+ public String getStreamKey() {
+ return streamKey;
+ }
+
+ public void setStreamKey(String v) {
+ streamKey = v;
+ }
+
+ public String getCredentialRef() {
+ return credentialRef;
+ }
+
+ public void setCredentialRef(String v) {
+ credentialRef = v;
+ }
+
+ public Integer getEnabled() {
+ return enabled;
+ }
+
+ public void setEnabled(Integer v) {
+ enabled = v;
+ }
+
+ public LocalDateTime getNextRunAt() {
+ return nextRunAt;
+ }
+
+ public void setNextRunAt(LocalDateTime v) {
+ nextRunAt = v;
+ }
+
+ public Integer getPollIntervalSeconds() {
+ return pollIntervalSeconds;
+ }
+
+ public void setPollIntervalSeconds(Integer v) {
+ pollIntervalSeconds = v;
+ }
+
+ public Integer getPageSize() {
+ return pageSize;
+ }
+
+ public void setPageSize(Integer v) {
+ pageSize = v;
+ }
+
+ public Integer getMaxPagesPerRun() {
+ return maxPagesPerRun;
+ }
+
+ public void setMaxPagesPerRun(Integer v) {
+ maxPagesPerRun = v;
+ }
+
+ public Integer getSchedulerPriority() {
+ return schedulerPriority;
+ }
+
+ public void setSchedulerPriority(Integer v) {
+ schedulerPriority = v;
+ }
+
+ public Integer getConnectTimeoutMs() {
+ return connectTimeoutMs;
+ }
+
+ public void setConnectTimeoutMs(Integer v) {
+ connectTimeoutMs = v;
+ }
+
+ public Integer getRequestTimeoutMs() {
+ return requestTimeoutMs;
+ }
+
+ public void setRequestTimeoutMs(Integer v) {
+ requestTimeoutMs = v;
+ }
+
+ public Integer getWindowSizeSeconds() {
+ return windowSizeSeconds;
+ }
+
+ public void setWindowSizeSeconds(Integer v) {
+ windowSizeSeconds = v;
+ }
+
+ public Integer getSafetyDelaySeconds() {
+ return safetyDelaySeconds;
+ }
+
+ public void setSafetyDelaySeconds(Integer v) {
+ safetyDelaySeconds = v;
+ }
+
+ public Integer getLookbackSeconds() {
+ return lookbackSeconds;
+ }
+
+ public void setLookbackSeconds(Integer v) {
+ lookbackSeconds = v;
+ }
+
+ public Long getMaxResponseBytes() {
+ return maxResponseBytes;
+ }
+
+ public void setMaxResponseBytes(Long v) {
+ maxResponseBytes = v;
+ }
+
+ public LocalDateTime getInitialStartTime() {
+ return initialStartTime;
+ }
+
+ public void setInitialStartTime(LocalDateTime v) {
+ initialStartTime = v;
+ }
+
+ public String getExtraConfigJson() {
+ return extraConfigJson;
+ }
+
+ public void setExtraConfigJson(String v) {
+ extraConfigJson = v;
+ }
+
+ public Long getConfigVersion() {
+ return configVersion;
+ }
+
+ public void setConfigVersion(Long v) {
+ configVersion = v;
+ }
+
+ public LocalDateTime getCreatedAt() {
+ return createdAt;
+ }
+
+ public void setCreatedAt(LocalDateTime v) {
+ createdAt = v;
+ }
+
+ public LocalDateTime getUpdatedAt() {
+ return updatedAt;
+ }
+
+ public void setUpdatedAt(LocalDateTime v) {
+ updatedAt = v;
+ }
+
+ public boolean isEnabled() {
+ return enabled != null && enabled == 1;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceCollectApiRun.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceCollectApiRun.java
new file mode 100644
index 0000000..64856f4
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/persistence/DeviceCollectApiRun.java
@@ -0,0 +1,254 @@
+package com.haobang.apipull.persistence;
+
+import java.time.LocalDateTime;
+
+/**
+ * {@code device_collect_api_run} 运行审计行模型。
+ *
+ *
同时记录拉取数量和已 durable 数量，便于区分“设备已返回”与“本地已可靠落盘”。 errorSummary 只能写入经过脱敏和截断的摘要，不保存设备响应原文。
+ */
+public class DeviceCollectApiRun {
+ private String runId;
+ private Integer collectTaskId;
+ private Integer deviceId;
+ private String adapterType;
+ private Integer adapterVersion;
+ private String streamKey;
+ private String instanceId;
+ private Long configVersion;
+ private Long fencingToken;
+ private String triggerType;
+ private String status;
+ private LocalDateTime logicalWindowStart;
+ private LocalDateTime logicalWindowEnd;
+ private LocalDateTime queryStart;
+ private LocalDateTime queryEnd;
+ private Integer pagesFetched = 0;
+ private Long recordsFetched = 0L;
+ private Long recordsDurable = 0L;
+ private Long recordsQuarantined = 0L;
+ private Long duplicateCandidates = 0L;
+ private Integer lastHttpStatus;
+ private String errorCode;
+ private String errorSummary;
+ private LocalDateTime scheduledAt;
+ private LocalDateTime startedAt;
+ private LocalDateTime finishedAt;
+ private LocalDateTime createdAt;
+
+ public String getRunId() {
+ return runId;
+ }
+
+ public void setRunId(String v) {
+ runId = v;
+ }
+
+ public Integer getCollectTaskId() {
+ return collectTaskId;
+ }
+
+ public void setCollectTaskId(Integer v) {
+ collectTaskId = v;
+ }
+
+ public Integer getDeviceId() {
+ return deviceId;
+ }
+
+ public void setDeviceId(Integer v) {
+ deviceId = v;
+ }
+
+ public String getAdapterType() {
+ return adapterType;
+ }
+
+ public void setAdapterType(String v) {
+ adapterType = v;
+ }
+
+ public Integer getAdapterVersion() {
+ return adapterVersion;
+ }
+
+ public void setAdapterVersion(Integer v) {
+ adapterVersion = v;
+ }
+
+ public String getStreamKey() {
+ return streamKey;
+ }
+
+ public void setStreamKey(String v) {
+ streamKey = v;
+ }
+
+ public String getInstanceId() {
+ return instanceId;
+ }
+
+ public void setInstanceId(String v) {
+ instanceId = v;
+ }
+
+ public Long getConfigVersion() {
+ return configVersion;
+ }
+
+ public void setConfigVersion(Long v) {
+ configVersion = v;
+ }
+
+ public Long getFencingToken() {
+ return fencingToken;
+ }
+
+ public void setFencingToken(Long v) {
+ fencingToken = v;
+ }
+
+ public String getTriggerType() {
+ return triggerType;
+ }
+
+ public void setTriggerType(String v) {
+ triggerType = v;
+ }
+
+ public String getStatus() {
+ return status;
+ }
+
+ public void setStatus(String v) {
+ status = v;
+ }
+
+ public LocalDateTime getLogicalWindowStart() {
+ return logicalWindowStart;
+ }
+
+ public void setLogicalWindowStart(LocalDateTime v) {
+ logicalWindowStart = v;
+ }
+
+ public LocalDateTime getLogicalWindowEnd() {
+ return logicalWindowEnd;
+ }
+
+ public void setLogicalWindowEnd(LocalDateTime v) {
+ logicalWindowEnd = v;
+ }
+
+ public LocalDateTime getQueryStart() {
+ return queryStart;
+ }
+
+ public void setQueryStart(LocalDateTime v) {
+ queryStart = v;
+ }
+
+ public LocalDateTime getQueryEnd() {
+ return queryEnd;
+ }
+
+ public void setQueryEnd(LocalDateTime v) {
+ queryEnd = v;
+ }
+
+ public Integer getPagesFetched() {
+ return pagesFetched;
+ }
+
+ public void setPagesFetched(Integer v) {
+ pagesFetched = v;
+ }
+
+ public Long getRecordsFetched() {
+ return recordsFetched;
+ }
+
+ public void setRecordsFetched(Long v) {
+ recordsFetched = v;
+ }
+
+ public Long getRecordsDurable() {
+ return recordsDurable;
+ }
+
+ public void setRecordsDurable(Long v) {
+ recordsDurable = v;
+ }
+
+ public Long getRecordsQuarantined() {
+ return recordsQuarantined;
+ }
+
+ public void setRecordsQuarantined(Long v) {
+ recordsQuarantined = v;
+ }
+
+ public Long getDuplicateCandidates() {
+ return duplicateCandidates;
+ }
+
+ public void setDuplicateCandidates(Long v) {
+ duplicateCandidates = v;
+ }
+
+ public Integer getLastHttpStatus() {
+ return lastHttpStatus;
+ }
+
+ public void setLastHttpStatus(Integer v) {
+ lastHttpStatus = v;
+ }
+
+ public String getErrorCode() {
+ return errorCode;
+ }
+
+ public void setErrorCode(String v) {
+ errorCode = v;
+ }
+
+ public String getErrorSummary() {
+ return errorSummary;
+ }
+
+ public void setErrorSummary(String v) {
+ errorSummary = v;
+ }
+
+ public LocalDateTime getScheduledAt() {
+ return scheduledAt;
+ }
+
+ public void setScheduledAt(LocalDateTime v) {
+ scheduledAt = v;
+ }
+
+ public LocalDateTime getStartedAt() {
+ return startedAt;
+ }
+
+ public void setStartedAt(LocalDateTime v) {
+ startedAt = v;
+ }
+
+ public LocalDateTime getFinishedAt() {
+ return finishedAt;
+ }
+
+ public void setFinishedAt(LocalDateTime v) {
+ finishedAt = v;
+ }
+
+ public LocalDateTime getCreatedAt() {
+ return createdAt;
+ }
+
+ public void setCreatedAt(LocalDateTime v) {
+ createdAt = v;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/security/SafeBaseUrlValidator.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/security/SafeBaseUrlValidator.java
new file mode 100644
index 0000000..4c0ebfe
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/security/SafeBaseUrlValidator.java
@@ -0,0 +1,69 @@
+package com.haobang.apipull.security;
+
+import com.haobang.apipull.ApiPullProperties;
+import com.haobang.apipull.adapter.AdapterException;
+import java.net.InetAddress;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.springframework.stereotype.Component;
+
+/**
+ * 设备基础地址的 SSRF 防护边界。
+ *
+ *
只有显式加入白名单的 HTTPS 主机才能访问；拒绝 user-info、query、fragment 和路径穿越。 主机采用精确匹配，不允许后缀或通配符匹配，避免相似域名绕过。TLS
+ * 证书链和主机名继续由 JVM 默认校验器验证。
+ */
+@Component
+public class SafeBaseUrlValidator {
+ private final Set allowedHosts;
+
+ public SafeBaseUrlValidator(ApiPullProperties properties) {
+ String configured = properties.getAllowedHosts();
+ allowedHosts =
+ configured == null || configured.trim().isEmpty()
+ ? Collections.emptySet()
+ : Arrays.stream(configured.split(","))
+ .map(String::trim)
+ .filter(s -> !s.isEmpty())
+ .map(String::toLowerCase)
+ .collect(Collectors.toCollection(HashSet::new));
+ }
+
+ /** 校验并返回规范化 URI；不会接受重定向后的新地址。 */
+ public URI validate(String value) {
+ try {
+ URI uri = new URI(value);
+ if (!"https".equalsIgnoreCase(uri.getScheme())
+ || uri.getHost() == null
+ || uri.getUserInfo() != null
+ || uri.getQuery() != null
+ || uri.getFragment() != null)
+ throw new AdapterException(
+ "SSRF_REJECTED", "baseUrl 必须是无 user-info/query/fragment 的 HTTPS URL");
+ String host = uri.getHost().toLowerCase();
+ if (allowedHosts.isEmpty() || !allowedHosts.contains(host))
+ throw new AdapterException("SSRF_REJECTED", "设备主机不在 device.api.allowed-hosts 中: " + host);
+ InetAddress[] addresses = InetAddress.getAllByName(host);
+ if (addresses.length == 0)
+ throw new AdapterException("DNS_RESOLUTION_FAILED", "设备主机无法解析: " + host);
+ String path = uri.getPath() == null ? "" : uri.getPath();
+ String rawPath = uri.getRawPath() == null ? "" : uri.getRawPath().toLowerCase();
+ if (path.contains("..")
+ || path.indexOf('\\') >= 0
+ || rawPath.contains("%2e")
+ || rawPath.contains("%5c"))
+ throw new AdapterException("SSRF_REJECTED", "baseUrl 路径包含非法穿越片段");
+ String normalizedPath =
+ path.endsWith("/") && path.length() > 1 ? path.substring(0, path.length() - 1) : path;
+ return new URI("https", null, host, uri.getPort(), normalizedPath, null, null);
+ } catch (AdapterException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AdapterException("SSRF_REJECTED", "baseUrl 非法", null, e);
+ }
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java
index 1859ad2..1bdb226 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java
@@ -72,8 +72,11 @@ public class AppProperties {
@Value("${syslog.wal.segment-max-bytes:268435456}")
private long syslogWalSegmentMaxBytes;
- /** 已完成段的保留天数；0 表示不自动删除。 */
- @Value("${syslog.wal.retention-days:0}")
+ /**
+ * 已完成并确认投递的 WAL 分段保留天数，默认 2 天；0 表示不自动删除。
+ * 启动时立即清理，运行期间随 checkpoint 推进最多每 30 分钟清理一次。
+ */
+ @Value("${syslog.wal.retention-days:2}")
private int syslogWalRetentionDays;
@Value("${syslog.wal.shutdown-timeout-ms:30000}")
@@ -86,12 +89,17 @@ public class AppProperties {
@Value("${syslog.wal.checkpoint-flush-interval-ms:200}")
private long syslogWalCheckpointFlushIntervalMs;
- @Value("${syslog.wal.disk-high-watermark-percent:80}")
+ /**
+ * WAL 所在文件系统的使用率保护线。系统会把该百分比换算为剩余容量，
+ * 再与 syslog.wal.min-free-bytes 比较，取较小值作为实际停止接入线。
+ */
+ @Value("${syslog.wal.disk-high-watermark-percent:90}")
private int syslogWalDiskHighWatermarkPercent;
- @Value("${syslog.wal.disk-critical-watermark-percent:90}")
- private int syslogWalDiskCriticalWatermarkPercent;
-
+ /**
+ * 实际停止接入线的容量上限，默认 10GiB。大磁盘最多预留 10GiB，
+ * 小磁盘则按使用率保护线对应的更小剩余容量处理。
+ */
@Value("${syslog.wal.min-free-bytes:10737418240}")
private long syslogWalMinFreeBytes;
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/DeviceContextResolver.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/DeviceContextResolver.java
index 1d5947c..b6bb28b 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/DeviceContextResolver.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/DeviceContextResolver.java
@@ -27,6 +27,7 @@ public class DeviceContextResolver {
private final DeviceUnknownService unknownService;
private final AppProperties properties;
private final ConcurrentHashMap cache = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap deviceIdCache = new ConcurrentHashMap<>();
private final Object[] locks = new Object[64];
private final LongAdder hits = new LongAdder();
private final LongAdder misses = new LongAdder();
@@ -84,14 +85,55 @@ public class DeviceContextResolver {
}
}
+ /**
+ * 按任务固化的设备主键解析 API_PULL 上下文。
+ *
+ * 不能复用 syslog 的 source_ip 路径，因为 API 的 HTTP 对端可能是负载均衡或网关。
+ * 缓存仍会重新验证设备是否归属当前探针，避免任务数据被错误迁移后继续发送 Kafka。
+ */
+ public Resolution resolveDeviceId(Integer deviceId) {
+ if (deviceId == null) return Resolution.terminal(Type.UNKNOWN_DEVICE, null, "deviceId 为空");
+ long now = System.currentTimeMillis();
+ CacheEntry cached = deviceIdCache.get(deviceId);
+ if (cached != null && cached.expiresAt > now) {
+ hits.increment();
+ return cached.resolution;
+ }
+ Object lock = locks[(deviceId.hashCode() & Integer.MAX_VALUE) % locks.length];
+ synchronized (lock) {
+ cached = deviceIdCache.get(deviceId);
+ if (cached != null && cached.expiresAt > now) {
+ hits.increment();
+ return cached.resolution;
+ }
+ misses.increment();
+ DeviceDevice device = deviceService.getByIdSafely(deviceId);
+ Resolution resolution = device == null
+ ? Resolution.terminal(Type.UNKNOWN_DEVICE, null, "设备不存在: deviceId=" + deviceId)
+ : validateAssignment(device);
+ deviceIdCache.put(deviceId, new CacheEntry(resolution,
+ now + properties.getSyslogDeviceContextCacheTtlMs()));
+ return resolution;
+ }
+ }
+
+ private Resolution validateAssignment(DeviceDevice device) {
+ Integer assignedCollector = device.getDeviceCollectId();
+ if (assignedCollector == null || assignedCollector != properties.getAppServiceDeviceCollectId()) {
+ return Resolution.terminal(Type.NOT_ASSIGNED, device,
+ "设备不属于当前采集探针: deviceId=" + device.getId());
+ }
+ return Resolution.registered(device);
+ }
+
public void invalidate(String sourceIp) {
if (sourceIp != null) cache.remove(sourceIp);
}
- public void invalidateAll() { cache.clear(); }
+ public void invalidateAll() { cache.clear(); deviceIdCache.clear(); }
public long getHitCount() { return hits.sum(); }
public long getMissCount() { return misses.sum(); }
- public int getCacheSize() { return cache.size(); }
+ public int getCacheSize() { return cache.size() + deviceIdCache.size(); }
/** 未知设备仍保留原业务登记能力，但最多每个本地缓存 TTL 执行一次，不占用网络线程。 */
private void recordUnknown(String sourceIp, String protocol) {
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/SyslogWalDispatcher.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/SyslogWalDispatcher.java
index 4c63f87..9417e14 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/SyslogWalDispatcher.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/SyslogWalDispatcher.java
@@ -25,7 +25,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
-/** 从 WAL 有界并发异步投递 Kafka，并只按连续 ACK 推进 WAL 游标。 */
+/**
+ * 从 WAL 有界并发异步投递 Kafka，并只按连续 ACK 推进 WAL 游标。
+ *
+ *
v2/v3 保持原有按 source_ip 识别设备和单时间参数的生产路径；v4 改为按可信 device_id
+ * 识别设备，并分别传递 receive_time/event_time。兼容分支留在 Dispatcher，Kafka 消息外层结构
+ * 不因接入方式改变。
+ */
@Component
public class SyslogWalDispatcher {
private static final Logger log = LoggerFactory.getLogger(SyslogWalDispatcher.class);
@@ -120,7 +126,12 @@ public class SyslogWalDispatcher {
}
final DeviceContextResolver.Resolution resolution;
try {
- resolution = deviceResolver.resolve(record.getSourceIp(), record.getProtocol());
+ // API 的 HTTP 对端可能是代理或网关；v4 必须使用任务写入 WAL 的 device_id。
+ if (record.getSchemaVersion() >= 4 && record.getDeviceId() != null) {
+ resolution = deviceResolver.resolveDeviceId(record.getDeviceId());
+ } else {
+ resolution = deviceResolver.resolve(record.getSourceIp(), record.getProtocol());
+ }
} catch (Throwable failure) {
scheduleRetry(record, attempt, "DEVICE_LOOKUP", failure);
return;
@@ -138,10 +149,31 @@ public class SyslogWalDispatcher {
}
final String receiveTime;
+ final String eventTime;
try {
receiveTime = LocalDateTime.parse(record.getReceiveTime(), WAL_TIME).format(HEADER_TIME);
- producer.messagePush(resolution.getDevice(), record.getRawMessage(), receiveTime,
- record.getEventId(), record.getPointTimeNs())
+ String walEventTime = record.getEventTime() == null
+ ? record.getReceiveTime()
+ : record.getEventTime();
+ eventTime = LocalDateTime.parse(walEventTime, WAL_TIME).format(HEADER_TIME);
+
+ // v2/v3 继续调用旧重载，确保升级前已落盘记录产生的 Kafka 消息完全不变。
+ java.util.concurrent.CompletableFuture deliveryFuture =
+ record.getSchemaVersion() >= 4
+ ? producer.messagePush(
+ resolution.getDevice(),
+ record.getRawMessage(),
+ receiveTime,
+ eventTime,
+ record.getEventId(),
+ record.getPointTimeNs())
+ : producer.messagePush(
+ resolution.getDevice(),
+ record.getRawMessage(),
+ receiveTime,
+ record.getEventId(),
+ record.getPointTimeNs());
+ deliveryFuture
.whenComplete((delivery, failure) -> {
if (failure == null) {
kafkaAcked.increment();
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/util/DeviceInfoUtil.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/util/DeviceInfoUtil.java
index 2e85ada..2bdab77 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/util/DeviceInfoUtil.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/util/DeviceInfoUtil.java
@@ -35,6 +35,7 @@ public class DeviceInfoUtil {
Map map = new LinkedHashMap<>();
map.put("receive_time",timeString);
+ map.put("event_time",timeString);
map.put("device_id", appProps.getAppServiceDeviceId());
map.put("device_name", appProps.getAppServiceDeviceName());
map.put("vendor", appProps.getAppServiceVendor());
@@ -65,6 +66,7 @@ public class DeviceInfoUtil {
String timeString = now.format(formatter);
Map map = new LinkedHashMap<>();
map.put("receive_time",timeString);
+ map.put("event_time",timeString);
map.put("device_id",deviceDevice.getId().toString());
map.put("device_name", deviceDevice.getName());
map.put("vendor", deviceDevice.getVendor());
@@ -95,16 +97,32 @@ public class DeviceInfoUtil {
String receive_time, String eventId,
AppProperties appProps)
{
- return getFullLogString(deviceDevice, strSyslog, receive_time, eventId, null, appProps);
+ return getFullLogString(deviceDevice, strSyslog, receive_time, receive_time,
+ eventId, null, appProps);
}
- /** Kafka 业务头只新增 point_time_ns；WAL schema_version 绝不进入业务载荷。 */
+ /**
+ * 现有 TCP/UDP 入口的事件时间等于接收时间；保留旧签名以兼容现有调用方。
+ */
public static String getFullLogString(DeviceDevice deviceDevice, String strSyslog,
String receive_time, String eventId, Long pointTimeNs,
AppProperties appProps)
+ {
+ return getFullLogString(deviceDevice, strSyslog, receive_time, receive_time,
+ eventId, pointTimeNs, appProps);
+ }
+
+ /**
+ * Kafka 业务头同时保留接收时间和事件时间；WAL schema_version 绝不进入业务载荷。
+ */
+ public static String getFullLogString(DeviceDevice deviceDevice, String strSyslog,
+ String receive_time, String event_time,
+ String eventId, Long pointTimeNs,
+ AppProperties appProps)
{
Map map = new LinkedHashMap<>();
map.put("receive_time",receive_time);
+ map.put("event_time",event_time);
if (eventId != null && !eventId.isEmpty()) {
map.put("event_id", eventId);
}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AdmissionStatus.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AdmissionStatus.java
new file mode 100644
index 0000000..d4757e7
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AdmissionStatus.java
@@ -0,0 +1,8 @@
+package com.haobang.wal;
+
+/** WAL 非阻塞接入结果。 ACCEPTED 表示事件进入内存队列但不代表已落盘；BACKPRESSURE 可重试；UNAVAILABLE 必须停止接入。 */
+public enum AdmissionStatus {
+ ACCEPTED,
+ BACKPRESSURE,
+ UNAVAILABLE
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AppendAttempt.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AppendAttempt.java
new file mode 100644
index 0000000..bcbcb70
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AppendAttempt.java
@@ -0,0 +1,39 @@
+package com.haobang.wal;
+
+import java.util.concurrent.CompletionStage;
+
+/**
+ * 一次非阻塞 WAL 接入尝试的结果。
+ *
+ * 只有 {@link AdmissionStatus#ACCEPTED} 携带 durable future；future 成功才表示该事件已经
+ * 完成文件和目录持久化屏障。被背压或不可用时不会产生一个永远无法完成的 future。
+ */
+public final class AppendAttempt {
+ private final AdmissionStatus status;
+ private final CompletionStage durableStage;
+
+ private AppendAttempt(AdmissionStatus status, CompletionStage durableStage) {
+ this.status = status;
+ this.durableStage = durableStage;
+ }
+
+ public static AppendAttempt accepted(CompletionStage stage) {
+ if (stage == null) throw new IllegalArgumentException("ACCEPTED 必须携带 durableStage");
+ return new AppendAttempt(AdmissionStatus.ACCEPTED, stage);
+ }
+
+ public static AppendAttempt rejected(AdmissionStatus status) {
+ if (status == null || status == AdmissionStatus.ACCEPTED) {
+ throw new IllegalArgumentException("rejected 只能使用 BACKPRESSURE 或 UNAVAILABLE");
+ }
+ return new AppendAttempt(status, null);
+ }
+
+ public AdmissionStatus getStatus() {
+ return status;
+ }
+
+ public CompletionStage getDurableStage() {
+ return durableStage;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/DefaultUnifiedIngressService.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/DefaultUnifiedIngressService.java
new file mode 100644
index 0000000..824dbdf
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/DefaultUnifiedIngressService.java
@@ -0,0 +1,18 @@
+package com.haobang.wal;
+
+import org.springframework.stereotype.Component;
+
+/** 统一接入端口的默认实现。 该薄适配层让 API 拉取执行器依赖稳定接口，同时保留现有 {@link SyslogWalService} 生命周期。 */
+@Component
+public class DefaultUnifiedIngressService implements UnifiedIngressService {
+ private final SyslogWalService wal;
+
+ public DefaultUnifiedIngressService(SyslogWalService wal) {
+ this.wal = wal;
+ }
+
+ @Override
+ public AppendAttempt tryAppend(IngressEvent event) {
+ return wal.tryAppendUnified(event);
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/DurableReceipt.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/DurableReceipt.java
new file mode 100644
index 0000000..6038637
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/DurableReceipt.java
@@ -0,0 +1,38 @@
+package com.haobang.wal;
+
+import java.time.Instant;
+
+/**
+ * {@code FileChannel.force(true)} 成功后才产生的事件级持久化凭据。
+ *
+ * receipt 只证明事件已可靠进入本地 WAL，不代表已经发送到 Kafka。API checkpoint 只依赖 这个边界，因此 Kafka 暂时不可用不会导致设备历史窗口被反复拉取。
+ */
+public final class DurableReceipt {
+ private final String eventId;
+ private final long walSequence;
+ private final int walSchemaVersion;
+ private final Instant durableAt;
+
+ public DurableReceipt(String eventId, long walSequence, int walSchemaVersion, Instant durableAt) {
+ this.eventId = eventId;
+ this.walSequence = walSequence;
+ this.walSchemaVersion = walSchemaVersion;
+ this.durableAt = durableAt;
+ }
+
+ public String getEventId() {
+ return eventId;
+ }
+
+ public long getWalSequence() {
+ return walSequence;
+ }
+
+ public int getWalSchemaVersion() {
+ return walSchemaVersion;
+ }
+
+ public Instant getDurableAt() {
+ return durableAt;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/IngressEvent.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/IngressEvent.java
new file mode 100644
index 0000000..4afc104
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/IngressEvent.java
@@ -0,0 +1,170 @@
+package com.haobang.wal;
+
+import java.nio.charset.Charset;
+import java.time.Instant;
+import java.util.Objects;
+
+/**
+ * TCP/UDP 推送与设备 API 拉取共用的不可变入站事件。
+ *
+ *
{@code receivedAt} 是本服务观察到事件的时间；{@code eventTime} 是日志真实发生时间。 推送模式缺少可靠设备时间时二者相同，API
+ * 模式则必须由厂商响应解析 eventTime。这样 Kafka 消费端始终使用同一字段语义，而无需根据接入方式分支处理。
+ */
+public final class IngressEvent {
+ private final String eventId;
+ private final Instant receivedAt;
+ private final Instant eventTime;
+ private final IngressType ingressType;
+ private final String transport;
+ private final Integer deviceId;
+ private final Integer collectTaskId;
+ private final String adapterType;
+ private final String externalEventId;
+ private final String sourceIp;
+ private final Integer sourcePort;
+ private final String contentType;
+ private final Charset charset;
+ private final String rawMessage;
+ private final Long pointTimeNs;
+
+ public IngressEvent(
+ String eventId,
+ Instant receivedAt,
+ Instant eventTime,
+ IngressType ingressType,
+ String transport,
+ Integer deviceId,
+ Integer collectTaskId,
+ String adapterType,
+ String externalEventId,
+ String sourceIp,
+ Integer sourcePort,
+ String contentType,
+ Charset charset,
+ String rawMessage,
+ Long pointTimeNs) {
+ this.eventId = requireText(eventId, "eventId");
+ this.receivedAt = Objects.requireNonNull(receivedAt, "receivedAt");
+ this.eventTime = Objects.requireNonNull(eventTime, "eventTime");
+ this.ingressType = Objects.requireNonNull(ingressType, "ingressType");
+ this.transport = requireText(transport, "transport");
+ this.contentType = requireText(contentType, "contentType");
+ this.charset = Objects.requireNonNull(charset, "charset");
+ this.rawMessage = Objects.requireNonNull(rawMessage, "rawMessage");
+ this.deviceId = deviceId;
+ this.collectTaskId = collectTaskId;
+ this.adapterType = adapterType;
+ this.externalEventId = externalEventId;
+ this.sourceIp = sourceIp;
+ this.sourcePort = sourcePort;
+ this.pointTimeNs = pointTimeNs;
+ validate();
+ }
+
+ private void validate() {
+ // API 事件必须带齐设备和外部主键，才能生成可追踪、可幂等的 WAL v4 记录。
+ if (ingressType == IngressType.API_PULL) {
+ if (deviceId == null
+ || collectTaskId == null
+ || isBlank(adapterType)
+ || isBlank(externalEventId)) {
+ throw new IllegalArgumentException(
+ "API_PULL 必须包含 deviceId、collectTaskId、adapterType 和 externalEventId");
+ }
+ if (!"UTF-8".equalsIgnoreCase(charset.name())) {
+ throw new IllegalArgumentException("API_PULL rawMessage 必须使用 UTF-8");
+ }
+ } else if (!eventTime.equals(receivedAt)) {
+ // 保留原 syslog 行为：没有解析日志内容中的时间时，以接收时刻作为事件时刻。
+ throw new IllegalArgumentException("SYSLOG_PUSH 的 eventTime 必须等于 receivedAt");
+ }
+ }
+
+ private static String requireText(String value, String name) {
+ if (isBlank(value)) throw new IllegalArgumentException(name + " 不能为空");
+ return value;
+ }
+
+ private static boolean isBlank(String value) {
+ return value == null || value.trim().isEmpty();
+ }
+
+ public IngressEvent withPointTimeNs(long value) {
+ return new IngressEvent(
+ eventId,
+ receivedAt,
+ eventTime,
+ ingressType,
+ transport,
+ deviceId,
+ collectTaskId,
+ adapterType,
+ externalEventId,
+ sourceIp,
+ sourcePort,
+ contentType,
+ charset,
+ rawMessage,
+ value);
+ }
+
+ public String getEventId() {
+ return eventId;
+ }
+
+ public Instant getReceivedAt() {
+ return receivedAt;
+ }
+
+ public Instant getEventTime() {
+ return eventTime;
+ }
+
+ public IngressType getIngressType() {
+ return ingressType;
+ }
+
+ public String getTransport() {
+ return transport;
+ }
+
+ public Integer getDeviceId() {
+ return deviceId;
+ }
+
+ public Integer getCollectTaskId() {
+ return collectTaskId;
+ }
+
+ public String getAdapterType() {
+ return adapterType;
+ }
+
+ public String getExternalEventId() {
+ return externalEventId;
+ }
+
+ public String getSourceIp() {
+ return sourceIp;
+ }
+
+ public Integer getSourcePort() {
+ return sourcePort;
+ }
+
+ public String getContentType() {
+ return contentType;
+ }
+
+ public Charset getCharset() {
+ return charset;
+ }
+
+ public String getRawMessage() {
+ return rawMessage;
+ }
+
+ public Long getPointTimeNs() {
+ return pointTimeNs;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/IngressType.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/IngressType.java
new file mode 100644
index 0000000..a7310b0
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/IngressType.java
@@ -0,0 +1,7 @@
+package com.haobang.wal;
+
+/** 日志进入统一可靠接入边界的方式；该值也决定 WAL Schema 的必填字段校验。 */
+public enum IngressType {
+ SYSLOG_PUSH,
+ API_PULL
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/PendingWalAppend.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/PendingWalAppend.java
new file mode 100644
index 0000000..90df38d
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/PendingWalAppend.java
@@ -0,0 +1,39 @@
+package com.haobang.wal;
+
+import java.util.concurrent.CompletableFuture;
+
+/** WAL 写线程队列中的事件及其待完成 durable future。 sequence/schemaVersion 由唯一写线程赋值，生产者线程不得提前推断 WAL 位置。 */
+final class PendingWalAppend {
+ private final IngressEvent event;
+ private final CompletableFuture durableFuture = new CompletableFuture<>();
+ private long sequence;
+ private int schemaVersion;
+
+ PendingWalAppend(IngressEvent event) {
+ this.event = event;
+ }
+
+ IngressEvent getEvent() {
+ return event;
+ }
+
+ CompletableFuture getDurableFuture() {
+ return durableFuture;
+ }
+
+ long getSequence() {
+ return sequence;
+ }
+
+ void setSequence(long sequence) {
+ this.sequence = sequence;
+ }
+
+ int getSchemaVersion() {
+ return schemaVersion;
+ }
+
+ void setSchemaVersion(int schemaVersion) {
+ this.schemaVersion = schemaVersion;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecord.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecord.java
index c169c4d..379fdbf 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecord.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecord.java
@@ -6,80 +6,285 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
- * WAL v2/v3 磁盘记录；segment/offset 仅用于运行时确认游标，不写入 JSON 正文。
+ * WAL v2/v3/v4 磁盘记录；segment/offset 仅用于运行时确认游标，不写入 JSON 正文。
*
- * 早期 v2 记录错误复用了 Redis 多态 ObjectMapper，因而额外包含 "@class"。这里仅忽略这个
- * 已知历史字段，使修复后的专用 WAL ObjectMapper 能无损恢复既有数据；其他未知字段仍保持
- * Jackson 默认的严格失败语义，避免悄悄接受不可识别的 WAL schema。
+ * v2 是原始 TCP/UDP 记录；v3 为推送日志增加 point_time_ns；v4 为 API 拉取增加 event_time、设备身份、适配器和外部事件 ID。读取端必须继续识别
+ * v2/v3，升级不能要求现场先清空 WAL。
+ *
+ *
早期 v2 记录错误复用了 Redis 多态 ObjectMapper，因而额外包含 "@class"。这里仅忽略这个 已知历史字段，使专用 WAL ObjectMapper
+ * 能无损恢复既有数据；其他未知字段仍保持严格失败语义， 避免悄悄接受不可识别的 WAL schema。
*/
@JsonIgnoreProperties(value = "@class")
public class SyslogWalRecord {
- @JsonProperty("schema_version")
- private int schemaVersion = 2;
- private long sequence;
- @JsonProperty("event_id")
- private String eventId;
- @JsonProperty("receive_time")
- private String receiveTime;
- @JsonProperty("point_time_ns")
- @JsonInclude(JsonInclude.Include.NON_NULL)
- private Long pointTimeNs;
- @JsonProperty("source_ip")
- private String sourceIp;
- @JsonProperty("source_port")
- private int sourcePort;
- private String protocol;
- private String charset;
- @JsonProperty("raw_length")
- private int rawLength;
- @JsonProperty("raw_sha256")
- private String rawSha256;
- @JsonProperty("raw_message")
- private String rawMessage;
+ @JsonProperty("schema_version")
+ private int schemaVersion = 2;
- @JsonIgnore
- private String segment;
- @JsonIgnore
- private long startOffset;
- @JsonIgnore
- private long endOffset;
- @JsonIgnore
- private boolean quarantinedInvalid;
- @JsonIgnore
- private String validationError;
+ private long sequence;
- public int getSchemaVersion() { return schemaVersion; }
- public void setSchemaVersion(int schemaVersion) { this.schemaVersion = schemaVersion; }
- public long getSequence() { return sequence; }
- public void setSequence(long sequence) { this.sequence = sequence; }
- public String getEventId() { return eventId; }
- public void setEventId(String eventId) { this.eventId = eventId; }
- public String getReceiveTime() { return receiveTime; }
- public void setReceiveTime(String receiveTime) { this.receiveTime = receiveTime; }
- public Long getPointTimeNs() { return pointTimeNs; }
- public void setPointTimeNs(Long pointTimeNs) { this.pointTimeNs = pointTimeNs; }
- public String getSourceIp() { return sourceIp; }
- public void setSourceIp(String sourceIp) { this.sourceIp = sourceIp; }
- public int getSourcePort() { return sourcePort; }
- public void setSourcePort(int sourcePort) { this.sourcePort = sourcePort; }
- public String getProtocol() { return protocol; }
- public void setProtocol(String protocol) { this.protocol = protocol; }
- public String getCharset() { return charset; }
- public void setCharset(String charset) { this.charset = charset; }
- public int getRawLength() { return rawLength; }
- public void setRawLength(int rawLength) { this.rawLength = rawLength; }
- public String getRawSha256() { return rawSha256; }
- public void setRawSha256(String rawSha256) { this.rawSha256 = rawSha256; }
- public String getRawMessage() { return rawMessage; }
- public void setRawMessage(String rawMessage) { this.rawMessage = rawMessage; }
- public String getSegment() { return segment; }
- public void setSegment(String segment) { this.segment = segment; }
- public long getStartOffset() { return startOffset; }
- public void setStartOffset(long startOffset) { this.startOffset = startOffset; }
- public long getEndOffset() { return endOffset; }
- public void setEndOffset(long endOffset) { this.endOffset = endOffset; }
- public boolean isQuarantinedInvalid() { return quarantinedInvalid; }
- public void setQuarantinedInvalid(boolean quarantinedInvalid) { this.quarantinedInvalid = quarantinedInvalid; }
- public String getValidationError() { return validationError; }
- public void setValidationError(String validationError) { this.validationError = validationError; }
+ @JsonProperty("event_id")
+ private String eventId;
+
+ @JsonProperty("receive_time")
+ private String receiveTime;
+
+ @JsonProperty("event_time")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private String eventTime;
+
+ @JsonProperty("point_time_ns")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private Long pointTimeNs;
+
+ @JsonProperty("source_ip")
+ private String sourceIp;
+
+ @JsonProperty("source_port")
+ private int sourcePort;
+
+ private String protocol;
+
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private String transport;
+
+ @JsonProperty("ingress_type")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private String ingressType;
+
+ @JsonProperty("device_id")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private Integer deviceId;
+
+ @JsonProperty("collect_task_id")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private Integer collectTaskId;
+
+ @JsonProperty("adapter_type")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private String adapterType;
+
+ @JsonProperty("external_event_id")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private String externalEventId;
+
+ @JsonProperty("content_type")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private String contentType;
+
+ private String charset;
+
+ @JsonProperty("raw_length")
+ private int rawLength;
+
+ @JsonProperty("raw_sha256")
+ private String rawSha256;
+
+ @JsonProperty("raw_message")
+ private String rawMessage;
+
+ // 以下字段来自扫描器的物理位置，只参与运行时 checkpoint，不属于磁盘 JSON 协议。
+ @JsonIgnore private String segment;
+ @JsonIgnore private long startOffset;
+ @JsonIgnore private long endOffset;
+ @JsonIgnore private boolean quarantinedInvalid;
+ @JsonIgnore private String validationError;
+
+ public int getSchemaVersion() {
+ return schemaVersion;
+ }
+
+ public void setSchemaVersion(int schemaVersion) {
+ this.schemaVersion = schemaVersion;
+ }
+
+ public long getSequence() {
+ return sequence;
+ }
+
+ public void setSequence(long sequence) {
+ this.sequence = sequence;
+ }
+
+ public String getEventId() {
+ return eventId;
+ }
+
+ public void setEventId(String eventId) {
+ this.eventId = eventId;
+ }
+
+ public String getReceiveTime() {
+ return receiveTime;
+ }
+
+ public void setReceiveTime(String receiveTime) {
+ this.receiveTime = receiveTime;
+ }
+
+ public String getEventTime() {
+ return eventTime;
+ }
+
+ public void setEventTime(String eventTime) {
+ this.eventTime = eventTime;
+ }
+
+ public Long getPointTimeNs() {
+ return pointTimeNs;
+ }
+
+ public void setPointTimeNs(Long pointTimeNs) {
+ this.pointTimeNs = pointTimeNs;
+ }
+
+ public String getSourceIp() {
+ return sourceIp;
+ }
+
+ public void setSourceIp(String sourceIp) {
+ this.sourceIp = sourceIp;
+ }
+
+ public int getSourcePort() {
+ return sourcePort;
+ }
+
+ public void setSourcePort(int sourcePort) {
+ this.sourcePort = sourcePort;
+ }
+
+ public String getProtocol() {
+ return protocol;
+ }
+
+ public void setProtocol(String protocol) {
+ this.protocol = protocol;
+ }
+
+ public String getTransport() {
+ return transport;
+ }
+
+ public void setTransport(String transport) {
+ this.transport = transport;
+ }
+
+ public String getIngressType() {
+ return ingressType;
+ }
+
+ public void setIngressType(String ingressType) {
+ this.ingressType = ingressType;
+ }
+
+ public Integer getDeviceId() {
+ return deviceId;
+ }
+
+ public void setDeviceId(Integer deviceId) {
+ this.deviceId = deviceId;
+ }
+
+ public Integer getCollectTaskId() {
+ return collectTaskId;
+ }
+
+ public void setCollectTaskId(Integer collectTaskId) {
+ this.collectTaskId = collectTaskId;
+ }
+
+ public String getAdapterType() {
+ return adapterType;
+ }
+
+ public void setAdapterType(String adapterType) {
+ this.adapterType = adapterType;
+ }
+
+ public String getExternalEventId() {
+ return externalEventId;
+ }
+
+ public void setExternalEventId(String externalEventId) {
+ this.externalEventId = externalEventId;
+ }
+
+ public String getContentType() {
+ return contentType;
+ }
+
+ public void setContentType(String contentType) {
+ this.contentType = contentType;
+ }
+
+ public String getCharset() {
+ return charset;
+ }
+
+ public void setCharset(String charset) {
+ this.charset = charset;
+ }
+
+ public int getRawLength() {
+ return rawLength;
+ }
+
+ public void setRawLength(int rawLength) {
+ this.rawLength = rawLength;
+ }
+
+ public String getRawSha256() {
+ return rawSha256;
+ }
+
+ public void setRawSha256(String rawSha256) {
+ this.rawSha256 = rawSha256;
+ }
+
+ public String getRawMessage() {
+ return rawMessage;
+ }
+
+ public void setRawMessage(String rawMessage) {
+ this.rawMessage = rawMessage;
+ }
+
+ public String getSegment() {
+ return segment;
+ }
+
+ public void setSegment(String segment) {
+ this.segment = segment;
+ }
+
+ public long getStartOffset() {
+ return startOffset;
+ }
+
+ public void setStartOffset(long startOffset) {
+ this.startOffset = startOffset;
+ }
+
+ public long getEndOffset() {
+ return endOffset;
+ }
+
+ public void setEndOffset(long endOffset) {
+ this.endOffset = endOffset;
+ }
+
+ public boolean isQuarantinedInvalid() {
+ return quarantinedInvalid;
+ }
+
+ public void setQuarantinedInvalid(boolean quarantinedInvalid) {
+ this.quarantinedInvalid = quarantinedInvalid;
+ }
+
+ public String getValidationError() {
+ return validationError;
+ }
+
+ public void setValidationError(String validationError) {
+ this.validationError = validationError;
+ }
}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecordValidator.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecordValidator.java
index fbfeaaa..eb507f6 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecordValidator.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecordValidator.java
@@ -1,81 +1,165 @@
package com.haobang.wal;
import com.fasterxml.jackson.databind.JsonNode;
-
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.ResolverStyle;
-/** WAL 所有扫描、恢复和派发路径共用的 v2/v3 严格校验。 */
+/**
+ * WAL 所有写入、扫描、恢复和派发路径共用的 v2/v3/v4 严格校验。
+ *
+ *
校验同时作用于原始 JSON 节点和反序列化对象：前者防止 Jackson 默认值掩盖字段缺失， 后者保证新写出的记录也满足同一协议。未知版本一律
+ * fail-closed，不能越过并推进游标。
+ */
final class SyslogWalRecordValidator {
- private static final ZoneId RECEIVE_ZONE = ZoneId.of("Asia/Shanghai");
- private static final DateTimeFormatter RECEIVE_FORMAT = DateTimeFormatter
- .ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS").withResolverStyle(ResolverStyle.STRICT);
+ private static final ZoneId RECEIVE_ZONE = ZoneId.of("Asia/Shanghai");
+ private static final DateTimeFormatter RECEIVE_FORMAT =
+ DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS")
+ .withResolverStyle(ResolverStyle.STRICT);
- private SyslogWalRecordValidator() { }
+ private SyslogWalRecordValidator() {}
- static void validateNode(JsonNode node) {
- JsonNode versionNode = node.get("schema_version");
- if (versionNode == null || !versionNode.isIntegralNumber() || !versionNode.canConvertToInt()) {
- throw new IllegalArgumentException("schema_version 缺失或不是整数");
- }
- int version = versionNode.intValue();
- JsonNode pointTimeNode = node.get("point_time_ns");
- if (version == 2) {
- if (pointTimeNode != null && !pointTimeNode.isNull()) {
- throw new IllegalArgumentException("WAL v2 禁止携带 point_time_ns");
- }
- return;
- }
- if (version != 3) {
- throw new IllegalArgumentException("不支持的 WAL 版本: " + version);
- }
- if (pointTimeNode == null || !pointTimeNode.isIntegralNumber() || !pointTimeNode.canConvertToLong()) {
- throw new IllegalArgumentException("WAL v3 point_time_ns 缺失、非数字或溢出");
- }
- validatePointTime(pointTimeNode.longValue(), text(node, "receive_time"));
+ static void validateNode(JsonNode node) {
+ JsonNode versionNode = node.get("schema_version");
+ if (versionNode == null || !versionNode.isIntegralNumber() || !versionNode.canConvertToInt()) {
+ throw new IllegalArgumentException("schema_version 缺失或不是整数");
}
-
- static void validateRecord(SyslogWalRecord record) {
- if (record.getSchemaVersion() == 2) {
- if (record.getPointTimeNs() != null) {
- throw new IllegalArgumentException("WAL v2 禁止携带 point_time_ns");
- }
- return;
- }
- if (record.getSchemaVersion() != 3) {
- throw new IllegalArgumentException("不支持的 WAL 版本: " + record.getSchemaVersion());
- }
- if (record.getPointTimeNs() == null) {
- throw new IllegalArgumentException("WAL v3 缺少 point_time_ns");
- }
- validatePointTime(record.getPointTimeNs(), record.getReceiveTime());
+ int version = versionNode.intValue();
+ JsonNode pointTimeNode = node.get("point_time_ns");
+ if (version == 2) {
+ if (pointTimeNode != null && !pointTimeNode.isNull()) {
+ throw new IllegalArgumentException("WAL v2 禁止携带 point_time_ns");
+ }
+ return;
}
-
- static long receiveTimeMillis(String receiveTime) {
- if (receiveTime == null || receiveTime.isEmpty()) {
- throw new IllegalArgumentException("receive_time 不能为空");
- }
- return LocalDateTime.parse(receiveTime, RECEIVE_FORMAT)
- .atZone(RECEIVE_ZONE).toInstant().toEpochMilli();
+ if (version == 4) {
+ // v4 必须自描述接入方式和事件时间，不能从 source_ip 或接收时间反向猜测。
+ requireText(node, "event_id");
+ requireText(node, "receive_time");
+ requireText(node, "event_time");
+ requireText(node, "ingress_type");
+ requireText(node, "transport");
+ requireText(node, "content_type");
+ requireText(node, "charset");
+ requireText(node, "raw_sha256");
+ requireText(node, "raw_message");
+ requirePositiveInt(node, "raw_length");
+ if (pointTimeNode == null
+ || !pointTimeNode.isIntegralNumber()
+ || !pointTimeNode.canConvertToLong()) {
+ throw new IllegalArgumentException("WAL v4 point_time_ns 缺失、非数字或溢出");
+ }
+ String ingressType = text(node, "ingress_type");
+ if (IngressType.API_PULL.name().equals(ingressType)) {
+ // API 的 HTTP 对端可能是网关，设备身份必须来自任务中固化的 device_id。
+ requirePositiveInt(node, "device_id");
+ requirePositiveInt(node, "collect_task_id");
+ requireText(node, "adapter_type");
+ requireText(node, "external_event_id");
+ } else if (!IngressType.SYSLOG_PUSH.name().equals(ingressType)) {
+ throw new IllegalArgumentException("WAL v4 ingress_type 非法: " + ingressType);
+ }
+ validatePointTime(pointTimeNode.longValue(), text(node, "event_time"), "event_time");
+ return;
}
-
- private static void validatePointTime(long pointTimeNs, String receiveTime) {
- if (pointTimeNs <= 0L) {
- throw new IllegalArgumentException("point_time_ns 必须为正整数");
- }
- long receiveTimeMillis = receiveTimeMillis(receiveTime);
- if (Math.floorDiv(pointTimeNs, PointTimeAllocator.NANOS_PER_MILLI) != receiveTimeMillis) {
- throw new IllegalArgumentException("point_time_ns 毫秒部分与 receive_time 不一致");
- }
+ if (version != 3) {
+ throw new IllegalArgumentException("不支持的 WAL 版本: " + version);
}
-
- private static String text(JsonNode node, String name) {
- JsonNode value = node.get(name);
- if (value == null || !value.isTextual()) {
- throw new IllegalArgumentException(name + " 缺失或不是字符串");
- }
- return value.textValue();
+ if (pointTimeNode == null
+ || !pointTimeNode.isIntegralNumber()
+ || !pointTimeNode.canConvertToLong()) {
+ throw new IllegalArgumentException("WAL v3 point_time_ns 缺失、非数字或溢出");
}
+ validatePointTime(pointTimeNode.longValue(), text(node, "receive_time"), "receive_time");
+ }
+
+ static void validateRecord(SyslogWalRecord record) {
+ if (record.getSchemaVersion() == 2) {
+ if (record.getPointTimeNs() != null) {
+ throw new IllegalArgumentException("WAL v2 禁止携带 point_time_ns");
+ }
+ return;
+ }
+ if (record.getSchemaVersion() == 4) {
+ require(record.getEventId(), "event_id");
+ require(record.getReceiveTime(), "receive_time");
+ require(record.getEventTime(), "event_time");
+ require(record.getIngressType(), "ingress_type");
+ require(record.getTransport(), "transport");
+ require(record.getContentType(), "content_type");
+ require(record.getCharset(), "charset");
+ require(record.getRawSha256(), "raw_sha256");
+ if (record.getPointTimeNs() == null)
+ throw new IllegalArgumentException("WAL v4 缺少 point_time_ns");
+ if (IngressType.API_PULL.name().equals(record.getIngressType())) {
+ if (record.getDeviceId() == null
+ || record.getDeviceId() <= 0
+ || record.getCollectTaskId() == null
+ || record.getCollectTaskId() <= 0) {
+ throw new IllegalArgumentException("WAL v4 API_PULL 缺少合法 device_id/collect_task_id");
+ }
+ require(record.getAdapterType(), "adapter_type");
+ require(record.getExternalEventId(), "external_event_id");
+ } else if (!IngressType.SYSLOG_PUSH.name().equals(record.getIngressType())) {
+ throw new IllegalArgumentException("WAL v4 ingress_type 非法: " + record.getIngressType());
+ }
+ validatePointTime(record.getPointTimeNs(), record.getEventTime(), "event_time");
+ return;
+ }
+ if (record.getSchemaVersion() != 3) {
+ throw new IllegalArgumentException("不支持的 WAL 版本: " + record.getSchemaVersion());
+ }
+ if (record.getPointTimeNs() == null) {
+ throw new IllegalArgumentException("WAL v3 缺少 point_time_ns");
+ }
+ validatePointTime(record.getPointTimeNs(), record.getReceiveTime(), "receive_time");
+ }
+
+ static long receiveTimeMillis(String receiveTime) {
+ if (receiveTime == null || receiveTime.isEmpty()) {
+ throw new IllegalArgumentException("receive_time 不能为空");
+ }
+ return LocalDateTime.parse(receiveTime, RECEIVE_FORMAT)
+ .atZone(RECEIVE_ZONE)
+ .toInstant()
+ .toEpochMilli();
+ }
+
+ private static void validatePointTime(long pointTimeNs, String time, String field) {
+ if (pointTimeNs <= 0L) {
+ throw new IllegalArgumentException("point_time_ns 必须为正整数");
+ }
+ // point_time_ns 的毫秒部分与事件时间绑定，纳秒尾数仅用于同毫秒内稳定排序。
+ long timeMillis = receiveTimeMillis(time);
+ if (Math.floorDiv(pointTimeNs, PointTimeAllocator.NANOS_PER_MILLI) != timeMillis) {
+ throw new IllegalArgumentException("point_time_ns 毫秒部分与 " + field + " 不一致");
+ }
+ }
+
+ private static void requireText(JsonNode node, String name) {
+ require(text(node, name), name);
+ }
+
+ private static void require(String value, String name) {
+ if (value == null || value.trim().isEmpty()) throw new IllegalArgumentException(name + " 不能为空");
+ }
+
+ private static void requirePositiveInt(JsonNode node, String name) {
+ JsonNode value = node.get(name);
+ if (value == null
+ || !value.isIntegralNumber()
+ || !value.canConvertToInt()
+ || value.intValue() <= 0) {
+ throw new IllegalArgumentException(name + " 缺失或不是正整数");
+ }
+ }
+
+ private static String text(JsonNode node, String name) {
+ JsonNode value = node.get(name);
+ if (value == null || !value.isTextual()) {
+ throw new IllegalArgumentException(name + " 缺失或不是字符串");
+ }
+ return value.textValue();
+ }
}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalService.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalService.java
index c365311..fd8ab4e 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalService.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalService.java
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.haobang.config.AppProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
@@ -47,9 +48,13 @@ import java.util.concurrent.atomic.LongAdder;
/**
* Kafka 接管前的本地持久化 WAL。
*
- *
网络线程只向有界队列提交事件；单写线程顺序追加并批量 force。Dispatcher 只读取已经
- * force 的记录，Kafka ACK 后按连续 sequence 推进持久化游标。崩溃恢复可能重复投递游标后
- * 的少量记录，但 event_id 不变，由下游幂等约束消除业务重复。
+ * 网络/API 线程只向有界队列提交事件；单写线程顺序追加并批量 force。Dispatcher 只读取已经
+ * force 的记录，Kafka ACK 后按连续 sequence 推进持久化游标。崩溃恢复可能重复投递游标后的
+ * 少量记录，但 event_id 不变，由下游幂等约束消除业务重复。
+ *
+ *
API 拉取使用事件级 durable receipt：事件进入队列只表示 accepted，只有覆盖该事件的
+ * {@code FileChannel.force(true)} 成功后 future 才完成。force 失败时 WAL 进入 fail-closed，所有
+ * 已接收但未 durable 的 future 异常完成，调用方因此不会错误推进设备 checkpoint。
*/
@Component
public class SyslogWalService {
@@ -57,12 +62,14 @@ public class SyslogWalService {
private static final DateTimeFormatter RECORD_TIME = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS");
private static final DateTimeFormatter FILE_TIME = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmssSSS");
private static final String SEGMENT_GLOB = "syslog-wal-*.jsonl";
+ static final long COMPLETED_CLEANUP_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(30L);
public enum AdmissionResult { ACCEPTED, BACKPRESSURE, UNAVAILABLE }
private final AppProperties properties;
private final ObjectMapper objectMapper;
private final Charset syslogCharset;
+ private final ForceOperation forceOperation;
private final PointTimeAllocator pointTimeAllocator = new PointTimeAllocator();
private final AtomicBoolean initialized = new AtomicBoolean();
private final AtomicLong nextSequence = new AtomicLong();
@@ -82,8 +89,14 @@ public class SyslogWalService {
private volatile long lastCheckpointFlushNanos;
private volatile DiskState diskState = DiskState.NORMAL;
private volatile long lastDiskCheckNanos;
+ private volatile long diskTotalBytes;
+ private volatile long diskUsableBytes;
+ private volatile long diskStopThresholdBytes;
+ private volatile long lastCompletedCleanupNanos;
- private ArrayBlockingQueue appendQueue;
+ private ArrayBlockingQueue appendQueue;
+ /** 已 write、尚未被一次成功 force 覆盖的事件；仅由写线程访问。 */
+ private final List pendingForce = new ArrayList<>();
private Thread writerThread;
private Path rootDirectory;
private Path activeDirectory;
@@ -100,7 +113,21 @@ public class SyslogWalService {
private long lastWrittenSequence;
private long lastForceNanos;
+ /**
+ * Spring 生产构造器。
+ *
+ * 另一个接收 {@link ForceOperation} 的包级构造器用于测试刷盘失败。存在多个构造器时必须
+ * 显式标记生产构造器，否则 Spring 会尝试寻找无参构造器并导致应用启动失败。
+ */
+ @Autowired
public SyslogWalService(AppProperties properties, ObjectMapper objectMapper) {
+ this(properties, objectMapper, FileChannel::force);
+ }
+
+ SyslogWalService(
+ AppProperties properties,
+ ObjectMapper objectMapper,
+ ForceOperation forceOperation) {
this.properties = properties;
/*
* WAL 是内部、固定 schema 的持久化协议，不能直接复用 Redis 缓存的多态 ObjectMapper。
@@ -115,6 +142,7 @@ public class SyslogWalService {
*/
this.objectMapper = objectMapper.copy().deactivateDefaultTyping();
this.syslogCharset = resolveSyslogCharset(properties.getSyslogCharset());
+ this.forceOperation = forceOperation;
}
@PostConstruct
@@ -147,7 +175,7 @@ public class SyslogWalService {
long maxSequence = Math.max(persistedCheckpoint.sequence, scanMaxSequence());
nextSequence.set(maxSequence);
durableSequence.set(maxSequence);
- cleanupCompletedSegments();
+ cleanupCompletedSegments(true);
refreshDiskState(true);
} catch (Exception failure) {
fatalFailure = failure;
@@ -163,9 +191,15 @@ public class SyslogWalService {
writerThread.setDaemon(true);
writerThread.start();
log.info("Syslog WAL 已启用: directory={}, charset={}, queueCapacity={}, highWatermark={}%, lowWatermark={}%, "
+ + "retentionDays={}, completedCleanupIntervalMinutes={}, "
+ + "diskUsedWatermark={}%, diskMaxFreeBytes={}, diskStopThresholdBytes={}, "
+ "flushRecords={}, flushIntervalMs={}, checkpointSequence={}",
rootDirectory, syslogCharset.name(), properties.getSyslogWalQueueCapacity(),
properties.getSyslogWalHighWatermarkPercent(), properties.getSyslogWalLowWatermarkPercent(),
+ properties.getSyslogWalRetentionDays(),
+ TimeUnit.NANOSECONDS.toMinutes(COMPLETED_CLEANUP_INTERVAL_NANOS),
+ properties.getSyslogWalDiskHighWatermarkPercent(), properties.getSyslogWalMinFreeBytes(),
+ diskStopThresholdBytes,
properties.getSyslogWalFlushRecords(), properties.getSyslogWalFlushIntervalMs(),
persistedCheckpoint.sequence);
}
@@ -174,15 +208,35 @@ public class SyslogWalService {
if (properties.isSyslogPointTimeWriteEnabled() && event.getPointTimeNs() == null) {
throw new IllegalArgumentException("point_time 写入已开启，SyslogIngressEvent 必须先固化 point_time_ns");
}
+ AppendAttempt attempt = tryAppendUnified(toUnified(event));
+ if (attempt.getStatus() == AdmissionStatus.ACCEPTED) return AdmissionResult.ACCEPTED;
+ if (attempt.getStatus() == AdmissionStatus.BACKPRESSURE) return AdmissionResult.BACKPRESSURE;
+ return AdmissionResult.UNAVAILABLE;
+ }
+
+ /**
+ * 非阻塞接入统一事件。
+ *
+ * 返回 ACCEPTED 只代表进入有界队列；调用方必须等待 durableStage 成功后才能提交自己的
+ * 上游进度。BACKPRESSURE 可以使用同一个事件对象重试，UNAVAILABLE 表示 WAL 已 fail-closed。
+ */
+ public AppendAttempt tryAppendUnified(IngressEvent event) {
+ if (event == null) throw new IllegalArgumentException("IngressEvent 不能为空");
+ if (event.getPointTimeNs() == null
+ && (event.getIngressType() == IngressType.API_PULL
+ || properties.isSyslogPointTimeWriteEnabled())) {
+ throw new IllegalArgumentException("IngressEvent 必须先通过 prepareForWal 固化 point_time_ns");
+ }
refreshDiskState(false);
if (!running || fatalFailure != null || diskState == DiskState.CRITICAL) {
- return AdmissionResult.UNAVAILABLE;
+ return AppendAttempt.rejected(AdmissionStatus.UNAVAILABLE);
}
- if (!appendQueue.offer(event)) {
+ PendingWalAppend pending = new PendingWalAppend(event);
+ if (!appendQueue.offer(pending)) {
admissionBackpressure.increment();
- return AdmissionResult.BACKPRESSURE;
+ return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE);
}
- return AdmissionResult.ACCEPTED;
+ return AppendAttempt.accepted(pending.getDurableFuture());
}
public boolean shouldPauseTcp() {
@@ -219,6 +273,10 @@ public class SyslogWalService {
}
public Throwable getFatalFailure() { return fatalFailure; }
public boolean isDiskCritical() { refreshDiskState(false); return diskState == DiskState.CRITICAL; }
+ public String getDiskState() { refreshDiskState(false); return diskState.name(); }
+ public long getDiskTotalBytes() { refreshDiskState(false); return diskTotalBytes; }
+ public long getDiskUsableBytes() { refreshDiskState(false); return diskUsableBytes; }
+ public long getDiskStopThresholdBytes() { refreshDiskState(false); return diskStopThresholdBytes; }
/** 每个事件只调用一次；TCP 背压重试必须复用返回对象，不能重新分配 Point 身份。 */
public SyslogIngressEvent prepareForWal(SyslogIngressEvent event) {
@@ -230,6 +288,35 @@ public class SyslogWalService {
return event.withPointTimeNs(pointTimeAllocator.allocate(receiveTimeMillis));
}
+ /**
+ * 为事件分配稳定 point_time_ns。
+ * 每个事件只调用一次，API 背压重试必须复用返回对象，避免同一事件身份在重试时变化。
+ */
+ public IngressEvent prepareIngressEvent(IngressEvent event) {
+ if (event.getPointTimeNs() != null) return event;
+ return event.withPointTimeNs(pointTimeAllocator.allocate(event.getEventTime().toEpochMilli()));
+ }
+
+ private IngressEvent toUnified(SyslogIngressEvent event) {
+ Instant received = event.getReceivedAt().atZone(ZoneId.of("Asia/Shanghai")).toInstant();
+ return new IngressEvent(
+ event.getEventId(),
+ received,
+ received,
+ IngressType.SYSLOG_PUSH,
+ event.getProtocol(),
+ null,
+ null,
+ null,
+ null,
+ event.getSourceIp(),
+ event.getSourcePort(),
+ "TEXT",
+ syslogCharset,
+ event.getRawMessage(),
+ event.getPointTimeNs());
+ }
+
public WalCursor openCursor() throws IOException {
return new WalCursor(persistedCheckpoint.copy());
}
@@ -303,8 +390,9 @@ public class SyslogWalService {
private void writerLoop() {
try {
while (running || !appendQueue.isEmpty()) {
- SyslogIngressEvent event = appendQueue.poll(properties.getSyslogWalFlushIntervalMs(), TimeUnit.MILLISECONDS);
- if (event != null) appendEvent(event);
+ PendingWalAppend pending = appendQueue.poll(
+ properties.getSyslogWalFlushIntervalMs(), TimeUnit.MILLISECONDS);
+ if (pending != null) appendEvent(pending);
long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastForceNanos);
if (recordsSinceForce > 0 && (recordsSinceForce >= properties.getSyslogWalFlushRecords()
|| elapsedMs >= properties.getSyslogWalFlushIntervalMs())) {
@@ -313,7 +401,9 @@ public class SyslogWalService {
flushCheckpointIfDue();
}
forceWriter();
- synchronized (checkpointMonitor) { flushCheckpointLocked(); }
+ synchronized (checkpointMonitor) {
+ flushCheckpointLocked();
+ }
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
failWal(interrupted);
@@ -324,18 +414,26 @@ public class SyslogWalService {
}
}
- private void appendEvent(SyslogIngressEvent event) throws Exception {
+ private void appendEvent(PendingWalAppend pending) throws Exception {
+ IngressEvent event = pending.getEvent();
long sequence = nextSequence.incrementAndGet();
+ pending.setSequence(sequence);
+ // 当前元素已经离开队列；先登记，确保序列化/write 异常也会异常完成 receipt。
+ pendingForce.add(pending);
SyslogWalRecord record = toRecord(sequence, event);
+ pending.setSchemaVersion(record.getSchemaVersion());
byte[] json = objectMapper.writeValueAsBytes(record);
byte[] line = new byte[json.length + 1];
System.arraycopy(json, 0, line, 0, json.length);
line[line.length - 1] = (byte) '\n';
- LocalDate receiveDate = event.getReceivedAt().toLocalDate();
+ LocalDate receiveDate = event.getReceivedAt()
+ .atZone(ZoneId.of("Asia/Shanghai"))
+ .toLocalDate();
if (writerChannel == null) openWriter(sequence, receiveDate);
- if (writerBytes > 0 && (writerBytes + line.length > properties.getSyslogWalSegmentMaxBytes()
- || !receiveDate.equals(writerDate))) {
+ if (writerBytes > 0
+ && (writerBytes + line.length > properties.getSyslogWalSegmentMaxBytes()
+ || !receiveDate.equals(writerDate))) {
forceWriter();
closeWriter();
openWriter(sequence, receiveDate);
@@ -346,10 +444,21 @@ public class SyslogWalService {
lastWrittenSequence = sequence;
}
- private SyslogWalRecord toRecord(long sequence, SyslogIngressEvent event) throws Exception {
- byte[] raw = event.getRawMessage().getBytes(syslogCharset);
+ private SyslogWalRecord toRecord(long sequence, IngressEvent event) throws Exception {
+ byte[] raw = event.getRawMessage().getBytes(event.getCharset());
SyslogWalRecord record = new SyslogWalRecord();
- if (properties.isSyslogPointTimeWriteEnabled()) {
+ if (event.getIngressType() == IngressType.API_PULL) {
+ record.setSchemaVersion(4);
+ record.setEventTime(formatWalTime(event.getEventTime()));
+ record.setPointTimeNs(event.getPointTimeNs());
+ record.setIngressType(event.getIngressType().name());
+ record.setTransport(event.getTransport());
+ record.setDeviceId(event.getDeviceId());
+ record.setCollectTaskId(event.getCollectTaskId());
+ record.setAdapterType(event.getAdapterType());
+ record.setExternalEventId(event.getExternalEventId());
+ record.setContentType(event.getContentType());
+ } else if (properties.isSyslogPointTimeWriteEnabled()) {
record.setSchemaVersion(3);
record.setPointTimeNs(event.getPointTimeNs());
} else {
@@ -357,11 +466,11 @@ public class SyslogWalService {
}
record.setSequence(sequence);
record.setEventId(event.getEventId());
- record.setReceiveTime(event.getReceivedAt().format(RECORD_TIME));
+ record.setReceiveTime(formatWalTime(event.getReceivedAt()));
record.setSourceIp(event.getSourceIp());
- record.setSourcePort(event.getSourcePort());
- record.setProtocol(event.getProtocol());
- record.setCharset(syslogCharset.name());
+ record.setSourcePort(event.getSourcePort() == null ? 0 : event.getSourcePort());
+ record.setProtocol(event.getTransport());
+ record.setCharset(event.getCharset().name());
record.setRawLength(raw.length);
record.setRawSha256(sha256(raw));
record.setRawMessage(event.getRawMessage());
@@ -369,22 +478,42 @@ public class SyslogWalService {
return record;
}
+ private String formatWalTime(Instant value) {
+ return LocalDateTime.ofInstant(value, ZoneId.of("Asia/Shanghai")).format(RECORD_TIME);
+ }
+
private void forceWriter() throws IOException {
if (writerChannel == null || recordsSinceForce == 0) return;
- writerChannel.force(false);
+ // receipt 代表可恢复边界，文件内容与长度元数据都必须先持久化。
+ forceOperation.force(writerChannel, true);
+ Instant durableAt = Instant.now();
durableSequence.set(lastWrittenSequence);
appended.add(recordsSinceForce);
+ for (PendingWalAppend pending : pendingForce) {
+ DurableReceipt receipt = new DurableReceipt(
+ pending.getEvent().getEventId(),
+ pending.getSequence(),
+ pending.getSchemaVersion(),
+ durableAt);
+ pending.getDurableFuture().complete(receipt);
+ }
+ pendingForce.clear();
recordsSinceForce = 0;
lastForceNanos = System.nanoTime();
- synchronized (durableMonitor) { durableMonitor.notifyAll(); }
+ synchronized (durableMonitor) {
+ durableMonitor.notifyAll();
+ }
}
private void openWriter(long startSequence, LocalDate date) throws IOException {
String name = String.format("syslog-wal-%020d-%s.jsonl", startSequence,
LocalDateTime.now().format(FILE_TIME));
writerPath = activeDirectory.resolve(name);
- writerChannel = FileChannel.open(writerPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
+ writerChannel = FileChannel.open(
+ writerPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
applyFilePermissions(writerPath);
+ // 新目录项也需要持久化；否则断电后文件数据虽 force，分段文件名仍可能丢失。
+ forceDirectoryBestEffort(activeDirectory);
writerDate = date;
writerBytes = 0L;
}
@@ -399,10 +528,26 @@ public class SyslogWalService {
private void failWal(Throwable failure) {
fatalFailure = failure;
running = false;
- synchronized (durableMonitor) { durableMonitor.notifyAll(); }
+ // 任何尚未 durable 的调用方都必须收到失败，不能留下永久等待或假成功的 receipt。
+ completePendingExceptionally(failure);
+ synchronized (durableMonitor) {
+ durableMonitor.notifyAll();
+ }
log.error("Syslog WAL 发生致命故障，接入已进入 fail-closed 状态", failure);
}
+ private void completePendingExceptionally(Throwable failure) {
+ for (PendingWalAppend pending : pendingForce) {
+ pending.getDurableFuture().completeExceptionally(failure);
+ }
+ pendingForce.clear();
+ if (appendQueue == null) return;
+ PendingWalAppend pending;
+ while ((pending = appendQueue.poll()) != null) {
+ pending.getDurableFuture().completeExceptionally(failure);
+ }
+ }
+
private int percentageCount(int percent) {
return Math.max(1, (int) Math.floor(properties.getSyslogWalQueueCapacity() * percent / 100.0));
}
@@ -428,10 +573,8 @@ public class SyslogWalService {
throw new IllegalArgumentException("WAL 水位必须满足 0 <= low < high < 100");
}
if (properties.getSyslogWalDiskHighWatermarkPercent() <= 0
- || properties.getSyslogWalDiskCriticalWatermarkPercent()
- <= properties.getSyslogWalDiskHighWatermarkPercent()
- || properties.getSyslogWalDiskCriticalWatermarkPercent() >= 100) {
- throw new IllegalArgumentException("WAL 磁盘水位必须满足 0 < high < critical < 100");
+ || properties.getSyslogWalDiskHighWatermarkPercent() >= 100) {
+ throw new IllegalArgumentException("WAL 磁盘使用率保护线必须满足 0 < high < 100");
}
if (properties.getSyslogWalRetentionDays() < 0 || properties.getSyslogWalMinFreeBytes() < 0) {
throw new IllegalArgumentException("WAL 保留天数和最小剩余字节不能小于 0");
@@ -557,16 +700,32 @@ public class SyslogWalService {
if (name.compareTo(checkpointSegment) >= 0 || segment.equals(writerPath)) continue;
Files.move(segment, completedDirectory.resolve(name), StandardCopyOption.REPLACE_EXISTING);
}
- cleanupCompletedSegments();
+ cleanupCompletedSegments(false);
}
- private void cleanupCompletedSegments() throws IOException {
+ private void cleanupCompletedSegments(boolean force) throws IOException {
int retentionDays = properties.getSyslogWalRetentionDays();
if (retentionDays <= 0) return;
+ long nowNanos = System.nanoTime();
+ if (!force && !isCompletedCleanupDue(lastCompletedCleanupNanos, nowNanos)) return;
+ lastCompletedCleanupNanos = nowNanos;
+
Instant cutoff = LocalDate.now().minusDays(retentionDays).atStartOfDay(ZoneId.systemDefault()).toInstant();
+ int deletedSegments = 0;
for (Path path : listSegments(completedDirectory)) {
- if (Files.getLastModifiedTime(path).toInstant().isBefore(cutoff)) Files.deleteIfExists(path);
+ if (Files.getLastModifiedTime(path).toInstant().isBefore(cutoff)
+ && Files.deleteIfExists(path)) {
+ deletedSegments++;
+ }
}
+ if (deletedSegments > 0) {
+ log.info("WAL 已完成分段清理完成: retentionDays={}, cutoff={}, deletedSegments={}",
+ retentionDays, cutoff, deletedSegments);
+ }
+ }
+
+ static boolean isCompletedCleanupDue(long lastCleanupNanos, long nowNanos) {
+ return nowNanos - lastCleanupNanos >= COMPLETED_CLEANUP_INTERVAL_NANOS;
}
private List listSegments(Path directory) throws IOException {
@@ -589,13 +748,20 @@ public class SyslogWalService {
long total = store.getTotalSpace();
long usable = store.getUsableSpace();
long usedPercent = total <= 0 ? 0 : ((total - usable) * 100L / total);
- DiskState next = usedPercent >= properties.getSyslogWalDiskCriticalWatermarkPercent()
- || usable < properties.getSyslogWalMinFreeBytes() ? DiskState.CRITICAL
- : usedPercent >= properties.getSyslogWalDiskHighWatermarkPercent()
- ? DiskState.HIGH : DiskState.NORMAL;
+ long stopThresholdBytes = calculateDiskStopThresholdBytes(total,
+ properties.getSyslogWalDiskHighWatermarkPercent(),
+ properties.getSyslogWalMinFreeBytes());
+ diskTotalBytes = total;
+ diskUsableBytes = usable;
+ diskStopThresholdBytes = stopThresholdBytes;
+ DiskState next = isDiskStopRequired(usable, stopThresholdBytes)
+ ? DiskState.CRITICAL : DiskState.NORMAL;
if (next != diskState) {
- log.warn("WAL 磁盘状态变化: previous={}, current={}, usedPercent={}%, usableBytes={}",
- diskState, next, usedPercent, usable);
+ log.warn("WAL 磁盘状态变化: previous={}, current={}, usedPercent={}%, usableBytes={}, "
+ + "stopThresholdBytes={}, configuredUsedPercent={}%, configuredMaxFreeBytes={}",
+ diskState, next, usedPercent, usable, stopThresholdBytes,
+ properties.getSyslogWalDiskHighWatermarkPercent(),
+ properties.getSyslogWalMinFreeBytes());
}
diskState = next;
} catch (IOException failure) {
@@ -605,6 +771,26 @@ public class SyslogWalService {
}
}
+ /**
+ * 将使用率保护线换算为剩余字节数，再与固定容量上限取较小值。
+ * 例如 2TiB 磁盘在 90% 时约剩 205GiB，最终按 10GiB 触发；
+ * 50GiB 磁盘在 90% 时剩 5GiB，最终按 5GiB 触发。
+ */
+ static long calculateDiskStopThresholdBytes(long totalBytes, int usedWatermarkPercent,
+ long maxFreeBytes) {
+ if (totalBytes <= 0 || maxFreeBytes <= 0) return 0L;
+ int remainingPercent = 100 - usedWatermarkPercent;
+ long quotient = totalBytes / 100L;
+ long remainder = totalBytes % 100L;
+ long percentageThreshold = quotient * remainingPercent
+ + (remainder * remainingPercent + 99L) / 100L;
+ return Math.min(percentageThreshold, maxFreeBytes);
+ }
+
+ static boolean isDiskStopRequired(long usableBytes, long stopThresholdBytes) {
+ return stopThresholdBytes > 0L && usableBytes <= stopThresholdBytes;
+ }
+
private String sha256(byte[] value) throws Exception {
byte[] digest = MessageDigest.getInstance("SHA-256").digest(value);
StringBuilder hex = new StringBuilder(digest.length * 2);
@@ -658,6 +844,7 @@ public class SyslogWalService {
if (writerThread.isAlive()) {
log.error("WAL 写线程未在超时内退出: remaining={}", appendQueue.size());
writerThread.interrupt();
+ completePendingExceptionally(new IOException("WAL 优雅关闭超时"));
}
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
@@ -830,5 +1017,11 @@ public class SyslogWalService {
private Checkpoint copy() { return new Checkpoint(sequence, segment, offset); }
}
- private enum DiskState { NORMAL, HIGH, CRITICAL }
+ private enum DiskState { NORMAL, CRITICAL }
+
+ /** 测试注入点，用于验证 force 失败时 receipt 和 fail-closed 语义。 */
+ @FunctionalInterface
+ interface ForceOperation {
+ void force(FileChannel channel, boolean metadata) throws IOException;
+ }
}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalStatusController.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalStatusController.java
index 7ccfb6b..c30c884 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalStatusController.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalStatusController.java
@@ -42,6 +42,10 @@ public class SyslogWalStatusController {
pointTimeAssigned.put("ingress", wal.getIngressPointTimeAssignedCount());
value.put("point_time_assigned_total", pointTimeAssigned);
value.put("wal_disk_critical", wal.isDiskCritical());
+ value.put("wal_disk_state", wal.getDiskState());
+ value.put("wal_disk_total_bytes", wal.getDiskTotalBytes());
+ value.put("wal_disk_usable_bytes", wal.getDiskUsableBytes());
+ value.put("wal_disk_stop_threshold_bytes", wal.getDiskStopThresholdBytes());
value.put("wal_fatal_error", wal.getFatalFailure() == null
? null : wal.getFatalFailure().toString());
value.put("dispatch_inflight", dispatcher.getInflight());
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/UnifiedIngressService.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/UnifiedIngressService.java
new file mode 100644
index 0000000..6d799a7
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/UnifiedIngressService.java
@@ -0,0 +1,7 @@
+package com.haobang.wal;
+
+/** 所有日志接入方式共享的可靠 WAL 边界。 方法必须非阻塞返回接入状态；持久化完成通过 {@link AppendAttempt} 中的 future 异步通知。 */
+public interface UnifiedIngressService {
+ /** 尝试接入单个事件，不在调用线程执行磁盘 force。 */
+ AppendAttempt tryAppend(IngressEvent event);
+}
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java b/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java
index 682cf66..f0c267a 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java
@@ -108,9 +108,22 @@ public class kafkaProducer {
String receiveTime,
String eventId,
Long pointTimeNs) {
+ return messagePush(deviceDevice, syslog, receiveTime, receiveTime, eventId, pointTimeNs);
+ }
+
+ /**
+ * 显式区分接收时间和事件时间。TCP/UDP 调用旧签名时两者相同，
+ * 后续 API 拉取可以在不改 Kafka 头结构的前提下传入设备事件时间。
+ */
+ public CompletableFuture messagePush(DeviceDevice deviceDevice,
+ String syslog,
+ String receiveTime,
+ String eventTime,
+ String eventId,
+ Long pointTimeNs) {
// [流程节点3.1] event_id 在接入边界生成并写入加密载荷，Retry/Replay 不再重新生成。
String fullMessage = DeviceInfoUtil.getFullLogString(
- deviceDevice, syslog, receiveTime, eventId, pointTimeNs, appProperties);
+ deviceDevice, syslog, receiveTime, eventTime, eventId, pointTimeNs, appProperties);
// [流程节点3.2] Kafka 中保存携带 event_id 的完整密文，原始正文不写普通应用日志。
String encrypted = Sm4Util.encryptCbc(fullMessage, appProperties.getSm4Key());
ProducerRecord record = buildRecord(deviceDevice.getId(), eventId, encrypted);
diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java b/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java
index f7ea833..3ae1317 100644
--- a/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java
+++ b/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java
@@ -123,10 +123,10 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler {
ctx.channel().config().setAutoRead(false);
autoReadPaused = true;
TCP_PAUSE_COUNT.increment();
- log.warn("TCP Syslog 已按 WAL 高水位暂停读取: channel={}, pendingFrames={}, walQueue={}/{}, "
- + "pauseTotal={}",
+ log.warn("TCP Syslog 已按 WAL 接入保护暂停读取: channel={}, pendingFrames={}, walQueue={}/{}, "
+ + "diskState={}, pauseTotal={}",
ctx.channel().remoteAddress(), pendingTcp.size(), wal.getQueueSize(), wal.getQueueCapacity(),
- TCP_PAUSE_COUNT.sum());
+ wal.getDiskState(), TCP_PAUSE_COUNT.sum());
}
scheduleDrain(ctx, wal);
}
diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application.properties b/haobang-security-dm/syslog-serve/src/main/resources/application.properties
index bcc35d0..b20a276 100644
--- a/haobang-security-dm/syslog-serve/src/main/resources/application.properties
+++ b/haobang-security-dm/syslog-serve/src/main/resources/application.properties
@@ -7,7 +7,8 @@ server.error.include-message=always
server.error.include-binding-errors=always
# Syslog Server Configuration
-syslog.server.enabled=${SYSLOG_SERVER_ENABLED:true}
+# 当前测试只验证设备 API 拉取，关闭 TCP/UDP Netty 监听；可用环境变量重新开启。
+syslog.server.enabled=${SYSLOG_SERVER_ENABLED:false}
syslog.tcp.port=514
syslog.udp.port=514
syslog.tcp.backlog=${SYSLOG_TCP_BACKLOG:4096}
@@ -37,12 +38,13 @@ syslog.wal.disk-retry-interval-ms=${SYSLOG_WAL_DISK_RETRY_INTERVAL_MS:1000}
syslog.wal.flush-records=${SYSLOG_WAL_FLUSH_RECORDS:200}
syslog.wal.flush-interval-ms=${SYSLOG_WAL_FLUSH_INTERVAL_MS:200}
syslog.wal.segment-max-bytes=${SYSLOG_WAL_SEGMENT_MAX_BYTES:268435456}
-syslog.wal.retention-days=${SYSLOG_WAL_RETENTION_DAYS:0}
+# 已完成并确认投递的 WAL 分段默认保留 2 天；启动时立即清理，运行期间最多每 30 分钟清理一次。
+syslog.wal.retention-days=${SYSLOG_WAL_RETENTION_DAYS:2}
syslog.wal.shutdown-timeout-ms=${SYSLOG_WAL_SHUTDOWN_TIMEOUT_MS:30000}
syslog.wal.checkpoint-flush-records=${SYSLOG_WAL_CHECKPOINT_FLUSH_RECORDS:200}
syslog.wal.checkpoint-flush-interval-ms=${SYSLOG_WAL_CHECKPOINT_FLUSH_INTERVAL_MS:200}
-syslog.wal.disk-high-watermark-percent=${SYSLOG_WAL_DISK_HIGH_WATERMARK_PERCENT:80}
-syslog.wal.disk-critical-watermark-percent=${SYSLOG_WAL_DISK_CRITICAL_WATERMARK_PERCENT:90}
+# 磁盘停止接入线取“达到 90% 使用率时的剩余容量”和 10GiB 中较小者。
+syslog.wal.disk-high-watermark-percent=${SYSLOG_WAL_DISK_HIGH_WATERMARK_PERCENT:90}
syslog.wal.min-free-bytes=${SYSLOG_WAL_MIN_FREE_BYTES:10737418240}
# 兼容阶段必须保持 false；全部实例具备 WAL v2/v3 reader 后再切换为 true。
syslog.point-time.write-enabled=${SYSLOG_POINT_TIME_WRITE_ENABLED:true}
@@ -99,6 +101,24 @@ mybatis.mapper-locations=classpath:mapper/*.xml
mybatis.type-aliases-package=com.common.entity
mybatis.configuration.map-underscore-to-camel-case=true
+# 当前测试环境开启设备 API 拉取调度。
+device.api.enabled=${DEVICE_API_ENABLED:true}
+# 设备 AppKey/AppSecret 由 device_collect_api_credential 表管理，配置文件不再保存厂商凭据。
+device.api.scheduler.instance-id=${DEVICE_API_INSTANCE_ID:${HOSTNAME:local}}
+device.api.scheduler.scan-interval-ms=${DEVICE_API_SCAN_INTERVAL_MS:5000}
+device.api.scheduler.lease-seconds=${DEVICE_API_LEASE_SECONDS:120}
+# 同一任务页内严格串行；不同设备可并发。同一设备主机默认仅一个在途请求。
+device.api.scheduler.global-concurrency=${DEVICE_API_GLOBAL_CONCURRENCY:4}
+device.api.scheduler.per-host-concurrency=${DEVICE_API_PER_HOST_CONCURRENCY:1}
+# 每次主机许可的目标分页时间片；Runner 会优先缩小时间窗口，完整提交后再让出主机。
+device.api.scheduler.max-pages-per-slice=${DEVICE_API_MAX_PAGES_PER_SLICE:20}
+# HTTP + 解析 + 本页 WAL durable 总耗时达到 1 秒即打印一条结构化 WARN。
+device.api.page.slow-threshold-ms=${DEVICE_API_SLOW_PAGE_THRESHOLD_MS:1000}
+device.api.page.backpressure-timeout-ms=${DEVICE_API_BACKPRESSURE_TIMEOUT_MS:120000}
+# 逗号分隔的设备 DNS 名或 IP；必须显式配置，防止管理接口被用作 SSRF 跳板。
+# NDR 测试设备证书仅声明 CN=wlza.com；通过宿主机/容器 hosts 将该域名固定映射到 192.168.4.19。
+device.api.allowed-hosts=${DEVICE_API_ALLOWED_HOSTS:192.168.4.19,wlza.com}
+
# 开发环境缓存配置
spring.redis.host=localhost
spring.redis.port=6379
diff --git a/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceApiCredentialMapper.xml b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceApiCredentialMapper.xml
new file mode 100644
index 0000000..4b57cd8
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceApiCredentialMapper.xml
@@ -0,0 +1,39 @@
+
+
+
+
+ INSERT INTO device_collect_api_credential
+ (credential_ref, device_id, adapter_type, credential_version, app_key, app_secret,
+ config_version, created_at, updated_at)
+ VALUES
+ (#{credentialRef}, #{deviceId}, #{adapterType}, #{credentialVersion}, #{appKey}, #{appSecret},
+ 1, SYSDATE, SYSDATE)
+
+
+
+ SELECT credential_ref, device_id, adapter_type, credential_version,
+ config_version, created_at, updated_at
+ FROM device_collect_api_credential
+ WHERE credential_ref = #{credentialRef}
+
+
+
+ SELECT credential_ref, device_id, adapter_type, credential_version, app_key, app_secret,
+ config_version, created_at, updated_at
+ FROM device_collect_api_credential
+ WHERE credential_ref = #{credentialRef}
+ AND device_id = #{deviceId}
+ AND adapter_type = #{adapterType}
+
+
+
+ UPDATE device_collect_api_credential
+ SET credential_version = #{credentialVersion},
+ app_key = #{appKey},
+ app_secret = #{appSecret},
+ config_version = config_version + 1,
+ updated_at = SYSDATE
+ WHERE credential_ref = #{credentialRef}
+ AND config_version = #{configVersion}
+
+
diff --git a/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceCollectApiMapper.xml b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceCollectApiMapper.xml
new file mode 100644
index 0000000..09de44c
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceCollectApiMapper.xml
@@ -0,0 +1,130 @@
+
+
+
+
+ INSERT INTO device_collect_api_config
+ (collect_task_id,device_id,adapter_type,adapter_version,base_url,stream_key,credential_ref,enabled,
+ next_run_at,poll_interval_seconds,page_size,max_pages_per_run,scheduler_priority,connect_timeout_ms,request_timeout_ms,
+ window_size_seconds,safety_delay_seconds,lookback_seconds,max_response_bytes,initial_start_time,
+ extra_config_json,config_version,created_at,updated_at)
+ VALUES
+ (#{collectTaskId},#{deviceId},#{adapterType},#{adapterVersion},#{baseUrl},#{streamKey},#{credentialRef},0,
+ NULL,#{pollIntervalSeconds},#{pageSize},#{maxPagesPerRun},#{schedulerPriority},#{connectTimeoutMs},#{requestTimeoutMs},
+ #{windowSizeSeconds},#{safetyDelaySeconds},#{lookbackSeconds},#{maxResponseBytes},#{initialStartTime},
+ #{extraConfigJson},1,SYSDATE,SYSDATE)
+
+
+ INSERT INTO device_collect_api_checkpoint
+ (collect_task_id,checkpoint_type,checkpoint_value,committed_window_end,adapter_type,adapter_version,
+ version,lease_owner,lease_expires_at,fencing_token,created_at,updated_at)
+ VALUES (#{collectTaskId},#{checkpointType},#{checkpointValue},#{committedWindowEnd},#{adapterType},
+ #{adapterVersion},1,NULL,NULL,0,SYSDATE,SYSDATE)
+
+
+ SELECT * FROM device_collect_api_config WHERE collect_task_id=#{taskId}
+
+
+ SELECT * FROM device_collect_api_checkpoint WHERE collect_task_id=#{taskId}
+
+
+ SELECT * FROM device_collect_api_config
+ WHERE enabled=1 AND (next_run_at IS NULL OR next_run_at <= SYSDATE)
+ ORDER BY CASE WHEN next_run_at IS NULL THEN 0 ELSE 1 END,
+ next_run_at, scheduler_priority DESC, collect_task_id
+
+
+ UPDATE device_collect_api_config SET base_url=#{baseUrl},credential_ref=#{credentialRef},
+ poll_interval_seconds=#{pollIntervalSeconds},page_size=#{pageSize},max_pages_per_run=#{maxPagesPerRun},
+ scheduler_priority=#{schedulerPriority},
+ connect_timeout_ms=#{connectTimeoutMs},request_timeout_ms=#{requestTimeoutMs},
+ window_size_seconds=#{windowSizeSeconds},safety_delay_seconds=#{safetyDelaySeconds},
+ lookback_seconds=#{lookbackSeconds},max_response_bytes=#{maxResponseBytes},
+ extra_config_json=#{extraConfigJson},config_version=config_version+1,updated_at=SYSDATE
+ WHERE collect_task_id=#{collectTaskId} AND config_version=#{configVersion} AND enabled=0
+
+
+ UPDATE device_collect_api_config SET enabled=0,next_run_at=NULL,config_version=config_version+1,updated_at=SYSDATE
+ WHERE collect_task_id=#{taskId} AND config_version=#{configVersion} AND enabled=1
+
+
+ UPDATE device_collect_api_config SET enabled=1,next_run_at=SYSDATE,config_version=config_version+1,updated_at=SYSDATE
+ WHERE collect_task_id=#{taskId} AND config_version=#{configVersion} AND enabled=0
+
+
+ UPDATE device_collect_api_checkpoint SET lease_owner=#{owner},
+ lease_expires_at=SYSDATE+(CAST(#{leaseSeconds} AS DECIMAL(18,6))/86400),
+ fencing_token=fencing_token+1,updated_at=SYSDATE
+ WHERE collect_task_id=#{taskId} AND (lease_owner IS NULL OR lease_expires_at < SYSDATE)
+ AND EXISTS (SELECT 1 FROM device_collect_api_config c
+ WHERE c.collect_task_id=device_collect_api_checkpoint.collect_task_id AND c.enabled=1
+ AND c.config_version=#{configVersion} AND (c.next_run_at IS NULL OR c.next_run_at <= SYSDATE))
+
+
+ UPDATE device_collect_api_checkpoint SET
+ lease_expires_at=SYSDATE+(CAST(#{leaseSeconds} AS DECIMAL(18,6))/86400),updated_at=SYSDATE
+ WHERE collect_task_id=#{taskId} AND lease_owner=#{owner} AND fencing_token=#{fencingToken}
+ AND lease_expires_at>=SYSDATE
+
+
+ UPDATE device_collect_api_checkpoint SET lease_owner=NULL,lease_expires_at=NULL,updated_at=SYSDATE
+ WHERE collect_task_id=#{taskId} AND lease_owner=#{owner} AND fencing_token=#{fencingToken}
+
+
+ UPDATE device_collect_api_checkpoint SET checkpoint_value=#{checkpointValue},
+ committed_window_end=#{committedWindowEnd},last_event_time=#{lastEventTime},
+ last_external_event_id=#{lastExternalEventId},version=version+1,updated_at=SYSDATE
+ WHERE collect_task_id=#{taskId} AND version=#{checkpointVersion} AND lease_owner=#{owner}
+ AND fencing_token=#{fencingToken} AND lease_expires_at>=SYSDATE
+ AND EXISTS (SELECT 1 FROM device_collect_api_config c
+ WHERE c.collect_task_id=device_collect_api_checkpoint.collect_task_id
+ AND c.enabled=1 AND c.config_version=#{configVersion})
+
+
+ UPDATE device_collect_api_checkpoint SET checkpoint_value=#{checkpointValue},
+ committed_window_end=#{committedWindowEnd},last_event_time=NULL,last_external_event_id=NULL,
+ version=version+1,updated_at=SYSDATE
+ WHERE collect_task_id=#{taskId} AND version=#{checkpointVersion}
+ AND (lease_owner IS NULL OR lease_expires_at<SYSDATE)
+ AND EXISTS (SELECT 1 FROM device_collect_api_config c WHERE c.collect_task_id=#{taskId} AND c.enabled=0)
+
+
+ UPDATE device_collect_api_config SET
+ next_run_at=SYSDATE+(CAST(#{delaySeconds} AS DECIMAL(18,6))/86400),updated_at=SYSDATE
+ WHERE collect_task_id=#{taskId} AND config_version=#{configVersion}
+
+
+ INSERT INTO device_collect_api_run
+ (run_id,collect_task_id,device_id,adapter_type,adapter_version,stream_key,instance_id,config_version,
+ fencing_token,trigger_type,status,logical_window_start,logical_window_end,query_start,query_end,
+ pages_fetched,records_fetched,records_durable,records_quarantined,duplicate_candidates,
+ last_http_status,error_code,error_summary,scheduled_at,started_at,finished_at,created_at)
+ VALUES (#{runId},#{collectTaskId},#{deviceId},#{adapterType},#{adapterVersion},#{streamKey},#{instanceId},
+ #{configVersion},#{fencingToken},#{triggerType},#{status},#{logicalWindowStart},#{logicalWindowEnd},
+ #{queryStart},#{queryEnd},#{pagesFetched},#{recordsFetched},#{recordsDurable},#{recordsQuarantined},
+ #{duplicateCandidates},#{lastHttpStatus},#{errorCode},#{errorSummary},#{scheduledAt},SYSDATE,
+ #{finishedAt},SYSDATE)
+
+
+ UPDATE device_collect_api_run SET status=#{status},logical_window_start=#{logicalWindowStart},
+ logical_window_end=#{logicalWindowEnd},query_start=#{queryStart},query_end=#{queryEnd},
+ pages_fetched=#{pagesFetched},records_fetched=#{recordsFetched},records_durable=#{recordsDurable},
+ records_quarantined=#{recordsQuarantined},duplicate_candidates=#{duplicateCandidates},
+ last_http_status=#{lastHttpStatus},error_code=#{errorCode},error_summary=#{errorSummary},
+ finished_at=#{finishedAt} WHERE run_id=#{runId}
+
+
+ SELECT * FROM device_collect_api_run WHERE run_id=#{runId}
+
+
+ SELECT * FROM (SELECT * FROM device_collect_api_run WHERE collect_task_id=#{taskId}
+ ORDER BY started_at DESC) WHERE ROWNUM=1
+
+
+ SELECT * FROM (SELECT r.*,ROW_NUMBER() OVER (ORDER BY started_at DESC) rn
+ FROM device_collect_api_run r WHERE collect_task_id=#{taskId}) x
+ WHERE rn>#{offset} AND rn<=#{offset}+#{pageSize} ORDER BY rn
+
+
+ SELECT COUNT(*) FROM device_collect_api_run WHERE collect_task_id=#{taskId}
+
+
diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiCredentialServiceTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiCredentialServiceTest.java
new file mode 100644
index 0000000..1dd787b
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiCredentialServiceTest.java
@@ -0,0 +1,67 @@
+package com.haobang.apipull;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.common.entity.DeviceDevice;
+import com.common.mapper.DeviceApiCredentialMapper;
+import com.common.service.DeviceDeviceService;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.haobang.apipull.adapter.AdapterRegistry;
+import com.haobang.apipull.adapter.ApiPullAdapter;
+import com.haobang.apipull.api.ApiPullDtos.ApiCredentialResponse;
+import com.haobang.apipull.api.ApiPullDtos.CreateApiCredentialRequest;
+import com.haobang.apipull.persistence.DeviceApiCredential;
+import com.haobang.config.AppProperties;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.api.Test;
+
+class ApiCredentialServiceTest {
+ @Test
+ void storesPlaintextButNeverReturnsAppSecret() throws Exception {
+ DeviceApiCredentialMapper mapper = mock(DeviceApiCredentialMapper.class);
+ DeviceDeviceService devices = mock(DeviceDeviceService.class);
+ AdapterRegistry adapters = mock(AdapterRegistry.class);
+ AppProperties app = new AppProperties();
+ app.setAppServiceDeviceCollectId(7);
+
+ DeviceDevice device = new DeviceDevice();
+ device.setId(126);
+ device.setDeviceCollectId(7);
+ when(devices.getByIdSafely(126)).thenReturn(device);
+ when(adapters.require("NDRSERVER_V1")).thenReturn(mock(ApiPullAdapter.class));
+
+ AtomicReference stored = new AtomicReference<>();
+ when(mapper.insert(any(DeviceApiCredential.class)))
+ .thenAnswer(
+ invocation -> {
+ stored.set(invocation.getArgument(0));
+ return 1;
+ });
+ when(mapper.selectByRef("ndr-v1")).thenAnswer(invocation -> stored.get());
+
+ ApiCredentialService service = new ApiCredentialService(mapper, devices, adapters, app);
+ ApiCredentialResponse response = service.create(request());
+
+ assertEquals("plain-secret", stored.get().getAppSecret());
+ String responseJson = new ObjectMapper().writeValueAsString(response);
+ assertFalse(responseJson.contains("app-key"));
+ assertFalse(responseJson.contains("appKey"));
+ assertFalse(responseJson.contains("plain-secret"));
+ assertFalse(responseJson.contains("appSecret"));
+ }
+
+ private CreateApiCredentialRequest request() {
+ CreateApiCredentialRequest request = new CreateApiCredentialRequest();
+ request.setCredentialRef("ndr-v1");
+ request.setDeviceId(126);
+ request.setAdapterType("NDRSERVER_V1");
+ request.setCredentialVersion("20260901-v1");
+ request.setAppKey("app-key");
+ request.setAppSecret("plain-secret");
+ return request;
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullContractTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullContractTest.java
new file mode 100644
index 0000000..480847a
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullContractTest.java
@@ -0,0 +1,71 @@
+package com.haobang.apipull;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import com.config.CacheConfig;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.haobang.apipull.api.ApiPullDtos.ApiPullTaskResponse;
+import com.haobang.apipull.api.ApiPullDtos.CreateApiPullTaskRequest;
+import java.io.InputStream;
+import org.apache.ibatis.builder.xml.XMLMapperBuilder;
+import org.apache.ibatis.session.Configuration;
+import org.junit.jupiter.api.Test;
+
+class ApiPullContractTest {
+ @Test
+ void webJsonNeverPublishesJavaClassMetadata() throws Exception {
+ ApiPullTaskResponse response = new ApiPullTaskResponse();
+ response.setTaskId(1);
+ response.setAdapterType("NDRSERVER_V1");
+ String json = new CacheConfig().webObjectMapper().writeValueAsString(response);
+ assertFalse(json.contains("@class"));
+ assertFalse(json.contains("com.haobang"));
+ }
+
+ @Test
+ void managementDtoRejectsUnknownSecretBearingFields() {
+ String body =
+ "{\"deviceId\":1,\"taskName\":\"x\",\"adapterType\":\"NDRSERVER_V1\",\"streamKey\":\"NDR_ALERT\","
+ + "\"baseUrl\":\"https://localhost/ndr/api\",\"credentialRef\":\"ref\",\"initialStartTime\":\"2026-08-31T10:00:00+08:00\","
+ + "\"extraConfig\":{},\"appSecret\":\"must-not-be-accepted\"}";
+ assertThrows(
+ Exception.class,
+ () ->
+ new ObjectMapper()
+ .findAndRegisterModules()
+ .readValue(body, CreateApiPullTaskRequest.class));
+ }
+
+ @Test
+ void mybatisMapperXmlIsSyntacticallyValidAndBindsEveryStatement() throws Exception {
+ Configuration configuration = new Configuration();
+ try (InputStream in =
+ getClass().getClassLoader().getResourceAsStream("mapper/DeviceCollectApiMapper.xml")) {
+ assertNotNull(in);
+ new XMLMapperBuilder(
+ in,
+ configuration,
+ "mapper/DeviceCollectApiMapper.xml",
+ configuration.getSqlFragments())
+ .parse();
+ }
+ try (InputStream in =
+ getClass().getClassLoader().getResourceAsStream("mapper/DeviceApiCredentialMapper.xml")) {
+ assertNotNull(in);
+ new XMLMapperBuilder(
+ in,
+ configuration,
+ "mapper/DeviceApiCredentialMapper.xml",
+ configuration.getSqlFragments())
+ .parse();
+ }
+ assertTrue(configuration.hasStatement("com.common.mapper.DeviceCollectApiMapper.acquireLease"));
+ assertTrue(
+ configuration.hasStatement("com.common.mapper.DeviceCollectApiMapper.commitCheckpoint"));
+ assertTrue(
+ configuration.hasStatement("com.common.mapper.DeviceCollectApiMapper.resetCheckpoint"));
+ assertTrue(
+ configuration.hasStatement("com.common.mapper.DeviceApiCredentialMapper.selectForLoad"));
+ assertTrue(configuration.hasStatement("com.common.mapper.DeviceApiCredentialMapper.updateCas"));
+ }
+}
diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullRunnerTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullRunnerTest.java
new file mode 100644
index 0000000..f61412e
--- /dev/null
+++ b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullRunnerTest.java
@@ -0,0 +1,316 @@
+package com.haobang.apipull;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.common.mapper.DeviceCollectApiMapper;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.haobang.apipull.adapter.AdapterRegistry;
+import com.haobang.apipull.adapter.ApiPullAdapter;
+import com.haobang.apipull.adapter.PullPage;
+import com.haobang.apipull.adapter.PullRequest;
+import com.haobang.apipull.adapter.PulledEvent;
+import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
+import com.haobang.apipull.persistence.DeviceCollectApiConfig;
+import com.haobang.apipull.persistence.DeviceCollectApiRun;
+import com.haobang.wal.AppendAttempt;
+import com.haobang.wal.DurableReceipt;
+import com.haobang.wal.IngressEvent;
+import com.haobang.wal.SyslogWalService;
+import com.haobang.wal.UnifiedIngressService;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.api.Test;
+
+class ApiPullRunnerTest {
+ private static final Instant WINDOW_START = Instant.parse("2026-08-31T02:00:00Z");
+
+ @Test
+ void checkpointWaitsForEveryDurableReceipt() throws Exception {
+ RunnerFixture fixture = new RunnerFixture();
+ CompletableFuture durable = new CompletableFuture<>();
+ CountDownLatch accepted = new CountDownLatch(1);
+
+ when(fixture.ingress.tryAppend(any()))
+ .thenAnswer(
+ invocation -> {
+ accepted.countDown();
+ return AppendAttempt.accepted(durable);
+ });
+ when(fixture.adapter.pull(any())).thenReturn(page(1L, 1, 0));
+
+ AtomicBoolean committed = new AtomicBoolean();
+ when(fixture.mapper.commitCheckpoint(
+ anyInt(),
+ anyLong(),
+ anyString(),
+ anyLong(),
+ anyLong(),
+ anyString(),
+ any(),
+ any(),
+ any()))
+ .thenAnswer(
+ invocation -> {
+ committed.set(true);
+ return 1;
+ });
+
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future future =
+ executor.submit(
+ () -> fixture.runner.execute(fixture.config, fixture.checkpoint, fixture.run));
+
+ try {
+ assertTrue(accepted.await(2, TimeUnit.SECONDS));
+ Thread.sleep(100L);
+ assertFalse(committed.get(), "事件仅入队、尚未完成 WAL force 时不能推进 checkpoint");
+
+ durable.complete(new DurableReceipt("event-0", 1L, 4, Instant.now()));
+ future.get(2, TimeUnit.SECONDS);
+
+ assertTrue(committed.get());
+ assertEquals("SUCCESS", fixture.run.getStatus());
+ assertEquals(Long.valueOf(1L), fixture.run.getRecordsDurable());
+ verify(fixture.adapter).pull(any(PullRequest.class));
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void splitsOversizedWindowBeforeWritingWalAndCommitsLeftSubwindow() {
+ RunnerFixture fixture = new RunnerFixture();
+ fixture.config.setPageSize(2);
+ fixture.config.setMaxPagesPerRun(2);
+
+ /*
+ * 计划的 60 秒窗口有 5 条，超过容量 2 × 2；拆为 30 秒后只有 3 条，
+ * 可以用两页完成。父窗口探测页的两条记录不能进入 WAL。
+ */
+ when(fixture.adapter.pull(any()))
+ .thenAnswer(
+ invocation -> {
+ PullRequest request = invocation.getArgument(0);
+ long windowSeconds =
+ Duration.between(WINDOW_START, request.getQueryEnd()).getSeconds();
+ if (windowSeconds == 60L) {
+ return page(5L, 2, request.getOffset());
+ }
+ if (request.getOffset() == 0L) {
+ return page(3L, 2, 0);
+ }
+ return page(3L, 1, request.getOffset());
+ });
+
+ AtomicReference committedEnd = new AtomicReference<>();
+ when(fixture.mapper.commitCheckpoint(
+ anyInt(),
+ anyLong(),
+ anyString(),
+ anyLong(),
+ anyLong(),
+ anyString(),
+ any(),
+ any(),
+ any()))
+ .thenAnswer(
+ invocation -> {
+ committedEnd.set(invocation.getArgument(6));
+ return 1;
+ });
+
+ fixture.runner.execute(fixture.config, fixture.checkpoint, fixture.run);
+
+ assertEquals("SUCCESS", fixture.run.getStatus());
+ assertEquals(Integer.valueOf(2), fixture.run.getPagesFetched());
+ assertEquals(Long.valueOf(3L), fixture.run.getRecordsFetched());
+ assertEquals(Long.valueOf(3L), fixture.run.getRecordsDurable());
+ assertEquals(
+ ApiPullTime.toDatabase(WINDOW_START.plusSeconds(30L)), committedEnd.get());
+ verify(fixture.adapter, times(3)).pull(any(PullRequest.class));
+ verify(fixture.ingress, times(3)).tryAppend(any(IngressEvent.class));
+ }
+
+ @Test
+ void schedulingSliceCanSplitBelowTheHardRunCapacity() {
+ RunnerFixture fixture = new RunnerFixture();
+ fixture.config.setPageSize(2);
+ fixture.config.setMaxPagesPerRun(100);
+ fixture.properties.setMaxPagesPerSlice(2);
+
+ when(fixture.adapter.pull(any()))
+ .thenAnswer(
+ invocation -> {
+ PullRequest request = invocation.getArgument(0);
+ long windowSeconds =
+ Duration.between(WINDOW_START, request.getQueryEnd()).getSeconds();
+ if (windowSeconds == 60L) {
+ return page(5L, 2, request.getOffset());
+ }
+ if (request.getOffset() == 0L) {
+ return page(3L, 2, 0);
+ }
+ return page(3L, 1, request.getOffset());
+ });
+
+ fixture.runner.execute(fixture.config, fixture.checkpoint, fixture.run);
+
+ assertEquals("SUCCESS", fixture.run.getStatus());
+ assertEquals(Long.valueOf(3L), fixture.run.getRecordsDurable());
+ verify(fixture.adapter, times(3)).pull(any(PullRequest.class));
+ verify(fixture.ingress, times(3)).tryAppend(any(IngressEvent.class));
+ }
+
+ @Test
+ void failsWithoutWritingWalWhenOneSecondWindowStillExceedsCapacity() {
+ RunnerFixture fixture = new RunnerFixture();
+ fixture.config.setPageSize(2);
+ fixture.config.setMaxPagesPerRun(2);
+ fixture.config.setWindowSizeSeconds(1);
+ when(fixture.adapter.pull(any())).thenReturn(page(5L, 2, 0));
+
+ fixture.runner.execute(fixture.config, fixture.checkpoint, fixture.run);
+
+ assertEquals("FAILED", fixture.run.getStatus());
+ assertEquals("MIN_WINDOW_CAPACITY_EXCEEDED", fixture.run.getErrorCode());
+ assertEquals(Long.valueOf(0L), fixture.run.getRecordsFetched());
+ assertEquals(Long.valueOf(0L), fixture.run.getRecordsDurable());
+ verify(fixture.ingress, never()).tryAppend(any(IngressEvent.class));
+ verify(fixture.mapper, never())
+ .commitCheckpoint(
+ anyInt(),
+ anyLong(),
+ anyString(),
+ anyLong(),
+ anyLong(),
+ anyString(),
+ any(),
+ any(),
+ any());
+ }
+
+ private static PullPage page(long total, int count, long offset) {
+ List