feat(syslog): 优化大日志通道并增加异常连续消费兜底

- 128 KiB 及以上日志默认进入大日志 Topic
- InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存
- 截断内容追加 HB_XDR_TRUNCATED 标记
- InfluxDB 写入失败时继续执行规则解析和达梦写入
- 未知或系统性异常优先写入普通/大日志 DLT
- DLT 写入失败时记录特殊告警并提交 offset，避免阻塞后续消息
- DLT 失败等待时间默认限制为 1 秒
- 增加连续性放行、DLT 成功及失败监控指标
- 更新 Docker Compose、v3 升级文档及相关测试

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java

@@ -23,9 +23,10 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 
 /**
- * 把当前事实事务的内存聚合结果直接写入小型统计表。
+ * 把当前事实事务的内存聚合结果写入同步投影或可靠异步投影入口。
  *
- * <p>调用方必须已经开启事实事务；本类不会新开事务，也不会吞掉任何数据库异常。</p>
+ * <p>DIRECT 默认只追加一条批次 Outbox；SHADOW/显式回退仍同步更新统计表。调用方必须
+ * 已经开启事实事务，本类不会吞掉任何数据库异常。</p>
  */
 @Slf4j
 @Service
@@ -63,6 +64,7 @@ public class DirectStatsProjectionWriter {
     private final StatsWriteModeProvider modeProvider;
     private final DirectStatsWriteMetrics metrics;
     private final StatsMaterializationPublisher materializationPublisher;
+    private final StatsProjectionOutboxWriter projectionOutboxWriter;
 
     @Value("${stats.incremental.read-enabled:false}")
     private boolean readEnabled;
@@ -91,11 +93,13 @@ public class DirectStatsProjectionWriter {
     public DirectStatsProjectionWriter(JdbcTemplate jdbc,
                                        StatsWriteModeProvider modeProvider,
                                        DirectStatsWriteMetrics metrics,
-                                       StatsMaterializationPublisher materializationPublisher) {
+                                       StatsMaterializationPublisher materializationPublisher,
+                                       StatsProjectionOutboxWriter projectionOutboxWriter) {
         this.jdbc = jdbc;
         this.modeProvider = modeProvider;
         this.metrics = metrics;
         this.materializationPublisher = materializationPublisher;
+        this.projectionOutboxWriter = projectionOutboxWriter;
     }
 
     public WriteResult writeStandardizedRows(List<Map<String, Object>> rows) {
@@ -160,6 +164,8 @@ public class DirectStatsProjectionWriter {
         status.put("today_stats_date_enabled", todayStatsDateEnabled);
         status.put("business_materialization_mode", asynchronousMaterialization()
                 ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS");
+        status.put("daily_projection_write_mode", asynchronousProjection()
+                ? "DURABLE_OUTBOX" : "SYNCHRONOUS_MERGE");
         return status;
     }
 
@@ -194,6 +200,24 @@ public class DirectStatsProjectionWriter {
         }
         requireTransaction();
 
+        if (asynchronousProjection()) {
+            try {
+                long outboxStarted = System.nanoTime();
+                projectionOutboxWriter.append(StatsProjectionOutboxEvent.fromAccumulator(accumulator));
+                long outboxMs = elapsedMs(outboxStarted);
+                int deviceGroups = effectiveDeviceEntries(accumulator).size();
+                int ruleGroups = effectiveRuleEntries(accumulator).size();
+                metrics.recordSuccess(accumulator.getInputRows(), deviceGroups, ruleGroups,
+                        aggregateMs, outboxMs, 0L);
+                return WriteResult.direct(accumulator.getInputRows(), deviceGroups, ruleGroups,
+                        aggregateMs, outboxMs, 0L);
+            } catch (RuntimeException failure) {
+                metrics.recordFailure();
+                throw new DirectStatsWriteException(
+                        "统计投影 Outbox 写入失败，事实事务必须整体回滚", failure);
+            }
+        }
+
         List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> devices =
                 effectiveDeviceEntries(accumulator);
         List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> rules =
@@ -244,6 +268,11 @@ public class DirectStatsProjectionWriter {
                     aggregateMs, 0L, 0L);
         }
         requireTransaction();
+        // Additive positive/negative changes commute in the serialized projection consumer. This
+        // keeps non-normal UPDATE statistics outside the fact transaction as well.
+        if (asynchronousProjection()) {
+            return write(accumulator, aggregateMs);
+        }
         List<DirectStatsAccumulator.DailyKey> keys = new ArrayList<>(accumulator.getDevices().keySet());
         try {
             long mergeStarted = System.nanoTime();
@@ -408,6 +437,11 @@ public class DirectStatsProjectionWriter {
                 && materializationPublisher.isEnabled();
     }
 
+    private boolean asynchronousProjection() {
+        return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT
+                && projectionOutboxWriter != null && projectionOutboxWriter.isEnabled();
+    }
+
     private List<StatsMaterializationCommand> materializationCommands(
             List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> devices,
             List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> rules) {

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmLargeTextLimiter.java

@@ -0,0 +1,89 @@
+package com.common.service;
+
+import com.common.entity.SyslogNonNormalMessage;
+import com.common.util.Utf8TextTruncator;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.LongAdder;
+
+/** 达梦持久化边界的大字段保护；0 表示兼容模式下不限制。 */
+@Component
+public class DmLargeTextLimiter {
+    private static final long MIN_ENABLED_LIMIT = 64L;
+
+    @Value("${dm.normal.large-field-max-utf8-bytes:0}")
+    private long normalMaxBytes;
+
+    @Value("${dm.normal.large-fields:http_req_body,http_resp_body,http_req_header,http_resp_header,detail,description,cmdline,shell_cmdline,parent_cmdline,mail_message}")
+    private String normalFieldsValue;
+
+    @Value("${dm.non-normal.large-field-max-utf8-bytes:0}")
+    private long nonNormalMaxBytes;
+
+    private volatile Set<String> normalFields = Collections.emptySet();
+    private final LongAdder normalFieldsTruncated = new LongAdder();
+    private final LongAdder nonNormalFieldsTruncated = new LongAdder();
+
+    @PostConstruct
+    public void validate() {
+        validateLimit(normalMaxBytes, "dm.normal.large-field-max-utf8-bytes");
+        validateLimit(nonNormalMaxBytes, "dm.non-normal.large-field-max-utf8-bytes");
+        Set<String> parsed = new LinkedHashSet<>();
+        if (normalFieldsValue != null) {
+            Arrays.stream(normalFieldsValue.split(","))
+                    .map(String::trim).filter(value -> !value.isEmpty()).forEach(parsed::add);
+        }
+        normalFields = Collections.unmodifiableSet(parsed);
+    }
+
+    private static void validateLimit(long value, String name) {
+        if (value < 0L || (value > 0L && value < MIN_ENABLED_LIMIT)) {
+            throw new IllegalArgumentException(name + " 必须为 0 或至少 " + MIN_ENABLED_LIMIT);
+        }
+    }
+
+    public void truncateNormalRow(Map<String, Object> row) {
+        if (row == null || normalMaxBytes == 0L) return;
+        for (String field : normalFields) {
+            Object value = row.get(field);
+            if (!(value instanceof String)) continue;
+            Utf8TextTruncator.Result result = Utf8TextTruncator.truncate((String) value, normalMaxBytes);
+            if (result.isTruncated()) {
+                row.put(field, result.getValue());
+                normalFieldsTruncated.increment();
+            }
+        }
+    }
+
+    public void truncateNonNormal(SyslogNonNormalMessage message) {
+        if (message == null || nonNormalMaxBytes == 0L) return;
+        message.setSyslogMessage(truncateNonNormalValue(message.getSyslogMessage()));
+        message.setHeaderMessage(truncateNonNormalValue(message.getHeaderMessage()));
+        message.setReasonDetail(truncateNonNormalValue(message.getReasonDetail()));
+    }
+
+    private String truncateNonNormalValue(String value) {
+        Utf8TextTruncator.Result result = Utf8TextTruncator.truncate(value, nonNormalMaxBytes);
+        if (result.isTruncated()) nonNormalFieldsTruncated.increment();
+        return result.getValue();
+    }
+
+    public long getNormalFieldsTruncated() { return normalFieldsTruncated.sum(); }
+    public long getNonNormalFieldsTruncated() { return nonNormalFieldsTruncated.sum(); }
+
+    /** 测试和非 Spring 兼容入口。 */
+    static DmLargeTextLimiter unlimited() {
+        DmLargeTextLimiter limiter = new DmLargeTextLimiter();
+        limiter.normalMaxBytes = 0L;
+        limiter.nonNormalMaxBytes = 0L;
+        limiter.validate();
+        return limiter;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java

@@ -174,8 +174,8 @@ public class DmStandardizedBatchExecutor {
                 lateAlarmBucketWriter.record(rows);
             }
 
-            // DIRECT/SHADOW 模式在当前事务内按设备、规则聚合写统计；LEGACY 模式会直接返回。
-            // 任一统计 SQL 失败都会回滚本事务的事实行，禁止产生无法定位的统计缺口。
+            // DIRECT 异步投影模式只在当前事实事务追加一条聚合 Outbox；独立 Kafka consumer
+            // 再更新日统计。SHADOW/回退模式仍保留原同步 MERGE。
             timing.statsResult = directStatsProjectionWriter.writeStandardizedRows(rows);
             return null;
         });
@@ -183,7 +183,7 @@ public class DmStandardizedBatchExecutor {
         transactionMetrics.recordSuccess(rows.size(), timing.factFlushMs, timing.commitMs, transactionMs);
         if (transactionLogEnabled && timing.statsResult != null && timing.statsResult.isDirect()) {
             log.info("DIRECT 达梦事务完成: factRows={}, deviceGroups={}, ruleGroups={}, aggregateMs={}, " +
-                            "factFlushMs={}, statsMergeMs={}, materializeMs={}, commitMs={}, transactionMs={}, retry={}",
+                            "factFlushMs={}, statsProjectionStageMs={}, materializeMs={}, commitMs={}, transactionMs={}, retry={}",
                     rows.size(), timing.statsResult.getDeviceGroups(), timing.statsResult.getRuleGroups(),
                     timing.statsResult.getAggregateMs(), timing.factFlushMs,
                     timing.statsResult.getMergeMs(), timing.statsResult.getMaterializeMs(),

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java

@@ -60,6 +60,7 @@ public class DmStandardizedBatchWriter {
     private final SyslogNormalDataService normalDataService;
     private final DmStandardizedBatchExecutor batchExecutor;
     private final Executor writerExecutor;
+    private final DmLargeTextLimiter largeTextLimiter;
 
     // 未配置时直接启用本次改造的批写路径；SINGLE 只作为现场故障回退开关保留。
     @Value("${dm.standardized-write.mode:BATCH}")
@@ -89,12 +90,22 @@ public class DmStandardizedBatchWriter {
     private final AtomicInteger consecutiveSystemFailures = new AtomicInteger();
     private volatile long circuitOpenUntilMs;
 
+    @org.springframework.beans.factory.annotation.Autowired
     public DmStandardizedBatchWriter(SyslogNormalDataService normalDataService,
                                      DmStandardizedBatchExecutor batchExecutor,
-                                     @Qualifier("dmStandardizedWriterExecutor") Executor writerExecutor) {
+                                     @Qualifier("dmStandardizedWriterExecutor") Executor writerExecutor,
+                                     DmLargeTextLimiter largeTextLimiter) {
         this.normalDataService = normalDataService;
         this.batchExecutor = batchExecutor;
         this.writerExecutor = writerExecutor;
+        this.largeTextLimiter = largeTextLimiter;
+    }
+
+    /** 保留已有单元测试和离线工具入口，默认不改变历史存储内容。 */
+    public DmStandardizedBatchWriter(SyslogNormalDataService normalDataService,
+                                     DmStandardizedBatchExecutor batchExecutor,
+                                     Executor writerExecutor) {
+        this(normalDataService, batchExecutor, writerExecutor, DmLargeTextLimiter.unlimited());
     }
 
     /**
@@ -148,6 +159,7 @@ public class DmStandardizedBatchWriter {
         List<WriteRequest<T>> prepared = new ArrayList<>(requests.size());
         for (WriteRequest<T> request : requests) {
             try {
+                largeTextLimiter.truncateNormalRow(request.getData());
                 normalDataService.prepareForInsert(request.getData());
                 prepared.add(request);
             } catch (RuntimeException invalidRow) {

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java

@@ -29,6 +29,7 @@ public class IngressStatsProjectionWriter {
     private final PlatformTransactionManager transactionManager;
     private final StatsWriteModeProvider modeProvider;
     private final StatsMaterializationPublisher materializationPublisher;
+    private final StatsProjectionOutboxWriter projectionOutboxWriter;
 
     @Value("${stats.ingress-metadata.enabled:false}")
     private boolean enabled;
@@ -50,11 +51,13 @@ public class IngressStatsProjectionWriter {
     public IngressStatsProjectionWriter(JdbcTemplate jdbc,
                                         PlatformTransactionManager transactionManager,
                                         StatsWriteModeProvider modeProvider,
-                                        StatsMaterializationPublisher materializationPublisher) {
+                                        StatsMaterializationPublisher materializationPublisher,
+                                        StatsProjectionOutboxWriter projectionOutboxWriter) {
         this.jdbc = jdbc;
         this.transactionManager = transactionManager;
         this.modeProvider = modeProvider;
         this.materializationPublisher = materializationPublisher;
+        this.projectionOutboxWriter = projectionOutboxWriter;
     }
 
     public int writeBatch(List<IngressStatsEvent> events) {
@@ -92,6 +95,8 @@ public class IngressStatsProjectionWriter {
                 ? "device_daily_stats+" + shadowDeviceTable : "device_daily_stats");
         status.put("business_materialization_mode", asynchronousMaterialization()
                 ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS");
+        status.put("daily_projection_write_mode", asynchronousProjection()
+                ? "DURABLE_OUTBOX" : "SYNCHRONOUS_MERGE");
         status.put("successful_batches", successfulBatches.get());
         status.put("received_events", receivedEvents.get());
         status.put("updated_groups", updatedGroups.get());
@@ -109,6 +114,10 @@ public class IngressStatsProjectionWriter {
         TransactionTemplate transaction = new TransactionTemplate(transactionManager);
         transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
         Integer updated = transaction.execute(status -> {
+            if (asynchronousProjection()) {
+                projectionOutboxWriter.append(StatsProjectionOutboxEvent.fromIngress(aggregation));
+                return aggregation.devices.size() + aggregation.collects.size();
+            }
             int groups = 0;
             boolean asyncMaterialization = asynchronousMaterialization();
             List<StatsMaterializationCommand> commands = asyncMaterialization
@@ -237,6 +246,11 @@ public class IngressStatsProjectionWriter {
                 && materializationPublisher.isEnabled();
     }
 
+    private boolean asynchronousProjection() {
+        return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT
+                && projectionOutboxWriter != null && projectionOutboxWriter.isEnabled();
+    }
+
     private static String safeTable(String table) {
         if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
             throw new IllegalStateException("非法入口设备统计表名: " + table);

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionDeltaConsumer.java

@@ -0,0 +1,386 @@
+package com.common.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.kafka.annotation.KafkaListener;
+import org.springframework.kafka.support.Acknowledgment;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionTemplate;
+
+import java.sql.Date;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** Applies durable projection deltas outside the fact transaction. */
+@Slf4j
+@Component
+public class StatsProjectionDeltaConsumer {
+    private static final String DEVICE_MERGE =
+            "MERGE INTO %s t USING (SELECT ? stat_date,? device_id,? parse_delta," +
+            "? non_log_delta,? receive_time FROM DUAL) s " +
+            "ON (t.stat_date=s.stat_date AND t.device_id=s.device_id) " +
+            "WHEN MATCHED THEN UPDATE SET t.parse_count=t.parse_count+s.parse_delta," +
+            "t.non_log_count=t.non_log_count+s.non_log_delta," +
+            "t.last_receive_time=CASE WHEN s.receive_time IS NULL THEN t.last_receive_time " +
+            "WHEN t.last_receive_time IS NULL OR t.last_receive_time<s.receive_time " +
+            "THEN s.receive_time ELSE t.last_receive_time END," +
+            "t.projection_seq=t.projection_seq+1,t.updated_at=SYSDATE " +
+            "WHEN NOT MATCHED THEN INSERT (stat_date,device_id,parse_count,non_log_count," +
+            "last_receive_time,projection_seq,updated_at) VALUES " +
+            "(s.stat_date,s.device_id,s.parse_delta,s.non_log_delta,s.receive_time,1,SYSDATE)";
+
+    private static final String RULE_MERGE =
+            "MERGE INTO %s t USING (SELECT ? stat_date,? normalize_rule_id,? max_log_time FROM DUAL) s " +
+            "ON (t.stat_date=s.stat_date AND t.normalize_rule_id=s.normalize_rule_id) " +
+            "WHEN MATCHED THEN UPDATE SET t.max_log_time=CASE " +
+            "WHEN t.max_log_time IS NULL OR t.max_log_time<s.max_log_time THEN s.max_log_time " +
+            "ELSE t.max_log_time END,t.projection_seq=t.projection_seq+1,t.updated_at=SYSDATE " +
+            "WHEN NOT MATCHED THEN INSERT (stat_date,normalize_rule_id,max_log_time,projection_seq,updated_at) " +
+            "VALUES (s.stat_date,s.normalize_rule_id,s.max_log_time,1,SYSDATE)";
+
+    private static final String COLLECT_MERGE =
+            "MERGE INTO device_collect_stats t USING (SELECT ? device_collect_id," +
+            "? first_success_time,? last_success_time FROM DUAL) s " +
+            "ON (t.device_collect_id=s.device_collect_id) " +
+            "WHEN MATCHED THEN UPDATE SET t.first_success_time=CASE " +
+            "WHEN s.first_success_time IS NULL THEN t.first_success_time " +
+            "WHEN t.first_success_time IS NULL OR t.first_success_time>s.first_success_time " +
+            "THEN s.first_success_time ELSE t.first_success_time END," +
+            "t.last_success_time=CASE WHEN s.last_success_time IS NULL THEN t.last_success_time " +
+            "WHEN t.last_success_time IS NULL OR t.last_success_time<s.last_success_time " +
+            "THEN s.last_success_time ELSE t.last_success_time END," +
+            "t.projection_seq=t.projection_seq+1,t.updated_at=SYSDATE " +
+            "WHEN NOT MATCHED THEN INSERT (device_collect_id,first_success_time,last_success_time," +
+            "last_fail_time,projection_seq,updated_at) VALUES " +
+            "(s.device_collect_id,s.first_success_time,s.last_success_time,NULL,1,SYSDATE)";
+
+    private final JdbcTemplate jdbc;
+    private final PlatformTransactionManager transactionManager;
+    private final ObjectMapper objectMapper;
+    private final StatsMaterializationPublisher materializationPublisher;
+
+    @Value("${stats.projection-async.enabled:true}") private boolean enabled;
+    @Value("${stats.projection-async.applied-table:stats_projection_applied_event}") private String appliedTable;
+    @Value("${stats.projection-async.outbox-table:stats_projection_outbox}") private String outboxTable;
+    @Value("${stats.direct.device-table:device_daily_stats}") private String deviceTable;
+    @Value("${stats.direct.rule-table:normalize_rule_daily_stats}") private String ruleTable;
+    @Value("${stats.incremental.transaction-timeout-seconds:45}") private int transactionTimeoutSeconds;
+    @Value("${stats.projection-async.max-concurrency-retries:3}") private int maxConcurrencyRetries;
+    @Value("${stats.projection-async.concurrency-retry-backoff-ms:50}") private long retryBackoffMs;
+
+    private final AtomicLong consumed = new AtomicLong();
+    private final AtomicLong appliedEvents = new AtomicLong();
+    private final AtomicLong duplicates = new AtomicLong();
+    private final AtomicLong deviceGroups = new AtomicLong();
+    private final AtomicLong ruleGroups = new AtomicLong();
+    private final AtomicLong collectGroups = new AtomicLong();
+    private final AtomicLong successfulBatches = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+    private final AtomicLong retries = new AtomicLong();
+    private volatile LocalDateTime lastSuccessTime;
+    private volatile LocalDateTime lastFailureTime;
+
+    public StatsProjectionDeltaConsumer(JdbcTemplate jdbc,
+                                        PlatformTransactionManager transactionManager,
+                                        ObjectMapper objectMapper,
+                                        StatsMaterializationPublisher materializationPublisher) {
+        this.jdbc = jdbc;
+        this.transactionManager = transactionManager;
+        this.objectMapper = objectMapper.copy();
+        this.objectMapper.deactivateDefaultTyping();
+        this.materializationPublisher = materializationPublisher;
+    }
+
+    @KafkaListener(
+            id = "syslog-stats-projection",
+            groupId = "${stats.projection-async.group-id:${spring.kafka.consumer.group-id}-stats-projection}",
+            topics = "${stats.projection-async.topic:${spring.kafka.consumer.topic}.STATS.PROJECTION}",
+            autoStartup = "${stats.projection-async.enabled:true}",
+            containerFactory = "statsProjectionKafkaListenerContainerFactory"
+    )
+    public void consume(List<ConsumerRecord<String, String>> records, Acknowledgment acknowledgment) {
+        if (records == null || records.isEmpty()) {
+            acknowledgment.acknowledge();
+            return;
+        }
+        consumed.addAndGet(records.size());
+        List<StatsProjectionOutboxEvent> events = parse(records);
+        long started = System.nanoTime();
+        try {
+            ApplyResult result = applyWithRetry(events);
+            acknowledgment.acknowledge();
+            appliedEvents.addAndGet(result.appliedEvents);
+            duplicates.addAndGet(result.duplicates);
+            deviceGroups.addAndGet(result.deviceGroups);
+            ruleGroups.addAndGet(result.ruleGroups);
+            collectGroups.addAndGet(result.collectGroups);
+            successfulBatches.incrementAndGet();
+            lastSuccessTime = LocalDateTime.now();
+            log.info("统计投影批次完成: received={}, appliedEvents={}, duplicates={}, " +
+                            "deviceGroups={}, ruleGroups={}, collectGroups={}, durationMs={}",
+                    records.size(), result.appliedEvents, result.duplicates,
+                    result.deviceGroups, result.ruleGroups, result.collectGroups,
+                    (System.nanoTime() - started) / 1_000_000L);
+        } catch (RuntimeException failure) {
+            failures.incrementAndGet();
+            lastFailureTime = LocalDateTime.now();
+            throw failure;
+        }
+    }
+
+    private List<StatsProjectionOutboxEvent> parse(List<ConsumerRecord<String, String>> records) {
+        Map<String, StatsProjectionOutboxEvent> unique = new LinkedHashMap<>();
+        for (ConsumerRecord<String, String> record : records) {
+            try {
+                if (!StatsProjectionOutboxEvent.KAFKA_KEY.equals(record.key())) {
+                    throw new IllegalArgumentException("Unexpected stats projection Kafka key: " + record.key());
+                }
+                StatsProjectionOutboxEvent event = objectMapper.readValue(
+                        record.value(), StatsProjectionOutboxEvent.class);
+                event.validate();
+                unique.put(event.getEventId(), event);
+            } catch (Exception invalid) {
+                throw new IllegalStateException("Invalid internal stats projection event at " +
+                        record.topic() + "-" + record.partition() + "@" + record.offset(), invalid);
+            }
+        }
+        return new ArrayList<>(unique.values());
+    }
+
+    ApplyResult applyWithRetry(List<StatsProjectionOutboxEvent> events) {
+        int attempt = 0;
+        while (true) {
+            try {
+                return applyOnce(events);
+            } catch (RuntimeException failure) {
+                if (!DirectStatsProjectionWriter.isConcurrentFailure(failure)
+                        || attempt >= Math.max(0, maxConcurrencyRetries)) {
+                    throw failure;
+                }
+                attempt++;
+                retries.incrementAndGet();
+                sleepBeforeRetry(attempt);
+            }
+        }
+    }
+
+    private ApplyResult applyOnce(List<StatsProjectionOutboxEvent> events) {
+        if (!enabled) throw new IllegalStateException("Stats projection consumer is disabled");
+        TransactionTemplate transaction = new TransactionTemplate(transactionManager);
+        transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
+        ApplyResult result = transaction.execute(status -> {
+            Set<String> existing = existingEventIds(events);
+            List<StatsProjectionOutboxEvent> fresh = new ArrayList<>();
+            for (StatsProjectionOutboxEvent event : events) {
+                if (!existing.contains(event.getEventId())) fresh.add(event);
+            }
+            ProjectionAggregation aggregation = aggregate(fresh);
+            if (!aggregation.devices.isEmpty()) {
+                jdbc.batchUpdate(String.format(DEVICE_MERGE,
+                        StatsProjectionOutboxWriter.safeTable(deviceTable)), aggregation.deviceParams());
+            }
+            if (!aggregation.rules.isEmpty()) {
+                jdbc.batchUpdate(String.format(RULE_MERGE,
+                        StatsProjectionOutboxWriter.safeTable(ruleTable)), aggregation.ruleParams());
+            }
+            if (!aggregation.collects.isEmpty()) {
+                jdbc.batchUpdate(COLLECT_MERGE, aggregation.collectParams());
+            }
+            if (!fresh.isEmpty()) {
+                List<Object[]> markers = new ArrayList<>(fresh.size());
+                for (StatsProjectionOutboxEvent event : fresh) {
+                    markers.add(new Object[]{event.getEventId(), event.getSchema()});
+                }
+                jdbc.batchUpdate("INSERT INTO " + StatsProjectionOutboxWriter.safeTable(appliedTable) +
+                        " (event_id,event_schema,applied_at) VALUES (?,?,SYSDATE)", markers);
+            }
+            if (!events.isEmpty()) {
+                List<Object[]> acknowledged = new ArrayList<>(events.size());
+                for (StatsProjectionOutboxEvent event : events) {
+                    acknowledged.add(new Object[]{event.getEventId()});
+                }
+                jdbc.batchUpdate("UPDATE " + StatsProjectionOutboxWriter.safeTable(outboxTable) +
+                        " SET status='APPLIED',applied_at=SYSDATE,next_attempt_at=NULL," +
+                        "updated_at=SYSDATE WHERE event_id=?", acknowledged);
+            }
+            Collection<StatsMaterializationCommand> dirty = aggregation.materializationCommands();
+            if (!dirty.isEmpty()) materializationPublisher.enqueueAfterCommit(dirty);
+            return new ApplyResult(fresh.size(), events.size() - fresh.size(),
+                    aggregation.devices.size(), aggregation.rules.size(), aggregation.collects.size());
+        });
+        return result == null ? new ApplyResult(0, events.size(), 0, 0, 0) : result;
+    }
+
+    private Set<String> existingEventIds(List<StatsProjectionOutboxEvent> events) {
+        if (events.isEmpty()) return Collections.emptySet();
+        Set<String> result = new HashSet<>();
+        int chunkSize = 900;
+        for (int start = 0; start < events.size(); start += chunkSize) {
+            int end = Math.min(events.size(), start + chunkSize);
+            StringBuilder placeholders = new StringBuilder();
+            Object[] ids = new Object[end - start];
+            for (int i = start; i < end; i++) {
+                if (placeholders.length() > 0) placeholders.append(',');
+                placeholders.append('?');
+                ids[i - start] = events.get(i).getEventId();
+            }
+            result.addAll(jdbc.queryForList("SELECT event_id FROM " +
+                    StatsProjectionOutboxWriter.safeTable(appliedTable) + " WHERE event_id IN (" +
+                    placeholders + ")", String.class, ids));
+        }
+        return result;
+    }
+
+    static ProjectionAggregation aggregate(List<StatsProjectionOutboxEvent> events) {
+        ProjectionAggregation result = new ProjectionAggregation();
+        for (StatsProjectionOutboxEvent event : events) {
+            event.validate();
+            for (StatsProjectionOutboxEvent.DeviceDelta delta : event.getDevices()) {
+                DeviceKey key = new DeviceKey(delta.date(), delta.getDeviceId());
+                DeviceAggregate value = result.devices.computeIfAbsent(key,
+                        ignored -> new DeviceAggregate());
+                value.parseDelta += delta.getParseDelta();
+                value.nonLogDelta += delta.getNonLogDelta();
+                LocalDateTime receiveTime = delta.receiveTime();
+                if (receiveTime != null && (value.lastReceiveTime == null
+                        || receiveTime.isAfter(value.lastReceiveTime))) {
+                    value.lastReceiveTime = receiveTime;
+                }
+            }
+            for (StatsProjectionOutboxEvent.RuleDelta delta : event.getRules()) {
+                RuleKey key = new RuleKey(delta.date(), delta.getNormalizeRuleId());
+                LocalDateTime value = delta.time();
+                LocalDateTime current = result.rules.get(key);
+                if (current == null || value.isAfter(current)) result.rules.put(key, value);
+            }
+            for (StatsProjectionOutboxEvent.CollectDelta delta : event.getCollects()) {
+                CollectAggregate value = result.collects.computeIfAbsent(delta.getDeviceCollectId(),
+                        ignored -> new CollectAggregate());
+                value.include(delta.firstTime());
+                value.include(delta.lastTime());
+            }
+        }
+        return result;
+    }
+
+    private void sleepBeforeRetry(int attempt) {
+        try {
+            Thread.sleep(Math.max(0L, retryBackoffMs) * attempt);
+        } catch (InterruptedException interrupted) {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException("Stats projection retry interrupted", interrupted);
+        }
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("enabled", enabled);
+        result.put("consumed_total", consumed.get());
+        result.put("applied_events_total", appliedEvents.get());
+        result.put("duplicate_events_total", duplicates.get());
+        result.put("device_groups_total", deviceGroups.get());
+        result.put("rule_groups_total", ruleGroups.get());
+        result.put("collect_groups_total", collectGroups.get());
+        result.put("successful_batches", successfulBatches.get());
+        result.put("failures_total", failures.get());
+        result.put("concurrency_retries_total", retries.get());
+        result.put("last_success_time", lastSuccessTime);
+        result.put("last_failure_time", lastFailureTime);
+        return result;
+    }
+
+    static final class ProjectionAggregation {
+        final Map<DeviceKey, DeviceAggregate> devices = new TreeMap<>();
+        final Map<RuleKey, LocalDateTime> rules = new TreeMap<>();
+        final Map<Long, CollectAggregate> collects = new TreeMap<>();
+
+        List<Object[]> deviceParams() {
+            List<Object[]> result = new ArrayList<>(devices.size());
+            for (Map.Entry<DeviceKey, DeviceAggregate> entry : devices.entrySet()) {
+                DeviceAggregate value = entry.getValue();
+                result.add(new Object[]{Date.valueOf(entry.getKey().date), entry.getKey().id,
+                        value.parseDelta, value.nonLogDelta, value.lastReceiveTime});
+            }
+            return result;
+        }
+
+        List<Object[]> ruleParams() {
+            List<Object[]> result = new ArrayList<>(rules.size());
+            for (Map.Entry<RuleKey, LocalDateTime> entry : rules.entrySet()) {
+                result.add(new Object[]{Date.valueOf(entry.getKey().date), entry.getKey().id,
+                        entry.getValue()});
+            }
+            return result;
+        }
+
+        List<Object[]> collectParams() {
+            List<Object[]> result = new ArrayList<>(collects.size());
+            for (Map.Entry<Long, CollectAggregate> entry : collects.entrySet()) {
+                result.add(new Object[]{entry.getKey(), entry.getValue().min, entry.getValue().max});
+            }
+            return result;
+        }
+
+        Collection<StatsMaterializationCommand> materializationCommands() {
+            Set<StatsMaterializationCommand> result = new LinkedHashSet<>();
+            for (DeviceKey key : devices.keySet()) result.add(StatsMaterializationCommand.device(key.date, key.id));
+            for (RuleKey key : rules.keySet()) result.add(StatsMaterializationCommand.rule(key.date, key.id));
+            for (Long id : collects.keySet()) result.add(StatsMaterializationCommand.collect(id));
+            return result;
+        }
+    }
+
+    static class DimensionKey implements Comparable<DimensionKey> {
+        final LocalDate date;
+        final long id;
+        DimensionKey(LocalDate date, long id) { this.date = date; this.id = id; }
+        @Override public int compareTo(DimensionKey other) {
+            int dateOrder = date.compareTo(other.date);
+            return dateOrder != 0 ? dateOrder : Long.compare(id, other.id);
+        }
+        @Override public boolean equals(Object other) {
+            if (this == other) return true;
+            if (!(other instanceof DimensionKey)) return false;
+            DimensionKey key = (DimensionKey) other;
+            return id == key.id && date.equals(key.date);
+        }
+        @Override public int hashCode() { return 31 * date.hashCode() + Long.hashCode(id); }
+    }
+    static final class DeviceKey extends DimensionKey { DeviceKey(LocalDate date, long id) { super(date, id); } }
+    static final class RuleKey extends DimensionKey { RuleKey(LocalDate date, long id) { super(date, id); } }
+    static final class DeviceAggregate { long parseDelta; long nonLogDelta; LocalDateTime lastReceiveTime; }
+    static final class CollectAggregate {
+        LocalDateTime min; LocalDateTime max;
+        void include(LocalDateTime time) {
+            if (time == null) return;
+            if (min == null || time.isBefore(min)) min = time;
+            if (max == null || time.isAfter(max)) max = time;
+        }
+    }
+
+    static final class ApplyResult {
+        final int appliedEvents; final int duplicates;
+        final int deviceGroups; final int ruleGroups; final int collectGroups;
+        ApplyResult(int appliedEvents, int duplicates, int deviceGroups, int ruleGroups, int collectGroups) {
+            this.appliedEvents = appliedEvents; this.duplicates = duplicates;
+            this.deviceGroups = deviceGroups; this.ruleGroups = ruleGroups; this.collectGroups = collectGroups;
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxEvent.java

@@ -0,0 +1,221 @@
+package com.common.service;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * A durable, batch-aggregated statistics delta written beside the fact rows.
+ *
+ * <p>The event is additive/max-min only, so events from different application nodes may be
+ * published in a different order without changing the final projection. Kafka duplicates are
+ * eliminated by {@code eventId} in the projection consumer transaction.</p>
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class StatsProjectionOutboxEvent {
+    public static final String SCHEMA = "SYSLOG_STATS_PROJECTION_V1";
+    public static final String KAFKA_KEY = "SYSLOG_STATS_PROJECTION";
+
+    private String schema = SCHEMA;
+    private String eventId = UUID.randomUUID().toString();
+    private List<DeviceDelta> devices = new ArrayList<>();
+    private List<RuleDelta> rules = new ArrayList<>();
+    private List<CollectDelta> collects = new ArrayList<>();
+
+    public static StatsProjectionOutboxEvent fromAccumulator(DirectStatsAccumulator accumulator) {
+        StatsProjectionOutboxEvent event = new StatsProjectionOutboxEvent();
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry
+                : accumulator.getDevices().entrySet()) {
+            DirectStatsAccumulator.DeviceDelta value = entry.getValue();
+            if (value.getParseDelta() != 0L || value.getNonLogDelta() != 0L) {
+                event.devices.add(DeviceDelta.counts(entry.getKey().getDate(),
+                        entry.getKey().getDimensionId(), value.getParseDelta(), value.getNonLogDelta()));
+            }
+        }
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry
+                : accumulator.getRules().entrySet()) {
+            if (entry.getValue().getMaxLogTime() != null) {
+                event.rules.add(RuleDelta.max(entry.getKey().getDate(),
+                        entry.getKey().getDimensionId(), entry.getValue().getMaxLogTime()));
+            }
+        }
+        return event;
+    }
+
+    static StatsProjectionOutboxEvent fromIngress(IngressStatsProjectionWriter.Aggregation aggregation) {
+        StatsProjectionOutboxEvent event = new StatsProjectionOutboxEvent();
+        for (Map.Entry<IngressStatsProjectionWriter.DeviceKey, LocalDateTime> entry
+                : aggregation.devices.entrySet()) {
+            event.devices.add(DeviceDelta.received(entry.getKey().date,
+                    entry.getKey().deviceId, entry.getValue()));
+        }
+        for (Map.Entry<Long, IngressStatsProjectionWriter.CollectTimes> entry
+                : aggregation.collects.entrySet()) {
+            event.collects.add(CollectDelta.range(entry.getKey(),
+                    entry.getValue().min, entry.getValue().max));
+        }
+        return event;
+    }
+
+    public void validate() {
+        if (!SCHEMA.equals(schema)) {
+            throw new IllegalArgumentException("Unsupported stats projection schema: " + schema);
+        }
+        if (eventId == null || eventId.trim().isEmpty()) {
+            throw new IllegalArgumentException("Stats projection event is missing eventId");
+        }
+        if (devices == null || rules == null || collects == null) {
+            throw new IllegalArgumentException("Stats projection event contains null collections");
+        }
+        if (devices.isEmpty() && rules.isEmpty() && collects.isEmpty()) {
+            throw new IllegalArgumentException("Stats projection event has no changes");
+        }
+        for (DeviceDelta delta : devices) delta.validate();
+        for (RuleDelta delta : rules) delta.validate();
+        for (CollectDelta delta : collects) delta.validate();
+    }
+
+    public boolean isEmpty() {
+        return devices.isEmpty() && rules.isEmpty() && collects.isEmpty();
+    }
+
+    public String getSchema() { return schema; }
+    public void setSchema(String schema) { this.schema = schema; }
+    public String getEventId() { return eventId; }
+    public void setEventId(String eventId) { this.eventId = eventId; }
+    public List<DeviceDelta> getDevices() { return devices; }
+    public void setDevices(List<DeviceDelta> devices) { this.devices = devices; }
+    public List<RuleDelta> getRules() { return rules; }
+    public void setRules(List<RuleDelta> rules) { this.rules = rules; }
+    public List<CollectDelta> getCollects() { return collects; }
+    public void setCollects(List<CollectDelta> collects) { this.collects = collects; }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class DeviceDelta {
+        private String statDate;
+        private long deviceId;
+        private long parseDelta;
+        private long nonLogDelta;
+        private String lastReceiveTime;
+
+        public DeviceDelta() {}
+
+        static DeviceDelta counts(LocalDate date, long deviceId, long parseDelta, long nonLogDelta) {
+            DeviceDelta result = new DeviceDelta();
+            result.statDate = date.toString();
+            result.deviceId = deviceId;
+            result.parseDelta = parseDelta;
+            result.nonLogDelta = nonLogDelta;
+            return result;
+        }
+
+        static DeviceDelta received(LocalDate date, long deviceId, LocalDateTime receiveTime) {
+            DeviceDelta result = counts(date, deviceId, 0L, 0L);
+            result.lastReceiveTime = receiveTime.toString();
+            return result;
+        }
+
+        void validate() {
+            date();
+            if (deviceId <= 0L) throw new IllegalArgumentException("Invalid device projection id");
+            if (parseDelta == 0L && nonLogDelta == 0L && receiveTime() == null) {
+                throw new IllegalArgumentException("Device projection delta is empty");
+            }
+        }
+
+        public LocalDate date() { return LocalDate.parse(statDate); }
+        public LocalDateTime receiveTime() {
+            return lastReceiveTime == null || lastReceiveTime.trim().isEmpty()
+                    ? null : LocalDateTime.parse(lastReceiveTime);
+        }
+        public String getStatDate() { return statDate; }
+        public void setStatDate(String statDate) { this.statDate = statDate; }
+        public long getDeviceId() { return deviceId; }
+        public void setDeviceId(long deviceId) { this.deviceId = deviceId; }
+        public long getParseDelta() { return parseDelta; }
+        public void setParseDelta(long parseDelta) { this.parseDelta = parseDelta; }
+        public long getNonLogDelta() { return nonLogDelta; }
+        public void setNonLogDelta(long nonLogDelta) { this.nonLogDelta = nonLogDelta; }
+        public String getLastReceiveTime() { return lastReceiveTime; }
+        public void setLastReceiveTime(String lastReceiveTime) { this.lastReceiveTime = lastReceiveTime; }
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class RuleDelta {
+        private String statDate;
+        private long normalizeRuleId;
+        private String maxLogTime;
+
+        public RuleDelta() {}
+
+        static RuleDelta max(LocalDate date, long ruleId, LocalDateTime maxLogTime) {
+            RuleDelta result = new RuleDelta();
+            result.statDate = date.toString();
+            result.normalizeRuleId = ruleId;
+            result.maxLogTime = maxLogTime.toString();
+            return result;
+        }
+
+        void validate() {
+            date();
+            if (normalizeRuleId <= 0L || time() == null) {
+                throw new IllegalArgumentException("Invalid rule projection delta");
+            }
+        }
+
+        public LocalDate date() { return LocalDate.parse(statDate); }
+        public LocalDateTime time() {
+            return maxLogTime == null || maxLogTime.trim().isEmpty()
+                    ? null : LocalDateTime.parse(maxLogTime);
+        }
+        public String getStatDate() { return statDate; }
+        public void setStatDate(String statDate) { this.statDate = statDate; }
+        public long getNormalizeRuleId() { return normalizeRuleId; }
+        public void setNormalizeRuleId(long normalizeRuleId) { this.normalizeRuleId = normalizeRuleId; }
+        public String getMaxLogTime() { return maxLogTime; }
+        public void setMaxLogTime(String maxLogTime) { this.maxLogTime = maxLogTime; }
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class CollectDelta {
+        private long deviceCollectId;
+        private String firstSuccessTime;
+        private String lastSuccessTime;
+
+        public CollectDelta() {}
+
+        static CollectDelta range(long collectId, LocalDateTime first, LocalDateTime last) {
+            CollectDelta result = new CollectDelta();
+            result.deviceCollectId = collectId;
+            result.firstSuccessTime = first == null ? null : first.toString();
+            result.lastSuccessTime = last == null ? null : last.toString();
+            return result;
+        }
+
+        void validate() {
+            if (deviceCollectId <= 0L || (firstTime() == null && lastTime() == null)) {
+                throw new IllegalArgumentException("Invalid collector projection delta");
+            }
+        }
+
+        public LocalDateTime firstTime() {
+            return firstSuccessTime == null || firstSuccessTime.trim().isEmpty()
+                    ? null : LocalDateTime.parse(firstSuccessTime);
+        }
+        public LocalDateTime lastTime() {
+            return lastSuccessTime == null || lastSuccessTime.trim().isEmpty()
+                    ? null : LocalDateTime.parse(lastSuccessTime);
+        }
+        public long getDeviceCollectId() { return deviceCollectId; }
+        public void setDeviceCollectId(long deviceCollectId) { this.deviceCollectId = deviceCollectId; }
+        public String getFirstSuccessTime() { return firstSuccessTime; }
+        public void setFirstSuccessTime(String firstSuccessTime) { this.firstSuccessTime = firstSuccessTime; }
+        public String getLastSuccessTime() { return lastSuccessTime; }
+        public void setLastSuccessTime(String lastSuccessTime) { this.lastSuccessTime = lastSuccessTime; }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxRelay.java

@@ -0,0 +1,151 @@
+package com.common.service;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.SendResult;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import org.springframework.util.concurrent.ListenableFuture;
+
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Relays durable outbox events to the dedicated projection topic.
+ *
+ * <p>Multiple application nodes may relay the same row during a race. The consumer's applied-event
+ * table makes that harmless, so the relay never needs a distributed lease that could stall the
+ * fact path.</p>
+ */
+@Slf4j
+@Component
+public class StatsProjectionOutboxRelay {
+    private final JdbcTemplate jdbc;
+    private final KafkaTemplate<String, String> kafkaTemplate;
+    private final AtomicBoolean running = new AtomicBoolean();
+    private final AtomicLong published = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+    private final AtomicLong cleaned = new AtomicLong();
+    private volatile long lastSuccessAt;
+    private volatile long lastFailureAt;
+
+    @Value("${stats.projection-async.enabled:true}") private boolean enabled;
+    @Value("${stats.projection-async.outbox-table:stats_projection_outbox}") private String outboxTable;
+    @Value("${stats.projection-async.applied-table:stats_projection_applied_event}") private String appliedTable;
+    @Value("${stats.projection-async.topic:${spring.kafka.consumer.topic}.STATS.PROJECTION}") private String topic;
+    @Value("${stats.projection-async.relay-batch-size:100}") private int batchSize;
+    @Value("${stats.projection-async.publish-timeout-ms:10000}") private long publishTimeoutMs;
+    @Value("${stats.projection-async.retry-backoff-ms:1000}") private long retryBackoffMs;
+    @Value("${stats.projection-async.redelivery-interval-ms:60000}") private long redeliveryIntervalMs;
+    @Value("${stats.projection-async.retention-days:14}") private int retentionDays;
+
+    public StatsProjectionOutboxRelay(JdbcTemplate jdbc, KafkaTemplate<String, String> kafkaTemplate) {
+        this.jdbc = jdbc;
+        this.kafkaTemplate = kafkaTemplate;
+    }
+
+    @Scheduled(fixedDelayString = "${stats.projection-async.relay-interval-ms:200}")
+    public void relay() {
+        if (!enabled || !running.compareAndSet(false, true)) return;
+        try {
+            for (OutboxRow row : loadDueRows()) {
+                publish(row);
+            }
+        } catch (RuntimeException failure) {
+            failures.incrementAndGet();
+            lastFailureAt = System.currentTimeMillis();
+            log.error("统计投影 Outbox 扫描失败，下轮自动重试", failure);
+        } finally {
+            running.set(false);
+        }
+    }
+
+    private List<OutboxRow> loadDueRows() {
+        String table = StatsProjectionOutboxWriter.safeTable(outboxTable);
+        String sql = "SELECT event_id,payload FROM (SELECT event_id,payload FROM " + table +
+                " WHERE status IN ('PENDING','FAILED','PUBLISHED') AND (next_attempt_at IS NULL " +
+                "OR next_attempt_at<=SYSDATE) ORDER BY created_at,event_id) WHERE ROWNUM<=?";
+        return jdbc.query(sql, ps -> ps.setInt(1, Math.max(1, batchSize)),
+                (rs, rowNum) -> new OutboxRow(rs.getString(1), rs.getString(2)));
+    }
+
+    private void publish(OutboxRow row) {
+        try {
+            ListenableFuture<SendResult<String, String>> future = kafkaTemplate.send(
+                    topic, StatsProjectionOutboxEvent.KAFKA_KEY, row.payload);
+            future.get(Math.max(1L, publishTimeoutMs), TimeUnit.MILLISECONDS);
+            jdbc.update("UPDATE " + StatsProjectionOutboxWriter.safeTable(outboxTable) +
+                            " SET status='PUBLISHED',published_at=SYSDATE,next_attempt_at=?,last_error=NULL," +
+                            "updated_at=SYSDATE WHERE event_id=? AND status<>'APPLIED'",
+                    new Timestamp(System.currentTimeMillis() + Math.max(1000L, redeliveryIntervalMs)),
+                    row.eventId);
+            published.incrementAndGet();
+            lastSuccessAt = System.currentTimeMillis();
+        } catch (InterruptedException interrupted) {
+            Thread.currentThread().interrupt();
+            markFailure(row, interrupted);
+        } catch (Exception failure) {
+            markFailure(row, failure);
+        }
+    }
+
+    private void markFailure(OutboxRow row, Exception failure) {
+        failures.incrementAndGet();
+        lastFailureAt = System.currentTimeMillis();
+        String message = String.valueOf(failure.getMessage());
+        if (message.length() > 1800) message = message.substring(0, 1800);
+        try {
+            jdbc.update("UPDATE " + StatsProjectionOutboxWriter.safeTable(outboxTable) +
+                            " SET status='FAILED',attempt_count=attempt_count+1,next_attempt_at=?," +
+                            "last_error=?,updated_at=SYSDATE WHERE event_id=? AND status<>'APPLIED'",
+                    new Timestamp(System.currentTimeMillis() + Math.max(0L, retryBackoffMs)),
+                    message, row.eventId);
+        } catch (RuntimeException markFailure) {
+            failure.addSuppressed(markFailure);
+        }
+        log.error("统计投影 Outbox 发布失败，将自动重试: eventId={}", row.eventId, failure);
+    }
+
+    @Scheduled(initialDelayString = "${stats.projection-async.cleanup-initial-delay-ms:3600000}",
+            fixedDelayString = "${stats.projection-async.cleanup-interval-ms:3600000}")
+    public void cleanup() {
+        if (!enabled) return;
+        long keepMillis = TimeUnit.DAYS.toMillis(Math.max(1, retentionDays));
+        Timestamp cutoff = new Timestamp(System.currentTimeMillis() - keepMillis);
+        int removed = jdbc.update("DELETE FROM " + StatsProjectionOutboxWriter.safeTable(outboxTable) +
+                " WHERE status='APPLIED' AND applied_at<?", cutoff);
+        removed += jdbc.update("DELETE FROM " + StatsProjectionOutboxWriter.safeTable(appliedTable) +
+                " WHERE applied_at<?", cutoff);
+        cleaned.addAndGet(removed);
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("enabled", enabled);
+        result.put("topic", topic);
+        result.put("running", running.get());
+        result.put("published_total", published.get());
+        result.put("failures_total", failures.get());
+        result.put("cleaned_rows_total", cleaned.get());
+        result.put("last_success_epoch_ms", lastSuccessAt == 0L ? null : lastSuccessAt);
+        result.put("last_failure_epoch_ms", lastFailureAt == 0L ? null : lastFailureAt);
+        return result;
+    }
+
+    private static final class OutboxRow {
+        private final String eventId;
+        private final String payload;
+        private OutboxRow(String eventId, String payload) {
+            this.eventId = eventId;
+            this.payload = payload;
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxWriter.java

@@ -0,0 +1,70 @@
+package com.common.service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** Writes one aggregated outbox row in the same transaction as the corresponding fact batch. */
+@Component
+public class StatsProjectionOutboxWriter {
+    private final JdbcTemplate jdbc;
+    private final ObjectMapper objectMapper;
+    private final AtomicLong events = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+
+    @Value("${stats.projection-async.enabled:true}") private boolean enabled;
+    @Value("${stats.projection-async.outbox-table:stats_projection_outbox}") private String outboxTable;
+
+    public StatsProjectionOutboxWriter(JdbcTemplate jdbc, ObjectMapper objectMapper) {
+        this.jdbc = jdbc;
+        this.objectMapper = objectMapper.copy();
+        this.objectMapper.deactivateDefaultTyping();
+    }
+
+    public boolean isEnabled() { return enabled; }
+
+    public int append(StatsProjectionOutboxEvent event) {
+        if (!enabled || event == null || event.isEmpty()) return 0;
+        if (!TransactionSynchronizationManager.isActualTransactionActive()) {
+            throw new IllegalStateException("Stats projection outbox must be written inside a transaction");
+        }
+        event.validate();
+        try {
+            int inserted = jdbc.update("INSERT INTO " + safeTable(outboxTable) +
+                            " (event_id,event_schema,payload,status,attempt_count,next_attempt_at," +
+                            "created_at,updated_at) VALUES (?,?,?,'PENDING',0,SYSDATE,SYSDATE,SYSDATE)",
+                    event.getEventId(), event.getSchema(), objectMapper.writeValueAsString(event));
+            events.incrementAndGet();
+            return inserted;
+        } catch (JsonProcessingException serializationFailure) {
+            failures.incrementAndGet();
+            throw new IllegalStateException("Failed to serialize stats projection outbox event", serializationFailure);
+        } catch (RuntimeException databaseFailure) {
+            failures.incrementAndGet();
+            throw databaseFailure;
+        }
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("enabled", enabled);
+        result.put("outbox_table", outboxTable);
+        result.put("events_written_total", events.get());
+        result.put("write_failures_total", failures.get());
+        return result;
+    }
+
+    static String safeTable(String table) {
+        if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
+            throw new IllegalStateException("Invalid stats projection table: " + table);
+        }
+        return table;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java

@@ -9,6 +9,7 @@ import com.common.mapper.SyslogNonNormalMessageMapper;
 import com.common.service.SyslogNonNormalMessageService;
 import com.common.service.StatsChangeCaptureService;
 import com.common.service.DirectStatsProjectionWriter;
+import com.common.service.DmLargeTextLimiter;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -40,6 +41,9 @@ public class SyslogNonNormalMessageServiceImpl
     @Autowired
     private DirectStatsProjectionWriter directStatsProjectionWriter;
 
+    @Autowired(required = false)
+    private DmLargeTextLimiter largeTextLimiter;
+
     @Value("${dm.non-normal-write.batch-size:500}")
     private int nonNormalBatchSize;
 
@@ -47,6 +51,7 @@ public class SyslogNonNormalMessageServiceImpl
     @Transactional(rollbackFor = Exception.class)
     public boolean saveMessage(SyslogNonNormalMessage message) {
         try {
+            truncateLargeFields(message);
             // 设置ID（如果未提供）
             if (message.getId() == null || message.getId().isEmpty()) {
                 message.setId(generateMessageId());
@@ -101,6 +106,7 @@ public class SyslogNonNormalMessageServiceImpl
         try {
             // 为每条记录设置默认值
             for (SyslogNonNormalMessage message : messages) {
+                truncateLargeFields(message);
                 if (message.getId() == null || message.getId().isEmpty()) {
                     message.setId(generateMessageId());
                 }
@@ -159,10 +165,15 @@ public class SyslogNonNormalMessageServiceImpl
         }
     }
 
+    private void truncateLargeFields(SyslogNonNormalMessage message) {
+        if (largeTextLimiter != null) largeTextLimiter.truncateNonNormal(message);
+    }
+
     @Override
     @Transactional(rollbackFor = Exception.class)
     public boolean updateMessage(SyslogNonNormalMessage message) {
         try {
+            truncateLargeFields(message);
             SyslogNonNormalMessage oldValue = getMessageById(message.getId());
             // 设置更新时间
             message.setUpdateTime(LocalDateTime.now());

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/Utf8TextTruncator.java

@@ -0,0 +1,88 @@
+package com.common.util;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+
+/** UTF-8 字节预算截断器；标记属于最终存储值的一部分，且永不切断 Unicode 字符。 */
+public final class Utf8TextTruncator {
+    private static final String MARKER_FORMAT = "...[HB_XDR_TRUNCATED original_utf8_bytes=%d]";
+
+    private Utf8TextTruncator() { }
+
+    public static Result truncate(String text, long maxUtf8Bytes) {
+        if (text == null) return new Result(null, false, 0L, 0L, null);
+        if (maxUtf8Bytes < 0L) throw new IllegalArgumentException("maxUtf8Bytes 不能小于 0");
+
+        byte[] original = text.getBytes(StandardCharsets.UTF_8);
+        long originalBytes = original.length;
+        if (maxUtf8Bytes == 0L || originalBytes <= maxUtf8Bytes) {
+            return new Result(text, false, originalBytes, originalBytes, null);
+        }
+
+        String marker = String.format(MARKER_FORMAT, originalBytes);
+        int markerBytes = marker.getBytes(StandardCharsets.UTF_8).length;
+        if (maxUtf8Bytes < markerBytes) {
+            throw new IllegalArgumentException("UTF-8 截断上限小于截断标记长度: limit="
+                    + maxUtf8Bytes + ", markerBytes=" + markerBytes);
+        }
+        long prefixBudget = maxUtf8Bytes - markerBytes;
+        int end = prefixEnd(text, prefixBudget);
+        String stored = text.substring(0, end) + marker;
+        return new Result(stored, true, originalBytes,
+                stored.getBytes(StandardCharsets.UTF_8).length, sha256(original));
+    }
+
+    private static int prefixEnd(String text, long byteBudget) {
+        long used = 0L;
+        int index = 0;
+        while (index < text.length()) {
+            int codePoint = text.codePointAt(index);
+            int width = utf8Width(codePoint);
+            if (used + width > byteBudget) break;
+            used += width;
+            index += Character.charCount(codePoint);
+        }
+        return index;
+    }
+
+    private static int utf8Width(int codePoint) {
+        if (codePoint <= 0x7f) return 1;
+        if (codePoint <= 0x7ff) return 2;
+        if (codePoint <= 0xffff) return 3;
+        return 4;
+    }
+
+    private static String sha256(byte[] bytes) {
+        try {
+            byte[] digest = MessageDigest.getInstance("SHA-256").digest(bytes);
+            StringBuilder hex = new StringBuilder(digest.length * 2);
+            for (byte item : digest) hex.append(String.format("%02x", item & 0xff));
+            return hex.toString();
+        } catch (Exception impossible) {
+            throw new IllegalStateException("JVM 不支持 SHA-256", impossible);
+        }
+    }
+
+    public static final class Result {
+        private final String value;
+        private final boolean truncated;
+        private final long originalUtf8Bytes;
+        private final long storedUtf8Bytes;
+        private final String originalSha256;
+
+        Result(String value, boolean truncated, long originalUtf8Bytes,
+               long storedUtf8Bytes, String originalSha256) {
+            this.value = value;
+            this.truncated = truncated;
+            this.originalUtf8Bytes = originalUtf8Bytes;
+            this.storedUtf8Bytes = storedUtf8Bytes;
+            this.originalSha256 = originalSha256;
+        }
+
+        public String getValue() { return value; }
+        public boolean isTruncated() { return truncated; }
+        public long getOriginalUtf8Bytes() { return originalUtf8Bytes; }
+        public long getStoredUtf8Bytes() { return storedUtf8Bytes; }
+        public String getOriginalSha256() { return originalSha256; }
+    }
+}