fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁

- 将设备、规则和采集任务业务统计改为专用 Kafka Topic 异步物化
- 使用维度 ID 作为 Kafka key，保证同一统计维度串行更新
- 物化过程读取小型统计表精确值，确保重复消费不会重复计数
- 增加提交后发布、失败重试及定时小表对账补偿
- 固定日统计聚合键写入顺序，完善达梦死锁识别和事务重试
- JSON 解析失败直接转为非标日志，避免进入 DLT 重试链路
- 非标日志使用独立异步队列，不再阻塞正常消息 Kafka ACK
- Kafka 升级脚本新增 STATS.MATERIALIZE Topic
- 增加统计物化状态指标、配置示例及单元测试

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java

@@ -11,6 +11,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.support.TransactionSynchronizationManager;
 
 import java.sql.Date;
+import java.sql.SQLException;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
@@ -61,6 +62,7 @@ public class DirectStatsProjectionWriter {
     private final JdbcTemplate jdbc;
     private final StatsWriteModeProvider modeProvider;
     private final DirectStatsWriteMetrics metrics;
+    private final StatsMaterializationPublisher materializationPublisher;
 
     @Value("${stats.incremental.read-enabled:false}")
     private boolean readEnabled;
@@ -88,10 +90,12 @@ public class DirectStatsProjectionWriter {
 
     public DirectStatsProjectionWriter(JdbcTemplate jdbc,
                                        StatsWriteModeProvider modeProvider,
-                                       DirectStatsWriteMetrics metrics) {
+                                       DirectStatsWriteMetrics metrics,
+                                       StatsMaterializationPublisher materializationPublisher) {
         this.jdbc = jdbc;
         this.modeProvider = modeProvider;
         this.metrics = metrics;
+        this.materializationPublisher = materializationPublisher;
     }
 
     public WriteResult writeStandardizedRows(List<Map<String, Object>> rows) {
@@ -154,6 +158,8 @@ public class DirectStatsProjectionWriter {
         status.put("device_table", selectedDeviceTable());
         status.put("rule_table", selectedRuleTable());
         status.put("today_stats_date_enabled", todayStatsDateEnabled);
+        status.put("business_materialization_mode", asynchronousMaterialization()
+                ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS");
         return status;
     }
 
@@ -202,8 +208,12 @@ public class DirectStatsProjectionWriter {
 
             if (modeProvider.materializesBusinessFields() && readEnabled) {
                 long materializeStarted = System.nanoTime();
-                materializeDevices(devices);
-                materializeRules(rules);
+                if (asynchronousMaterialization()) {
+                    materializationPublisher.enqueueAfterCommit(materializationCommands(devices, rules));
+                } else {
+                    materializeDevices(devices);
+                    materializeRules(rules);
+                }
                 materializeMs = elapsedMs(materializeStarted);
             }
             metrics.recordSuccess(accumulator.getInputRows(), devices.size(), rules.size(),
@@ -216,7 +226,7 @@ public class DirectStatsProjectionWriter {
                     aggregateMs, mergeMs, materializeMs);
         } catch (RuntimeException failure) {
             metrics.recordFailure();
-            if (isConcurrentWrite(failure)) {
+            if (isConcurrentFailure(failure)) {
                 throw new ConcurrentStatsWriteException("直接统计聚合键并发写入失败，可安全回滚后有限重试", failure);
             }
             throw new DirectStatsWriteException("直接统计写入失败，事实事务必须整体回滚", failure);
@@ -261,7 +271,11 @@ public class DirectStatsProjectionWriter {
             long materializeMs = 0L;
             if (modeProvider.materializesBusinessFields() && readEnabled) {
                 long materializeStarted = System.nanoTime();
-                materializeDeviceKeys(keys);
+                if (asynchronousMaterialization()) {
+                    materializationPublisher.enqueueAfterCommit(deviceMaterializationCommands(keys));
+                } else {
+                    materializeDeviceKeys(keys);
+                }
                 materializeMs = elapsedMs(materializeStarted);
             }
             metrics.recordSuccess(accumulator.getInputRows(), keys.size(), 0,
@@ -270,7 +284,7 @@ public class DirectStatsProjectionWriter {
                     aggregateMs, mergeMs, materializeMs);
         } catch (RuntimeException failure) {
             metrics.recordFailure();
-            if (isConcurrentWrite(failure)) {
+            if (isConcurrentFailure(failure)) {
                 throw new ConcurrentStatsWriteException("直接统计局部分组并发写入失败，可安全回滚后有限重试", failure);
             }
             throw new DirectStatsWriteException("直接统计局部分组重算失败，事实事务必须整体回滚", failure);
@@ -389,6 +403,41 @@ public class DirectStatsProjectionWriter {
                         "WHERE id=? AND del_flag='0'", parameters);
     }
 
+    private boolean asynchronousMaterialization() {
+        return modeProvider.materializesBusinessFields() && readEnabled
+                && materializationPublisher.isEnabled();
+    }
+
+    private List<StatsMaterializationCommand> materializationCommands(
+            List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> devices,
+            List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> rules) {
+        List<StatsMaterializationCommand> commands = new ArrayList<>(devices.size() + rules.size());
+        LocalDate today = LocalDate.now(ZoneId.of(businessZone));
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry : devices) {
+            DirectStatsAccumulator.DailyKey key = entry.getKey();
+            if (today.equals(key.getDate())) {
+                commands.add(StatsMaterializationCommand.device(key.getDate(), key.getDimensionId()));
+            }
+        }
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry : rules) {
+            DirectStatsAccumulator.DailyKey key = entry.getKey();
+            commands.add(StatsMaterializationCommand.rule(key.getDate(), key.getDimensionId()));
+        }
+        return commands;
+    }
+
+    private List<StatsMaterializationCommand> deviceMaterializationCommands(
+            List<DirectStatsAccumulator.DailyKey> keys) {
+        List<StatsMaterializationCommand> commands = new ArrayList<>(keys.size());
+        LocalDate today = LocalDate.now(ZoneId.of(businessZone));
+        for (DirectStatsAccumulator.DailyKey key : keys) {
+            if (today.equals(key.getDate())) {
+                commands.add(StatsMaterializationCommand.device(key.getDate(), key.getDimensionId()));
+            }
+        }
+        return commands;
+    }
+
     private String selectedDeviceTable() {
         return modeProvider.writesShadowProjection() ? shadowDeviceTable : directDeviceTable;
     }
@@ -410,7 +459,12 @@ public class DirectStatsProjectionWriter {
         }
     }
 
-    private static boolean isConcurrentWrite(Throwable failure) {
+    /**
+     * 识别 Spring 与不同 JDBC 驱动对并发冲突的各种包装形式。
+     * 达梦可能把死锁翻译成 DataIntegrityViolationException，并仅在内层
+     * BatchUpdateException 的 SQLState/消息中保留 Deadlock 信息。
+     */
+    public static boolean isConcurrentFailure(Throwable failure) {
         Throwable current = failure;
         while (current != null) {
             if (current instanceof DuplicateKeyException
@@ -418,6 +472,18 @@ public class DirectStatsProjectionWriter {
                     || current instanceof DeadlockLoserDataAccessException) {
                 return true;
             }
+            if (current instanceof SQLException) {
+                String sqlState = ((SQLException) current).getSQLState();
+                if (sqlState != null && sqlState.startsWith("40")) {
+                    return true;
+                }
+            }
+            String message = String.valueOf(current.getMessage())
+                    .toLowerCase(java.util.Locale.ROOT);
+            if (message.contains("deadlock") || message.contains("死锁")
+                    || message.contains("unique constraint") || message.contains("duplicate key")) {
+                return true;
+            }
             current = current.getCause();
         }
         return false;

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java

@@ -2,9 +2,6 @@ package com.common.service;
 
 import com.common.entity.IngressStatsEvent;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.dao.DuplicateKeyException;
-import org.springframework.dao.CannotAcquireLockException;
-import org.springframework.dao.DeadlockLoserDataAccessException;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.PlatformTransactionManager;
@@ -15,6 +12,7 @@ import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.util.Comparator;
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -30,6 +28,7 @@ public class IngressStatsProjectionWriter {
     private final JdbcTemplate jdbc;
     private final PlatformTransactionManager transactionManager;
     private final StatsWriteModeProvider modeProvider;
+    private final StatsMaterializationPublisher materializationPublisher;
 
     @Value("${stats.ingress-metadata.enabled:false}")
     private boolean enabled;
@@ -50,10 +49,12 @@ public class IngressStatsProjectionWriter {
 
     public IngressStatsProjectionWriter(JdbcTemplate jdbc,
                                         PlatformTransactionManager transactionManager,
-                                        StatsWriteModeProvider modeProvider) {
+                                        StatsWriteModeProvider modeProvider,
+                                        StatsMaterializationPublisher materializationPublisher) {
         this.jdbc = jdbc;
         this.transactionManager = transactionManager;
         this.modeProvider = modeProvider;
+        this.materializationPublisher = materializationPublisher;
     }
 
     public int writeBatch(List<IngressStatsEvent> events) {
@@ -89,6 +90,8 @@ public class IngressStatsProjectionWriter {
         status.put("read_enabled", readEnabled);
         status.put("target_device_table", modeProvider.writesShadowProjection()
                 ? "device_daily_stats+" + shadowDeviceTable : "device_daily_stats");
+        status.put("business_materialization_mode", asynchronousMaterialization()
+                ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS");
         status.put("successful_batches", successfulBatches.get());
         status.put("received_events", receivedEvents.get());
         status.put("updated_groups", updatedGroups.get());
@@ -99,9 +102,7 @@ public class IngressStatsProjectionWriter {
     }
 
     private static boolean retryableConcurrentWrite(RuntimeException failure) {
-        return failure instanceof DuplicateKeyException
-                || failure instanceof CannotAcquireLockException
-                || failure instanceof DeadlockLoserDataAccessException;
+        return DirectStatsProjectionWriter.isConcurrentFailure(failure);
     }
 
     private int writeAggregation(Aggregation aggregation) {
@@ -109,6 +110,11 @@ public class IngressStatsProjectionWriter {
         transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
         Integer updated = transaction.execute(status -> {
             int groups = 0;
+            boolean asyncMaterialization = asynchronousMaterialization();
+            List<StatsMaterializationCommand> commands = asyncMaterialization
+                    ? new ArrayList<>(aggregation.devices.size() + aggregation.collects.size())
+                    : null;
+            LocalDate today = LocalDate.now(ZoneId.of(businessZone));
             for (Map.Entry<DeviceKey, LocalDateTime> entry : aggregation.devices.entrySet()) {
                 DeviceKey key = entry.getKey();
                 LocalDateTime receiveTime = entry.getValue();
@@ -118,11 +124,15 @@ public class IngressStatsProjectionWriter {
                 if (modeProvider.writesShadowProjection()) {
                     mergeDeviceReceive(safeTable(shadowDeviceTable), key, receiveTime);
                 }
-                if (readEnabled && LocalDate.now(ZoneId.of(businessZone)).equals(key.date)) {
-                    jdbc.update("UPDATE device_device SET last_receive_time=CASE " +
-                                    "WHEN last_receive_time IS NULL OR last_receive_time<? THEN ? ELSE last_receive_time END," +
-                                    "updated_at=SYSDATE WHERE id=?",
-                            receiveTime, receiveTime, key.deviceId);
+                if (readEnabled && today.equals(key.date)) {
+                    if (asyncMaterialization) {
+                        commands.add(StatsMaterializationCommand.device(key.date, key.deviceId));
+                    } else {
+                        jdbc.update("UPDATE device_device SET last_receive_time=CASE " +
+                                        "WHEN last_receive_time IS NULL OR last_receive_time<? THEN ? ELSE last_receive_time END," +
+                                        "updated_at=SYSDATE WHERE id=?",
+                                receiveTime, receiveTime, key.deviceId);
+                    }
                 }
                 groups++;
             }
@@ -142,14 +152,21 @@ public class IngressStatsProjectionWriter {
                                 "(s.device_collect_id,s.first_success_time,s.last_success_time,NULL,0,SYSDATE)",
                         entry.getKey(), times.min, times.max);
                 if (readEnabled) {
-                    jdbc.update("UPDATE device_collect_task SET " +
-                                    "first_time=CASE WHEN first_time IS NULL OR first_time>? THEN ? ELSE first_time END," +
-                                    "last_success_time=CASE WHEN last_success_time IS NULL OR last_success_time<? " +
-                                    "THEN ? ELSE last_success_time END,updated_at=SYSDATE WHERE id=?",
-                            times.min, times.min, times.max, times.max, entry.getKey());
+                    if (asyncMaterialization) {
+                        commands.add(StatsMaterializationCommand.collect(entry.getKey()));
+                    } else {
+                        jdbc.update("UPDATE device_collect_task SET " +
+                                        "first_time=CASE WHEN first_time IS NULL OR first_time>? THEN ? ELSE first_time END," +
+                                        "last_success_time=CASE WHEN last_success_time IS NULL OR last_success_time<? " +
+                                        "THEN ? ELSE last_success_time END,updated_at=SYSDATE WHERE id=?",
+                                times.min, times.min, times.max, times.max, entry.getKey());
+                    }
                 }
                 groups++;
             }
+            if (asyncMaterialization) {
+                materializationPublisher.enqueueAfterCommit(commands);
+            }
             return groups;
         });
         return updated == null ? 0 : updated;
@@ -215,6 +232,11 @@ public class IngressStatsProjectionWriter {
                 Date.valueOf(key.date), key.deviceId, receiveTime);
     }
 
+    private boolean asynchronousMaterialization() {
+        return modeProvider.materializesBusinessFields() && readEnabled
+                && materializationPublisher.isEnabled();
+    }
+
     private static String safeTable(String table) {
         if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
             throw new IllegalStateException("非法入口设备统计表名: " + table);

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/LateAlarmBucketWriter.java

@@ -10,9 +10,6 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.dao.CannotAcquireLockException;
-import org.springframework.dao.DeadlockLoserDataAccessException;
-import org.springframework.dao.DuplicateKeyException;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
 
@@ -116,20 +113,7 @@ public class LateAlarmBucketWriter {
   }
 
   private boolean isConcurrentWrite(Throwable failure) {
-    Throwable current = failure;
-    while (current != null) {
-      if (current instanceof DuplicateKeyException
-          || current instanceof CannotAcquireLockException
-          || current instanceof DeadlockLoserDataAccessException) {
-        return true;
-      }
-      String message = String.valueOf(current.getMessage()).toLowerCase(java.util.Locale.ROOT);
-      if (message.contains("unique constraint") || message.contains("duplicate key")) {
-        return true;
-      }
-      current = current.getCause();
-    }
-    return false;
+    return DirectStatsProjectionWriter.isConcurrentFailure(failure);
   }
 
   private static final class BucketKey {

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsMaterializationCommand.java

@@ -0,0 +1,63 @@
+package com.common.service;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.time.LocalDate;
+import java.util.Objects;
+
+/** 统计物化脏键；通知只要求复制当前精确值，不携带会被重复累加的 delta。 */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class StatsMaterializationCommand {
+    public static final String SCHEMA = "SYSLOG_STATS_MATERIALIZE_V1";
+    public enum Type { DEVICE, RULE, COLLECT }
+
+    private String schema = SCHEMA;
+    private Type type;
+    private long dimensionId;
+    private String statDate;
+
+    public StatsMaterializationCommand() {}
+
+    private StatsMaterializationCommand(Type type, long dimensionId, LocalDate statDate) {
+        this.type = Objects.requireNonNull(type, "type");
+        this.dimensionId = dimensionId;
+        this.statDate = statDate == null ? null : statDate.toString();
+    }
+
+    public static StatsMaterializationCommand device(LocalDate date, long id) {
+        return new StatsMaterializationCommand(Type.DEVICE, id, date);
+    }
+    public static StatsMaterializationCommand rule(LocalDate date, long id) {
+        return new StatsMaterializationCommand(Type.RULE, id, date);
+    }
+    public static StatsMaterializationCommand collect(long id) {
+        return new StatsMaterializationCommand(Type.COLLECT, id, null);
+    }
+
+    public void validate() {
+        if (!SCHEMA.equals(schema)) throw new IllegalArgumentException("不支持的统计物化协议: " + schema);
+        if (type == null || dimensionId <= 0L) throw new IllegalArgumentException("统计物化命令缺少合法维度");
+        if (type != Type.COLLECT) date();
+    }
+
+    public String key() {
+        validate();
+        return type.name() + ":" + dimensionId;
+    }
+
+    public LocalDate date() {
+        if (statDate == null || statDate.trim().isEmpty()) {
+            throw new IllegalArgumentException("统计物化命令缺少 statDate");
+        }
+        return LocalDate.parse(statDate);
+    }
+
+    public String getSchema() { return schema; }
+    public void setSchema(String schema) { this.schema = schema; }
+    public Type getType() { return type; }
+    public void setType(Type type) { this.type = type; }
+    public long getDimensionId() { return dimensionId; }
+    public void setDimensionId(long dimensionId) { this.dimensionId = dimensionId; }
+    public String getStatDate() { return statDate; }
+    public void setStatDate(String statDate) { this.statDate = statDate; }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsMaterializationConsumer.java

@@ -0,0 +1,255 @@
+package com.common.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.kafka.annotation.KafkaListener;
+import org.springframework.kafka.support.Acknowledgment;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionTemplate;
+
+import java.sql.Date;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * 消费统计脏键并从小型聚合表复制精确值到业务表。
+ *
+ * <p>Topic key 是维度类型和 ID；Kafka 保证同一维度只落在一个分区，消费组保证同一时刻只由
+ * 一个实例处理该分区。命令不携带 delta，因此重复投递和故障重放不会造成重复计数。</p>
+ */
+@Slf4j
+@Component
+public class StatsMaterializationConsumer {
+    private static final String DEVICE_MATERIALIZE_WITH_DATE =
+            "MERGE INTO device_device d USING (SELECT stat_date,device_id,parse_count,non_log_count," +
+            "last_receive_time FROM %s WHERE stat_date=? AND device_id=?) s " +
+            "ON (d.id=s.device_id) WHEN MATCHED THEN UPDATE SET " +
+            "d.today_parse_count=s.parse_count,d.today_non_log_count=s.non_log_count," +
+            "d.today_stats_date=s.stat_date,d.last_receive_time=CASE " +
+            "WHEN s.last_receive_time IS NULL THEN d.last_receive_time " +
+            "WHEN d.last_receive_time IS NULL OR d.last_receive_time<s.last_receive_time " +
+            "THEN s.last_receive_time ELSE d.last_receive_time END,d.updated_at=SYSDATE";
+
+    private static final String DEVICE_MATERIALIZE_WITHOUT_DATE =
+            "MERGE INTO device_device d USING (SELECT stat_date,device_id,parse_count,non_log_count," +
+            "last_receive_time FROM %s WHERE stat_date=? AND device_id=?) s " +
+            "ON (d.id=s.device_id) WHEN MATCHED THEN UPDATE SET " +
+            "d.today_parse_count=s.parse_count,d.today_non_log_count=s.non_log_count," +
+            "d.last_receive_time=CASE WHEN s.last_receive_time IS NULL THEN d.last_receive_time " +
+            "WHEN d.last_receive_time IS NULL OR d.last_receive_time<s.last_receive_time " +
+            "THEN s.last_receive_time ELSE d.last_receive_time END,d.updated_at=SYSDATE";
+
+    private static final String RULE_MATERIALIZE =
+            "MERGE INTO dm_normalize_rule r USING (SELECT normalize_rule_id," +
+            "MAX(max_log_time) max_log_time FROM %s WHERE normalize_rule_id=? " +
+            "GROUP BY normalize_rule_id) s ON (r.id=s.normalize_rule_id AND r.del_flag='0') " +
+            "WHEN MATCHED THEN UPDATE SET r.first_data_saved_at=CASE " +
+            "WHEN s.max_log_time IS NULL THEN r.first_data_saved_at " +
+            "WHEN r.first_data_saved_at IS NULL OR r.first_data_saved_at<s.max_log_time " +
+            "THEN s.max_log_time ELSE r.first_data_saved_at END,r.updated_at=SYSDATE";
+
+    private static final String COLLECT_MATERIALIZE =
+            "MERGE INTO device_collect_task c USING (SELECT device_collect_id,first_success_time," +
+            "last_success_time FROM device_collect_stats WHERE device_collect_id=?) s " +
+            "ON (c.id=s.device_collect_id) WHEN MATCHED THEN UPDATE SET " +
+            "c.first_time=CASE WHEN s.first_success_time IS NULL THEN c.first_time " +
+            "WHEN c.first_time IS NULL OR c.first_time>s.first_success_time " +
+            "THEN s.first_success_time ELSE c.first_time END," +
+            "c.last_success_time=CASE WHEN s.last_success_time IS NULL THEN c.last_success_time " +
+            "WHEN c.last_success_time IS NULL OR c.last_success_time<s.last_success_time " +
+            "THEN s.last_success_time ELSE c.last_success_time END,c.updated_at=SYSDATE";
+
+    private final JdbcTemplate jdbc;
+    private final PlatformTransactionManager transactionManager;
+    private final ObjectMapper objectMapper;
+    private final StatsWriteModeProvider modeProvider;
+
+    @Value("${stats.incremental.read-enabled:false}") private boolean readEnabled;
+    @Value("${stats.direct.device-table:device_daily_stats}") private String deviceTable;
+    @Value("${stats.direct.rule-table:normalize_rule_daily_stats}") private String ruleTable;
+    @Value("${stats.direct.today-stats-date-enabled:true}") private boolean todayStatsDateEnabled;
+    @Value("${stats.business-zone:Asia/Shanghai}") private String businessZone;
+    @Value("${stats.incremental.transaction-timeout-seconds:45}") private int transactionTimeoutSeconds;
+    @Value("${stats.materialization.max-concurrency-retries:3}") private int maxConcurrencyRetries;
+    @Value("${stats.materialization.concurrency-retry-backoff-ms:50}") private long retryBackoffMs;
+
+    private final AtomicLong consumed = new AtomicLong();
+    private final AtomicLong invalid = new AtomicLong();
+    private final AtomicLong applied = new AtomicLong();
+    private final AtomicLong successfulBatches = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+    private final AtomicLong retries = new AtomicLong();
+    private volatile LocalDateTime lastSuccessTime;
+    private volatile LocalDateTime lastFailureTime;
+
+    public StatsMaterializationConsumer(JdbcTemplate jdbc,
+                                        PlatformTransactionManager transactionManager,
+                                        ObjectMapper objectMapper,
+                                        StatsWriteModeProvider modeProvider) {
+        this.jdbc = jdbc;
+        this.transactionManager = transactionManager;
+        this.objectMapper = objectMapper.copy();
+        this.objectMapper.deactivateDefaultTyping();
+        this.modeProvider = modeProvider;
+    }
+
+    @KafkaListener(
+            id = "syslog-stats-materializer",
+            groupId = "${stats.materialization.group-id:${spring.kafka.consumer.group-id}-stats-materialize}",
+            topics = "${stats.materialization.topic:${spring.kafka.consumer.topic}.STATS.MATERIALIZE}",
+            autoStartup = "${stats.materialization.enabled:true}",
+            containerFactory = "statsMaterializationKafkaListenerContainerFactory"
+    )
+    public void consume(List<ConsumerRecord<String, String>> records, Acknowledgment acknowledgment) {
+        if (records == null || records.isEmpty()) {
+            acknowledgment.acknowledge();
+            return;
+        }
+        consumed.addAndGet(records.size());
+        Map<String, StatsMaterializationCommand> commands = parseAndCoalesce(records);
+        if (commands.isEmpty()) {
+            acknowledgment.acknowledge();
+            return;
+        }
+        try {
+            int count = materializeWithRetry(commands.values());
+            acknowledgment.acknowledge();
+            applied.addAndGet(count);
+            successfulBatches.incrementAndGet();
+            lastSuccessTime = LocalDateTime.now();
+            log.info("统计物化批次完成: received={}, coalesced={}, applied={}, invalidTotal={}",
+                    records.size(), commands.size(), count, invalid.get());
+        } catch (RuntimeException failure) {
+            failures.incrementAndGet();
+            lastFailureTime = LocalDateTime.now();
+            throw failure;
+        }
+    }
+
+    private Map<String, StatsMaterializationCommand> parseAndCoalesce(
+            List<ConsumerRecord<String, String>> records) {
+        Map<String, StatsMaterializationCommand> result = new TreeMap<>();
+        for (ConsumerRecord<String, String> record : records) {
+            try {
+                StatsMaterializationCommand command = objectMapper.readValue(
+                        record.value(), StatsMaterializationCommand.class);
+                command.validate();
+                String key = command.key();
+                if (record.key() == null || !key.equals(record.key())) {
+                    throw new IllegalArgumentException("Kafka key 与统计物化命令不一致");
+                }
+                result.put(key, command);
+            } catch (Exception badCommand) {
+                invalid.incrementAndGet();
+                log.error("忽略非法内部统计物化命令: topic={}, partition={}, offset={}, key={}",
+                        record.topic(), record.partition(), record.offset(), record.key(), badCommand);
+            }
+        }
+        return result;
+    }
+
+    int materializeWithRetry(Collection<StatsMaterializationCommand> commands) {
+        int attempt = 0;
+        while (true) {
+            try {
+                return materializeOnce(commands);
+            } catch (RuntimeException failure) {
+                if (!DirectStatsProjectionWriter.isConcurrentFailure(failure)
+                        || attempt >= Math.max(0, maxConcurrencyRetries)) {
+                    throw failure;
+                }
+                attempt++;
+                retries.incrementAndGet();
+                sleepBeforeRetry(attempt);
+            }
+        }
+    }
+
+    private int materializeOnce(Collection<StatsMaterializationCommand> commands) {
+        if (!modeProvider.materializesBusinessFields() || !readEnabled) {
+            throw new IllegalStateException("统计物化消费者只允许在 DIRECT 且 read-enabled 模式运行");
+        }
+        final LocalDate today = LocalDate.now(ZoneId.of(businessZone));
+        final List<Object[]> deviceParams = new ArrayList<>();
+        final List<Object[]> ruleParams = new ArrayList<>();
+        final List<Object[]> collectParams = new ArrayList<>();
+        for (StatsMaterializationCommand command : commands) {
+            command.validate();
+            switch (command.getType()) {
+                case DEVICE:
+                    // yesterday 的迟到事实只保存在日统计表，不允许覆盖今日展示字段。
+                    if (today.equals(command.date())) {
+                        deviceParams.add(new Object[]{Date.valueOf(today), command.getDimensionId()});
+                    }
+                    break;
+                case RULE:
+                    // 按 rule id 从所有日统计取 MAX，命令即使跨日期合并也不会丢失较大值。
+                    ruleParams.add(new Object[]{command.getDimensionId()});
+                    break;
+                case COLLECT:
+                    collectParams.add(new Object[]{command.getDimensionId()});
+                    break;
+                default:
+                    throw new IllegalArgumentException("未知统计物化命令类型: " + command.getType());
+            }
+        }
+        TransactionTemplate transaction = new TransactionTemplate(transactionManager);
+        transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
+        Integer count = transaction.execute(status -> {
+            if (!deviceParams.isEmpty()) {
+                String template = todayStatsDateEnabled
+                        ? DEVICE_MATERIALIZE_WITH_DATE : DEVICE_MATERIALIZE_WITHOUT_DATE;
+                jdbc.batchUpdate(String.format(template, safeTable(deviceTable)), deviceParams);
+            }
+            if (!ruleParams.isEmpty()) {
+                jdbc.batchUpdate(String.format(RULE_MATERIALIZE, safeTable(ruleTable)), ruleParams);
+            }
+            if (!collectParams.isEmpty()) {
+                jdbc.batchUpdate(COLLECT_MATERIALIZE, collectParams);
+            }
+            return deviceParams.size() + ruleParams.size() + collectParams.size();
+        });
+        return count == null ? 0 : count;
+    }
+
+    private void sleepBeforeRetry(int attempt) {
+        try {
+            Thread.sleep(Math.max(0L, retryBackoffMs) * attempt);
+        } catch (InterruptedException interrupted) {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException("统计物化死锁重试等待被中断", interrupted);
+        }
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("consumed_total", consumed.get());
+        result.put("invalid_total", invalid.get());
+        result.put("applied_total", applied.get());
+        result.put("successful_batches", successfulBatches.get());
+        result.put("failures_total", failures.get());
+        result.put("concurrency_retries_total", retries.get());
+        result.put("last_success_time", lastSuccessTime);
+        result.put("last_failure_time", lastFailureTime);
+        return result;
+    }
+
+    private static String safeTable(String table) {
+        if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
+            throw new IllegalStateException("非法统计物化源表名: " + table);
+        }
+        return table;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsMaterializationPublisher.java

@@ -0,0 +1,128 @@
+package com.common.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.SendResult;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.support.TransactionSynchronizationAdapter;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+import org.springframework.util.concurrent.ListenableFuture;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** 事务提交后合并统计脏键并批量发布；失败的 key 留在内存等待下轮，小表校准负责崩溃窗口。 */
+@Slf4j
+@Component
+public class StatsMaterializationPublisher {
+    private final KafkaTemplate<String, String> kafkaTemplate;
+    private final ObjectMapper objectMapper;
+    private final ConcurrentMap<String, StatsMaterializationCommand> pending = new ConcurrentHashMap<>();
+    private final AtomicBoolean flushing = new AtomicBoolean();
+    private final AtomicLong enqueued = new AtomicLong();
+    private final AtomicLong published = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+    private volatile long lastSuccessAt;
+    private volatile long lastFailureAt;
+
+    @Value("${stats.materialization.enabled:true}") private boolean enabled;
+    @Value("${stats.materialization.topic:${spring.kafka.consumer.topic}.STATS.MATERIALIZE}") private String topic;
+    @Value("${stats.materialization.publish-timeout-ms:10000}") private long publishTimeoutMs;
+    @Value("${stats.materialization.publish-max-batch-size:1000}") private int maxBatchSize;
+
+    public StatsMaterializationPublisher(KafkaTemplate<String, String> kafkaTemplate, ObjectMapper objectMapper) {
+        this.kafkaTemplate = kafkaTemplate;
+        this.objectMapper = objectMapper.copy();
+        this.objectMapper.deactivateDefaultTyping();
+    }
+
+    public boolean isEnabled() { return enabled; }
+
+    public void enqueueAfterCommit(Collection<StatsMaterializationCommand> commands) {
+        if (!enabled || commands == null || commands.isEmpty()) return;
+        if (!TransactionSynchronizationManager.isSynchronizationActive()) {
+            throw new IllegalStateException("统计物化通知必须在 Spring 事务内登记");
+        }
+        List<StatsMaterializationCommand> snapshot = new ArrayList<>(commands);
+        TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() {
+            @Override public void afterCommit() { enqueueNow(snapshot); }
+        });
+    }
+
+    public void enqueueNow(Collection<StatsMaterializationCommand> commands) {
+        if (!enabled || commands == null) return;
+        for (StatsMaterializationCommand command : commands) {
+            if (command == null) continue;
+            pending.put(command.key(), command);
+            enqueued.incrementAndGet();
+        }
+    }
+
+    @Scheduled(fixedDelayString = "${stats.materialization.publish-interval-ms:200}")
+    public void flush() {
+        if (!enabled || pending.isEmpty() || !flushing.compareAndSet(false, true)) return;
+        try {
+            List<StatsMaterializationCommand> snapshot = new ArrayList<>(pending.values());
+            snapshot.sort(Comparator.comparing(StatsMaterializationCommand::key));
+            int size = Math.min(snapshot.size(), Math.max(1, maxBatchSize));
+            List<PendingSend> sends = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                StatsMaterializationCommand command = snapshot.get(i);
+                sends.add(new PendingSend(command, kafkaTemplate.send(
+                        topic, command.key(), objectMapper.writeValueAsString(command))));
+            }
+            long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(Math.max(1L, publishTimeoutMs));
+            for (PendingSend send : sends) {
+                long remaining = deadline - System.nanoTime();
+                if (remaining <= 0L) throw new java.util.concurrent.TimeoutException("统计物化通知发布超时");
+                send.future.get(remaining, TimeUnit.NANOSECONDS);
+                if (pending.remove(send.command.key(), send.command)) published.incrementAndGet();
+            }
+            lastSuccessAt = System.currentTimeMillis();
+        } catch (InterruptedException interrupted) {
+            Thread.currentThread().interrupt();
+            recordFailure(interrupted);
+        } catch (Exception failure) {
+            recordFailure(failure);
+        } finally {
+            flushing.set(false);
+        }
+    }
+
+    private void recordFailure(Exception failure) {
+        failures.incrementAndGet();
+        lastFailureAt = System.currentTimeMillis();
+        log.error("统计物化通知发布失败，脏键保留等待重试: pending={}", pending.size(), failure);
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("enabled", enabled); result.put("topic", topic);
+        result.put("pending_keys", pending.size()); result.put("enqueued_total", enqueued.get());
+        result.put("published_total", published.get()); result.put("publish_failures_total", failures.get());
+        result.put("last_success_epoch_ms", lastSuccessAt == 0L ? null : lastSuccessAt);
+        result.put("last_failure_epoch_ms", lastFailureAt == 0L ? null : lastFailureAt);
+        return result;
+    }
+
+    private static final class PendingSend {
+        private final StatsMaterializationCommand command;
+        private final ListenableFuture<SendResult<String, String>> future;
+        private PendingSend(StatsMaterializationCommand command,
+                            ListenableFuture<SendResult<String, String>> future) {
+            this.command = command; this.future = future;
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsMaterializationReconciler.java

@@ -0,0 +1,154 @@
+package com.common.service;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.sql.Date;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * 扫描三个小型聚合表与业务展示字段的差异，补偿“数据库已提交、进程却在发布通知前退出”的窗口。
+ * 不扫描 syslog 事实宽表，不参与事实消费事务。
+ */
+@Slf4j
+@Component
+public class StatsMaterializationReconciler {
+    private final JdbcTemplate jdbc;
+    private final StatsWriteModeProvider modeProvider;
+    private final StatsMaterializationPublisher publisher;
+    private final AtomicBoolean running = new AtomicBoolean();
+    private final AtomicLong runs = new AtomicLong();
+    private final AtomicLong detected = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+    private volatile LocalDateTime lastSuccessTime;
+    private volatile LocalDateTime lastFailureTime;
+
+    @Value("${stats.materialization.enabled:true}") private boolean enabled;
+    @Value("${stats.incremental.read-enabled:false}") private boolean readEnabled;
+    @Value("${stats.direct.device-table:device_daily_stats}") private String deviceTable;
+    @Value("${stats.direct.rule-table:normalize_rule_daily_stats}") private String ruleTable;
+    @Value("${stats.business-zone:Asia/Shanghai}") private String businessZone;
+
+    public StatsMaterializationReconciler(JdbcTemplate jdbc,
+                                          StatsWriteModeProvider modeProvider,
+                                          StatsMaterializationPublisher publisher) {
+        this.jdbc = jdbc;
+        this.modeProvider = modeProvider;
+        this.publisher = publisher;
+    }
+
+    @Scheduled(initialDelayString = "${stats.materialization.reconcile-initial-delay-ms:30000}",
+            fixedDelayString = "${stats.materialization.reconcile-interval-ms:60000}")
+    public void reconcile() {
+        if (!enabled || !readEnabled || !modeProvider.materializesBusinessFields()
+                || !running.compareAndSet(false, true)) {
+            return;
+        }
+        try {
+            List<StatsMaterializationCommand> commands = findDifferences();
+            publisher.enqueueNow(commands);
+            runs.incrementAndGet();
+            detected.addAndGet(commands.size());
+            lastSuccessTime = LocalDateTime.now();
+            if (!commands.isEmpty()) {
+                log.warn("统计物化对账发现差异并已重新投递: dirtyKeys={}", commands.size());
+            }
+        } catch (RuntimeException failure) {
+            failures.incrementAndGet();
+            lastFailureTime = LocalDateTime.now();
+            log.error("统计物化小表对账失败，下个周期自动重试", failure);
+        } finally {
+            running.set(false);
+        }
+    }
+
+    List<StatsMaterializationCommand> findDifferences() {
+        LocalDate today = LocalDate.now(ZoneId.of(businessZone));
+        List<StatsMaterializationCommand> commands = new ArrayList<>();
+        String safeDeviceTable = safeTable(deviceTable);
+        String safeRuleTable = safeTable(ruleTable);
+
+        List<Map<String, Object>> devices = jdbc.queryForList(
+                "SELECT s.device_id dimension_id FROM " + safeDeviceTable + " s " +
+                        "JOIN device_device d ON d.id=s.device_id WHERE s.stat_date=? AND (" +
+                        "NVL(d.today_parse_count,0)<>NVL(s.parse_count,0) OR " +
+                        "NVL(d.today_non_log_count,0)<>NVL(s.non_log_count,0) OR " +
+                        "d.today_stats_date IS NULL OR d.today_stats_date<>s.stat_date OR " +
+                        "(s.last_receive_time IS NOT NULL AND (d.last_receive_time IS NULL " +
+                        "OR d.last_receive_time<s.last_receive_time)))",
+                Date.valueOf(today));
+        for (Map<String, Object> row : devices) {
+            commands.add(StatsMaterializationCommand.device(today, longValue(row, "dimension_id")));
+        }
+
+        List<Map<String, Object>> rules = jdbc.queryForList(
+                "SELECT s.normalize_rule_id dimension_id FROM (SELECT normalize_rule_id," +
+                        "MAX(max_log_time) max_log_time FROM " + safeRuleTable +
+                        " WHERE max_log_time IS NOT NULL GROUP BY normalize_rule_id) s " +
+                        "JOIN dm_normalize_rule r ON r.id=s.normalize_rule_id AND r.del_flag='0' " +
+                        "WHERE r.first_data_saved_at IS NULL OR r.first_data_saved_at<s.max_log_time");
+        for (Map<String, Object> row : rules) {
+            commands.add(StatsMaterializationCommand.rule(today, longValue(row, "dimension_id")));
+        }
+
+        List<Map<String, Object>> collects = jdbc.queryForList(
+                "SELECT s.device_collect_id dimension_id FROM device_collect_stats s " +
+                        "JOIN device_collect_task c ON c.id=s.device_collect_id WHERE " +
+                        "(s.first_success_time IS NOT NULL AND (c.first_time IS NULL " +
+                        "OR c.first_time>s.first_success_time)) OR " +
+                        "(s.last_success_time IS NOT NULL AND (c.last_success_time IS NULL " +
+                        "OR c.last_success_time<s.last_success_time))");
+        for (Map<String, Object> row : collects) {
+            commands.add(StatsMaterializationCommand.collect(longValue(row, "dimension_id")));
+        }
+        return commands;
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("enabled", enabled);
+        result.put("runs_total", runs.get());
+        result.put("dirty_keys_detected_total", detected.get());
+        result.put("failures_total", failures.get());
+        result.put("running", running.get());
+        result.put("last_success_time", lastSuccessTime);
+        result.put("last_failure_time", lastFailureTime);
+        return result;
+    }
+
+    private static long longValue(Map<String, Object> row, String name) {
+        Object value = row.get(name);
+        if (value == null) value = row.get(name.toUpperCase(Locale.ROOT));
+        if (value == null) {
+            for (Map.Entry<String, Object> entry : row.entrySet()) {
+                if (entry.getKey().equalsIgnoreCase(name)) {
+                    value = entry.getValue();
+                    break;
+                }
+            }
+        }
+        if (!(value instanceof Number)) {
+            throw new IllegalStateException("统计物化对账结果缺少数字列 " + name + ": " + value);
+        }
+        return ((Number) value).longValue();
+    }
+
+    private static String safeTable(String table) {
+        if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
+            throw new IllegalStateException("非法统计物化对账表名: " + table);
+        }
+        return table;
+    }
+}