feat(syslog): 完成高吞吐可靠链路与统计存储优化

- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程
- 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标
- 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读
- 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换
- 完善告警处理、配置校验、运行监控及测试覆盖

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/Alarm.java

@@ -74,5 +74,10 @@ public class Alarm {
     private String[] httpReqBody;
     private String[] httpRespHeader;
     private String[] httpRespBody;
+    /** 规则告警幂等与追踪元数据。 */
+    private String ruleId;
+    private String ruleVersion;
+    private LocalDateTime windowStart;
+    private String groupKeyHash;
 
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/AlarmVisit.java

@@ -79,4 +79,9 @@ public class AlarmVisit {
     private String dtype;
     private String originField;
     private String originLog;
-}
+    /** 算法告警幂等与追踪元数据。 */
+    private String algorithmId;
+    private String algorithmVersion;
+    private String eventId;
+    private String resultType;
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceDevice.java

@@ -2,6 +2,7 @@ package com.common.entity;
 
 
 import java.time.LocalDateTime;
+import java.time.LocalDate;
 import java.time.OffsetDateTime;
 public class DeviceDevice {
     public Integer id;
@@ -38,6 +39,7 @@ public class DeviceDevice {
     public String managerName;
     public Integer todayParseCount;
     public Integer todayNonLogCount;
+    public LocalDate todayStatsDate;
     public Long createDept;
     public Integer deviceCollectId;
 
@@ -315,6 +317,14 @@ public class DeviceDevice {
         this.todayNonLogCount = todayNonLogCount;
     }
 
+    public LocalDate getTodayStatsDate() {
+        return todayStatsDate;
+    }
+
+    public void setTodayStatsDate(LocalDate todayStatsDate) {
+        this.todayStatsDate = todayStatsDate;
+    }
+
     public Long getCreateDept() {
         return createDept;
     }
@@ -368,9 +378,10 @@ public class DeviceDevice {
                 ", managerName='" + managerName + '\'' +
                 ", todayParseCount=" + todayParseCount +
                 ", todayNonLogCount=" + todayNonLogCount +
+                ", todayStatsDate=" + todayStatsDate +
                 ", createDept=" + createDept +
                 ", deviceCollectId=" + deviceCollectId +
                 '}';
     }
 
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/IngressStatsEvent.java

@@ -0,0 +1,24 @@
+package com.common.entity;
+
+import java.time.LocalDateTime;
+
+/** Kafka 入口元数据形成的轻量接收状态事件，不包含原始 Syslog 正文。 */
+public final class IngressStatsEvent {
+    private final String eventId;
+    private final Long deviceId;
+    private final Long deviceCollectId;
+    private final LocalDateTime receiveTime;
+
+    public IngressStatsEvent(String eventId, Long deviceId, Long deviceCollectId,
+                             LocalDateTime receiveTime) {
+        this.eventId = eventId;
+        this.deviceId = deviceId;
+        this.deviceCollectId = deviceCollectId;
+        this.receiveTime = receiveTime;
+    }
+
+    public String getEventId() { return eventId; }
+    public Long getDeviceId() { return deviceId; }
+    public Long getDeviceCollectId() { return deviceCollectId; }
+    public LocalDateTime getReceiveTime() { return receiveTime; }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/StatsChangeRecord.java

@@ -0,0 +1,33 @@
+package com.common.entity;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+
+/** 统计投影所需的最小事实变化，不包含原始日志正文。 */
+@Data
+@Builder
+public class StatsChangeRecord {
+    private Long changeSeq;
+    private String changeId;
+    private String sourceType;
+    private String sourceRowId;
+    private String operationType;
+    private LocalDate oldStatDate;
+    private LocalDate newStatDate;
+    private Long oldDeviceId;
+    private Long newDeviceId;
+    private Long oldCollectId;
+    private Long newCollectId;
+    private Long oldRuleId;
+    private Long newRuleId;
+    private LocalDateTime oldEventTime;
+    private LocalDateTime newEventTime;
+    private LocalDateTime oldCollectTime;
+    private LocalDateTime newCollectTime;
+    private Integer oldPushSuccess;
+    private Integer newPushSuccess;
+    private LocalDateTime committedAt;
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/SyslogNormalAlarm.java

@@ -376,8 +376,9 @@ public class SyslogNormalAlarm {
     private String destinationSystem;
     private String etlHost;
     private Integer normalizeRuleId;
+    private String ruleVersion;
     private String normalizeRuleName;
     private String syslogUuid;
     private String syslogTopic;
 
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/SyslogNormalData.java

@@ -375,7 +375,8 @@ public class SyslogNormalData {
     private String destinationSystem;
     private String etlHost;
     private Integer normalizeRuleId;
+    private String ruleVersion;
     private String normalizeRuleName;
     private String syslogUuid;
     private String syslogTopic;
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/AlarmMapper.java

@@ -8,6 +8,12 @@ import java.util.List;
 @Mapper
 public interface AlarmMapper {
 
+    @Select({"<script>",
+            "SELECT id FROM alarm WHERE id IN",
+            "<foreach collection='ids' item='id' open='(' separator=',' close=')'>#{id}</foreach>",
+            "</script>"})
+    List<String> selectExistingIds(@Param("ids") List<String> ids);
+
     /**
      * 批量插入告警数据
      */
@@ -19,7 +25,8 @@ public interface AlarmMapper {
             "attack_port, victim_port, attack_method, etl_time, log_count, ",
             "attack_chain_phase, disposition_advice, attack_direction, ",
             "judged_state, disposed_state, attack_result, fall, payload,  dns_info, engine_type,   " ,
-            "http_req_header  , http_req_body,http_resp_header , http_resp_body ",
+            "http_req_header  , http_req_body,http_resp_header , http_resp_body, ",
+            "rule_id, rule_version, window_start, group_key_hash ",
             ") VALUES ",
             "<foreach collection='list' item='item' separator=','>",
             "( #{item.id}, #{item.createdAt}, #{item.alarmName}, #{item.alarmLevel}, ",
@@ -41,7 +48,8 @@ public interface AlarmMapper {
             "#{item.httpReqHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
             "#{item.httpReqBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
             "#{item.httpRespHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
-            "#{item.httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler})",
+            "#{item.httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
+            "#{item.ruleId}, #{item.ruleVersion}, #{item.windowStart}, #{item.groupKeyHash})",
             "</foreach>",
             "</script>"})
     void batchInsert(@Param("list") List<Alarm> alarmList);
@@ -56,7 +64,8 @@ public interface AlarmMapper {
             "attack_port, victim_port, attack_method, etl_time, log_count, " +
             "attack_chain_phase, disposition_advice, attack_direction, " +
             "judged_state, disposed_state, attack_result, fall, payload, dns_info, engine_type,  " +
-            "http_req_header  , http_req_body,http_resp_header , http_resp_body " +
+            "http_req_header  , http_req_body,http_resp_header , http_resp_body, " +
+            "rule_id, rule_version, window_start, group_key_hash " +
             ") VALUES (" +
             "#{id}, #{createdAt}, #{alarmName}, #{alarmLevel}, " +
             "#{alarmType}, #{alarmMajorType}, #{alarmMinorType}, #{alarmAreaId}, " +
@@ -76,7 +85,8 @@ public interface AlarmMapper {
             "#{httpReqHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
             "#{httpReqBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
             "#{httpRespHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
-            "#{httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}   " +
+            "#{httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
+            "#{ruleId}, #{ruleVersion}, #{windowStart}, #{groupKeyHash} " +
             ")")
     void insert(Alarm alarm);
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/AlarmVisitMapper.java

@@ -24,7 +24,8 @@ public interface AlarmVisitMapper {
             "attack_chain_phase, disposition_advice, attack_direction, ",
             "judged_state, disposed_state, attack_result, fall, payload, dns_info, engine_type,  " ,
             "http_req_header  , http_req_body,http_resp_header , http_resp_body, ",
-            "reason, dtype, origin_field, origin_log ",
+            "reason, dtype, origin_field, origin_log, ",
+            "algorithm_id, algorithm_version, event_id, result_type ",
             ") VALUES ",
             "<foreach collection='list' item='item' separator=','>",
             "(#{item.id}, #{item.createdAt}, #{item.alarmName}, #{item.alarmLevel}, ",
@@ -47,7 +48,8 @@ public interface AlarmVisitMapper {
             "#{item.httpReqBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
             "#{item.httpRespHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
             "#{item.httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
-            "#{item.reason}, #{item.dtype}, #{item.originField}, #{item.originLog}) ",
+            "#{item.reason}, #{item.dtype}, #{item.originField}, #{item.originLog}, ",
+            "#{item.algorithmId}, #{item.algorithmVersion}, #{item.eventId}, #{item.resultType}) ",
             "</foreach>",
             "</script>"})
     void batchInsert(@Param("list") List<AlarmVisit> alarmList);
@@ -63,7 +65,8 @@ public interface AlarmVisitMapper {
             "attack_chain_phase, disposition_advice, attack_direction, " +
             "judged_state, disposed_state, attack_result, fall, payload, dns_info,engine_type, " +
             "http_req_header  , http_req_body,http_resp_header , http_resp_body, " +
-            "reason, dtype, origin_field, origin_log " +
+            "reason, dtype, origin_field, origin_log, " +
+            "algorithm_id, algorithm_version, event_id, result_type " +
             ") VALUES (" +
             "#{id}, #{createdAt}, #{alarmName}, #{alarmLevel}, " +
             "#{alarmType}, #{alarmMajorType}, #{alarmMinorType}, #{alarmAreaId}, " +
@@ -84,7 +87,8 @@ public interface AlarmVisitMapper {
             "#{httpReqBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
             "#{httpRespHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
             "#{httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
-            "#{reason}, #{dtype}, #{originField}, #{originLog} " +
+            "#{reason}, #{dtype}, #{originField}, #{originLog}, " +
+            "#{algorithmId}, #{algorithmVersion}, #{eventId}, #{resultType} " +
             ")")
     void insert(AlarmVisit alarm);
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DmNormalizeRuleMapper.java

@@ -22,6 +22,8 @@ public interface DmNormalizeRuleMapper {
      */
     List<Map<String, Object>> selectByDeviceId(@Param("id") Long id);
 
+    List<Long> selectDeviceIdsByRuleId(@Param("ruleId") Long ruleId);
+
     /**
      * 根据ID更新数据标准化规则
      * @param rule 数据标准化规则对象
@@ -43,4 +45,4 @@ public interface DmNormalizeRuleMapper {
      * @return 更新影响的行数
      */
     int updateByIdAndTenant(DmNormalizeRule rule);
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SecExceptionAlgorithmMapper.java

@@ -13,6 +13,7 @@ public interface SecExceptionAlgorithmMapper {
     @Select("SELECT * FROM sec_exception_algorithm WHERE status = 1 AND del_flag = '0'")
     List<SecExceptionAlgorithm> findEnabledAlgorithms();
 
-    @Select("SELECT * FROM sec_exception_algorithm WHERE id = #{id}")
+    // findById 的 SQL 统一由 SecExceptionAlgorithmMapper.xml 提供。两个消费模块保持一致，
+    // 防止任一容器启动时再次打印 duplicate mapped statement 错误。
     SecExceptionAlgorithm findById(@Param("id") Long id);
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/StatsChangeLogMapper.java

@@ -0,0 +1,47 @@
+package com.common.mapper;
+
+import com.common.entity.StatsChangeRecord;
+import org.apache.ibatis.annotations.Insert;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import java.time.LocalDateTime;
+
+@Mapper
+public interface StatsChangeLogMapper {
+
+    @Insert("INSERT INTO syslog_stats_change_log (" +
+            "change_seq,change_id,source_type,source_row_id,operation_type," +
+            "old_stat_date,new_stat_date,old_device_id,new_device_id," +
+            "old_collect_id,new_collect_id,old_rule_id,new_rule_id," +
+            "old_event_time,new_event_time,old_collect_time,new_collect_time," +
+            "old_push_success,new_push_success,committed_at" +
+            ") VALUES (seq_syslog_stats_change.NEXTVAL,#{changeId},#{sourceType},#{sourceRowId},#{operationType}," +
+            "#{oldStatDate},#{newStatDate},#{oldDeviceId},#{newDeviceId}," +
+            "#{oldCollectId},#{newCollectId},#{oldRuleId},#{newRuleId}," +
+            "#{oldEventTime},#{newEventTime},#{oldCollectTime},#{newCollectTime}," +
+            "#{oldPushSuccess},#{newPushSuccess},SYSDATE)")
+    int insert(StatsChangeRecord record);
+
+    @Insert("INSERT INTO stats_projection_pending_change(projection_name,change_seq,change_id,created_at) " +
+            "SELECT 'SYSLOG_STATS',change_seq,change_id,SYSDATE FROM syslog_stats_change_log " +
+            "WHERE change_id=#{changeId}")
+    int insertPending(StatsChangeRecord record);
+
+    @Insert("INSERT INTO syslog_stats_change_log (change_seq,change_id,source_type,source_row_id," +
+            "operation_type,old_stat_date,old_device_id,old_collect_id,old_event_time,old_collect_time," +
+            "old_push_success,committed_at) SELECT seq_syslog_stats_change.NEXTVAL," +
+            "'DEVICE_RECEIVE:'||TO_CHAR(id)||':DELETE','DEVICE_RECEIVE',TO_CHAR(id),'DELETE'," +
+            "TRUNC(receive_time),device_id,device_collect_id,receive_time,created_at,push_success,SYSDATE " +
+            "FROM device_receive_log WHERE created_at BETWEEN #{startTime} AND #{endTime}")
+    int captureDeviceReceiveDeletes(@Param("startTime") LocalDateTime startTime,
+                                    @Param("endTime") LocalDateTime endTime);
+
+    @Insert("INSERT INTO stats_projection_pending_change(projection_name,change_seq,change_id,created_at) " +
+            "SELECT 'SYSLOG_STATS',l.change_seq,l.change_id,SYSDATE FROM syslog_stats_change_log l " +
+            "WHERE l.source_type='DEVICE_RECEIVE' AND l.operation_type='DELETE' " +
+            "AND EXISTS (SELECT 1 FROM device_receive_log d WHERE d.created_at BETWEEN #{startTime} AND #{endTime} " +
+            "AND TO_CHAR(d.id)=l.source_row_id) AND NOT EXISTS (SELECT 1 FROM stats_projection_pending_change p " +
+            "WHERE p.projection_name='SYSLOG_STATS' AND p.change_seq=l.change_seq)")
+    int enqueueDeviceReceiveDeletes(@Param("startTime") LocalDateTime startTime,
+                                    @Param("endTime") LocalDateTime endTime);
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java

@@ -83,4 +83,10 @@ public interface SyslogNormalAlarmMapper {
             @Param("endTime") LocalDateTime endTime,
             @Param("offset") int offset,
             @Param("pageSize") int pageSize);
-}
+
+    @Select("SELECT COUNT(1) FROM syslog_normal_alarm " +
+            "WHERE log_time = #{logTime, jdbcType=TIMESTAMP} " +
+            "AND id = #{id}")
+    int countByLogTimeAndId(@Param("logTime") LocalDateTime logTime,
+                            @Param("id") String id);
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java

@@ -100,4 +100,10 @@ public interface SyslogNormalDataMapper {
 
     @Select("SELECT * FROM syslog_normal_data WHERE id = #{id}")
     SyslogNormalData findById(@Param("id") String id);
-}
+
+    @Select("SELECT COUNT(1) FROM syslog_normal_data " +
+            "WHERE log_time = #{logTime, jdbcType=TIMESTAMP} " +
+            "AND id = #{id}")
+    int countByLogTimeAndId(@Param("logTime") LocalDateTime logTime,
+                            @Param("id") String id);
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DeviceCollectTaskService.java

@@ -1,28 +1,36 @@
 package com.common.service;
 
+import com.Modules.NormalData.cache.NormalizationCacheInvalidationService;
+import com.Modules.NormalData.cache.NormalizationLookupFacade;
 import com.common.entity.DeviceCollectTask;
 import com.common.mapper.DeviceCollectTaskMapper;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.cache.annotation.CacheConfig;
-import org.springframework.cache.annotation.CacheEvict;
 import org.springframework.cache.annotation.Cacheable;
-import org.springframework.cache.annotation.CachePut;
 import org.springframework.stereotype.Service;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import java.util.List;
 
 @Service
 @CacheConfig(cacheNames = "collectTask")
 public class DeviceCollectTaskService {
+    private static final Logger logger = LoggerFactory.getLogger(DeviceCollectTaskService.class);
 
     @Autowired
     private DeviceCollectTaskMapper deviceCollectTaskMapper;
 
+    @Autowired
+    private NormalizationCacheInvalidationService cacheInvalidationService;
+
+    @Autowired
+    private NormalizationLookupFacade normalizationLookupFacade;
+
     /**
      * 根据ID查询采集任务 - 缓存
      */
-    @Cacheable(key = "'id:' + #id")
     public DeviceCollectTask getById(Integer id) {
-        return deviceCollectTaskMapper.selectById(id);
+        return normalizationLookupFacade.collectTask(id);
     }
 
     /**
@@ -58,51 +66,73 @@ public class DeviceCollectTaskService {
     /**
      * 新增采集任务 - 清除相关缓存
      */
-    @CacheEvict(allEntries = true)
     public int create(DeviceCollectTask task) {
-        return deviceCollectTaskMapper.insert(task);
+        int rows = deviceCollectTaskMapper.insert(task);
+        if (rows > 0) invalidateCollectTaskSafely(task.getId(), task.getDeviceId());
+        return rows;
     }
 
     /**
      * 更新采集任务 - 更新缓存
      */
-    @CachePut(key = "'id:' + #task.id")
-    @CacheEvict(key = "'device:' + #task.deviceId")
     public DeviceCollectTask update(DeviceCollectTask task) {
+        DeviceCollectTask before = deviceCollectTaskMapper.selectById(task.getId());
         deviceCollectTaskMapper.update(task);
-        return deviceCollectTaskMapper.selectById(task.getId());
+        DeviceCollectTask updated = deviceCollectTaskMapper.selectById(task.getId());
+        invalidateCollectTaskSafely(task.getId(),
+                before == null ? null : before.getDeviceId(),
+                updated == null ? task.getDeviceId() : updated.getDeviceId());
+        return updated;
     }
 
     /**
      * 删除采集任务 - 清除缓存
      */
-    @CacheEvict(key = "'id:' + #id")
     public int delete(Integer id) {
-        return deviceCollectTaskMapper.deleteById(id);
+        DeviceCollectTask before = deviceCollectTaskMapper.selectById(id);
+        int rows = deviceCollectTaskMapper.deleteById(id);
+        if (rows > 0) invalidateCollectTaskSafely(id, before == null ? null : before.getDeviceId());
+        return rows;
     }
 
     /**
      * 标记任务成功 - 更新缓存
      */
-    @CacheEvict(key = "'id:' + #id")
     public int markSuccess(Integer id) {
-        return deviceCollectTaskMapper.updateSuccessStatus(id);
+        DeviceCollectTask before = deviceCollectTaskMapper.selectById(id);
+        int rows = deviceCollectTaskMapper.updateSuccessStatus(id);
+        if (rows > 0) invalidateCollectTaskSafely(id, before == null ? null : before.getDeviceId());
+        return rows;
     }
 
     /**
      * 标记任务失败 - 更新缓存
      */
-    @CacheEvict(key = "'id:' + #id")
     public int markFailed(Integer id) {
-        return deviceCollectTaskMapper.updateFailedStatus(id);
+        DeviceCollectTask before = deviceCollectTaskMapper.selectById(id);
+        int rows = deviceCollectTaskMapper.updateFailedStatus(id);
+        if (rows > 0) invalidateCollectTaskSafely(id, before == null ? null : before.getDeviceId());
+        return rows;
     }
 
     /**
      * 更新EPM指标 - 更新缓存
      */
-    @CacheEvict(key = "'id:' + #id")
     public int updateEpm(Integer id, Integer epm) {
-        return deviceCollectTaskMapper.updateEpm(id, epm);
+        DeviceCollectTask before = deviceCollectTaskMapper.selectById(id);
+        int rows = deviceCollectTaskMapper.updateEpm(id, epm);
+        if (rows > 0) invalidateCollectTaskSafely(id, before == null ? null : before.getDeviceId());
+        return rows;
+    }
+
+    private void invalidateCollectTaskSafely(Integer id, Integer... deviceIds) {
+        try {
+            if (id != null) cacheInvalidationService.invalidateCollectTask(id);
+            cacheInvalidationService.invalidateCollectTaskViews(deviceIds);
+        } catch (RuntimeException cacheFailure) {
+            // 数据库事实已经成功，缓存失效失败只影响 TTL 窗口内的新鲜度。
+            logger.error("采集任务缓存失效失败: id={}", id, cacheFailure);
+        }
     }
 
     /**
@@ -133,4 +163,4 @@ public class DeviceCollectTaskService {
     public int countByDeviceId(Integer deviceId) {
         return deviceCollectTaskMapper.countByDeviceId(deviceId);
     }
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DeviceDeviceService.java

@@ -31,4 +31,4 @@ public interface DeviceDeviceService {
     List<DeviceDevice> getMonitoringDevices();
 
     List<DeviceDevice> getActiveDevices();
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsAccumulator.java

@@ -0,0 +1,242 @@
+package com.common.service;
+
+import com.common.entity.SyslogNonNormalMessage;
+
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+/**
+ * 单个达梦事实事务内的临时统计聚合器。实例不跨事务共享，也不承担持久化职责。
+ */
+public final class DirectStatsAccumulator {
+
+    private final SortedMap<DailyKey, DeviceDelta> devices = new TreeMap<>();
+    private final SortedMap<DailyKey, RuleDelta> rules = new TreeMap<>();
+    private final ZoneId businessZone;
+    private int inputRows;
+
+    public DirectStatsAccumulator() {
+        this(ZoneId.systemDefault());
+    }
+
+    public DirectStatsAccumulator(ZoneId businessZone) {
+        this.businessZone = Objects.requireNonNull(businessZone, "businessZone");
+    }
+
+    public DirectStatsAccumulator addStandardizedRows(List<Map<String, Object>> rows) {
+        if (rows != null) {
+            for (Map<String, Object> row : rows) {
+                addStandardizedRow(row);
+            }
+        }
+        return this;
+    }
+
+    public DirectStatsAccumulator addStandardizedRow(Map<String, Object> row) {
+        if (row == null) {
+            return this;
+        }
+        LocalDate statDate = dateValue(row.get("created_at"), businessZone);
+        Long deviceId = longValue(row.get("device_id"));
+        Long ruleId = longValue(row.get("normalize_rule_id"));
+        LocalDateTime logTime = timeValue(row.get("log_time"), businessZone);
+        if (statDate == null) {
+            throw new IllegalArgumentException("标准化统计缺少 created_at");
+        }
+        if (deviceId != null) {
+            device(statDate, deviceId).parseDelta++;
+        }
+        if (ruleId != null) {
+            rules.computeIfAbsent(new DailyKey(statDate, ruleId), ignored -> new RuleDelta())
+                    .include(logTime);
+        }
+        inputRows++;
+        return this;
+    }
+
+    public DirectStatsAccumulator addNonNormalRows(List<SyslogNonNormalMessage> messages) {
+        if (messages != null) {
+            for (SyslogNonNormalMessage message : messages) {
+                addNonNormalInsert(message);
+            }
+        }
+        return this;
+    }
+
+    public DirectStatsAccumulator addNonNormalInsert(SyslogNonNormalMessage message) {
+        if (message == null) {
+            return this;
+        }
+        LocalDate statDate = dateValue(message.getCreatedAt(), businessZone);
+        Long deviceId = longValue(message.getDeviceId());
+        if (statDate == null) {
+            throw new IllegalArgumentException("非标统计缺少 created_at");
+        }
+        if (deviceId != null) {
+            device(statDate, deviceId).nonLogDelta++;
+        }
+        inputRows++;
+        return this;
+    }
+
+    /** UPDATE 等价于撤销旧维度贡献，再应用合并后的新维度贡献。 */
+    public DirectStatsAccumulator addNonNormalUpdate(SyslogNonNormalMessage oldValue,
+                                                     SyslogNonNormalMessage patch) {
+        if (oldValue == null || patch == null) {
+            return this;
+        }
+        LocalDate oldDate = dateValue(oldValue.getCreatedAt(), businessZone);
+        Long oldDeviceId = longValue(oldValue.getDeviceId());
+        LocalDate newDate = dateValue(patch.getCreatedAt() != null
+                ? patch.getCreatedAt() : oldValue.getCreatedAt(), businessZone);
+        Long newDeviceId = longValue(patch.getDeviceId() != null
+                ? patch.getDeviceId() : oldValue.getDeviceId());
+        if (oldDate != null && oldDeviceId != null) {
+            device(oldDate, oldDeviceId).nonLogDelta--;
+        }
+        if (newDate != null && newDeviceId != null) {
+            device(newDate, newDeviceId).nonLogDelta++;
+        }
+        inputRows++;
+        return this;
+    }
+
+    private DeviceDelta device(LocalDate date, long deviceId) {
+        return devices.computeIfAbsent(new DailyKey(date, deviceId), ignored -> new DeviceDelta());
+    }
+
+    public SortedMap<DailyKey, DeviceDelta> getDevices() {
+        return devices;
+    }
+
+    public SortedMap<DailyKey, RuleDelta> getRules() {
+        return rules;
+    }
+
+    public int getInputRows() {
+        return inputRows;
+    }
+
+    public boolean isEmpty() {
+        return devices.isEmpty() && rules.isEmpty();
+    }
+
+    static Long longValue(Object value) {
+        if (value == null) {
+            return null;
+        }
+        return value instanceof Number
+                ? ((Number) value).longValue()
+                : Long.valueOf(value.toString());
+    }
+
+    static LocalDate dateValue(Object value) {
+        return dateValue(value, ZoneId.systemDefault());
+    }
+
+    static LocalDate dateValue(Object value, ZoneId zone) {
+        LocalDateTime time = timeValue(value, zone);
+        return time == null ? null : time.toLocalDate();
+    }
+
+    static LocalDateTime timeValue(Object value) {
+        return timeValue(value, ZoneId.systemDefault());
+    }
+
+    static LocalDateTime timeValue(Object value, ZoneId zone) {
+        if (value == null) {
+            return null;
+        }
+        if (value instanceof LocalDateTime) {
+            return (LocalDateTime) value;
+        }
+        if (value instanceof LocalDate) {
+            return ((LocalDate) value).atStartOfDay();
+        }
+        if (value instanceof OffsetDateTime) {
+            return ((OffsetDateTime) value).atZoneSameInstant(zone).toLocalDateTime();
+        }
+        if (value instanceof Timestamp) {
+            return ((Timestamp) value).toLocalDateTime();
+        }
+        if (value instanceof Date) {
+            Instant instant = Instant.ofEpochMilli(((Date) value).getTime());
+            return LocalDateTime.ofInstant(instant, zone);
+        }
+        throw new IllegalArgumentException("不支持的统计时间类型: " + value.getClass().getName());
+    }
+
+    public static final class DailyKey implements Comparable<DailyKey> {
+        private final LocalDate date;
+        private final long dimensionId;
+
+        public DailyKey(LocalDate date, long dimensionId) {
+            this.date = Objects.requireNonNull(date, "date");
+            this.dimensionId = dimensionId;
+        }
+
+        public LocalDate getDate() {
+            return date;
+        }
+
+        public long getDimensionId() {
+            return dimensionId;
+        }
+
+        @Override
+        public int compareTo(DailyKey other) {
+            int dateOrder = date.compareTo(other.date);
+            return dateOrder != 0 ? dateOrder : Long.compare(dimensionId, other.dimensionId);
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (this == other) return true;
+            if (!(other instanceof DailyKey)) return false;
+            DailyKey key = (DailyKey) other;
+            return dimensionId == key.dimensionId && date.equals(key.date);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(date, dimensionId);
+        }
+    }
+
+    public static final class DeviceDelta {
+        private long parseDelta;
+        private long nonLogDelta;
+
+        public long getParseDelta() {
+            return parseDelta;
+        }
+
+        public long getNonLogDelta() {
+            return nonLogDelta;
+        }
+    }
+
+    public static final class RuleDelta {
+        private LocalDateTime maxLogTime;
+
+        private void include(LocalDateTime value) {
+            if (value != null && (maxLogTime == null || value.isAfter(maxLogTime))) {
+                maxLogTime = value;
+            }
+        }
+
+        public LocalDateTime getMaxLogTime() {
+            return maxLogTime;
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsDayRolloverService.java

@@ -0,0 +1,86 @@
+package com.common.service;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionTemplate;
+
+import java.sql.Date;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * DIRECT 事实写入节点负责的设备今日计数日切。
+ *
+ * <p>这里只更新小型设备维度表，不扫描任何 Syslog 事实表。显式传入业务日期，避免数据库会话时区
+ * 与 {@code stats.business-zone} 不一致时提前或延后清零。</p>
+ */
+@Slf4j
+@Service
+public class DirectStatsDayRolloverService {
+    private final JdbcTemplate jdbc;
+    private final PlatformTransactionManager transactionManager;
+    private final StatsWriteModeProvider modeProvider;
+
+    @Value("${stats.business-zone:Asia/Shanghai}")
+    private String businessZone;
+
+    @Value("${stats.incremental.transaction-timeout-seconds:45}")
+    private int transactionTimeoutSeconds;
+
+    private volatile LocalDate completedDate;
+    private volatile LocalDateTime lastSuccessTime;
+    private volatile LocalDateTime lastFailureTime;
+    private volatile int lastUpdatedDevices;
+
+    public DirectStatsDayRolloverService(JdbcTemplate jdbc,
+                                         PlatformTransactionManager transactionManager,
+                                         StatsWriteModeProvider modeProvider) {
+        this.jdbc = jdbc;
+        this.transactionManager = transactionManager;
+        this.modeProvider = modeProvider;
+    }
+
+    @Scheduled(cron = "${stats.direct.day-rollover-check-cron:5 * * * * ?}")
+    public void scheduledRollover() {
+        if (modeProvider.getMode() != StatsWriteModeProvider.Mode.DIRECT) return;
+        LocalDate today = LocalDate.now(ZoneId.of(businessZone));
+        if (today.equals(completedDate)) return;
+        try {
+            TransactionTemplate transaction = new TransactionTemplate(transactionManager);
+            transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
+            Date businessDate = Date.valueOf(today);
+            Integer updated = transaction.execute(status -> jdbc.update(
+                    "UPDATE device_device SET today_parse_count=0,today_non_log_count=0," +
+                            "today_stats_date=?,updated_at=SYSDATE " +
+                            "WHERE today_stats_date IS NULL OR today_stats_date<?",
+                    businessDate, businessDate));
+            lastUpdatedDevices = updated == null ? 0 : updated;
+            completedDate = today;
+            lastSuccessTime = LocalDateTime.now();
+            log.info("DIRECT 设备今日计数日切完成: businessDate={}, resetDevices={}",
+                    today, lastUpdatedDevices);
+        } catch (RuntimeException failure) {
+            lastFailureTime = LocalDateTime.now();
+            // completedDate 不推进；下一分钟自动重试。
+            log.error("DIRECT 设备今日计数日切失败，将在下一检查周期重试: businessDate={}", today, failure);
+        }
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> status = new LinkedHashMap<>();
+        status.put("write_mode", modeProvider.getMode().name());
+        status.put("business_zone", businessZone);
+        status.put("completed_date", completedDate);
+        status.put("last_updated_devices", lastUpdatedDevices);
+        status.put("last_success_time", lastSuccessTime);
+        status.put("last_failure_time", lastFailureTime);
+        return status;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java

@@ -0,0 +1,495 @@
+package com.common.service;
+
+import com.common.entity.SyslogNonNormalMessage;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.dao.CannotAcquireLockException;
+import org.springframework.dao.DeadlockLoserDataAccessException;
+import org.springframework.dao.DuplicateKeyException;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+
+import java.sql.Date;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+/**
+ * 把当前事实事务的内存聚合结果直接写入小型统计表。
+ *
+ * <p>调用方必须已经开启事实事务；本类不会新开事务，也不会吞掉任何数据库异常。</p>
+ */
+@Slf4j
+@Service
+public class DirectStatsProjectionWriter {
+
+    private static final String DEVICE_MERGE_TEMPLATE =
+            "MERGE INTO %s t USING (SELECT ? stat_date,? device_id,? parse_delta,? non_log_delta FROM DUAL) s " +
+            "ON (t.stat_date=s.stat_date AND t.device_id=s.device_id) " +
+            "WHEN MATCHED THEN UPDATE SET t.parse_count=t.parse_count+s.parse_delta," +
+            "t.non_log_count=t.non_log_count+s.non_log_delta,t.updated_at=SYSDATE " +
+            "WHEN NOT MATCHED THEN INSERT (stat_date,device_id,parse_count,non_log_count," +
+            "last_receive_time,projection_seq,updated_at) " +
+            "VALUES (s.stat_date,s.device_id,s.parse_delta,s.non_log_delta,NULL,0,SYSDATE)";
+
+    private static final String RULE_MERGE_TEMPLATE =
+            "MERGE INTO %s t USING (SELECT ? stat_date,? normalize_rule_id,? max_log_time FROM DUAL) s " +
+            "ON (t.stat_date=s.stat_date AND t.normalize_rule_id=s.normalize_rule_id) " +
+            "WHEN MATCHED THEN UPDATE SET t.max_log_time=CASE " +
+            "WHEN s.max_log_time IS NULL THEN t.max_log_time " +
+            "WHEN t.max_log_time IS NULL OR t.max_log_time<s.max_log_time THEN s.max_log_time " +
+            "ELSE t.max_log_time END,t.updated_at=SYSDATE " +
+            "WHEN NOT MATCHED THEN INSERT (stat_date,normalize_rule_id,max_log_time,projection_seq,updated_at) " +
+            "VALUES (s.stat_date,s.normalize_rule_id,s.max_log_time,0,SYSDATE)";
+
+    private static final String DEVICE_EXACT_MERGE_TEMPLATE =
+            "MERGE INTO %s t USING (SELECT ? stat_date,? device_id,? parse_count,? non_log_count FROM DUAL) s " +
+            "ON (t.stat_date=s.stat_date AND t.device_id=s.device_id) " +
+            "WHEN MATCHED THEN UPDATE SET t.parse_count=s.parse_count," +
+            "t.non_log_count=s.non_log_count,t.updated_at=SYSDATE " +
+            "WHEN NOT MATCHED THEN INSERT (stat_date,device_id,parse_count,non_log_count," +
+            "last_receive_time,projection_seq,updated_at) " +
+            "VALUES (s.stat_date,s.device_id,s.parse_count,s.non_log_count,NULL,0,SYSDATE)";
+
+    private final JdbcTemplate jdbc;
+    private final StatsWriteModeProvider modeProvider;
+    private final DirectStatsWriteMetrics metrics;
+
+    @Value("${stats.incremental.read-enabled:false}")
+    private boolean readEnabled;
+
+    @Value("${stats.direct.device-table:device_daily_stats}")
+    private String directDeviceTable;
+
+    @Value("${stats.direct.rule-table:normalize_rule_daily_stats}")
+    private String directRuleTable;
+
+    @Value("${stats.direct.shadow-device-table:device_daily_stats_direct_shadow}")
+    private String shadowDeviceTable;
+
+    @Value("${stats.direct.shadow-rule-table:normalize_rule_daily_stats_direct_shadow}")
+    private String shadowRuleTable;
+
+    @Value("${stats.direct.control-table:stats_direct_control}")
+    private String controlTable;
+
+    @Value("${stats.direct.today-stats-date-enabled:true}")
+    private boolean todayStatsDateEnabled;
+
+    @Value("${stats.business-zone:Asia/Shanghai}")
+    private String businessZone;
+
+    public DirectStatsProjectionWriter(JdbcTemplate jdbc,
+                                       StatsWriteModeProvider modeProvider,
+                                       DirectStatsWriteMetrics metrics) {
+        this.jdbc = jdbc;
+        this.modeProvider = modeProvider;
+        this.metrics = metrics;
+    }
+
+    public WriteResult writeStandardizedRows(List<Map<String, Object>> rows) {
+        if (!shouldBuildAggregation()) {
+            handleFactOnlyMode();
+            return WriteResult.inactive(rows == null ? 0 : rows.size());
+        }
+        long aggregateStarted = System.nanoTime();
+        DirectStatsAccumulator accumulator = accumulator().addStandardizedRows(rows);
+        return write(accumulator, elapsedMs(aggregateStarted));
+    }
+
+    public WriteResult writeStandardizedRow(Map<String, Object> row) {
+        if (!shouldBuildAggregation()) {
+            handleFactOnlyMode();
+            return WriteResult.inactive(row == null ? 0 : 1);
+        }
+        long aggregateStarted = System.nanoTime();
+        DirectStatsAccumulator accumulator = accumulator().addStandardizedRow(row);
+        return write(accumulator, elapsedMs(aggregateStarted));
+    }
+
+    public WriteResult writeNonNormalRows(List<SyslogNonNormalMessage> messages) {
+        if (!shouldBuildAggregation()) {
+            handleFactOnlyMode();
+            return WriteResult.inactive(messages == null ? 0 : messages.size());
+        }
+        long aggregateStarted = System.nanoTime();
+        DirectStatsAccumulator accumulator = accumulator().addNonNormalRows(messages);
+        return write(accumulator, elapsedMs(aggregateStarted));
+    }
+
+    public WriteResult writeNonNormalInsert(SyslogNonNormalMessage message) {
+        if (!shouldBuildAggregation()) {
+            handleFactOnlyMode();
+            return WriteResult.inactive(message == null ? 0 : 1);
+        }
+        long aggregateStarted = System.nanoTime();
+        DirectStatsAccumulator accumulator = accumulator().addNonNormalInsert(message);
+        return write(accumulator, elapsedMs(aggregateStarted));
+    }
+
+    public WriteResult writeNonNormalUpdate(SyslogNonNormalMessage oldValue, SyslogNonNormalMessage patch) {
+        if (!shouldBuildAggregation()) {
+            handleFactOnlyMode();
+            return WriteResult.inactive(oldValue == null || patch == null ? 0 : 1);
+        }
+        long aggregateStarted = System.nanoTime();
+        DirectStatsAccumulator accumulator = accumulator().addNonNormalUpdate(oldValue, patch);
+        return writeExactDeviceGroups(accumulator, elapsedMs(aggregateStarted));
+    }
+
+    public void recordTransactionRetry() {
+        metrics.recordRetry();
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> status = new LinkedHashMap<>(metrics.snapshot(modeProvider.getMode()));
+        status.put("read_enabled", readEnabled);
+        status.put("device_table", selectedDeviceTable());
+        status.put("rule_table", selectedRuleTable());
+        status.put("today_stats_date_enabled", todayStatsDateEnabled);
+        return status;
+    }
+
+    private boolean shouldBuildAggregation() {
+        return modeProvider.writesDirectProjection();
+    }
+
+    private DirectStatsAccumulator accumulator() {
+        return new DirectStatsAccumulator(ZoneId.of(businessZone));
+    }
+
+    private void handleFactOnlyMode() {
+        if (!modeProvider.requiresRebuild()) {
+            return;
+        }
+        requireTransaction();
+        metrics.markRebuildRequired();
+        String table = safeTable(controlTable);
+        jdbc.update("MERGE INTO " + table + " t USING " +
+                        "(SELECT 'SYSLOG_STATS' control_name FROM DUAL) s " +
+                        "ON (t.control_name=s.control_name) " +
+                        "WHEN MATCHED THEN UPDATE SET t.rebuild_required=1,t.updated_at=SYSDATE " +
+                        "WHEN NOT MATCHED THEN INSERT (control_name,rebuild_required,updated_at) " +
+                        "VALUES (s.control_name,1,SYSDATE)");
+    }
+
+    private WriteResult write(DirectStatsAccumulator accumulator, long aggregateMs) {
+        if (accumulator.isEmpty()) {
+            metrics.recordSuccess(accumulator.getInputRows(), 0, 0, aggregateMs, 0L, 0L);
+            return WriteResult.direct(accumulator.getInputRows(), 0, 0,
+                    aggregateMs, 0L, 0L);
+        }
+        requireTransaction();
+
+        List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> devices =
+                effectiveDeviceEntries(accumulator);
+        List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> rules =
+                effectiveRuleEntries(accumulator);
+        long mergeMs;
+        long materializeMs = 0L;
+        try {
+            long mergeStarted = System.nanoTime();
+            mergeDevices(devices);
+            mergeRules(rules);
+            mergeMs = elapsedMs(mergeStarted);
+
+            if (modeProvider.materializesBusinessFields() && readEnabled) {
+                long materializeStarted = System.nanoTime();
+                materializeDevices(devices);
+                materializeRules(rules);
+                materializeMs = elapsedMs(materializeStarted);
+            }
+            metrics.recordSuccess(accumulator.getInputRows(), devices.size(), rules.size(),
+                    aggregateMs, mergeMs, materializeMs);
+            log.debug("直接统计事务块完成: factRows={}, deviceGroups={}, ruleGroups={}, " +
+                            "aggregateMs={}, statsMergeMs={}, materializeMs={}",
+                    accumulator.getInputRows(), devices.size(), rules.size(),
+                    aggregateMs, mergeMs, materializeMs);
+            return WriteResult.direct(accumulator.getInputRows(), devices.size(), rules.size(),
+                    aggregateMs, mergeMs, materializeMs);
+        } catch (RuntimeException failure) {
+            metrics.recordFailure();
+            if (isConcurrentWrite(failure)) {
+                throw new ConcurrentStatsWriteException("直接统计聚合键并发写入失败，可安全回滚后有限重试", failure);
+            }
+            throw new DirectStatsWriteException("直接统计写入失败，事实事务必须整体回滚", failure);
+        }
+    }
+
+    /**
+     * UPDATE/物理 DELETE 属于低频管理路径。负 delta 可能移除当前最大值或命中未建立历史基线的日期，
+     * 因此直接从事实表精确重算受影响的小分组，不能把负数插入一个不存在的聚合键。
+     */
+    private WriteResult writeExactDeviceGroups(DirectStatsAccumulator accumulator, long aggregateMs) {
+        if (accumulator.getDevices().isEmpty()) {
+            metrics.recordSuccess(accumulator.getInputRows(), 0, 0, aggregateMs, 0L, 0L);
+            return WriteResult.direct(accumulator.getInputRows(), 0, 0,
+                    aggregateMs, 0L, 0L);
+        }
+        requireTransaction();
+        List<DirectStatsAccumulator.DailyKey> keys = new ArrayList<>(accumulator.getDevices().keySet());
+        try {
+            long mergeStarted = System.nanoTime();
+            List<Object[]> exactValues = new ArrayList<>(keys.size());
+            for (DirectStatsAccumulator.DailyKey key : keys) {
+                LocalDateTime start = key.getDate().atStartOfDay();
+                LocalDateTime end = key.getDate().plusDays(1).atStartOfDay();
+                Map<String, Object> counts = jdbc.queryForMap(
+                        "SELECT " +
+                                "((SELECT COUNT(*) FROM syslog_normal_data " +
+                                "WHERE created_at>=? AND created_at<? AND device_id=?) + " +
+                                " (SELECT COUNT(*) FROM syslog_normal_alarm " +
+                                "WHERE created_at>=? AND created_at<? AND device_id=?)) parse_count," +
+                                "(SELECT COUNT(*) FROM syslog_non_normal_message " +
+                                "WHERE created_at>=? AND created_at<? AND device_id=?) non_log_count FROM DUAL",
+                        start, end, key.getDimensionId(),
+                        start, end, key.getDimensionId(),
+                        start, end, key.getDimensionId());
+                exactValues.add(new Object[]{Date.valueOf(key.getDate()), key.getDimensionId(),
+                        number(counts, "parse_count"), number(counts, "non_log_count")});
+            }
+            jdbc.batchUpdate(String.format(DEVICE_EXACT_MERGE_TEMPLATE,
+                    safeTable(selectedDeviceTable())), exactValues);
+            long mergeMs = elapsedMs(mergeStarted);
+            long materializeMs = 0L;
+            if (modeProvider.materializesBusinessFields() && readEnabled) {
+                long materializeStarted = System.nanoTime();
+                materializeDeviceKeys(keys);
+                materializeMs = elapsedMs(materializeStarted);
+            }
+            metrics.recordSuccess(accumulator.getInputRows(), keys.size(), 0,
+                    aggregateMs, mergeMs, materializeMs);
+            return WriteResult.direct(accumulator.getInputRows(), keys.size(), 0,
+                    aggregateMs, mergeMs, materializeMs);
+        } catch (RuntimeException failure) {
+            metrics.recordFailure();
+            if (isConcurrentWrite(failure)) {
+                throw new ConcurrentStatsWriteException("直接统计局部分组并发写入失败，可安全回滚后有限重试", failure);
+            }
+            throw new DirectStatsWriteException("直接统计局部分组重算失败，事实事务必须整体回滚", failure);
+        }
+    }
+
+    private List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>>
+    effectiveDeviceEntries(DirectStatsAccumulator accumulator) {
+        List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> result =
+                new ArrayList<>();
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry
+                : accumulator.getDevices().entrySet()) {
+            DirectStatsAccumulator.DeviceDelta delta = entry.getValue();
+            if (delta.getParseDelta() != 0L || delta.getNonLogDelta() != 0L) {
+                result.add(entry);
+            }
+        }
+        return result;
+    }
+
+    private List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>>
+    effectiveRuleEntries(DirectStatsAccumulator accumulator) {
+        List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> result =
+                new ArrayList<>();
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry
+                : accumulator.getRules().entrySet()) {
+            if (entry.getValue().getMaxLogTime() != null) {
+                result.add(entry);
+            }
+        }
+        return result;
+    }
+
+    private void mergeDevices(
+            List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> entries) {
+        if (entries.isEmpty()) return;
+        List<Object[]> parameters = new ArrayList<>(entries.size());
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry : entries) {
+            parameters.add(new Object[]{Date.valueOf(entry.getKey().getDate()),
+                    entry.getKey().getDimensionId(), entry.getValue().getParseDelta(),
+                    entry.getValue().getNonLogDelta()});
+        }
+        jdbc.batchUpdate(String.format(DEVICE_MERGE_TEMPLATE, safeTable(selectedDeviceTable())), parameters);
+    }
+
+    private void mergeRules(
+            List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> entries) {
+        if (entries.isEmpty()) return;
+        List<Object[]> parameters = new ArrayList<>(entries.size());
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry : entries) {
+            parameters.add(new Object[]{Date.valueOf(entry.getKey().getDate()),
+                    entry.getKey().getDimensionId(), entry.getValue().getMaxLogTime()});
+        }
+        jdbc.batchUpdate(String.format(RULE_MERGE_TEMPLATE, safeTable(selectedRuleTable())), parameters);
+    }
+
+    private void materializeDevices(
+            List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> entries) {
+        List<DirectStatsAccumulator.DailyKey> keys = new ArrayList<>(entries.size());
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry : entries) {
+            keys.add(entry.getKey());
+        }
+        materializeDeviceKeys(keys);
+    }
+
+    private void materializeDeviceKeys(List<DirectStatsAccumulator.DailyKey> keys) {
+        LocalDate today = LocalDate.now(ZoneId.of(businessZone));
+        List<Object[]> parameters = new ArrayList<>();
+        for (DirectStatsAccumulator.DailyKey key : keys) {
+            if (today.equals(key.getDate())) {
+                long deviceId = key.getDimensionId();
+                parameters.add(new Object[]{Date.valueOf(today), deviceId,
+                        Date.valueOf(today), deviceId, Date.valueOf(today), deviceId});
+            }
+        }
+        if (parameters.isEmpty()) return;
+        String deviceTable = safeTable(selectedDeviceTable());
+        String dateAssignment = todayStatsDateEnabled ? ",today_stats_date=?" : "";
+        String sql = "UPDATE device_device SET " +
+                "today_parse_count=(SELECT parse_count FROM " + deviceTable +
+                " WHERE stat_date=? AND device_id=?)," +
+                "today_non_log_count=(SELECT non_log_count FROM " + deviceTable +
+                " WHERE stat_date=? AND device_id=?)" + dateAssignment +
+                ",updated_at=SYSDATE WHERE id=?";
+        if (!todayStatsDateEnabled) {
+            List<Object[]> withoutDateColumn = new ArrayList<>(parameters.size());
+            for (Object[] row : parameters) {
+                withoutDateColumn.add(new Object[]{row[0], row[1], row[2], row[3], row[5]});
+            }
+            jdbc.batchUpdate(sql, withoutDateColumn);
+        } else {
+            jdbc.batchUpdate(sql, parameters);
+        }
+    }
+
+    private void materializeRules(
+            List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> entries) {
+        SortedMap<Long, LocalDateTime> maxima = new TreeMap<>();
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry : entries) {
+            long ruleId = entry.getKey().getDimensionId();
+            LocalDateTime candidate = entry.getValue().getMaxLogTime();
+            LocalDateTime current = maxima.get(ruleId);
+            if (current == null || candidate.isAfter(current)) {
+                maxima.put(ruleId, candidate);
+            }
+        }
+        if (maxima.isEmpty()) return;
+        List<Object[]> parameters = new ArrayList<>(maxima.size());
+        for (Map.Entry<Long, LocalDateTime> entry : maxima.entrySet()) {
+            parameters.add(new Object[]{entry.getValue(), entry.getValue(), entry.getValue(), entry.getKey()});
+        }
+        jdbc.batchUpdate("UPDATE dm_normalize_rule SET first_data_saved_at=CASE " +
+                        "WHEN ? IS NULL THEN first_data_saved_at " +
+                        "WHEN first_data_saved_at IS NULL OR first_data_saved_at<? THEN ? " +
+                        "ELSE first_data_saved_at END,updated_at=SYSDATE " +
+                        "WHERE id=? AND del_flag='0'", parameters);
+    }
+
+    private String selectedDeviceTable() {
+        return modeProvider.writesShadowProjection() ? shadowDeviceTable : directDeviceTable;
+    }
+
+    private String selectedRuleTable() {
+        return modeProvider.writesShadowProjection() ? shadowRuleTable : directRuleTable;
+    }
+
+    private static String safeTable(String table) {
+        if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
+            throw new IllegalStateException("非法统计表名: " + table);
+        }
+        return table;
+    }
+
+    private static void requireTransaction() {
+        if (!TransactionSynchronizationManager.isActualTransactionActive()) {
+            throw new IllegalStateException("直接统计必须与事实写入位于同一个 Spring 达梦事务");
+        }
+    }
+
+    private static boolean isConcurrentWrite(Throwable failure) {
+        Throwable current = failure;
+        while (current != null) {
+            if (current instanceof DuplicateKeyException
+                    || current instanceof CannotAcquireLockException
+                    || current instanceof DeadlockLoserDataAccessException) {
+                return true;
+            }
+            current = current.getCause();
+        }
+        return false;
+    }
+
+    private static long elapsedMs(long startedNanos) {
+        return (System.nanoTime() - startedNanos) / 1_000_000L;
+    }
+
+    private static long number(Map<String, Object> row, String name) {
+        Object value = row.get(name);
+        if (value == null) value = row.get(name.toUpperCase(java.util.Locale.ROOT));
+        if (value == null) {
+            for (Map.Entry<String, Object> entry : row.entrySet()) {
+                if (entry.getKey().equalsIgnoreCase(name)) {
+                    value = entry.getValue();
+                    break;
+                }
+            }
+        }
+        return value == null ? 0L : ((Number) value).longValue();
+    }
+
+    public static class DirectStatsWriteException extends RuntimeException {
+        DirectStatsWriteException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
+    public static final class ConcurrentStatsWriteException extends DirectStatsWriteException {
+        ConcurrentStatsWriteException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
+    /** 单个事实事务块的统计写入结果，用于批次 INFO 日志和事务耗时观测。 */
+    public static final class WriteResult {
+        private final boolean direct;
+        private final int inputRows;
+        private final int deviceGroups;
+        private final int ruleGroups;
+        private final long aggregateMs;
+        private final long mergeMs;
+        private final long materializeMs;
+
+        private WriteResult(boolean direct, int inputRows, int deviceGroups, int ruleGroups,
+                            long aggregateMs, long mergeMs, long materializeMs) {
+            this.direct = direct;
+            this.inputRows = inputRows;
+            this.deviceGroups = deviceGroups;
+            this.ruleGroups = ruleGroups;
+            this.aggregateMs = aggregateMs;
+            this.mergeMs = mergeMs;
+            this.materializeMs = materializeMs;
+        }
+
+        static WriteResult inactive(int inputRows) {
+            return new WriteResult(false, inputRows, 0, 0, 0L, 0L, 0L);
+        }
+
+        static WriteResult direct(int inputRows, int deviceGroups, int ruleGroups,
+                                  long aggregateMs, long mergeMs, long materializeMs) {
+            return new WriteResult(true, inputRows, deviceGroups, ruleGroups,
+                    aggregateMs, mergeMs, materializeMs);
+        }
+
+        public boolean isDirect() { return direct; }
+        public int getInputRows() { return inputRows; }
+        public int getDeviceGroups() { return deviceGroups; }
+        public int getRuleGroups() { return ruleGroups; }
+        public long getAggregateMs() { return aggregateMs; }
+        public long getMergeMs() { return mergeMs; }
+        public long getMaterializeMs() { return materializeMs; }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsWriteMetrics.java

@@ -0,0 +1,68 @@
+package com.common.service;
+
+import org.springframework.stereotype.Component;
+
+import java.time.LocalDateTime;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** 直接统计写入的进程级低成本指标。 */
+@Component
+public class DirectStatsWriteMetrics {
+    private final AtomicLong batches = new AtomicLong();
+    private final AtomicLong inputRows = new AtomicLong();
+    private final AtomicLong deviceGroups = new AtomicLong();
+    private final AtomicLong ruleGroups = new AtomicLong();
+    private final AtomicLong aggregateDurationMs = new AtomicLong();
+    private final AtomicLong mergeDurationMs = new AtomicLong();
+    private final AtomicLong materializeDurationMs = new AtomicLong();
+    private final AtomicLong retries = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+    private volatile boolean rebuildRequired;
+    private volatile LocalDateTime lastSuccessTime;
+    private volatile LocalDateTime lastFailureTime;
+
+    void recordSuccess(int rows, int devices, int rules, long aggregateMs,
+                       long mergeMs, long materializeMs) {
+        batches.incrementAndGet();
+        inputRows.addAndGet(rows);
+        deviceGroups.addAndGet(devices);
+        ruleGroups.addAndGet(rules);
+        aggregateDurationMs.addAndGet(aggregateMs);
+        mergeDurationMs.addAndGet(mergeMs);
+        materializeDurationMs.addAndGet(materializeMs);
+        lastSuccessTime = LocalDateTime.now();
+    }
+
+    void recordRetry() {
+        retries.incrementAndGet();
+    }
+
+    void recordFailure() {
+        failures.incrementAndGet();
+        lastFailureTime = LocalDateTime.now();
+    }
+
+    void markRebuildRequired() {
+        rebuildRequired = true;
+    }
+
+    public Map<String, Object> snapshot(StatsWriteModeProvider.Mode mode) {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("write_mode", mode.name());
+        result.put("batches", batches.get());
+        result.put("input_rows", inputRows.get());
+        result.put("device_groups", deviceGroups.get());
+        result.put("rule_groups", ruleGroups.get());
+        result.put("aggregate_duration_ms", aggregateDurationMs.get());
+        result.put("merge_duration_ms", mergeDurationMs.get());
+        result.put("materialize_duration_ms", materializeDurationMs.get());
+        result.put("transaction_retries", retries.get());
+        result.put("failures", failures.get());
+        result.put("rebuild_required", rebuildRequired);
+        result.put("last_success_time", lastSuccessTime);
+        result.put("last_failure_time", lastFailureTime);
+        return result;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmDirectTransactionMetrics.java

@@ -0,0 +1,63 @@
+package com.common.service;
+
+import org.springframework.stereotype.Component;
+
+import java.time.LocalDateTime;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** 达梦事实与直接统计同事务写入的累计耗时指标。 */
+@Component
+public class DmDirectTransactionMetrics {
+    private final AtomicLong transactions = new AtomicLong();
+    private final AtomicLong factRows = new AtomicLong();
+    private final AtomicLong factFlushMs = new AtomicLong();
+    private final AtomicLong factFlushMaxMs = new AtomicLong();
+    private final AtomicLong commitMs = new AtomicLong();
+    private final AtomicLong commitMaxMs = new AtomicLong();
+    private final AtomicLong transactionMs = new AtomicLong();
+    private final AtomicLong transactionMaxMs = new AtomicLong();
+    private final AtomicLong retries = new AtomicLong();
+    private volatile LocalDateTime lastSuccessTime;
+
+    void recordSuccess(int rows, long flushDurationMs, long commitDurationMs,
+                       long transactionDurationMs) {
+        transactions.incrementAndGet();
+        factRows.addAndGet(rows);
+        factFlushMs.addAndGet(flushDurationMs);
+        updateMax(factFlushMaxMs, flushDurationMs);
+        commitMs.addAndGet(commitDurationMs);
+        updateMax(commitMaxMs, commitDurationMs);
+        transactionMs.addAndGet(transactionDurationMs);
+        updateMax(transactionMaxMs, transactionDurationMs);
+        lastSuccessTime = LocalDateTime.now();
+    }
+
+    void recordRetry() {
+        retries.incrementAndGet();
+    }
+
+    public Map<String, Object> snapshot() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("transactions", transactions.get());
+        result.put("fact_rows", factRows.get());
+        result.put("fact_flush_duration_ms_total", factFlushMs.get());
+        result.put("fact_flush_duration_ms_max", factFlushMaxMs.get());
+        result.put("commit_duration_ms_total", commitMs.get());
+        result.put("commit_duration_ms_max", commitMaxMs.get());
+        result.put("transaction_duration_ms_total", transactionMs.get());
+        result.put("transaction_duration_ms_max", transactionMaxMs.get());
+        result.put("transaction_retries", retries.get());
+        result.put("last_success_time", lastSuccessTime);
+        return result;
+    }
+
+    private static void updateMax(AtomicLong target, long candidate) {
+        long current;
+        do {
+            current = target.get();
+            if (candidate <= current) return;
+        } while (!target.compareAndSet(current, candidate));
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmNormalizeRuleService.java

@@ -1,5 +1,9 @@
 package com.common.service;
 
+import com.Modules.NormalData.compiled.CompiledRuleCompiler;
+import com.Modules.NormalData.compiled.RuleSource;
+import com.Modules.NormalData.cache.NormalizationLookupFacade;
+import com.Modules.NormalData.cache.NormalizeRuleChangedEvent;
 import com.common.entity.XdrHoneypot;
 import com.common.mapper.XdrHoneypotMapper;
 import com.common.util.MyBatisUtil;
@@ -11,13 +15,17 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.cache.annotation.CacheConfig;
 import org.springframework.cache.annotation.Cacheable;
+import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 
 import com.common.mapper.DmNormalizeRuleMapper;
 import com.common.entity.DmNormalizeRule;
 
 import java.util.List;
 import java.util.Map;
+import java.util.ArrayList;
+import java.util.Collections;
 
 
 
@@ -28,6 +36,15 @@ public class DmNormalizeRuleService {
     @Autowired
     private DmNormalizeRuleMapper dmNormalizeRuleMapper;
 
+    @Autowired
+    private CompiledRuleCompiler compiledRuleCompiler;
+
+    @Autowired
+    private NormalizationLookupFacade normalizationLookupFacade;
+
+    @Autowired
+    private ApplicationEventPublisher eventPublisher;
+
     private static final Logger logger = LoggerFactory.getLogger(DmNormalizeRuleService.class);
 
 
@@ -48,22 +65,32 @@ public class DmNormalizeRuleService {
     /**
      * 根据ID更新规则
      */
+    @Transactional
     public boolean updateById(DmNormalizeRule rule) {
+        validateUpdate(rule, null);
         // 设置更新时间
         rule.setUpdatedAt(java.time.LocalDateTime.now());
         rule.setUpdateTime(java.time.LocalDateTime.now());
+        List<Long> affectedDevices = dmNormalizeRuleMapper.selectDeviceIdsByRuleId(rule.getId());
         int result = dmNormalizeRuleMapper.updateById(rule);
+        if (result > 0) eventPublisher.publishEvent(
+                new NormalizeRuleChangedEvent(rule.getId(), affectedDevices, "UPDATE"));
         return result > 0;
     }
 
     /**
      * 根据ID和租户更新规则
      */
+    @Transactional
     public boolean updateByIdAndTenant(DmNormalizeRule rule) {
+        validateUpdate(rule, rule.getTenantId());
         // 设置更新时间
         rule.setUpdatedAt(java.time.LocalDateTime.now());
         rule.setUpdateTime(java.time.LocalDateTime.now());
+        List<Long> affectedDevices = dmNormalizeRuleMapper.selectDeviceIdsByRuleId(rule.getId());
         int result = dmNormalizeRuleMapper.updateByIdAndTenant(rule);
+        if (result > 0) eventPublisher.publishEvent(
+                new NormalizeRuleChangedEvent(rule.getId(), affectedDevices, "UPDATE_TENANT"));
         return result > 0;
     }
     /**
@@ -90,12 +117,30 @@ public class DmNormalizeRuleService {
      * @param id
      * @return
      */
-    @Cacheable(key = "'deviceId:' + #id")
-    public List<Map<String, Object>> selectByDeviceIdAuto( Long id)
+    public List<RuleSource> selectByDeviceIdAuto(Long id)
     {
-        System.out.println("调用selectByDeviceIdAuto 方法,id:"+id);
-        List<Map<String, Object>>  ruleMap=dmNormalizeRuleMapper.selectByDeviceId(id);
-        return convertClobToString(ruleMap);
+        return normalizationLookupFacade.rules(id).getRules();
+    }
+
+    private void validateUpdate(DmNormalizeRule patch, String tenantId) {
+        if (patch == null || patch.getId() == null) {
+            throw new IllegalArgumentException("规则更新必须提供 id");
+        }
+        DmNormalizeRule current = tenantId == null
+                ? dmNormalizeRuleMapper.selectById(patch.getId())
+                : dmNormalizeRuleMapper.selectByIdAndTenant(patch.getId(), tenantId);
+        if (current == null) throw new IllegalArgumentException("待更新规则不存在: " + patch.getId());
+        RuleSource merged = new RuleSource(patch.getId(), first(patch.getName(), current.getName()),
+                first(patch.getDisplayName(), current.getDisplayName()),
+                first(patch.getDataType(), current.getDataType()),
+                first(patch.getRuleContent(), current.getRuleContent()),
+                patch.getIsRunning() == null ? current.getIsRunning() : patch.getIsRunning(),
+                first(patch.getDelFlag(), current.getDelFlag()));
+        compiledRuleCompiler.compile(merged);
+    }
+
+    private static String first(String preferred, String fallback) {
+        return preferred == null ? fallback : preferred;
     }
 
     /**
@@ -132,4 +177,4 @@ public class DmNormalizeRuleService {
         return list;
     }
 
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java

@@ -0,0 +1,171 @@
+package com.common.service;
+
+import com.common.mapper.SyslogNormalAlarmMapper;
+import com.common.mapper.SyslogNormalDataMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.ibatis.session.ExecutorType;
+import org.apache.ibatis.session.SqlSessionFactory;
+import org.mybatis.spring.SqlSessionTemplate;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionTemplate;
+import org.springframework.transaction.support.TransactionSynchronizationAdapter;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 在一个达梦小事务中执行真正的 MyBatis JDBC batch。
+ *
+ * <p>这个类只负责数据库执行，不负责错误分类、拆分或 Kafka 状态。上层写入器保证传入的所有行
+ * 属于同一张表和同一个动态列签名，因此 MyBatis BatchExecutor 可以复用同一个 PreparedStatement。
+ * 每个 {@link #execute(boolean, List, int)} 调用都会创建独立事务；失败时整个小批次回滚，上层随后
+ * 决定二分定位还是按系统故障停止。</p>
+ */
+@Component
+@Slf4j
+public class DmStandardizedBatchExecutor {
+
+    private final SqlSessionTemplate batchSqlSession;
+    private final PlatformTransactionManager transactionManager;
+    private final StatsChangeCaptureService statsChangeCaptureService;
+    private final DirectStatsProjectionWriter directStatsProjectionWriter;
+    private final DmDirectTransactionMetrics transactionMetrics;
+
+    @org.springframework.beans.factory.annotation.Value("${stats.direct.transaction-log-enabled:false}")
+    private boolean transactionLogEnabled;
+
+    @org.springframework.beans.factory.annotation.Value("${stats.direct.max-concurrency-retries:1}")
+    private int maxConcurrencyRetries;
+
+    @org.springframework.beans.factory.annotation.Value("${stats.direct.concurrency-retry-backoff-ms:20}")
+    private long concurrencyRetryBackoffMs;
+
+    public DmStandardizedBatchExecutor(SqlSessionFactory sqlSessionFactory,
+                                       PlatformTransactionManager transactionManager,
+                                       StatsChangeCaptureService statsChangeCaptureService,
+                                       DirectStatsProjectionWriter directStatsProjectionWriter,
+                                       DmDirectTransactionMetrics transactionMetrics) {
+        // SqlSessionTemplate 本身线程安全。ExecutorType.BATCH 会把相同 SQL 的参数暂存在当前事务会话中，
+        // 直到 flushStatements()/commit 时才真正交给 JDBC 驱动批量执行。
+        this.batchSqlSession = new SqlSessionTemplate(sqlSessionFactory, ExecutorType.BATCH);
+        this.transactionManager = transactionManager;
+        this.statsChangeCaptureService = statsChangeCaptureService;
+        this.directStatsProjectionWriter = directStatsProjectionWriter;
+        this.transactionMetrics = transactionMetrics;
+    }
+
+    /**
+     * 执行一个同构小批次。
+     *
+     * @param alarmTable               true 写 syslog_normal_alarm，false 写 syslog_normal_data
+     * @param rows                     已按相同动态列集合分组的行
+     * @param transactionTimeoutSeconds 当前小事务硬超时
+     */
+    public void execute(boolean alarmTable, List<Map<String, Object>> rows,
+                        int transactionTimeoutSeconds) {
+        if (rows == null || rows.isEmpty()) {
+            return;
+        }
+
+        int attempt = 0;
+        while (true) {
+            try {
+                executeTransaction(alarmTable, rows, transactionTimeoutSeconds, attempt);
+                return;
+            } catch (DirectStatsProjectionWriter.ConcurrentStatsWriteException concurrentFailure) {
+                if (attempt >= Math.max(0, maxConcurrencyRetries)) {
+                    throw concurrentFailure;
+                }
+                attempt++;
+                directStatsProjectionWriter.recordTransactionRetry();
+                transactionMetrics.recordRetry();
+                backoff(attempt);
+            }
+        }
+    }
+
+    private void executeTransaction(boolean alarmTable, List<Map<String, Object>> rows,
+                                    int transactionTimeoutSeconds, int retries) {
+
+        TransactionTemplate transaction = new TransactionTemplate(transactionManager);
+        transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
+        long transactionStarted = System.nanoTime();
+        final TransactionTiming timing = new TransactionTiming();
+        transaction.execute(status -> {
+            TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() {
+                @Override
+                public void beforeCommit(boolean readOnly) {
+                    timing.commitStarted = System.nanoTime();
+                }
+
+                @Override
+                public void afterCommit() {
+                    timing.commitMs = elapsedMs(timing.commitStarted);
+                }
+            });
+            if (alarmTable) {
+                SyslogNormalAlarmMapper mapper = batchSqlSession.getMapper(SyslogNormalAlarmMapper.class);
+                for (Map<String, Object> row : rows) {
+                    // BatchExecutor 下这里不会逐条提交；返回值也不是最终影响行数，不能据此判断成功。
+                    mapper.insertDynamic(row);
+                }
+            } else {
+                SyslogNormalDataMapper mapper = batchSqlSession.getMapper(SyslogNormalDataMapper.class);
+                for (Map<String, Object> row : rows) {
+                    mapper.insertDynamic(row);
+                }
+            }
+
+            // 与事实行共用当前达梦事务；任一变更日志写入失败都会使整批事实回滚。
+            statsChangeCaptureService.captureStandardizedBatch(batchSqlSession, rows, alarmTable);
+
+            // 先 flush 事实和兼容期变更日志，让唯一键、类型和分区错误在统计增量前暴露。
+            long flushStarted = System.nanoTime();
+            batchSqlSession.flushStatements();
+            timing.factFlushMs = elapsedMs(flushStarted);
+
+            // DIRECT/SHADOW 模式在当前事务内按设备、规则聚合写统计；LEGACY 模式会直接返回。
+            // 任一统计 SQL 失败都会回滚本事务的事实行，禁止产生无法定位的统计缺口。
+            timing.statsResult = directStatsProjectionWriter.writeStandardizedRows(rows);
+            return null;
+        });
+        long transactionMs = elapsedMs(transactionStarted);
+        transactionMetrics.recordSuccess(rows.size(), timing.factFlushMs, timing.commitMs, transactionMs);
+        if (transactionLogEnabled && timing.statsResult != null && timing.statsResult.isDirect()) {
+            log.info("DIRECT 达梦事务完成: factRows={}, deviceGroups={}, ruleGroups={}, aggregateMs={}, " +
+                            "factFlushMs={}, statsMergeMs={}, materializeMs={}, commitMs={}, transactionMs={}, retry={}",
+                    rows.size(), timing.statsResult.getDeviceGroups(), timing.statsResult.getRuleGroups(),
+                    timing.statsResult.getAggregateMs(), timing.factFlushMs,
+                    timing.statsResult.getMergeMs(), timing.statsResult.getMaterializeMs(),
+                    timing.commitMs, transactionMs, retries);
+        }
+    }
+
+    public Map<String, Object> status() {
+        return transactionMetrics.snapshot();
+    }
+
+    private static long elapsedMs(long startedNanos) {
+        return startedNanos == 0L ? 0L : (System.nanoTime() - startedNanos) / 1_000_000L;
+    }
+
+    private static final class TransactionTiming {
+        private long factFlushMs;
+        private long commitStarted;
+        private long commitMs;
+        private DirectStatsProjectionWriter.WriteResult statsResult;
+    }
+
+    private void backoff(int attempt) {
+        long delay = Math.max(0L, concurrencyRetryBackoffMs) * attempt;
+        if (delay == 0L) return;
+        try {
+            Thread.sleep(delay);
+        } catch (InterruptedException interrupted) {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException("直接统计并发重试等待被中断", interrupted);
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java

@@ -0,0 +1,555 @@
+package com.common.service;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.dao.CannotAcquireLockException;
+import org.springframework.dao.DataAccessResourceFailureException;
+import org.springframework.dao.DataIntegrityViolationException;
+import org.springframework.dao.QueryTimeoutException;
+import org.springframework.dao.TransientDataAccessResourceException;
+import org.springframework.dao.TypeMismatchDataAccessException;
+import org.springframework.stereotype.Component;
+import org.springframework.jdbc.CannotGetJdbcConnectionException;
+
+import java.net.ConnectException;
+import java.net.SocketException;
+import java.sql.SQLDataException;
+import java.sql.SQLIntegrityConstraintViolationException;
+import java.sql.SQLNonTransientConnectionException;
+import java.sql.SQLRecoverableException;
+import java.sql.SQLException;
+import java.sql.SQLTimeoutException;
+import java.sql.SQLTransientConnectionException;
+import java.sql.Timestamp;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * 把一个 Kafka poll 产生的动态标准化结果转换为有限、可隔离的达梦批量事务。
+ *
+ * <p>写入器遵守三条不能破坏的可靠性规则：</p>
+ *
+ * <ol>
+ *   <li>正常行按目标表、分区日期和动态列签名分组后批量提交；</li>
+ *   <li>数据类错误回滚当前小批次并有限二分，最终只把坏行交给 Retry/DLT；</li>
+ *   <li>连接、超时、提交未知等系统故障绝不降级为大量单条请求，而是立即失败并保留 Kafka offset。</li>
+ * </ol>
+ */
+@Slf4j
+@Component
+public class DmStandardizedBatchWriter {
+
+    private static final String MODE_BATCH = "BATCH";
+
+    private final SyslogNormalDataService normalDataService;
+    private final DmStandardizedBatchExecutor batchExecutor;
+    private final Executor writerExecutor;
+
+    // 未配置时直接启用本次改造的批写路径；SINGLE 只作为现场故障回退开关保留。
+    @Value("${dm.standardized-write.mode:BATCH}")
+    private String mode;
+
+    @Value("${dm.standardized-write.batch-size:100}")
+    private int batchSize;
+
+    @Value("${dm.standardized-write.transaction-timeout-seconds:45}")
+    private int transactionTimeoutSeconds;
+
+    @Value("${dm.standardized-write.kafka-batch-deadline-ms:90000}")
+    private long kafkaBatchDeadlineMs;
+
+    @Value("${dm.standardized-write.max-split-depth:10}")
+    private int maxSplitDepth;
+
+    @Value("${dm.standardized-write.max-fallback-operations:100}")
+    private int maxFallbackOperations;
+
+    @Value("${dm.standardized-write.circuit-breaker-failure-threshold:3}")
+    private int circuitBreakerFailureThreshold;
+
+    @Value("${dm.standardized-write.circuit-breaker-backoff-ms:30000}")
+    private long circuitBreakerBackoffMs;
+
+    private final AtomicInteger consecutiveSystemFailures = new AtomicInteger();
+    private volatile long circuitOpenUntilMs;
+
+    public DmStandardizedBatchWriter(SyslogNormalDataService normalDataService,
+                                     DmStandardizedBatchExecutor batchExecutor,
+                                     @Qualifier("dmStandardizedWriterExecutor") Executor writerExecutor) {
+        this.normalDataService = normalDataService;
+        this.batchExecutor = batchExecutor;
+        this.writerExecutor = writerExecutor;
+    }
+
+    /**
+     * 写入当前 Kafka 批次的全部标准化行。
+     *
+     * <p>返回值只包含可归因到具体原始消息的数据错误。系统故障会直接抛出
+     * {@link SystemicWriteException}，调用方必须让整个 Kafka 批次不 ACK。</p>
+     */
+    public <T> WriteReport<T> write(List<WriteRequest<T>> requests) {
+        WriteReport<T> report = new WriteReport<>();
+        if (requests == null || requests.isEmpty()) {
+            return report;
+        }
+
+        ensureCircuitAllowsAttempt();
+        long deadlineMs = System.currentTimeMillis() + Math.max(1L, kafkaBatchDeadlineMs);
+        IsolationBudget budget = new IsolationBudget(deadlineMs,
+                Math.max(0, maxSplitDepth), Math.max(0, maxFallbackOperations));
+
+        try {
+            List<WriteRequest<T>> prepared = prepareRows(requests, report);
+            if (prepared.isEmpty()) {
+                consecutiveSystemFailures.set(0);
+                return report;
+            }
+
+            if (!MODE_BATCH.equalsIgnoreCase(String.valueOf(mode).trim())) {
+                // 安全开关：SINGLE 模式仍走统一的数据校验和错误分类，便于灰度比较和快速回滚。
+                for (WriteRequest<T> request : prepared) {
+                    writeSingle(request, report);
+                }
+            } else {
+                writeGroupedBatches(prepared, report, budget);
+            }
+
+            consecutiveSystemFailures.set(0);
+            return report;
+        } catch (SystemicWriteException systemicFailure) {
+            recordSystemicFailure(systemicFailure);
+            throw systemicFailure;
+        } catch (RuntimeException unexpectedFailure) {
+            SystemicWriteException systemicFailure = new SystemicWriteException(
+                    "达梦标准化批写出现未确认异常，禁止提交 Kafka offset", unexpectedFailure);
+            recordSystemicFailure(systemicFailure);
+            throw systemicFailure;
+        }
+    }
+
+    /** 在任何 SQL 分组前执行与单条路径相同的校验，坏行不应拖累同批正常数据。 */
+    private <T> List<WriteRequest<T>> prepareRows(List<WriteRequest<T>> requests, WriteReport<T> report) {
+        List<WriteRequest<T>> prepared = new ArrayList<>(requests.size());
+        for (WriteRequest<T> request : requests) {
+            try {
+                normalDataService.prepareForInsert(request.getData());
+                prepared.add(request);
+            } catch (RuntimeException invalidRow) {
+                report.recordFailure(request.getSource(), new PermanentRowWriteException(
+                        "标准化结果未通过写入前校验", invalidRow));
+            }
+        }
+        return prepared;
+    }
+
+    /**
+     * 相同目标表、log_time 日期和非空列集合才会进入同一批次。
+     * LinkedHashMap 保持稳定遍历顺序，便于测试和现场日志复现。
+     */
+    private <T> void writeGroupedBatches(List<WriteRequest<T>> requests, WriteReport<T> report,
+                                         IsolationBudget budget) {
+        Map<BatchKey, List<WriteRequest<T>>> groups = new LinkedHashMap<>();
+        for (WriteRequest<T> request : requests) {
+            try {
+                BatchKey key = batchKey(request.getData());
+                groups.computeIfAbsent(key, ignored -> new ArrayList<>()).add(request);
+            } catch (RuntimeException invalidGroupingKey) {
+                report.recordFailure(request.getSource(), new PermanentRowWriteException(
+                        "标准化结果缺少稳定分区时间或列签名无效", invalidGroupingKey));
+            }
+        }
+
+        List<CompletableFuture<WriteReport<T>>> futures = new ArrayList<>(groups.size());
+        try {
+            for (Map.Entry<BatchKey, List<WriteRequest<T>>> entry : groups.entrySet()) {
+                futures.add(CompletableFuture.supplyAsync(() -> {
+                    WriteReport<T> groupReport = new WriteReport<>();
+                    writeGroup(entry.getKey(), entry.getValue(), groupReport, budget);
+                    return groupReport;
+                }, writerExecutor));
+            }
+
+            CompletableFuture<Void> allGroups = CompletableFuture.allOf(
+                    futures.toArray(new CompletableFuture[0]));
+            long remainingMs = budget.remainingMs();
+            if (remainingMs <= 0L) {
+                throw new TimeoutException("达梦批写在任务提交后已超过 Kafka 批次 deadline");
+            }
+            allGroups.get(remainingMs, TimeUnit.MILLISECONDS);
+            for (CompletableFuture<WriteReport<T>> future : futures) {
+                report.merge(future.join());
+            }
+        } catch (RejectedExecutionException rejected) {
+            cancelOutstanding(futures);
+            throw new SystemicWriteException("达梦批写线程池已满，保留 Kafka offset 进行背压", rejected);
+        } catch (TimeoutException timeout) {
+            cancelOutstanding(futures);
+            throw new SystemicWriteException("达梦批写超过 Kafka 批次 deadline，禁止降级为逐条请求", timeout);
+        } catch (InterruptedException interrupted) {
+            cancelOutstanding(futures);
+            Thread.currentThread().interrupt();
+            throw new SystemicWriteException("达梦批写等待被中断，写入结果未确认", interrupted);
+        } catch (ExecutionException executionFailure) {
+            cancelOutstanding(futures);
+            Throwable cause = executionFailure.getCause();
+            if (cause instanceof SystemicWriteException) {
+                throw (SystemicWriteException) cause;
+            }
+            throw new SystemicWriteException("达梦批写任务异常结束，写入结果未确认", cause);
+        }
+    }
+
+    private <T> void writeGroup(BatchKey key, List<WriteRequest<T>> requests,
+                                WriteReport<T> report, IsolationBudget budget) {
+        int effectiveBatchSize = Math.max(1, batchSize);
+        for (int start = 0; start < requests.size(); start += effectiveBatchSize) {
+            budget.ensureTimeRemaining();
+            int end = Math.min(start + effectiveBatchSize, requests.size());
+            writeChunkWithIsolation(key, new ArrayList<>(requests.subList(start, end)),
+                    report, budget, 0);
+        }
+    }
+
+    /**
+     * 一个失败小事务只拆失败路径。系统故障绝不进入递归，避免一次达梦故障被放大成 N 次请求。
+     */
+    private <T> void writeChunkWithIsolation(BatchKey key, List<WriteRequest<T>> chunk,
+                                             WriteReport<T> report, IsolationBudget budget, int depth) {
+        budget.ensureTimeRemaining();
+        try {
+            List<Map<String, Object>> rows = new ArrayList<>(chunk.size());
+            for (WriteRequest<T> request : chunk) {
+                rows.add(request.getData());
+            }
+            batchExecutor.execute(key.alarmTable, rows, transactionTimeoutSeconds);
+            report.recordSuccessfulRows(chunk.size());
+        } catch (RuntimeException batchFailure) {
+            if (isSystemic(batchFailure)) {
+                throw new SystemicWriteException(
+                        "达梦连接、超时或事务状态异常；禁止把失败批次放大为逐条写入", batchFailure);
+            }
+
+            if (chunk.size() == 1) {
+                // 批执行缩小到一行后改走原单条服务。该服务保留唯一冲突存在性确认逻辑，
+                // 能把 Retry/Replay 已写入的确定性 ID 当作幂等成功。
+                writeSingle(chunk.get(0), report);
+                return;
+            }
+
+            if (!budget.tryConsumeFallback(depth)) {
+                RuntimeException limitedFailure = new RetryableRowWriteException(
+                        "批写失败隔离达到次数、深度或时间上限，转入受限 Retry，禁止无限拆分", batchFailure);
+                for (WriteRequest<T> request : chunk) {
+                    report.recordFailure(request.getSource(), limitedFailure);
+                }
+                return;
+            }
+
+            int middle = chunk.size() / 2;
+            writeChunkWithIsolation(key, new ArrayList<>(chunk.subList(0, middle)),
+                    report, budget, depth + 1);
+            writeChunkWithIsolation(key, new ArrayList<>(chunk.subList(middle, chunk.size())),
+                    report, budget, depth + 1);
+        }
+    }
+
+    /**
+     * 单条写入只用于 SINGLE 模式或失败隔离的叶子节点。第一条系统性错误会立即抛出，
+     * 调用方不会继续尝试同批剩余行，因此不会形成“100 条批写超时后再超时 100 次”的风暴。
+     */
+    private <T> void writeSingle(WriteRequest<T> request, WriteReport<T> report) {
+        try {
+            normalDataService.insertDynamic(request.getData());
+            report.recordSuccessfulRows(1);
+        } catch (RuntimeException singleFailure) {
+            if (isSystemic(singleFailure)) {
+                throw new SystemicWriteException("达梦单条写入发生系统性故障，立即停止后续降级", singleFailure);
+            }
+            RuntimeException routedFailure = isPermanentData(singleFailure)
+                    ? new PermanentRowWriteException("标准化结果包含达梦无法接受的永久数据错误", singleFailure)
+                    : new RetryableRowWriteException("标准化结果单条写入失败，进入有限 Retry", singleFailure);
+            report.recordFailure(request.getSource(), routedFailure);
+        }
+    }
+
+    private BatchKey batchKey(Map<String, Object> data) {
+        boolean alarmTable = normalDataService.isAlarmData(data);
+        LocalDate partitionDate = partitionDate(data.get("log_time"));
+
+        // MyBatis XML 的动态列判断只包含非 null 值，所以签名也只记录非 null 列。
+        // TreeSet 提供固定顺序，避免 HashMap 插入顺序不同却被错误拆成多个批次。
+        Set<String> columns = new TreeSet<>();
+        for (Map.Entry<String, Object> entry : data.entrySet()) {
+            if (entry.getValue() != null) {
+                columns.add(entry.getKey());
+            }
+        }
+        if (columns.isEmpty()) {
+            throw new IllegalArgumentException("标准化结果没有可写入列");
+        }
+        return new BatchKey(alarmTable, partitionDate, String.join(",", columns));
+    }
+
+    private LocalDate partitionDate(Object value) {
+        if (value instanceof LocalDateTime) {
+            return ((LocalDateTime) value).toLocalDate();
+        }
+        if (value instanceof Timestamp) {
+            return ((Timestamp) value).toLocalDateTime().toLocalDate();
+        }
+        if (value instanceof Date) {
+            return new Timestamp(((Date) value).getTime()).toLocalDateTime().toLocalDate();
+        }
+        throw new IllegalArgumentException("log_time 不是受支持的稳定时间类型: "
+                + (value == null ? "null" : value.getClass().getName()));
+    }
+
+    /** 连接、超时、锁/资源故障属于整库级问题，不能用逐条降级处理。 */
+    private boolean isSystemic(Throwable failure) {
+        Throwable current = failure;
+        while (current != null) {
+            if (current instanceof CannotGetJdbcConnectionException
+                    || current instanceof DirectStatsProjectionWriter.DirectStatsWriteException
+                    || current instanceof DataAccessResourceFailureException
+                    || current instanceof QueryTimeoutException
+                    || current instanceof TransientDataAccessResourceException
+                    || current instanceof CannotAcquireLockException
+                    || current instanceof SQLTimeoutException
+                    || current instanceof SQLTransientConnectionException
+                    || current instanceof SQLNonTransientConnectionException
+                    || current instanceof SQLRecoverableException
+                    || current instanceof ConnectException
+                    || current instanceof SocketException
+                    || current instanceof TimeoutException) {
+                return true;
+            }
+            if (current instanceof SQLException) {
+                String sqlState = ((SQLException) current).getSQLState();
+                // SQLState 08 表示连接异常，40 表示事务回滚/并发冲突，HYT 表示 ODBC/JDBC 超时。
+                // 这些错误无法限定到某一行，必须停止整批而不是逐条放大。
+                if (sqlState != null && (sqlState.startsWith("08")
+                        || sqlState.startsWith("40") || sqlState.startsWith("HYT"))) {
+                    return true;
+                }
+            }
+            String message = String.valueOf(current.getMessage()).toLowerCase(Locale.ROOT);
+            if (containsAny(message, "connection refused", "connection closed", "broken pipe",
+                    "socket", "network", "timeout", "timed out", "database unavailable",
+                    "连接失败", "连接关闭", "网络", "超时", "数据库不可用")) {
+                return true;
+            }
+            current = current.getCause();
+        }
+        return false;
+    }
+
+    /** 明确的数据和约束问题可直接进入永久失败终态；未知单行错误先走有限 Retry。 */
+    private boolean isPermanentData(Throwable failure) {
+        Throwable current = failure;
+        while (current != null) {
+            if (current instanceof DataIntegrityViolationException
+                    || current instanceof TypeMismatchDataAccessException
+                    || current instanceof SQLDataException
+                    || current instanceof SQLIntegrityConstraintViolationException
+                    || current instanceof IllegalArgumentException) {
+                return true;
+            }
+            if (current instanceof SQLException) {
+                String sqlState = ((SQLException) current).getSQLState();
+                // SQLState 22 是数据转换/范围错误，23 是完整性约束错误；两者都可以在
+                // 回滚后通过二分稳定定位到具体行。
+                if (sqlState != null && (sqlState.startsWith("22") || sqlState.startsWith("23"))) {
+                    return true;
+                }
+            }
+            String message = String.valueOf(current.getMessage()).toLowerCase(Locale.ROOT);
+            if (containsAny(message, "value too long", "data too long", "truncat", "invalid number",
+                    "data conversion", "not null", "check constraint", "unique constraint",
+                    "duplicate key", "violate constraint", "字段长度", "数据超长", "类型转换",
+                    "非空约束", "检查约束", "唯一约束")) {
+                return true;
+            }
+            current = current.getCause();
+        }
+        return false;
+    }
+
+    private boolean containsAny(String text, String... tokens) {
+        for (String token : tokens) {
+            if (text.contains(token)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private void ensureCircuitAllowsAttempt() {
+        long remaining = circuitOpenUntilMs - System.currentTimeMillis();
+        if (remaining > 0L) {
+            throw new SystemicWriteException("达梦批写熔断器处于退避期，剩余 " + remaining + "ms", null);
+        }
+    }
+
+    private void recordSystemicFailure(SystemicWriteException failure) {
+        int failures = consecutiveSystemFailures.incrementAndGet();
+        if (failures >= Math.max(1, circuitBreakerFailureThreshold)) {
+            circuitOpenUntilMs = System.currentTimeMillis() + Math.max(1L, circuitBreakerBackoffMs);
+            log.error("达梦标准化写入连续系统故障，熔断并停止试写: failures={}, backoffMs={}",
+                    failures, circuitBreakerBackoffMs, failure);
+        }
+    }
+
+    private void cancelOutstanding(List<? extends CompletableFuture<?>> futures) {
+        for (CompletableFuture<?> future : futures) {
+            if (!future.isDone()) {
+                future.cancel(true);
+            }
+        }
+    }
+
+    /** 一条待写记录及其原 Kafka 消息关联对象。 */
+    public static final class WriteRequest<T> {
+        private final T source;
+        private final Map<String, Object> data;
+
+        public WriteRequest(T source, Map<String, Object> data) {
+            this.source = Objects.requireNonNull(source, "source");
+            this.data = Objects.requireNonNull(data, "data");
+        }
+
+        public T getSource() {
+            return source;
+        }
+
+        public Map<String, Object> getData() {
+            return data;
+        }
+    }
+
+    /**
+     * 可归因到具体消息的失败集合。相同源消息产生多行时只保留第一项失败，调用方据此只发布一次 Retry/DLT。
+     */
+    public static final class WriteReport<T> {
+        private final Map<T, RuntimeException> failures = new LinkedHashMap<>();
+        private int successfulRows;
+
+        private synchronized void recordFailure(T source, RuntimeException failure) {
+            failures.putIfAbsent(source, failure);
+        }
+
+        private synchronized void recordSuccessfulRows(int rows) {
+            successfulRows += rows;
+        }
+
+        private synchronized void merge(WriteReport<T> other) {
+            successfulRows += other.successfulRows;
+            for (Map.Entry<T, RuntimeException> failure : other.failures.entrySet()) {
+                failures.putIfAbsent(failure.getKey(), failure.getValue());
+            }
+        }
+
+        public Map<T, RuntimeException> getFailures() {
+            return Collections.unmodifiableMap(failures);
+        }
+
+        public int getSuccessfulRows() {
+            return successfulRows;
+        }
+    }
+
+    /** 已确认只影响某一行，现有失败路由会把它直接送入 DLT。 */
+    public static class PermanentRowWriteException extends RuntimeException {
+        public PermanentRowWriteException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
+    /** 无法确认永久性的单行错误，现有失败路由会先进入有限 Retry，耗尽后再进 DLT。 */
+    public static class RetryableRowWriteException extends RuntimeException {
+        public RetryableRowWriteException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
+    /** 影响范围无法限定到单行，调用方必须保留原 Kafka offset。 */
+    public static class SystemicWriteException extends RuntimeException {
+        public SystemicWriteException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
+    private static final class BatchKey {
+        private final boolean alarmTable;
+        private final LocalDate partitionDate;
+        private final String columnSignature;
+
+        private BatchKey(boolean alarmTable, LocalDate partitionDate, String columnSignature) {
+            this.alarmTable = alarmTable;
+            this.partitionDate = partitionDate;
+            this.columnSignature = columnSignature;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (this == other) return true;
+            if (!(other instanceof BatchKey)) return false;
+            BatchKey that = (BatchKey) other;
+            return alarmTable == that.alarmTable
+                    && Objects.equals(partitionDate, that.partitionDate)
+                    && Objects.equals(columnSignature, that.columnSignature);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(alarmTable, partitionDate, columnSignature);
+        }
+    }
+
+    /** 所有递归分支共享同一预算，防止多条坏数据把一个 Kafka 批次拖成无限同步重试。 */
+    private static final class IsolationBudget {
+        private final long deadlineMs;
+        private final int maxDepth;
+        private final int maxOperations;
+        private final AtomicInteger operations = new AtomicInteger();
+
+        private IsolationBudget(long deadlineMs, int maxDepth, int maxOperations) {
+            this.deadlineMs = deadlineMs;
+            this.maxDepth = maxDepth;
+            this.maxOperations = maxOperations;
+        }
+
+        private boolean tryConsumeFallback(int currentDepth) {
+            return currentDepth < maxDepth
+                    && System.currentTimeMillis() < deadlineMs
+                    && operations.incrementAndGet() <= maxOperations;
+        }
+
+        private long remainingMs() {
+            return deadlineMs - System.currentTimeMillis();
+        }
+
+        private void ensureTimeRemaining() {
+            if (remainingMs() <= 0L) {
+                throw new SystemicWriteException("达梦批写超过 Kafka 批次 deadline", new TimeoutException());
+            }
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/InfluxSyslogService.java

@@ -1,209 +1,150 @@
 package com.common.service;
 
- import com.influx.InfluxDBClient;
+import com.influx.InfluxDBClient;
+import com.influx.raw.RawInfluxMetrics;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
+import java.time.Duration;
 import java.time.Instant;
-import java.time.LocalDateTime;
+import java.time.LocalDate;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
-import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
- import java.util.ArrayList;
- import org.springframework.util.CollectionUtils;
- import java.util.Collections;
- import java.util.HashMap;
 
+/** V1/V2 原始 Syslog 双读、去重和响应兼容层。 */
 @Slf4j
 @Service
 public class InfluxSyslogService {
+    private static final ZoneId SHANGHAI = ZoneId.of("Asia/Shanghai");
+    private static final DateTimeFormatter TABLE_DATE = DateTimeFormatter.ofPattern("yyyyMMdd");
 
-    @Autowired
-    private com.influx.InfluxDBClient influxDBClient;
+    private final InfluxDBClient influxDBClient;
+    private final RawInfluxMetrics metrics;
 
-    private static final DateTimeFormatter TABLE_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
-    private static final DateTimeFormatter INPUT_DATE_FORMATTER = DateTimeFormatter.ISO_DATE_TIME;
+    public InfluxSyslogService(InfluxDBClient influxDBClient, RawInfluxMetrics metrics) {
+        this.influxDBClient = influxDBClient;
+        this.metrics = metrics;
+    }
 
-    /**
-     * 接口1：按时间范围查询（最长一天）
-     */
     public List<Map<String, Object>> queryByTimeRange(String startTime, String endTime) {
-        try {
-            Instant start = parseInstant(startTime);
-            Instant end = parseInstant(endTime);
-
-            validateTimeRange(start, end);
-
-            // 生成需要查询的表名（处理跨天情况）
-            List<String> measurementNames = generateMeasurementNames(start, end);
-            List<Map<String, Object>> allResults = new ArrayList<>();
-
-            for (String measurement : measurementNames) {
-                List<Map<String, Object>> results = influxDBClient.queryByTimeRange(measurement, start, end);
-                allResults.addAll(results);
-            }
-
-            log.info("按时间范围查询成功，查询表: {}, 返回记录数: {}", measurementNames, allResults.size());
-            return allResults;
-
-        } catch (Exception e) {
-            log.error("按时间范围查询失败: {}", e.getMessage());
-            throw new RuntimeException("查询失败: " + e.getMessage(), e);
+        Instant start = parseInstant(startTime), end = parseInstant(endTime);
+        validateTimeRange(start, end);
+        List<Map<String, Object>> rows = new ArrayList<>();
+        for (String measurement : generateMeasurementNames(start, end)) {
+            rows.addAll(influxDBClient.queryByTimeRange(measurement, start, end));
         }
+        return merge(rows);
     }
 
-    /**
-     * 接口2：按UUID查询详细日志
-     */
     public List<Map<String, Object>> queryByUuid(String uuid) {
-        try {
-            // 从UUID中提取日期部分
-            String datePart = extractDateFromUuid(uuid);
-            String measurement = "syslog_security_" + datePart;
-
-            List<Map<String, Object>> results = influxDBClient.queryByUuid(measurement, uuid);
-
-            log.info("按UUID查询成功，表: {}, UUID: {}, 返回记录数: {}", measurement, uuid, results.size());
-            return results;
-
-        } catch (Exception e) {
-            log.error("按UUID查询失败: {}", e.getMessage());
-            throw new RuntimeException("查询失败: " + e.getMessage(), e);
-        }
+        String datePart = extractDateFromUuid(uuid);
+        return merge(influxDBClient.queryByUuid("syslog_security_" + datePart, uuid));
     }
 
-
-
-    /**
-     * 接口3：按设备ID和时间范围查询
-     */
     public List<Map<String, Object>> queryByDeviceIdAndTime(String deviceId, String startTime, String endTime) {
-        try {
-            Instant start = parseInstant(startTime);
-            Instant end = parseInstant(endTime);
-
-            validateTimeRange(start, end);
-
-            List<String> measurementNames = generateMeasurementNames(start, end);
-            List<Map<String, Object>> allResults = new ArrayList<>();
-
-            for (String measurement : measurementNames) {
-                List<Map<String, Object>> results = influxDBClient.queryByDeviceIdAndTime(measurement, deviceId, start, end);
-                allResults.addAll(results);
-            }
-
-            log.info("按设备ID查询成功，设备: {}, 返回记录数: {}", deviceId, allResults.size());
-            return allResults;
-
-        } catch (Exception e) {
-            log.error("按设备ID查询失败: {}", e.getMessage());
-            throw new RuntimeException("查询失败: " + e.getMessage(), e);
+        if (deviceId == null || deviceId.trim().isEmpty()) throw new IllegalArgumentException("设备ID不能为空");
+        Instant start = parseInstant(startTime), end = parseInstant(endTime);
+        validateTimeRange(start, end);
+        List<Map<String, Object>> rows = new ArrayList<>();
+        for (String measurement : generateMeasurementNames(start, end)) {
+            rows.addAll(influxDBClient.queryByDeviceIdAndTime(measurement, deviceId, start, end));
         }
+        return merge(rows);
     }
 
-    /**
-     * 接口4：模糊查询message内容
-     */
     public List<Map<String, Object>> queryByMessageContent(String startTime, String endTime, String keyword) {
-        try {
-            Instant start = parseInstant(startTime);
-            Instant end = parseInstant(endTime);
-
-            validateTimeRange(start, end);
-            validateKeyword(keyword);
-
-            List<String> measurementNames = generateMeasurementNames(start, end);
-            List<Map<String, Object>> allResults = new ArrayList<>();
-
-            for (String measurement : measurementNames) {
-                List<Map<String, Object>> results = influxDBClient.queryByMessageContent(measurement, start, end, keyword);
-                allResults.addAll(results);
-            }
-
-            log.info("模糊查询成功，关键字: {}, 返回记录数: {}", keyword, allResults.size());
-            return allResults;
-
-        } catch (Exception e) {
-            log.error("模糊查询失败: {}", e.getMessage());
-            throw new RuntimeException("查询失败: " + e.getMessage(), e);
+        Instant start = parseInstant(startTime), end = parseInstant(endTime);
+        validateTimeRange(start, end); validateKeyword(keyword);
+        List<Map<String, Object>> rows = new ArrayList<>();
+        for (String measurement : generateMeasurementNames(start, end)) {
+            rows.addAll(influxDBClient.queryByMessageContent(measurement, start, end, keyword));
         }
+        return merge(rows);
     }
+
     public Map<String, List<Map<String, Object>>> queryByUuids(List<String> uuids) {
-        if (CollectionUtils.isEmpty(uuids)) {
-            return Collections.emptyMap();
-        }
-        Map<String, List<Map<String, Object>>> resultMap = new HashMap<>();
-        for (String uuid : uuids) {
-            // 查询单个UUID的日志
-            List<Map<String, Object>> logList = queryByUuid(uuid);
-            resultMap.put(uuid, logList);
-        }
-        return resultMap;
+        if (CollectionUtils.isEmpty(uuids)) return Collections.emptyMap();
+        Map<String, List<Map<String, Object>>> result = new LinkedHashMap<>();
+        for (String uuid : uuids) result.put(uuid, queryByUuid(uuid));
+        return result;
+    }
+
+    /** 按 uuid 去重；跨 schema 冲突优先 V2，同 schema 重复选择更早的 _time。 */
+    List<Map<String, Object>> merge(List<Map<String, Object>> rows) {
+        Map<String, List<Map<String, Object>>> grouped = new LinkedHashMap<>();
+        List<Map<String, Object>> withoutUuid = new ArrayList<>();
+        for (Map<String, Object> row : rows) {
+            Object uuid = row.get("uuid");
+            if (uuid == null || String.valueOf(uuid).isEmpty()) { withoutUuid.add(copyForResponse(row)); continue; }
+            grouped.computeIfAbsent(String.valueOf(uuid), ignored -> new ArrayList<>()).add(row);
+        }
+        List<Map<String, Object>> result = new ArrayList<>(withoutUuid);
+        for (List<Map<String, Object>> candidates : grouped.values()) {
+            candidates.sort(Comparator
+                    .comparing((Map<String, Object> row) -> !"V2".equals(row.get("__raw_schema")))
+                    .thenComparing(this::rowTime));
+            Map<String, Object> selected = candidates.get(0);
+            if (candidates.size() > 1) {
+                metrics.dualReadDuplicate();
+                for (int i = 1; i < candidates.size(); i++) {
+                    if (contentConflicts(selected, candidates.get(i))) { metrics.dualReadConflict(); break; }
+                }
+            }
+            result.add(copyForResponse(selected));
+        }
+        result.sort(Comparator.comparing(this::receiveSortValue).reversed()
+                .thenComparing(row -> String.valueOf(row.get("uuid"))));
+        return result;
+    }
+
+    private boolean contentConflicts(Map<String, Object> left, Map<String, Object> right) {
+        return !java.util.Objects.equals(left.get("message"), right.get("message"))
+                || !java.util.Objects.equals(left.get("receive_time"), right.get("receive_time"));
+    }
+
+    private Map<String, Object> copyForResponse(Map<String, Object> row) {
+        Map<String, Object> copy = new LinkedHashMap<>(row); copy.remove("__raw_schema"); return copy;
+    }
+    private Instant rowTime(Map<String, Object> row) {
+        Object value = row.get("time"); if (value instanceof Instant) return (Instant) value; return Instant.MAX;
+    }
+    private String receiveSortValue(Map<String, Object> row) {
+        Object value = row.get("receive_time"); return value == null ? "" : String.valueOf(value);
     }
-    /**
-     * 从UUID中提取日期部分
-     */
     private String extractDateFromUuid(String uuid) {
-        if (uuid == null || uuid.length() < 8) {
-            throw new IllegalArgumentException("UUID格式不正确，无法提取日期");
-        }
-        return uuid.substring(0, 8);
+        if (uuid == null || !uuid.matches("\\d{8}-.+")) throw new IllegalArgumentException("UUID格式不正确，无法提取日期");
+        LocalDate.parse(uuid.substring(0, 8), DateTimeFormatter.BASIC_ISO_DATE); return uuid.substring(0, 8);
     }
-
-    /**
-     * 解析时间字符串为Instant
-     */
-    private Instant parseInstant(String timeString) {
-        try {
-            return Instant.parse(timeString);
-        } catch (Exception e) {
-            throw new IllegalArgumentException("时间格式不正确，请使用ISO格式，例如: 2023-11-20T00:00:00Z");
-        }
+    private Instant parseInstant(String value) {
+        try { return Instant.parse(value); }
+        catch (Exception failure) { throw new IllegalArgumentException("时间格式不正确，请使用ISO Instant格式", failure); }
     }
-
-    /**
-     * 验证时间范围（最长一天）
-     */
     private void validateTimeRange(Instant start, Instant end) {
-        long hours = ChronoUnit.HOURS.between(start, end);
-        if (hours > 24) {
+        if (!start.isBefore(end)) throw new IllegalArgumentException("开始时间必须早于结束时间");
+        if (Duration.between(start, end).compareTo(Duration.ofHours(24)) > 0)
             throw new IllegalArgumentException("查询时间范围不能超过24小时");
-        }
-        if (start.isAfter(end)) {
-            throw new IllegalArgumentException("开始时间不能晚于结束时间");
-        }
     }
-
-    /**
-     * 验证搜索关键字
-     */
     private void validateKeyword(String keyword) {
-        if (keyword == null || keyword.trim().isEmpty()) {
-            throw new IllegalArgumentException("搜索关键字不能为空");
-        }
-        if (keyword.length() > 32) {
-            throw new IllegalArgumentException("搜索关键字长度不能超过32个字符");
-        }
+        if (keyword == null || keyword.trim().isEmpty()) throw new IllegalArgumentException("搜索关键字不能为空");
+        if (keyword.length() > 32) throw new IllegalArgumentException("搜索关键字长度不能超过32个字符");
     }
 
-    /**
-     * 生成需要查询的表名列表（处理跨天情况）
-     */
-    private List<String> generateMeasurementNames(Instant start, Instant end) {
-        List<String> measurements = new ArrayList<>();
-
-        LocalDateTime startDate = LocalDateTime.ofInstant(start, ZoneId.of("UTC"));
-        LocalDateTime endDate = LocalDateTime.ofInstant(end, ZoneId.of("UTC"));
-
-        LocalDateTime currentDate = startDate;
-        while (!currentDate.toLocalDate().isAfter(endDate.toLocalDate())) {
-            String dateStr = currentDate.format(TABLE_DATE_FORMATTER);
-            measurements.add("syslog_security_" + dateStr);
-            currentDate = currentDate.plusDays(1);
+    /** Flux stop 为排他边界，因此精确上海零点不额外枚举下一日 measurement。 */
+    static List<String> generateMeasurementNames(Instant start, Instant endExclusive) {
+        LocalDate first = start.atZone(SHANGHAI).toLocalDate();
+        LocalDate last = endExclusive.minusNanos(1L).atZone(SHANGHAI).toLocalDate();
+        List<String> names = new ArrayList<>();
+        for (LocalDate date = first; !date.isAfter(last); date = date.plusDays(1)) {
+            names.add("syslog_security_" + TABLE_DATE.format(date));
         }
-
-        return measurements;
+        return names;
     }
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java

@@ -0,0 +1,224 @@
+package com.common.service;
+
+import com.common.entity.IngressStatsEvent;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.dao.DuplicateKeyException;
+import org.springframework.dao.CannotAcquireLockException;
+import org.springframework.dao.DeadlockLoserDataAccessException;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionTemplate;
+
+import java.sql.Date;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * 把 Kafka 入口元数据按设备/探针批内聚合后幂等写入小型统计投影。
+ * 不写逐事件接收日志；失败时抛出，让调用方禁止提交 Kafka offset。
+ */
+@Service
+public class IngressStatsProjectionWriter {
+    private final JdbcTemplate jdbc;
+    private final PlatformTransactionManager transactionManager;
+    private final StatsWriteModeProvider modeProvider;
+
+    @Value("${stats.ingress-metadata.enabled:false}")
+    private boolean enabled;
+    @Value("${stats.incremental.read-enabled:false}")
+    private boolean readEnabled;
+    @Value("${stats.incremental.transaction-timeout-seconds:45}")
+    private int transactionTimeoutSeconds;
+    @Value("${stats.direct.shadow-device-table:device_daily_stats_direct_shadow}")
+    private String shadowDeviceTable;
+    @Value("${stats.business-zone:Asia/Shanghai}")
+    private String businessZone;
+    private final AtomicLong successfulBatches = new AtomicLong();
+    private final AtomicLong receivedEvents = new AtomicLong();
+    private final AtomicLong updatedGroups = new AtomicLong();
+    private final AtomicLong failedBatches = new AtomicLong();
+    private volatile LocalDateTime lastSuccessTime;
+    private volatile LocalDateTime lastFailureTime;
+
+    public IngressStatsProjectionWriter(JdbcTemplate jdbc,
+                                        PlatformTransactionManager transactionManager,
+                                        StatsWriteModeProvider modeProvider) {
+        this.jdbc = jdbc;
+        this.transactionManager = transactionManager;
+        this.modeProvider = modeProvider;
+    }
+
+    public int writeBatch(List<IngressStatsEvent> events) {
+        if (!enabled || events == null || events.isEmpty()) return 0;
+        final Aggregation aggregation = aggregate(events);
+        if (aggregation.devices.isEmpty() && aggregation.collects.isEmpty()) return 0;
+
+        try {
+            int groups;
+            try {
+                groups = writeAggregation(aggregation);
+            } catch (RuntimeException concurrentWrite) {
+                if (!retryableConcurrentWrite(concurrentWrite)) throw concurrentWrite;
+                // 新设备/探针在多节点首次出现时，两个 MERGE 可能同时走 NOT MATCHED。
+                // 锁冲突或失败事务已整体回滚；重试后的 MIN/MAX 更新仍保持幂等。
+                groups = writeAggregation(aggregation);
+            }
+            successfulBatches.incrementAndGet();
+            receivedEvents.addAndGet(events.size());
+            updatedGroups.addAndGet(groups);
+            lastSuccessTime = LocalDateTime.now();
+            return groups;
+        } catch (RuntimeException failure) {
+            failedBatches.incrementAndGet();
+            lastFailureTime = LocalDateTime.now();
+            throw failure;
+        }
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> status = new LinkedHashMap<>();
+        status.put("enabled", enabled);
+        status.put("read_enabled", readEnabled);
+        status.put("target_device_table", modeProvider.writesShadowProjection()
+                ? "device_daily_stats+" + shadowDeviceTable : "device_daily_stats");
+        status.put("successful_batches", successfulBatches.get());
+        status.put("received_events", receivedEvents.get());
+        status.put("updated_groups", updatedGroups.get());
+        status.put("failed_batches", failedBatches.get());
+        status.put("last_success_time", lastSuccessTime);
+        status.put("last_failure_time", lastFailureTime);
+        return status;
+    }
+
+    private static boolean retryableConcurrentWrite(RuntimeException failure) {
+        return failure instanceof DuplicateKeyException
+                || failure instanceof CannotAcquireLockException
+                || failure instanceof DeadlockLoserDataAccessException;
+    }
+
+    private int writeAggregation(Aggregation aggregation) {
+        TransactionTemplate transaction = new TransactionTemplate(transactionManager);
+        transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
+        Integer updated = transaction.execute(status -> {
+            int groups = 0;
+            for (Map.Entry<DeviceKey, LocalDateTime> entry : aggregation.devices.entrySet()) {
+                DeviceKey key = entry.getKey();
+                LocalDateTime receiveTime = entry.getValue();
+                // 接收时间是 MIN/MAX 幂等字段。SHADOW 期间正式表仍需对外服务，因此同时写正式和影子；
+                // 与可累加的 parse/non-log 不同，重复 MAX 不会造成双计数。
+                mergeDeviceReceive("device_daily_stats", key, receiveTime);
+                if (modeProvider.writesShadowProjection()) {
+                    mergeDeviceReceive(safeTable(shadowDeviceTable), key, receiveTime);
+                }
+                if (readEnabled && LocalDate.now(ZoneId.of(businessZone)).equals(key.date)) {
+                    jdbc.update("UPDATE device_device SET last_receive_time=CASE " +
+                                    "WHEN last_receive_time IS NULL OR last_receive_time<? THEN ? ELSE last_receive_time END," +
+                                    "updated_at=SYSDATE WHERE id=?",
+                            receiveTime, receiveTime, key.deviceId);
+                }
+                groups++;
+            }
+            for (Map.Entry<Long, CollectTimes> entry : aggregation.collects.entrySet()) {
+                CollectTimes times = entry.getValue();
+                jdbc.update("MERGE INTO device_collect_stats t USING " +
+                                "(SELECT ? device_collect_id,? first_success_time,? last_success_time FROM DUAL) s " +
+                                "ON (t.device_collect_id=s.device_collect_id) " +
+                                "WHEN MATCHED THEN UPDATE SET t.first_success_time=CASE " +
+                                "WHEN t.first_success_time IS NULL OR t.first_success_time>s.first_success_time " +
+                                "THEN s.first_success_time ELSE t.first_success_time END," +
+                                "t.last_success_time=CASE WHEN t.last_success_time IS NULL " +
+                                "OR t.last_success_time<s.last_success_time THEN s.last_success_time " +
+                                "ELSE t.last_success_time END,t.updated_at=SYSDATE " +
+                                "WHEN NOT MATCHED THEN INSERT (device_collect_id,first_success_time,last_success_time," +
+                                "last_fail_time,projection_seq,updated_at) VALUES " +
+                                "(s.device_collect_id,s.first_success_time,s.last_success_time,NULL,0,SYSDATE)",
+                        entry.getKey(), times.min, times.max);
+                if (readEnabled) {
+                    jdbc.update("UPDATE device_collect_task SET " +
+                                    "first_time=CASE WHEN first_time IS NULL OR first_time>? THEN ? ELSE first_time END," +
+                                    "last_success_time=CASE WHEN last_success_time IS NULL OR last_success_time<? " +
+                                    "THEN ? ELSE last_success_time END,updated_at=SYSDATE WHERE id=?",
+                            times.min, times.min, times.max, times.max, entry.getKey());
+                }
+                groups++;
+            }
+            return groups;
+        });
+        return updated == null ? 0 : updated;
+    }
+
+    static Aggregation aggregate(List<IngressStatsEvent> events) {
+        Aggregation result = new Aggregation();
+        for (IngressStatsEvent event : events) {
+            if (event == null || event.getReceiveTime() == null) continue;
+            if (event.getDeviceId() != null) {
+                DeviceKey key = new DeviceKey(event.getReceiveTime().toLocalDate(), event.getDeviceId());
+                LocalDateTime current = result.devices.get(key);
+                if (current == null || event.getReceiveTime().isAfter(current)) {
+                    result.devices.put(key, event.getReceiveTime());
+                }
+            }
+            if (event.getDeviceCollectId() != null) {
+                CollectTimes times = result.collects.get(event.getDeviceCollectId());
+                if (times == null) result.collects.put(event.getDeviceCollectId(),
+                        new CollectTimes(event.getReceiveTime(), event.getReceiveTime()));
+                else {
+                    if (event.getReceiveTime().isBefore(times.min)) times.min = event.getReceiveTime();
+                    if (event.getReceiveTime().isAfter(times.max)) times.max = event.getReceiveTime();
+                }
+            }
+        }
+        return result;
+    }
+
+    static final class Aggregation {
+        // 所有节点使用相同顺序取得投影行锁，避免不同 Kafka 批次顺序造成交叉等待。
+        final Map<DeviceKey, LocalDateTime> devices = new TreeMap<>(
+                Comparator.comparing((DeviceKey key) -> key.date)
+                        .thenComparingLong(key -> key.deviceId));
+        final Map<Long, CollectTimes> collects = new TreeMap<>();
+    }
+    static final class DeviceKey {
+        final LocalDate date; final long deviceId;
+        DeviceKey(LocalDate date, long deviceId) { this.date = date; this.deviceId = deviceId; }
+        @Override public boolean equals(Object other) {
+            if (this == other) return true;
+            if (!(other instanceof DeviceKey)) return false;
+            DeviceKey key = (DeviceKey) other;
+            return deviceId == key.deviceId && date.equals(key.date);
+        }
+        @Override public int hashCode() { return 31 * date.hashCode() + Long.hashCode(deviceId); }
+    }
+    static final class CollectTimes {
+        LocalDateTime min; LocalDateTime max;
+        CollectTimes(LocalDateTime min, LocalDateTime max) { this.min = min; this.max = max; }
+    }
+
+    private void mergeDeviceReceive(String table, DeviceKey key, LocalDateTime receiveTime) {
+        jdbc.update("MERGE INTO " + safeTable(table) + " t USING " +
+                        "(SELECT ? stat_date,? device_id,? receive_time FROM DUAL) s " +
+                        "ON (t.stat_date=s.stat_date AND t.device_id=s.device_id) " +
+                        "WHEN MATCHED THEN UPDATE SET t.last_receive_time=CASE " +
+                        "WHEN t.last_receive_time IS NULL OR t.last_receive_time<s.receive_time " +
+                        "THEN s.receive_time ELSE t.last_receive_time END,t.updated_at=SYSDATE " +
+                        "WHEN NOT MATCHED THEN INSERT (stat_date,device_id,parse_count,non_log_count," +
+                        "last_receive_time,projection_seq,updated_at) VALUES " +
+                        "(s.stat_date,s.device_id,0,0,s.receive_time,0,SYSDATE)",
+                Date.valueOf(key.date), key.deviceId, receiveTime);
+    }
+
+    private static String safeTable(String table) {
+        if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
+            throw new IllegalStateException("非法入口设备统计表名: " + table);
+        }
+        return table;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/LogDataCompleteService.java

@@ -51,7 +51,10 @@ public class LogDataCompleteService {
             JSONArray completeColumns = ruleJson.getJSONArray("complete_columns");
 
             if (completeColumns == null || completeColumns.isEmpty()) {
-                log.warn("补全规则为空");
+                // 空补全规则表示当前规则不需要补全，是正常业务分支。
+                // 本方法位于逐消息、逐规则的消费热路径，避免输出 WARN/INFO
+                // 造成日志刷屏和同步日志 I/O 开销。
+                log.debug("泛化规则未配置数据补全条件，跳过补全");
                 return false;
             }
 
@@ -255,4 +258,4 @@ public class LogDataCompleteService {
 
 
 
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/LogDataFilterService.java

@@ -4,13 +4,11 @@ package com.common.service;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
-import com.common.mapper.SyslogNormalDataMapper;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
-import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
 
@@ -30,7 +28,10 @@ public class LogDataFilterService {
                 JSONArray filtersParams = filtersParamsObj.getJSONArray("expressions");
 
                 if (filtersParams == null || filtersParams.isEmpty()) {
-                    log.warn("泛化规则-数据过滤规则为空，默认不处理!");
+                    // 空过滤规则表示当前规则不需要过滤，是正常业务分支。
+                    // 本方法位于逐消息、逐规则的消费热路径，禁止在此打印 WARN/INFO，
+                    // 否则高吞吐场景会产生海量同步日志 I/O，显著降低 Kafka 消费速度。
+                    log.debug("泛化规则未配置数据过滤条件，跳过过滤");
                     return false;
                 }
                 // 遍历所有过滤组，只要满足任一过滤组即可
@@ -201,4 +202,4 @@ public class LogDataFilterService {
     }
 
 
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsChangeCaptureService.java

@@ -0,0 +1,215 @@
+package com.common.service;
+
+import com.common.entity.DeviceReceiveLog;
+import com.common.entity.StatsChangeRecord;
+import com.common.entity.SyslogNonNormalMessage;
+import com.common.mapper.StatsChangeLogMapper;
+import org.mybatis.spring.SqlSessionTemplate;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * 在事实写入事务中同步追加最小变更记录。开关关闭时不访问增量统计表，便于先部署代码再执行 DDL。
+ */
+@Service
+public class StatsChangeCaptureService {
+
+    private final StatsChangeLogMapper mapper;
+    private final StatsWriteModeProvider modeProvider;
+
+    @Value("${stats.incremental.capture-enabled:false}")
+    private boolean enabled;
+
+    /** 旧接收明细的逐行捕获可能放大历史清理，客户确认前默认关闭但保留实现。 */
+    @Value("${stats.device-receive.change-capture-enabled:false}")
+    private boolean deviceReceiveCaptureEnabled;
+
+    public StatsChangeCaptureService(StatsChangeLogMapper mapper,
+                                     StatsWriteModeProvider modeProvider) {
+        this.mapper = mapper;
+        this.modeProvider = modeProvider;
+    }
+
+    public boolean isEnabled() {
+        return enabled && modeProvider.writesLegacyChangeLog();
+    }
+
+    public void captureStandardizedInsert(Map<String, Object> row, boolean alarm) {
+        if (isEnabled()) {
+            insertChange(mapper, standardizedInsert(row, alarm));
+        }
+    }
+
+    /** BatchExecutor 必须使用同一个 BATCH SqlSession，避免同一事务切换 ExecutorType。 */
+    public void captureStandardizedBatch(SqlSessionTemplate batchSession,
+                                         List<Map<String, Object>> rows,
+                                         boolean alarm) {
+        if (!isEnabled() || rows == null || rows.isEmpty()) {
+            return;
+        }
+        StatsChangeLogMapper batchMapper = batchSession.getMapper(StatsChangeLogMapper.class);
+        List<StatsChangeRecord> changes = new java.util.ArrayList<>(rows.size());
+        for (Map<String, Object> row : rows) {
+            StatsChangeRecord change = standardizedInsert(row, alarm);
+            changes.add(change);
+            batchMapper.insert(change);
+        }
+        for (StatsChangeRecord change : changes) {
+            batchMapper.insertPending(change);
+        }
+    }
+
+    public void captureNonNormalInsert(SyslogNonNormalMessage message) {
+        if (!isEnabled()) {
+            return;
+        }
+        String rowId = requiredText(message.getId(), "非标日志缺少主键");
+        insertChange(mapper, StatsChangeRecord.builder()
+                .changeId("NON_NORMAL:" + rowId + ":INSERT")
+                .sourceType("NON_NORMAL")
+                .sourceRowId(rowId)
+                .operationType("INSERT")
+                .newStatDate(dateValue(message.getCreatedAt()))
+                .newDeviceId(longValue(message.getDeviceId()))
+                .newEventTime(message.getLogTime())
+                .build());
+    }
+
+    public void captureNonNormalUpdate(SyslogNonNormalMessage oldValue,
+                                       SyslogNonNormalMessage newValue) {
+        if (!isEnabled() || oldValue == null || newValue == null) {
+            return;
+        }
+        String rowId = requiredText(oldValue.getId(), "非标日志缺少主键");
+        insertChange(mapper, StatsChangeRecord.builder()
+                .changeId("NON_NORMAL:" + rowId + ":UPDATE:" + UUID.randomUUID())
+                .sourceType("NON_NORMAL")
+                .sourceRowId(rowId)
+                .operationType("UPDATE")
+                .oldStatDate(dateValue(oldValue.getCreatedAt()))
+                .newStatDate(dateValue(newValue.getCreatedAt() != null
+                        ? newValue.getCreatedAt() : oldValue.getCreatedAt()))
+                .oldDeviceId(longValue(oldValue.getDeviceId()))
+                .newDeviceId(longValue(newValue.getDeviceId() != null
+                        ? newValue.getDeviceId() : oldValue.getDeviceId()))
+                .oldEventTime(oldValue.getLogTime())
+                .newEventTime(newValue.getLogTime() != null
+                        ? newValue.getLogTime() : oldValue.getLogTime())
+                .build());
+    }
+
+    public void captureDeviceReceiveInsert(DeviceReceiveLog log) {
+        if (!isEnabled() || !deviceReceiveCaptureEnabled) {
+            return;
+        }
+        String rowId = log.getId() == null ? UUID.randomUUID().toString() : log.getId().toString();
+        insertChange(mapper, StatsChangeRecord.builder()
+                .changeId("DEVICE_RECEIVE:" + rowId + ":INSERT")
+                .sourceType("DEVICE_RECEIVE")
+                .sourceRowId(rowId)
+                .operationType("INSERT")
+                .newStatDate(dateValue(log.getReceiveTime() != null
+                        ? log.getReceiveTime() : log.getCreatedAt()))
+                .newDeviceId(longValue(log.getDeviceId()))
+                .newCollectId(longValue(log.getDeviceCollectId()))
+                .newEventTime(log.getReceiveTime())
+                .newCollectTime(log.getCreatedAt())
+                .newPushSuccess(booleanValue(log.getPushSuccess()))
+                .build());
+    }
+
+    public void captureDeviceReceiveBatch(List<DeviceReceiveLog> logs) {
+        if (!isEnabled() || !deviceReceiveCaptureEnabled || logs == null) {
+            return;
+        }
+        for (DeviceReceiveLog log : logs) {
+            captureDeviceReceiveInsert(log);
+        }
+    }
+
+    public void captureDeviceReceiveDeletes(LocalDateTime startTime, LocalDateTime endTime) {
+        if (isEnabled() && deviceReceiveCaptureEnabled) {
+            mapper.captureDeviceReceiveDeletes(startTime, endTime);
+            mapper.enqueueDeviceReceiveDeletes(startTime, endTime);
+        }
+    }
+
+    private void insertChange(StatsChangeLogMapper targetMapper, StatsChangeRecord change) {
+        targetMapper.insert(change);
+        targetMapper.insertPending(change);
+    }
+
+    private StatsChangeRecord standardizedInsert(Map<String, Object> row, boolean alarm) {
+        String source = alarm ? "NORMAL_ALARM" : "NORMAL_DATA";
+        String rowId = requiredText(row.get("id"), "标准化结果缺少确定性 id");
+        return StatsChangeRecord.builder()
+                .changeId(source + ":" + rowId + ":INSERT")
+                .sourceType(source)
+                .sourceRowId(rowId)
+                .operationType("INSERT")
+                .newStatDate(dateValue(row.get("created_at")))
+                .newDeviceId(longValue(row.get("device_id")))
+                .newRuleId(longValue(row.get("normalize_rule_id")))
+                .newEventTime(timeValue(row.get("log_time")))
+                .build();
+    }
+
+    private static String requiredText(Object value, String message) {
+        if (value == null || value.toString().trim().isEmpty()) {
+            throw new IllegalArgumentException(message);
+        }
+        return value.toString();
+    }
+
+    private static Long longValue(Object value) {
+        if (value == null) {
+            return null;
+        }
+        return value instanceof Number
+                ? ((Number) value).longValue()
+                : Long.valueOf(value.toString());
+    }
+
+    private static Integer booleanValue(Boolean value) {
+        return value == null ? null : (value ? 1 : 0);
+    }
+
+    private static LocalDate dateValue(Object value) {
+        LocalDateTime time = timeValue(value);
+        return time == null ? null : time.toLocalDate();
+    }
+
+    private static LocalDateTime timeValue(Object value) {
+        if (value == null) {
+            return null;
+        }
+        if (value instanceof LocalDateTime) {
+            return (LocalDateTime) value;
+        }
+        if (value instanceof LocalDate) {
+            return ((LocalDate) value).atStartOfDay();
+        }
+        if (value instanceof OffsetDateTime) {
+            return ((OffsetDateTime) value).atZoneSameInstant(ZoneId.systemDefault()).toLocalDateTime();
+        }
+        if (value instanceof Timestamp) {
+            return ((Timestamp) value).toLocalDateTime();
+        }
+        if (value instanceof Date) {
+            Instant instant = Instant.ofEpochMilli(((Date) value).getTime());
+            return LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
+        }
+        throw new IllegalArgumentException("不支持的统计时间类型: " + value.getClass().getName());
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsWriteModeProvider.java

@@ -0,0 +1,63 @@
+package com.common.service;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.Locale;
+
+/**
+ * 统计写入模式的唯一解释入口，避免多个布尔开关组合出双计数或无人维护统计的状态。
+ */
+@Component
+public class StatsWriteModeProvider {
+
+    public enum Mode {
+        LEGACY_CHANGE_LOG,
+        DIRECT_SHADOW,
+        DIRECT,
+        FACT_ONLY_REBUILD_REQUIRED
+    }
+
+    private final Mode mode;
+
+    public StatsWriteModeProvider(@Value("${stats.write-mode:LEGACY_CHANGE_LOG}") String configuredMode) {
+        this.mode = parse(configuredMode);
+    }
+
+    public Mode getMode() {
+        return mode;
+    }
+
+    public boolean writesLegacyChangeLog() {
+        return mode == Mode.LEGACY_CHANGE_LOG || mode == Mode.DIRECT_SHADOW;
+    }
+
+    public boolean writesDirectProjection() {
+        return mode == Mode.DIRECT || mode == Mode.DIRECT_SHADOW;
+    }
+
+    public boolean writesShadowProjection() {
+        return mode == Mode.DIRECT_SHADOW;
+    }
+
+    public boolean materializesBusinessFields() {
+        return mode == Mode.DIRECT;
+    }
+
+    public boolean requiresRebuild() {
+        return mode == Mode.FACT_ONLY_REBUILD_REQUIRED;
+    }
+
+    static Mode parse(String configuredMode) {
+        String normalized = configuredMode == null
+                ? ""
+                : configuredMode.trim().toUpperCase(Locale.ROOT);
+        try {
+            return Mode.valueOf(normalized);
+        } catch (IllegalArgumentException invalidMode) {
+            throw new IllegalStateException("不支持的 stats.write-mode: " + configuredMode
+                    + "，允许值为 LEGACY_CHANGE_LOG、DIRECT_SHADOW、DIRECT、FACT_ONLY_REBUILD_REQUIRED",
+                    invalidMode);
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java

@@ -10,11 +10,14 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
+import org.springframework.dao.DataAccessException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.time.LocalDateTime;
 
 @Service
 public class SyslogNormalDataService {
@@ -23,6 +26,10 @@ public class SyslogNormalDataService {
     private SyslogNormalDataMapper syslogNormalDataMapper;
     @Autowired
     private SyslogNormalAlarmMapper syslogNormalAlarmMapper;
+    @Autowired
+    private StatsChangeCaptureService statsChangeCaptureService;
+    @Autowired
+    private DirectStatsProjectionWriter directStatsProjectionWriter;
 
 
     /**
@@ -30,11 +37,64 @@ public class SyslogNormalDataService {
      */
     @Transactional
     public int insertDynamic(Map<String, Object> dataMap) {
-        // 设置必要字段的默认值
-        if (!dataMap.containsKey("created_at")) {
+        // [流程节点7] 根据标准化结果分类落库：安全事件写入 syslog_normal_alarm，访问事件写入 syslog_normal_data。
+        // [SYSLOG-ALARM][DBBATCH] 每条标准化结果单独开启事务并执行动态 INSERT，会产生大量数据库往返、SQL 解析和提交开销；
+        // 高峰期 DB 变慢又会放大 Kafka 批次超时。建议按目标表和固定列集合聚合批次写入，并用 event_id + 规则版本唯一键保证重试幂等。
+        prepareForInsert(dataMap);
+
+        // [流程节点7.2] event_level 非空走安全事件表；为空走访问事件表，这是两条后续分析链路的分叉点。
+        boolean alarmData = isAlarmData(dataMap);
+        try {
+            int inserted = alarmData
+                    ? syslogNormalAlarmMapper.insertDynamic(dataMap)
+                    : syslogNormalDataMapper.insertDynamic(dataMap);
+            if (inserted > 0) {
+                statsChangeCaptureService.captureStandardizedInsert(dataMap, alarmData);
+                directStatsProjectionWriter.writeStandardizedRow(dataMap);
+            }
+            return inserted;
+        } catch (DataAccessException dataAccessFailure) {
+            if (isUniqueConstraintViolation(dataAccessFailure)
+                    && idempotentResultExists(dataMap, alarmData)) {
+                // 达梦可能不会把唯一约束冲突翻译为 DuplicateKeyException。确认原结果存在后按幂等成功处理，
+                // 只保留一条简洁告警，不打印异常栈，也不进入非标、Retry 或 DLT。
+                logger.warn("标准化结果重复，已按幂等成功处理: table={}, eventId={}, ruleId={}, ruleVersion={}",
+                        alarmData ? "syslog_normal_alarm" : "syslog_normal_data",
+                        dataMap.get("syslog_uuid"), dataMap.get("normalize_rule_id"), dataMap.get("rule_version"));
+                return 0;
+            }
+            throw dataAccessFailure;
+        }
+    }
+
+    /**
+     * 在单条和批量写入之前执行完全相同的数据准备与关键字段校验。
+     *
+     * <p>批写器必须在分组前调用本方法。这样无效 IP、稳定时间和幂等字段的处理不会因为
+     * SINGLE/BATCH 模式不同而产生两套数据口径。本方法只修改当前行 Map，不访问数据库，
+     * 也不开启事务。</p>
+     */
+    public void prepareForInsert(Map<String, Object> dataMap) {
+        if (dataMap == null) {
+            throw new IllegalArgumentException("标准化写入数据不能为空");
+        }
+
+        // [流程节点7.1] 补齐入库时间并校验关键字段，保证动态 SQL 使用的数据满足基础约束。
+        boolean idempotentWrite = dataMap.get("id") != null && dataMap.get("rule_version") != null;
+        if (!dataMap.containsKey("created_at") || dataMap.get("created_at") == null) {
+            if (idempotentWrite) {
+                throw new IllegalArgumentException("幂等标准化写入缺少稳定 created_at");
+            }
+            // 仅供历史非 Syslog 调用入口兼容。活动 Syslog 主链路会在上游把 created_at
+            // 固定为首次 receive_time。
             dataMap.put("created_at", new java.util.Date());
         }
-        if (!dataMap.containsKey("log_time")) {
+        if (!dataMap.containsKey("log_time") || dataMap.get("log_time") == null) {
+            if (idempotentWrite) {
+                throw new IllegalArgumentException("幂等标准化写入缺少稳定分区键 log_time");
+            }
+            // 仅供历史非 Syslog 调用入口兼容。log_time 是现场标准化表的真实分区键；活动 Syslog
+            // 主链路会在上游把它固定为首次 receive_time，保证 Retry/Replay 进入同一分区并命中本地唯一键。
             dataMap.put("log_time", new java.util.Date());
         }
         if (dataMap.containsKey("tc_hostip")) {
@@ -43,15 +103,78 @@ public class SyslogNormalDataService {
                 String hostip=dataMap.get("tc_hostip").toString();
                 logger.error("tc_hostip 字段非IP类型，值为：{}，将赋值为null",hostip);
                 dataMap.remove("tc_hostip");
-
             }
         }
-        //判断是否包含event_level (事件级别）,区分告警日志和访问日志，分别进入不同的标准化数据表
-        if (dataMap != null && dataMap.get("event_level") != null) {
-            return syslogNormalAlarmMapper.insertDynamic(dataMap);
+    }
+
+    /**
+     * 批写器和单条写入共用同一个目标表判断，避免两种模式产生不同分流结果。
+     *
+     * <p>这里仍保持现有业务口径：event_level 非空写告警表，否则写普通表。这个口径本身
+     * 可能把“缺失 event_level 的安全事件”分到普通表，但它是另一个业务改造事项，不能在
+     * 本次批写优化中顺带改变。</p>
+     */
+    public boolean isAlarmData(Map<String, Object> dataMap) {
+        return dataMap != null && dataMap.get("event_level") != null;
+    }
+
+    /**
+     * 批量失败缩小到单条后，用于确认唯一冲突是否属于已成功写入的幂等重投。
+     */
+    public boolean isIdempotentResultPresent(Map<String, Object> dataMap) {
+        return idempotentResultExists(dataMap, isAlarmData(dataMap));
+    }
+
+    /** 暴露统一的唯一冲突识别，供批写失败隔离逻辑复用。 */
+    public boolean isUniqueViolation(Throwable failure) {
+        return isUniqueConstraintViolation(failure);
+    }
+
+    private boolean idempotentResultExists(Map<String, Object> dataMap, boolean alarmData) {
+        LocalDateTime logTime = localDateTimeValue(dataMap.get("log_time"));
+        String id = stringValue(dataMap.get("id"));
+        if (logTime == null || id == null) {
+            return false;
         }
-        else
-            return syslogNormalDataMapper.insertDynamic(dataMap);
+        if (alarmData) {
+            return syslogNormalAlarmMapper.countByLogTimeAndId(logTime, id) > 0;
+        }
+        return syslogNormalDataMapper.countByLogTimeAndId(logTime, id) > 0;
+    }
+
+    private boolean isUniqueConstraintViolation(Throwable failure) {
+        Throwable current = failure;
+        while (current != null) {
+            String message = String.valueOf(current.getMessage()).toLowerCase(Locale.ROOT);
+            if (message.contains("violate unique constraint")
+                    || message.contains("unique constraint")
+                    || message.contains("duplicate key")) {
+                return true;
+            }
+            current = current.getCause();
+        }
+        return false;
+    }
+
+    private LocalDateTime localDateTimeValue(Object value) {
+        if (value instanceof LocalDateTime) {
+            return (LocalDateTime) value;
+        }
+        if (value instanceof java.sql.Timestamp) {
+            return ((java.sql.Timestamp) value).toLocalDateTime();
+        }
+        if (value instanceof java.util.Date) {
+            return new java.sql.Timestamp(((java.util.Date) value).getTime()).toLocalDateTime();
+        }
+        return null;
+    }
+
+    private String stringValue(Object value) {
+        if (value == null) {
+            return null;
+        }
+        String result = value.toString();
+        return result.trim().isEmpty() ? null : result;
     }
 
     /**
@@ -66,7 +189,12 @@ public class SyslogNormalDataService {
         if (entity.getLogTime() == null) {
             entity.setLogTime(java.time.LocalDateTime.now());
         }
-        return syslogNormalDataMapper.insertByEntity(entity);
+        int inserted = syslogNormalDataMapper.insertByEntity(entity);
+        if (inserted > 0) {
+            Map<String, Object> row = convertEntityToMap(entity);
+            directStatsProjectionWriter.writeStandardizedRow(row);
+        }
+        return inserted;
     }
 
     /**
@@ -87,7 +215,14 @@ public class SyslogNormalDataService {
                 data.put("log_time", new java.util.Date());
             }
         }
-        return syslogNormalDataMapper.batchInsert(dataList);
+        // 旧 mapper 的 batchInsert 只写固定的少数字段，DIRECT 下会造成“统计有 device/rule，
+        // 事实行却丢了 device/rule”的永久差异。主 Kafka 链路使用 DmStandardizedBatchExecutor；
+        // 这个低频兼容 API 逐条复用完整动态 INSERT，但仍处于本方法的同一个 Spring 事务中。
+        int inserted = 0;
+        for (Map<String, Object> data : dataList) {
+            inserted += insertDynamic(data);
+        }
+        return inserted;
     }
 
     /**
@@ -129,4 +264,4 @@ public class SyslogNormalDataService {
     private String camelToSnake(String str) {
         return str.replaceAll("([a-z])([A-Z])", "$1_$2").toLowerCase();
     }
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/AnalysisRuleServiceImpl.java

@@ -69,6 +69,7 @@ public class AnalysisRuleServiceImpl implements AnalysisRuleService {
 
     @Override
     public Map<String, Object> executeRealtimeRule(String ruleId) {
+        // [流程节点8] 加载指定实时规则并转交实时分析引擎执行；该入口也供手工触发和调度器调用。
         log.info("执行实时分析规则: ruleId={}", ruleId);
 
         // 查询规则
@@ -93,14 +94,21 @@ public class AnalysisRuleServiceImpl implements AnalysisRuleService {
             // 执行单个规则
             List<AnalysisAnalysisRule> ruleList = new ArrayList<>();
             ruleList.add(rule);
+            // [流程节点8.3] 把已校验的规则交给实时分析引擎；返回结果供调用方判断本次窗口是否成功。
             List<Map<String, Object>> results = realtimeAnalysisEngine.executeRules(ruleList);
 
             Map<String, Object> result = new HashMap<>();
-            result.put("success", true);
+            boolean allCompleted = !results.isEmpty() && results.stream()
+                    .allMatch(item -> "success".equalsIgnoreCase(String.valueOf(item.get("status"))));
+            // [SYSLOG-ALARM][RULECP][已修复] 内部任一窗口失败都会向调度器返回 success=false，禁止推进计划时间。
+            result.put("success", allCompleted);
             result.put("ruleId", ruleId);
             result.put("ruleName", rule.getRuleName());
             result.put("resultCount", results.size());
             result.put("results", results);
+            if (!allCompleted) {
+                result.put("message", "规则窗口未完整成功，成功水位保持不变");
+            }
             return result;
         } catch (Exception e) {
             log.error("执行实时分析规则失败: ruleId={}", ruleId, e);

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/DeviceDeviceServiceImpl.java

@@ -1,9 +1,11 @@
 package com.common.service.impl;
 
 
+import com.Modules.NormalData.cache.NormalizationLookupFacade;
 import com.common.entity.DeviceDevice;
 import com.common.mapper.DeviceDeviceMapper;
 import com.common.service.DeviceDeviceService;
+import com.common.util.SafeCacheUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -11,7 +13,6 @@ import java.util.List;
 import java.util.Map;
 import org.springframework.cache.annotation.CacheConfig;
 import org.springframework.cache.annotation.Cacheable;
-import com.common.util.SafeCacheUtil;
 
 @CacheConfig(cacheNames = "device")
 @Service
@@ -21,18 +22,19 @@ public class DeviceDeviceServiceImpl implements DeviceDeviceService {
     private DeviceDeviceMapper deviceDeviceMapper;
 
 
+    // 仅保留给尚未迁移的 IP 列表查询。
     @Autowired
     private SafeCacheUtil safeCacheUtil;
 
+    @Autowired
+    private NormalizationLookupFacade normalizationLookupFacade;
+
     /**
      * 使用安全的缓存方法
      */
-    @Cacheable(  key = "'device:id:' +#id")
     @Override
     public DeviceDevice getByIdSafely(Integer id) {
-        String cacheKey = "device:id:" + id;
-        return safeCacheUtil.getSafe(cacheKey, DeviceDevice.class,
-                () -> deviceDeviceMapper.selectById(id));
+        return normalizationLookupFacade.device(id);
     }
 
     /**
@@ -49,7 +51,6 @@ public class DeviceDeviceServiceImpl implements DeviceDeviceService {
     @Cacheable(  key = "'device:id:' +#id")
     @Override
     public DeviceDevice getById(Integer id) {
-        System.out.println("exec deviceDeviceMapper.selectById :" + id.toString());
         return deviceDeviceMapper.selectById(id);
     }
 
@@ -100,4 +101,4 @@ public class DeviceDeviceServiceImpl implements DeviceDeviceService {
     public List<DeviceDevice> getActiveDevices() {
         return deviceDeviceMapper.selectActiveDevices();
     }
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/DeviceReceiveLogServiceImpl.java

@@ -4,6 +4,7 @@ package com.common.service.impl;
 import com.common.entity.DeviceReceiveLog;
 import com.common.mapper.DeviceReceiveLogMapper;
 import com.common.service.DeviceReceiveLogService;
+import com.common.service.StatsChangeCaptureService;
 import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
 import lombok.RequiredArgsConstructor;
@@ -21,6 +22,7 @@ import java.util.Map;
 public class DeviceReceiveLogServiceImpl implements DeviceReceiveLogService {
 
     private final DeviceReceiveLogMapper deviceReceiveLogMapper;
+    private final StatsChangeCaptureService statsChangeCaptureService;
 
     @Override
     @Transactional(rollbackFor = Exception.class)
@@ -35,6 +37,7 @@ public class DeviceReceiveLogServiceImpl implements DeviceReceiveLogService {
         }
 
         deviceReceiveLogMapper.insert(log);
+        statsChangeCaptureService.captureDeviceReceiveInsert(log);
        // log.info("插入设备接收日志成功，ID: {}", log.getId());
         return log.getId();
     }
@@ -65,6 +68,7 @@ public class DeviceReceiveLogServiceImpl implements DeviceReceiveLogService {
             });
 
             int inserted = deviceReceiveLogMapper.batchInsert(batchList);
+            statsChangeCaptureService.captureDeviceReceiveBatch(batchList);
             totalInserted += inserted;
             log.info("批量插入进度: {}/{}", end, logs.size());
         }
@@ -142,10 +146,9 @@ public class DeviceReceiveLogServiceImpl implements DeviceReceiveLogService {
         }
 
 
-        int deleted = deviceReceiveLogMapper.deleteByTimeRange(
-                LocalDateTime.of(2025, 11, 1, 0, 0,0) , // 默认2025年11月
-                beforeTime
-        );
+        LocalDateTime startTime = LocalDateTime.of(2025, 11, 1, 0, 0, 0);
+        statsChangeCaptureService.captureDeviceReceiveDeletes(startTime, beforeTime);
+        int deleted = deviceReceiveLogMapper.deleteByTimeRange(startTime, beforeTime);
 
         log.info("删除{}天前的日志，共删除{}条", beforeTime, deleted);
         return deleted;
@@ -158,4 +161,4 @@ public class DeviceReceiveLogServiceImpl implements DeviceReceiveLogService {
         }
         return deviceReceiveLogMapper.selectRecent(limit);
     }
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java

@@ -1,5 +1,6 @@
 package com.common.service.impl;
 
+import com.alibaba.fastjson.JSON;
 import com.common.entity.*;
 import com.common.mapper.*;
 import com.common.service.AnalysisEngine;
@@ -12,6 +13,7 @@ import org.springframework.stereotype.Service;
 
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
+import java.nio.charset.StandardCharsets;
 import java.util.*;
 
 /**
@@ -62,11 +64,16 @@ public class RealtimeAnalysisEngine implements AnalysisEngine {
 
     @Override
     public Map<String, Object> executeRule(AnalysisAnalysisRule rule) {
+        // [流程节点9] 生成规则 SQL，执行过滤、分组和时间窗口计算，并把规则告警写入最终结果表 alarm。
         String batchNo = generateBatchNo();
         //LocalDateTime startTime = LocalDateTime.now();
         //LocalDateTime dataEndTime = startTime;
         //LocalDateTime dataStartTime = startTime.minusMinutes(30); // 默认查询最近30分钟
         // 时间点取整分钟，避免秒级时间导致窗口计算不一致
+        // [SYSLOG-ALARM][RULECP] 以当前机器时间回看固定窗口不是可靠 checkpoint：停机、调度延迟、数据库故障或标准化积压后，
+        // 当前时间仍向前走，未成功窗口不会自然补跑；不同节点时钟偏差也会产生重叠/空洞。
+        // 建议从持久化的“最后连续成功 window_end”计算下一窗口，并为迟到数据配置 allowed lateness。
+        // [流程节点9.1] 根据规则窗口配置计算本次查询的 window_start 和 window_end。
         LocalDateTime startTime = LocalDateTime.now().withSecond(0).withNano(0);
         LocalDateTime dataEndTime = startTime;
 
@@ -117,6 +124,7 @@ public class RealtimeAnalysisEngine implements AnalysisEngine {
             // 更新规则状态为运行中
             ruleMapper.updateTaskStatus(rule.getRuleId(), "running", 1L);
 
+            // [流程节点9.2] 加载字段、WHERE、过滤、GROUP BY 和 HAVING 配置，组成完整规则快照。
             // 加载规则配置
             List<AnalysisField> fields = fieldMapper.selectByRuleId(rule.getRuleId());
             List<AnalysisWhereCondition> whereConditions = whereConditionMapper.selectByRuleId(rule.getRuleId());
@@ -136,6 +144,7 @@ public class RealtimeAnalysisEngine implements AnalysisEngine {
             }
             ***/
 
+            // [流程节点9.3] 将规则配置和时间范围转换为实际执行 SQL；SQL 中的过滤和分组决定候选告警数量。
             // 生成SQL
             String sql = sqlGeneratorService.generateSql(
                     rule,
@@ -151,6 +160,7 @@ public class RealtimeAnalysisEngine implements AnalysisEngine {
 
             log.info("生成的SQL: {}", sql);
 
+            // [流程节点9.4] 在标准化安全事件数据上执行 SQL，得到窗口内聚合后的告警候选结果。
             // 执行SQL
             List<Map<String, Object>> queryResult = jdbcTemplate.queryForList(sql);
             result.put("queryResult", queryResult);
@@ -158,15 +168,19 @@ public class RealtimeAnalysisEngine implements AnalysisEngine {
             // 处理结果，生成告警
             long alarmCount = 0;
             if (!queryResult.isEmpty()) {
-                List<Alarm> alarms = convertToAlarms(rule, queryResult);
+                // [流程节点9.5] 将每个查询结果组转换为 Alarm 实体，补充规则、时间窗口和告警属性。
+                List<Alarm> alarms = convertToAlarms(rule, queryResult, dataStartTime);
                 if (!alarms.isEmpty()) {
                     //String tableName = "alarm_" + dataStartTime.format(DateTimeFormatter.ofPattern("yyyyMMdd"));
                     String tableName = "alarm";
-                    alarmMapper.batchInsert( alarms);
+                    // [SYSLOG-ALARM][AIDEMP][部分修复] 稳定 ID + 已存在集合过滤保证同一规则结果重跑不重复；该兼容模块仍需部署方确认唯一调度实例。
+                    // [流程节点9.6] 写入最终规则告警表 alarm；该数量应与本窗口可落库告警组数一致。
+                    insertMissingAlarms(alarms);
                     alarmCount = alarms.size();
                 }
             }
 
+            // [流程节点9.7] 记录输入数、告警数和 COMPLETED 状态，作为窗口执行结果和后续对账依据。
             // 更新任务历史
             LocalDateTime endTime = LocalDateTime.now();
             long durationSeconds = java.time.Duration.between(startTime, endTime).getSeconds();
@@ -190,6 +204,8 @@ public class RealtimeAnalysisEngine implements AnalysisEngine {
                     rule.getRuleId(), queryResult.size(), alarmCount);
 
         } catch (Exception e) {
+            // [SYSLOG-ALARM][RULECP] 捕获后只返回 status=failed 而不抛异常，上层若只看“调用未抛错”会误判成功并推进窗口。
+            // 建议使用强类型结果并强制上层校验，或抛出异常；失败窗口保持 FAILED/RETRYING。
             log.error("规则执行失败: ruleId={}", rule.getRuleId(), e);
 
             // 更新任务历史
@@ -445,12 +461,13 @@ public class RealtimeAnalysisEngine implements AnalysisEngine {
     /**
      * 转换查询结果为告警对象
      */
-    private List<Alarm> convertToAlarms(AnalysisAnalysisRule rule, List<Map<String, Object>> queryResult) {
+    private List<Alarm> convertToAlarms(AnalysisAnalysisRule rule, List<Map<String, Object>> queryResult,
+                                        LocalDateTime windowStart) {
         List<Alarm> alarms = new ArrayList<>();
 
         for (Map<String, Object> row : queryResult) {
             Alarm alarm = Alarm.builder()
-                    .id(UUID.randomUUID().toString())
+                    .id(stableAlarmId(rule, row, windowStart))
                     .createdAt(LocalDateTime.now())
                     .updatedAt(LocalDateTime.now())
                     .alarmName(rule.getRuleName())
@@ -470,6 +487,10 @@ public class RealtimeAnalysisEngine implements AnalysisEngine {
                     .comment(buildComment(row))
                     .attackChainPhase(getIntegerArray(-1))
                     .judgedState(0)
+                    .ruleId(rule.getRuleId())
+                    .ruleVersion(String.valueOf(rule.getVersion()))
+                    .windowStart(windowStart)
+                    .groupKeyHash(stableAlarmId(rule, row, windowStart))
                     .build();
 
             // 映射查询结果字段到告警对象
@@ -635,6 +656,21 @@ public class RealtimeAnalysisEngine implements AnalysisEngine {
         return alarms;
     }
 
+    private String stableAlarmId(AnalysisAnalysisRule rule, Map<String, Object> row, LocalDateTime windowStart) {
+        String material = rule.getRuleId() + ":" + String.valueOf(rule.getVersion()) + ":" + windowStart + ":"
+                + JSON.toJSONString(new TreeMap<>(row));
+        return UUID.nameUUIDFromBytes(material.getBytes(StandardCharsets.UTF_8)).toString();
+    }
+
+    private void insertMissingAlarms(List<Alarm> candidates) {
+        List<String> ids = new ArrayList<>();
+        for (Alarm alarm : candidates) ids.add(alarm.getId());
+        Set<String> existing = new HashSet<>(alarmMapper.selectExistingIds(ids));
+        List<Alarm> missing = new ArrayList<>();
+        for (Alarm alarm : candidates) if (!existing.contains(alarm.getId())) missing.add(alarm);
+        if (!missing.isEmpty()) alarmMapper.batchInsert(missing);
+    }
+
     private byte[] getBytesValue(Object value) {
         if (value == null) {
             return null;

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/RuleExecutionTimeServiceImpl.java

@@ -23,6 +23,9 @@ import  java.time.temporal.ChronoUnit;
 @Service
 public class RuleExecutionTimeServiceImpl implements RuleExecutionTimeService {
 
+    // [SYSLOG-ALARM][REDISCP] 这里只保存“下次计划执行时间”，没有保存最后连续成功的 window_start/window_end、状态和重试次数。
+    // 调度时间不是业务成功水位：即使 SQL/写库失败，next_execute 仍可能前移。建议以数据库 checkpoint 状态机为事实源，
+    // Redis 仅做缓存/租约；checkpoint 至少包含 rule_id、rule_version、窗口、COMPLETED/FAILED 和计数。
     private static final String REDIS_KEY_PREFIX = "rule:next_execute:";
     private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
 
@@ -55,6 +58,9 @@ public class RuleExecutionTimeServiceImpl implements RuleExecutionTimeService {
         String key = REDIS_KEY_PREFIX + ruleId;
         String value = nextTime.format(DATE_FORMATTER);
 
+        // [SYSLOG-ALARM][REDISCP] Redis 可以配置 RDB/AOF 持久化，但当前 key 主动设置 5 天 TTL，过期、淘汰、故障恢复或误删后会返回 null。
+        // 调度器随后按“当前时间”重新计算，不能知道历史漏跑窗口；而且 Redis set 与数据库告警写入不在同一事务中。
+        // 建议成功 checkpoint 不设置短 TTL，或持久化到数据库，并在告警提交成功后以 CAS/事务方式原子推进。
         // 设置过期时间：5天
         redisTemplate.opsForValue().set(key, value, Duration.ofDays(5));
 
@@ -69,6 +75,8 @@ public class RuleExecutionTimeServiceImpl implements RuleExecutionTimeService {
         String ruleId = rule.getRuleId();
         String key = REDIS_KEY_PREFIX + ruleId;
 
+        // [SYSLOG-ALARM][DISTLOCK] hasKey + 后续 set 是两个 Redis 命令，不具备原子互斥语义；多实例可同时判断不存在并执行同一窗口。
+        // 若 Redis 用作分布式锁，应使用 SET key value NX PX，并实现 owner、续租和 fencing token；更推荐数据库唯一约束兜底。
         // 如果已存在，则不覆盖（防止重启后重新计算）
         if (redisTemplate.hasKey(key)) {
             log.info("规则执行时间已存在，跳过初始化，ruleId={}", ruleId);

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java

@@ -22,6 +22,8 @@ import java.util.List;
 @Service
 public class SqlGeneratorServiceImpl implements SqlGeneratorService {
 
+    // [SYSLOG-ALARM][ROUTE] 实时规则默认只查询 syslog_normal_alarm；若标准化因 event_level 为空而误写
+    // syslog_normal_data，该事件不会成为实时规则候选。建议数据路由改为显式 event_category，并对两表数量做阶段对账。
     private static final String DEFAULT_TABLE_NAME = "syslog_normal_alarm";
 
     @Override
@@ -311,6 +313,9 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService {
             return "";
         }
 
+        // [SYSLOG-ALARM][RULESEM] GROUP BY/时间窗口会把多条原始 Syslog 合并为一个告警组，这是业务降噪语义，不应直接认定为丢失。
+        // 建议规则界面明确“单事件/聚合”模式，并在结果保留 log_count、origin_event_ids、group_key 和 window_start/end，
+        // 使客户能解释“32 条输入为什么只有十几组”；HAVING/WHERE 被过滤数量也应单独计数。
         return "GROUP BY " + String.join(", ", groupByFields);
     }
 

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java

@@ -7,6 +7,8 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.common.entity.SyslogNonNormalMessage;
 import com.common.mapper.SyslogNonNormalMessageMapper;
 import com.common.service.SyslogNonNormalMessageService;
+import com.common.service.StatsChangeCaptureService;
+import com.common.service.DirectStatsProjectionWriter;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -30,6 +32,12 @@ public class SyslogNonNormalMessageServiceImpl
     @Autowired
     private SyslogNonNormalMessageMapper messageMapper;
 
+    @Autowired
+    private StatsChangeCaptureService statsChangeCaptureService;
+
+    @Autowired
+    private DirectStatsProjectionWriter directStatsProjectionWriter;
+
     @Override
     @Transactional(rollbackFor = Exception.class)
     public boolean saveMessage(SyslogNonNormalMessage message) {
@@ -66,7 +74,12 @@ public class SyslogNonNormalMessageServiceImpl
             }
 
             // 执行插入
-            return this.save(message);
+            boolean inserted = this.save(message);
+            if (inserted) {
+                statsChangeCaptureService.captureNonNormalInsert(message);
+                directStatsProjectionWriter.writeNonNormalInsert(message);
+            }
+            return inserted;
         } catch (Exception e) {
             log.error("保存非标日志失败: {}", e.getMessage(), e);
             throw new RuntimeException("保存非标日志失败", e);
@@ -113,7 +126,16 @@ public class SyslogNonNormalMessageServiceImpl
             }
 
             // 批量插入
-            return messageMapper.batchInsert(messages);
+            int inserted = messageMapper.batchInsert(messages);
+            if (inserted > 0) {
+                if (statsChangeCaptureService.isEnabled()) {
+                    for (SyslogNonNormalMessage message : messages) {
+                        statsChangeCaptureService.captureNonNormalInsert(message);
+                    }
+                }
+                directStatsProjectionWriter.writeNonNormalRows(messages);
+            }
+            return inserted;
         } catch (Exception e) {
             log.error("批量保存非标日志失败: {}", e.getMessage(), e);
             throw new RuntimeException("批量保存非标日志失败", e);
@@ -124,6 +146,7 @@ public class SyslogNonNormalMessageServiceImpl
     @Transactional(rollbackFor = Exception.class)
     public boolean updateMessage(SyslogNonNormalMessage message) {
         try {
+            SyslogNonNormalMessage oldValue = getMessageById(message.getId());
             // 设置更新时间
             message.setUpdateTime(LocalDateTime.now());
 
@@ -132,7 +155,12 @@ public class SyslogNonNormalMessageServiceImpl
             wrapper.eq(SyslogNonNormalMessage::getId, message.getId())
                     .eq(SyslogNonNormalMessage::getDelFlag, "0");
 
-            return this.update(message, wrapper);
+            boolean updated = this.update(message, wrapper);
+            if (updated) {
+                statsChangeCaptureService.captureNonNormalUpdate(oldValue, message);
+                directStatsProjectionWriter.writeNonNormalUpdate(oldValue, message);
+            }
+            return updated;
         } catch (Exception e) {
             log.error("更新非标日志失败: {}", e.getMessage(), e);
             throw new RuntimeException("更新非标日志失败", e);
@@ -216,4 +244,4 @@ public class SyslogNonNormalMessageServiceImpl
         // 使用UUID生成唯一ID
         return   UUID.randomUUID().toString().replace("-", "").substring(0, 20);
     }
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/IngressReceiveTimeUtil.java

@@ -0,0 +1,44 @@
+package com.common.util;
+
+import java.time.DateTimeException;
+import java.time.LocalDateTime;
+
+/** 解析接入端固定的 yyyyMMddHHmmssSSS 接收时间，兼容生产使用的 Java 8。 */
+public final class IngressReceiveTimeUtil {
+
+    private static final int RECEIVE_TIME_LENGTH = 17;
+
+    private IngressReceiveTimeUtil() {
+    }
+
+    public static LocalDateTime parse(String value) {
+        if (value == null) {
+            throw new IllegalArgumentException("receive_time 不能为空");
+        }
+        String text = value.trim();
+        if (text.length() != RECEIVE_TIME_LENGTH) {
+            throw new IllegalArgumentException("receive_time 必须为17位数字");
+        }
+        for (int i = 0; i < text.length(); i++) {
+            if (!Character.isDigit(text.charAt(i))) {
+                throw new IllegalArgumentException("receive_time 包含非数字字符");
+            }
+        }
+        try {
+            return LocalDateTime.of(
+                    integer(text, 0, 4),
+                    integer(text, 4, 6),
+                    integer(text, 6, 8),
+                    integer(text, 8, 10),
+                    integer(text, 10, 12),
+                    integer(text, 12, 14),
+                    integer(text, 14, 17) * 1_000_000);
+        } catch (DateTimeException invalidDate) {
+            throw new IllegalArgumentException("receive_time 日期时间无效", invalidDate);
+        }
+    }
+
+    private static int integer(String value, int start, int end) {
+        return Integer.parseInt(value.substring(start, end));
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/JsonParser.java

@@ -144,8 +144,7 @@ public class JsonParser  {
             LinkedHashMap<String, Object> resultMap = new LinkedHashMap<>();
             if(!isValidJson(jsonStr))
             {
-                System.out.println("parseJsonToFlatMap() json str:"+jsonStr);
-                System.out.println("isValidJson(string) is false");
+                logger.warn("parseJsonToFlatMap 收到无效JSON，长度={}", jsonStr == null ? 0 : jsonStr.length());
                 return null;
             }
 
@@ -188,8 +187,7 @@ public class JsonParser  {
             }
 
         } catch (Exception e) {
-            System.out.println("Exception ex.message:"+ e.getMessage());
-
+            logger.error("flattenJson 处理异常: path={}, reason={}", currentPath, e.getMessage());
             throw new RuntimeException("flattenJson 处理异常", e);
         }
     }

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/NestedJsonParserUtil.java

@@ -53,9 +53,8 @@ public class NestedJsonParserUtil {
         try {
             // 1. 预处理JSON字符串
            // String processedJson = preprocessJsonString(jsonStr, allowUnquoted, allowSingleQuotes);
-            String processedJson =jsonStr;
-
-            System.out.println("processedJson:"+processedJson);
+            String processedJson = jsonStr;
+            logger.trace("准备解析JSON，长度={}", processedJson.length());
             // 2. 深度验证嵌套结构
             //validateNestedStructure(processedJson, maxDepth);
             //ObjectMapper mapper = new ObjectMapper(); // 这里使用Jackson
@@ -73,7 +72,7 @@ public class NestedJsonParserUtil {
             return postProcessParsedData(result, maxDepth);
 
         } catch (JsonProcessingException e) {
-            logger.error("JSON解析失败json:{} 错误消息： {}",jsonStr, e.getMessage());
+            logger.warn("JSON解析失败，长度={}, 原因={}", jsonStr.length(), e.getMessage());
 
             // 5. 尝试修复并重新解析
             try {
@@ -84,13 +83,13 @@ public class NestedJsonParserUtil {
                     return postProcessParsedData(result, maxDepth);
                 }
             } catch (Exception ex) {
-                logger.error("JSON修复后解析仍然失败", ex);
+                logger.debug("JSON修复后解析仍然失败: {}", ex.getMessage());
             }
             return null;
             //return Collections.emptyMap();
 
         } catch (Exception e) {
-            logger.error("JSON解析发生未知异常", e);
+            logger.warn("JSON解析发生未知异常，长度={}, 原因={}", jsonStr.length(), e.getMessage());
             return null;
         }
     }
@@ -507,4 +506,4 @@ public class NestedJsonParserUtil {
     }
 
 
-}
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/SafeCacheUtil.java

@@ -106,4 +106,13 @@ public class SafeCacheUtil {
             return loader.get();
         }
     }
-}
+
+    /** 缓存失效失败只记录为 false，不覆盖已经成功提交的数据库事实。 */
+    public boolean evictSafely(String key) {
+        try {
+            return Boolean.TRUE.equals(redisTemplate.delete(key));
+        } catch (Exception ignored) {
+            return false;
+        }
+    }
+}