feat(syslog): 优化大日志通道并增加异常连续消费兜底

- 128 KiB 及以上日志默认进入大日志 Topic
- InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存
- 截断内容追加 HB_XDR_TRUNCATED 标记
- InfluxDB 写入失败时继续执行规则解析和达梦写入
- 未知或系统性异常优先写入普通/大日志 DLT
- DLT 写入失败时记录特殊告警并提交 offset，避免阻塞后续消息
- DLT 失败等待时间默认限制为 1 秒
- 增加连续性放行、DLT 成功及失败监控指标
- 更新 Docker Compose、v3 升级文档及相关测试

haobang-security-dm/.gitattributes

@@ -0,0 +1 @@
+upgrade-doc/v3/kafka/*.sh text eol=lf

haobang-security-dm/syslog-consumer-rule/docker-compose/docker-compose-consumer-rule-prod.yaml

@@ -55,8 +55,20 @@ services:
       - SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest
       - SPRING_KAFKA_CONSUMER_ENABLE_AUTO_COMMIT=false
       - SPRING_KAFKA_CONSUMER_TOPIC=agent-01-syslog-topic
+      - KAFKA_CONSUMER_LARGE_TOPIC=agent-01-syslog-topic.large
       # consumer-rule 是活跃原始日志消费节点，多实例通过同一消费组分担分区。
       - SYSLOG_RAW_CONSUMER_ENABLED=true
+      # 大日志的 poll、fetch、批次和规则计算线程池均独立且有界。
+      - SYSLOG_LARGE_LISTENER_ENABLED=true
+      - SYSLOG_LARGE_LISTENER_CONCURRENCY=1
+      - SYSLOG_LARGE_LISTENER_MAX_POLL_RECORDS=5
+      - SYSLOG_LARGE_LISTENER_MAX_PARTITION_FETCH_BYTES=33554432
+      - SYSLOG_LARGE_LISTENER_FETCH_MAX_BYTES=67108864
+      - SYSLOG_LARGE_PROCESSING_BATCH_MAX_BYTES=8388608
+      - SYSLOG_LARGE_PROCESSOR_CORE_POOL_SIZE=1
+      - SYSLOG_LARGE_PROCESSOR_MAX_POOL_SIZE=2
+      - SYSLOG_LARGE_PROCESSOR_QUEUE_CAPACITY=10
+      - KAFKA_RELIABILITY_MAX_REQUEST_SIZE=33554432
       # Retry/Replay 与 Raw 隔离，避免未到期重试占用实时消费线程。
       - SYSLOG_RELIABILITY_CONSUMER_GROUP_ID=agent-01-syslog-group-dm-retry
       - SYSLOG_RELIABILITY_CONSUMER_ENABLED=true
@@ -79,6 +91,10 @@ services:
       - INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED=true
       - INFLUXDB_RAW_QUERY_MAX_ROWS=10000
       - INFLUXDB_RAW_QUERY_MAX_RESPONSE_BYTES=10485760
+      - INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES=1048576
+      - INFLUXDB_RAW_WRITE_BATCH_MAX_BYTES=8388608
+      - DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES=65536
+      - DM_NON_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES=262144
       - INFLUXDB_BATCH_SIZE=1000
       - INFLUXDB_FLUSH_INTERVAL=1000
       - INFLUXDB_CONNECTION_TIMEOUT=30s

haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java

@@ -43,6 +43,7 @@ import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.time.LocalDateTime;
+import java.nio.charset.StandardCharsets;
 
 import com.common.util.Sm4Util;
 @Slf4j
@@ -53,12 +54,19 @@ public class SysLogProcessor {
     @Qualifier("logNormalProcessorExecutor")
     private Executor taskExecutor;
 
+    @Autowired
+    @Qualifier("largeLogProcessorExecutor")
+    private Executor largeTaskExecutor;
+
     @Value("${app.processor.batch-size:100}")
     private int batchSize;
 
     @Value("${app.processor.process-timeout-ms:120000}")
     private long processTimeoutMs;
 
+    @Value("${syslog.large-listener.processing-batch-max-bytes:8388608}")
+    private long largeProcessingBatchMaxBytes;
+
     @Autowired
     private AppProperties appProperties;
 
@@ -132,7 +140,47 @@ public class SysLogProcessor {
             containerFactory = "rawKafkaListenerContainerFactory"
     )
     public void consumeAndParallelProcess(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
-        consumeBatch(records, ack, false, "RAW");
+        consumeBatch(records, ack, false, "RAW", taskExecutor);
+    }
+
+    @KafkaListener(
+            id = "syslog-consumer-rule-raw-large",
+            groupId = "${syslog.large-listener.group-id:${spring.kafka.consumer.group-id}-large}",
+            autoStartup = "${syslog.large-listener.enabled:true}",
+            topics = "${spring.kafka.consumer.large-topic:agent-syslog-topic.large}",
+            containerFactory = "largeRawKafkaListenerContainerFactory"
+    )
+    public void consumeLargeMessages(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
+        consumeLargeInByteChunks(records, ack, false, "RAW_LARGE");
+    }
+
+    @KafkaListener(
+            id = "syslog-consumer-rule-large-retry-replay",
+            groupId = "${syslog.large-reliability.consumer-group-id:${spring.kafka.consumer.group-id}-large-retry}",
+            autoStartup = "${syslog.large-reliability.consumer.enabled:true}",
+            topics = {
+                    "${syslog.large-reliability.retry-1m-topic:${spring.kafka.consumer.large-topic}.RETRY.1M}",
+                    "${syslog.large-reliability.retry-10m-topic:${spring.kafka.consumer.large-topic}.RETRY.10M}",
+                    "${syslog.large-reliability.retry-1h-topic:${spring.kafka.consumer.large-topic}.RETRY.1H}",
+                    "${syslog.large-reliability.replay-topic:${spring.kafka.consumer.large-topic}.REPLAY}"
+            },
+            containerFactory = "largeRawKafkaListenerContainerFactory"
+    )
+    public void consumeLargeRetryAndReplay(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
+        consumeLargeInByteChunks(records, ack, true, "RETRY_REPLAY_LARGE");
+    }
+
+    private void consumeLargeInByteChunks(List<ConsumerRecord<String, String>> records,
+                                          Acknowledgment ack, boolean enforceRetryDue, String lane) {
+        if (records.isEmpty()) {
+            ack.acknowledge();
+            return;
+        }
+        for (List<ConsumerRecord<String, String>> chunk : partitionRecordsByBytes(
+                records, Math.max(1L, largeProcessingBatchMaxBytes))) {
+            processBatch(chunk, enforceRetryDue, lane, largeTaskExecutor);
+        }
+        ack.acknowledge();
     }
 
     @KafkaListener(
@@ -148,7 +196,7 @@ public class SysLogProcessor {
             concurrency = "${syslog.reliability.listener.concurrency:1}"
     )
     public void consumeRetryAndReplay(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
-        consumeBatch(records, ack, true, "RETRY_REPLAY");
+        consumeBatch(records, ack, true, "RETRY_REPLAY", taskExecutor);
     }
 
     /**
@@ -157,11 +205,17 @@ public class SysLogProcessor {
      * 其写库结果不参与 Kafka ACK。
      */
     private void consumeBatch(List<ConsumerRecord<String, String>> records, Acknowledgment ack,
-                              boolean enforceRetryDue, String lane) {
+                              boolean enforceRetryDue, String lane, Executor normalizationExecutor) {
         if (records.isEmpty()) {
             ack.acknowledge();
             return;
         }
+        processBatch(records, enforceRetryDue, lane, normalizationExecutor);
+        ack.acknowledge();
+    }
+
+    private void processBatch(List<ConsumerRecord<String, String>> records,
+                              boolean enforceRetryDue, String lane, Executor normalizationExecutor) {
         long batchStartedAt = System.currentTimeMillis();
         long prepareStartedAt = batchStartedAt;
         BatchMetrics metrics = new BatchMetrics();
@@ -189,10 +243,9 @@ public class SysLogProcessor {
         long influxMs = System.currentTimeMillis() - influxStartedAt;
 
         long normalizeStartedAt = System.currentTimeMillis();
-        normalizePreparedMessages(persistedMessages, metrics);
+        normalizePreparedMessages(persistedMessages, metrics, normalizationExecutor);
         long normalizeMs = System.currentTimeMillis() - normalizeStartedAt;
 
-        ack.acknowledge();
         totalProcessed.addAndGet(records.size());
         log.info("Syslog批次完成: lane={}, topicPartitions={}, records={}, prepared={}, persisted={}, "
                         + "success={}, filtered={}, retry={}, dlt={}, normalizedRows={}, dmSuccessfulRows={}, "
@@ -218,6 +271,32 @@ public class SysLogProcessor {
                 System.currentTimeMillis() - batchStartedAt);
     }
 
+    static List<List<ConsumerRecord<String, String>>> partitionRecordsByBytes(
+            List<ConsumerRecord<String, String>> records, long maxBytes) {
+        if (maxBytes <= 0L) throw new IllegalArgumentException("maxBytes 必须大于 0");
+        List<List<ConsumerRecord<String, String>>> result = new ArrayList<>();
+        List<ConsumerRecord<String, String>> current = new ArrayList<>();
+        long currentBytes = 0L;
+        for (ConsumerRecord<String, String> record : records) {
+            String value = record.value();
+            long bytes = value == null ? 0L : value.getBytes(StandardCharsets.UTF_8).length;
+            if (!current.isEmpty() && bytes > maxBytes - currentBytes) {
+                result.add(current);
+                current = new ArrayList<>();
+                currentBytes = 0L;
+            }
+            current.add(record);
+            currentBytes = Math.addExact(currentBytes, bytes);
+            if (currentBytes >= maxBytes) {
+                result.add(current);
+                current = new ArrayList<>();
+                currentBytes = 0L;
+            }
+        }
+        if (!current.isEmpty()) result.add(current);
+        return result;
+    }
+
     /**
      * 方案二：分批多线程处理
      * 将大批次拆分为小批次进行并行处理
@@ -422,6 +501,16 @@ public class SysLogProcessor {
         if (messages.isEmpty()) {
             return messages;
         }
+        List<PreparedMessage> persisted = new ArrayList<>(messages.size());
+        long configuredBatchBytes = rawStorageProperties.getWriteBatchMaxBytes();
+        long effectiveBatchBytes = configuredBatchBytes > 0L ? configuredBatchBytes : Long.MAX_VALUE;
+        for (List<PreparedMessage> chunk : partitionPreparedByPointBytes(messages, effectiveBatchBytes)) {
+            persisted.addAll(persistRawChunk(chunk, metrics));
+        }
+        return persisted;
+    }
+
+    private List<PreparedMessage> persistRawChunk(List<PreparedMessage> messages, BatchMetrics metrics) {
         List<Point> points = new ArrayList<>(messages.size());
         for (PreparedMessage message : messages) {
             points.add(message.point);
@@ -452,6 +541,31 @@ public class SysLogProcessor {
         }
     }
 
+    static List<List<PreparedMessage>> partitionPreparedByPointBytes(
+            List<PreparedMessage> messages, long maxBytes) {
+        if (maxBytes <= 0L) throw new IllegalArgumentException("InfluxDB 批次字节上限必须大于 0");
+        List<List<PreparedMessage>> result = new ArrayList<>();
+        List<PreparedMessage> current = new ArrayList<>();
+        long currentBytes = 0L;
+        for (PreparedMessage message : messages) {
+            long bytes = message.point.toLineProtocol().getBytes(StandardCharsets.UTF_8).length + 1L;
+            if (!current.isEmpty() && bytes > maxBytes - currentBytes) {
+                result.add(current);
+                current = new ArrayList<>();
+                currentBytes = 0L;
+            }
+            current.add(message);
+            currentBytes = Math.addExact(currentBytes, bytes);
+            if (currentBytes >= maxBytes) {
+                result.add(current);
+                current = new ArrayList<>();
+                currentBytes = 0L;
+            }
+        }
+        if (!current.isEmpty()) result.add(current);
+        return result;
+    }
+
     /**
      * 历史单条兼容入口：原文已可靠持久化后，执行标准化并立即落库。
      * 活动 Kafka 批处理链路使用 prepareNormalization()，避免在计算线程中逐条开启达梦事务。
@@ -484,6 +598,11 @@ public class SysLogProcessor {
      * 批内有界并行标准化。任务异常必须汇总回 listener；超时或系统性失败时整批不 ACK。
      */
     void normalizePreparedMessages(List<PreparedMessage> messages, BatchMetrics metrics) {
+        normalizePreparedMessages(messages, metrics, taskExecutor);
+    }
+
+    private void normalizePreparedMessages(List<PreparedMessage> messages, BatchMetrics metrics,
+                                           Executor normalizationExecutor) {
         if (messages.isEmpty()) {
             return;
         }
@@ -509,7 +628,7 @@ public class SysLogProcessor {
                         return new PreparedNormalization(message, null,
                                 routeFailure(message.record, message.context, failure));
                     }
-                }, taskExecutor));
+                }, normalizationExecutor));
             }
             CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
                     .get(processTimeoutMs, TimeUnit.MILLISECONDS);

haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SyslogFailureRouter.java

@@ -53,6 +53,16 @@ public class SyslogFailureRouter {
     private String retry1hTopic;
     @Value("${syslog.reliability.dlt-topic:${spring.kafka.consumer.topic}.DLT}")
     private String dltTopic;
+    @Value("${spring.kafka.consumer.large-topic:${spring.kafka.consumer.topic}.large}")
+    private String largeOriginalTopic;
+    @Value("${syslog.large-reliability.retry-1m-topic:${spring.kafka.consumer.large-topic}.RETRY.1M}")
+    private String largeRetry1mTopic;
+    @Value("${syslog.large-reliability.retry-10m-topic:${spring.kafka.consumer.large-topic}.RETRY.10M}")
+    private String largeRetry10mTopic;
+    @Value("${syslog.large-reliability.retry-1h-topic:${spring.kafka.consumer.large-topic}.RETRY.1H}")
+    private String largeRetry1hTopic;
+    @Value("${syslog.large-reliability.dlt-topic:${spring.kafka.consumer.large-topic}.DLT}")
+    private String largeDltTopic;
     @Value("${syslog.reliability.publish-timeout-ms:30000}")
     private long publishTimeoutMs;
     @Value("${syslog.reliability.consumer-version:unknown}")
@@ -129,7 +139,11 @@ public class SyslogFailureRouter {
         int nextAttempt = Math.max(0, context.getAttemptCount()) + 1;
         boolean permanent = category == FailureCategory.PERMANENT_DATA;
         boolean exhausted = nextAttempt > 3;
-        String targetTopic = (permanent || exhausted) ? dltTopic : retryTopic(nextAttempt);
+        boolean largeLane = largeOriginalTopic != null
+                && largeOriginalTopic.equals(context.getOriginalTopic());
+        String targetTopic = (permanent || exhausted)
+                ? (largeLane ? largeDltTopic : dltTopic)
+                : retryTopic(nextAttempt, largeLane);
         Instant nextRetryAt = (permanent || exhausted) ? null : Instant.now().plusSeconds(delaySeconds(nextAttempt));
 
         FailureEnvelope source = readEnvelope(record.value());
@@ -256,10 +270,10 @@ public class SyslogFailureRouter {
         return record.timestamp() > 0 ? Instant.ofEpochMilli(record.timestamp()) : Instant.now();
     }
 
-    private String retryTopic(int attempt) {
-        if (attempt == 1) return retry1mTopic;
-        if (attempt == 2) return retry10mTopic;
-        return retry1hTopic;
+    private String retryTopic(int attempt, boolean largeLane) {
+        if (attempt == 1) return largeLane ? largeRetry1mTopic : retry1mTopic;
+        if (attempt == 2) return largeLane ? largeRetry10mTopic : retry10mTopic;
+        return largeLane ? largeRetry1hTopic : retry1hTopic;
     }
 
     private long delaySeconds(int attempt) {

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java

@@ -23,9 +23,10 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 
 /**
- * 把当前事实事务的内存聚合结果直接写入小型统计表。
+ * 把当前事实事务的内存聚合结果写入同步投影或可靠异步投影入口。
  *
- * <p>调用方必须已经开启事实事务；本类不会新开事务，也不会吞掉任何数据库异常。</p>
+ * <p>DIRECT 默认只追加一条批次 Outbox；SHADOW/显式回退仍同步更新统计表。调用方必须
+ * 已经开启事实事务，本类不会吞掉任何数据库异常。</p>
  */
 @Slf4j
 @Service
@@ -63,6 +64,7 @@ public class DirectStatsProjectionWriter {
     private final StatsWriteModeProvider modeProvider;
     private final DirectStatsWriteMetrics metrics;
     private final StatsMaterializationPublisher materializationPublisher;
+    private final StatsProjectionOutboxWriter projectionOutboxWriter;
 
     @Value("${stats.incremental.read-enabled:false}")
     private boolean readEnabled;
@@ -91,11 +93,13 @@ public class DirectStatsProjectionWriter {
     public DirectStatsProjectionWriter(JdbcTemplate jdbc,
                                        StatsWriteModeProvider modeProvider,
                                        DirectStatsWriteMetrics metrics,
-                                       StatsMaterializationPublisher materializationPublisher) {
+                                       StatsMaterializationPublisher materializationPublisher,
+                                       StatsProjectionOutboxWriter projectionOutboxWriter) {
         this.jdbc = jdbc;
         this.modeProvider = modeProvider;
         this.metrics = metrics;
         this.materializationPublisher = materializationPublisher;
+        this.projectionOutboxWriter = projectionOutboxWriter;
     }
 
     public WriteResult writeStandardizedRows(List<Map<String, Object>> rows) {
@@ -160,6 +164,8 @@ public class DirectStatsProjectionWriter {
         status.put("today_stats_date_enabled", todayStatsDateEnabled);
         status.put("business_materialization_mode", asynchronousMaterialization()
                 ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS");
+        status.put("daily_projection_write_mode", asynchronousProjection()
+                ? "DURABLE_OUTBOX" : "SYNCHRONOUS_MERGE");
         return status;
     }
 
@@ -194,6 +200,24 @@ public class DirectStatsProjectionWriter {
         }
         requireTransaction();
 
+        if (asynchronousProjection()) {
+            try {
+                long outboxStarted = System.nanoTime();
+                projectionOutboxWriter.append(StatsProjectionOutboxEvent.fromAccumulator(accumulator));
+                long outboxMs = elapsedMs(outboxStarted);
+                int deviceGroups = effectiveDeviceEntries(accumulator).size();
+                int ruleGroups = effectiveRuleEntries(accumulator).size();
+                metrics.recordSuccess(accumulator.getInputRows(), deviceGroups, ruleGroups,
+                        aggregateMs, outboxMs, 0L);
+                return WriteResult.direct(accumulator.getInputRows(), deviceGroups, ruleGroups,
+                        aggregateMs, outboxMs, 0L);
+            } catch (RuntimeException failure) {
+                metrics.recordFailure();
+                throw new DirectStatsWriteException(
+                        "统计投影 Outbox 写入失败，事实事务必须整体回滚", failure);
+            }
+        }
+
         List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> devices =
                 effectiveDeviceEntries(accumulator);
         List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> rules =
@@ -244,6 +268,11 @@ public class DirectStatsProjectionWriter {
                     aggregateMs, 0L, 0L);
         }
         requireTransaction();
+        // Additive positive/negative changes commute in the serialized projection consumer. This
+        // keeps non-normal UPDATE statistics outside the fact transaction as well.
+        if (asynchronousProjection()) {
+            return write(accumulator, aggregateMs);
+        }
         List<DirectStatsAccumulator.DailyKey> keys = new ArrayList<>(accumulator.getDevices().keySet());
         try {
             long mergeStarted = System.nanoTime();
@@ -408,6 +437,11 @@ public class DirectStatsProjectionWriter {
                 && materializationPublisher.isEnabled();
     }
 
+    private boolean asynchronousProjection() {
+        return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT
+                && projectionOutboxWriter != null && projectionOutboxWriter.isEnabled();
+    }
+
     private List<StatsMaterializationCommand> materializationCommands(
             List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> devices,
             List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> rules) {

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmLargeTextLimiter.java

@@ -0,0 +1,89 @@
+package com.common.service;
+
+import com.common.entity.SyslogNonNormalMessage;
+import com.common.util.Utf8TextTruncator;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.LongAdder;
+
+/** 达梦持久化边界的大字段保护；0 表示兼容模式下不限制。 */
+@Component
+public class DmLargeTextLimiter {
+    private static final long MIN_ENABLED_LIMIT = 64L;
+
+    @Value("${dm.normal.large-field-max-utf8-bytes:0}")
+    private long normalMaxBytes;
+
+    @Value("${dm.normal.large-fields:http_req_body,http_resp_body,http_req_header,http_resp_header,detail,description,cmdline,shell_cmdline,parent_cmdline,mail_message}")
+    private String normalFieldsValue;
+
+    @Value("${dm.non-normal.large-field-max-utf8-bytes:0}")
+    private long nonNormalMaxBytes;
+
+    private volatile Set<String> normalFields = Collections.emptySet();
+    private final LongAdder normalFieldsTruncated = new LongAdder();
+    private final LongAdder nonNormalFieldsTruncated = new LongAdder();
+
+    @PostConstruct
+    public void validate() {
+        validateLimit(normalMaxBytes, "dm.normal.large-field-max-utf8-bytes");
+        validateLimit(nonNormalMaxBytes, "dm.non-normal.large-field-max-utf8-bytes");
+        Set<String> parsed = new LinkedHashSet<>();
+        if (normalFieldsValue != null) {
+            Arrays.stream(normalFieldsValue.split(","))
+                    .map(String::trim).filter(value -> !value.isEmpty()).forEach(parsed::add);
+        }
+        normalFields = Collections.unmodifiableSet(parsed);
+    }
+
+    private static void validateLimit(long value, String name) {
+        if (value < 0L || (value > 0L && value < MIN_ENABLED_LIMIT)) {
+            throw new IllegalArgumentException(name + " 必须为 0 或至少 " + MIN_ENABLED_LIMIT);
+        }
+    }
+
+    public void truncateNormalRow(Map<String, Object> row) {
+        if (row == null || normalMaxBytes == 0L) return;
+        for (String field : normalFields) {
+            Object value = row.get(field);
+            if (!(value instanceof String)) continue;
+            Utf8TextTruncator.Result result = Utf8TextTruncator.truncate((String) value, normalMaxBytes);
+            if (result.isTruncated()) {
+                row.put(field, result.getValue());
+                normalFieldsTruncated.increment();
+            }
+        }
+    }
+
+    public void truncateNonNormal(SyslogNonNormalMessage message) {
+        if (message == null || nonNormalMaxBytes == 0L) return;
+        message.setSyslogMessage(truncateNonNormalValue(message.getSyslogMessage()));
+        message.setHeaderMessage(truncateNonNormalValue(message.getHeaderMessage()));
+        message.setReasonDetail(truncateNonNormalValue(message.getReasonDetail()));
+    }
+
+    private String truncateNonNormalValue(String value) {
+        Utf8TextTruncator.Result result = Utf8TextTruncator.truncate(value, nonNormalMaxBytes);
+        if (result.isTruncated()) nonNormalFieldsTruncated.increment();
+        return result.getValue();
+    }
+
+    public long getNormalFieldsTruncated() { return normalFieldsTruncated.sum(); }
+    public long getNonNormalFieldsTruncated() { return nonNormalFieldsTruncated.sum(); }
+
+    /** 测试和非 Spring 兼容入口。 */
+    static DmLargeTextLimiter unlimited() {
+        DmLargeTextLimiter limiter = new DmLargeTextLimiter();
+        limiter.normalMaxBytes = 0L;
+        limiter.nonNormalMaxBytes = 0L;
+        limiter.validate();
+        return limiter;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java

@@ -174,8 +174,8 @@ public class DmStandardizedBatchExecutor {
                 lateAlarmBucketWriter.record(rows);
             }
 
-            // DIRECT/SHADOW 模式在当前事务内按设备、规则聚合写统计；LEGACY 模式会直接返回。
-            // 任一统计 SQL 失败都会回滚本事务的事实行，禁止产生无法定位的统计缺口。
+            // DIRECT 异步投影模式只在当前事实事务追加一条聚合 Outbox；独立 Kafka consumer
+            // 再更新日统计。SHADOW/回退模式仍保留原同步 MERGE。
             timing.statsResult = directStatsProjectionWriter.writeStandardizedRows(rows);
             return null;
         });
@@ -183,7 +183,7 @@ public class DmStandardizedBatchExecutor {
         transactionMetrics.recordSuccess(rows.size(), timing.factFlushMs, timing.commitMs, transactionMs);
         if (transactionLogEnabled && timing.statsResult != null && timing.statsResult.isDirect()) {
             log.info("DIRECT 达梦事务完成: factRows={}, deviceGroups={}, ruleGroups={}, aggregateMs={}, " +
-                            "factFlushMs={}, statsMergeMs={}, materializeMs={}, commitMs={}, transactionMs={}, retry={}",
+                            "factFlushMs={}, statsProjectionStageMs={}, materializeMs={}, commitMs={}, transactionMs={}, retry={}",
                     rows.size(), timing.statsResult.getDeviceGroups(), timing.statsResult.getRuleGroups(),
                     timing.statsResult.getAggregateMs(), timing.factFlushMs,
                     timing.statsResult.getMergeMs(), timing.statsResult.getMaterializeMs(),

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java

@@ -60,6 +60,7 @@ public class DmStandardizedBatchWriter {
     private final SyslogNormalDataService normalDataService;
     private final DmStandardizedBatchExecutor batchExecutor;
     private final Executor writerExecutor;
+    private final DmLargeTextLimiter largeTextLimiter;
 
     // 未配置时直接启用本次改造的批写路径；SINGLE 只作为现场故障回退开关保留。
     @Value("${dm.standardized-write.mode:BATCH}")
@@ -89,12 +90,22 @@ public class DmStandardizedBatchWriter {
     private final AtomicInteger consecutiveSystemFailures = new AtomicInteger();
     private volatile long circuitOpenUntilMs;
 
+    @org.springframework.beans.factory.annotation.Autowired
     public DmStandardizedBatchWriter(SyslogNormalDataService normalDataService,
                                      DmStandardizedBatchExecutor batchExecutor,
-                                     @Qualifier("dmStandardizedWriterExecutor") Executor writerExecutor) {
+                                     @Qualifier("dmStandardizedWriterExecutor") Executor writerExecutor,
+                                     DmLargeTextLimiter largeTextLimiter) {
         this.normalDataService = normalDataService;
         this.batchExecutor = batchExecutor;
         this.writerExecutor = writerExecutor;
+        this.largeTextLimiter = largeTextLimiter;
+    }
+
+    /** 保留已有单元测试和离线工具入口，默认不改变历史存储内容。 */
+    public DmStandardizedBatchWriter(SyslogNormalDataService normalDataService,
+                                     DmStandardizedBatchExecutor batchExecutor,
+                                     Executor writerExecutor) {
+        this(normalDataService, batchExecutor, writerExecutor, DmLargeTextLimiter.unlimited());
     }
 
     /**
@@ -148,6 +159,7 @@ public class DmStandardizedBatchWriter {
         List<WriteRequest<T>> prepared = new ArrayList<>(requests.size());
         for (WriteRequest<T> request : requests) {
             try {
+                largeTextLimiter.truncateNormalRow(request.getData());
                 normalDataService.prepareForInsert(request.getData());
                 prepared.add(request);
             } catch (RuntimeException invalidRow) {

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java

@@ -29,6 +29,7 @@ public class IngressStatsProjectionWriter {
     private final PlatformTransactionManager transactionManager;
     private final StatsWriteModeProvider modeProvider;
     private final StatsMaterializationPublisher materializationPublisher;
+    private final StatsProjectionOutboxWriter projectionOutboxWriter;
 
     @Value("${stats.ingress-metadata.enabled:false}")
     private boolean enabled;
@@ -50,11 +51,13 @@ public class IngressStatsProjectionWriter {
     public IngressStatsProjectionWriter(JdbcTemplate jdbc,
                                         PlatformTransactionManager transactionManager,
                                         StatsWriteModeProvider modeProvider,
-                                        StatsMaterializationPublisher materializationPublisher) {
+                                        StatsMaterializationPublisher materializationPublisher,
+                                        StatsProjectionOutboxWriter projectionOutboxWriter) {
         this.jdbc = jdbc;
         this.transactionManager = transactionManager;
         this.modeProvider = modeProvider;
         this.materializationPublisher = materializationPublisher;
+        this.projectionOutboxWriter = projectionOutboxWriter;
     }
 
     public int writeBatch(List<IngressStatsEvent> events) {
@@ -92,6 +95,8 @@ public class IngressStatsProjectionWriter {
                 ? "device_daily_stats+" + shadowDeviceTable : "device_daily_stats");
         status.put("business_materialization_mode", asynchronousMaterialization()
                 ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS");
+        status.put("daily_projection_write_mode", asynchronousProjection()
+                ? "DURABLE_OUTBOX" : "SYNCHRONOUS_MERGE");
         status.put("successful_batches", successfulBatches.get());
         status.put("received_events", receivedEvents.get());
         status.put("updated_groups", updatedGroups.get());
@@ -109,6 +114,10 @@ public class IngressStatsProjectionWriter {
         TransactionTemplate transaction = new TransactionTemplate(transactionManager);
         transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
         Integer updated = transaction.execute(status -> {
+            if (asynchronousProjection()) {
+                projectionOutboxWriter.append(StatsProjectionOutboxEvent.fromIngress(aggregation));
+                return aggregation.devices.size() + aggregation.collects.size();
+            }
             int groups = 0;
             boolean asyncMaterialization = asynchronousMaterialization();
             List<StatsMaterializationCommand> commands = asyncMaterialization
@@ -237,6 +246,11 @@ public class IngressStatsProjectionWriter {
                 && materializationPublisher.isEnabled();
     }
 
+    private boolean asynchronousProjection() {
+        return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT
+                && projectionOutboxWriter != null && projectionOutboxWriter.isEnabled();
+    }
+
     private static String safeTable(String table) {
         if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
             throw new IllegalStateException("非法入口设备统计表名: " + table);

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionDeltaConsumer.java

@@ -0,0 +1,386 @@
+package com.common.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.kafka.annotation.KafkaListener;
+import org.springframework.kafka.support.Acknowledgment;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionTemplate;
+
+import java.sql.Date;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** Applies durable projection deltas outside the fact transaction. */
+@Slf4j
+@Component
+public class StatsProjectionDeltaConsumer {
+    private static final String DEVICE_MERGE =
+            "MERGE INTO %s t USING (SELECT ? stat_date,? device_id,? parse_delta," +
+            "? non_log_delta,? receive_time FROM DUAL) s " +
+            "ON (t.stat_date=s.stat_date AND t.device_id=s.device_id) " +
+            "WHEN MATCHED THEN UPDATE SET t.parse_count=t.parse_count+s.parse_delta," +
+            "t.non_log_count=t.non_log_count+s.non_log_delta," +
+            "t.last_receive_time=CASE WHEN s.receive_time IS NULL THEN t.last_receive_time " +
+            "WHEN t.last_receive_time IS NULL OR t.last_receive_time<s.receive_time " +
+            "THEN s.receive_time ELSE t.last_receive_time END," +
+            "t.projection_seq=t.projection_seq+1,t.updated_at=SYSDATE " +
+            "WHEN NOT MATCHED THEN INSERT (stat_date,device_id,parse_count,non_log_count," +
+            "last_receive_time,projection_seq,updated_at) VALUES " +
+            "(s.stat_date,s.device_id,s.parse_delta,s.non_log_delta,s.receive_time,1,SYSDATE)";
+
+    private static final String RULE_MERGE =
+            "MERGE INTO %s t USING (SELECT ? stat_date,? normalize_rule_id,? max_log_time FROM DUAL) s " +
+            "ON (t.stat_date=s.stat_date AND t.normalize_rule_id=s.normalize_rule_id) " +
+            "WHEN MATCHED THEN UPDATE SET t.max_log_time=CASE " +
+            "WHEN t.max_log_time IS NULL OR t.max_log_time<s.max_log_time THEN s.max_log_time " +
+            "ELSE t.max_log_time END,t.projection_seq=t.projection_seq+1,t.updated_at=SYSDATE " +
+            "WHEN NOT MATCHED THEN INSERT (stat_date,normalize_rule_id,max_log_time,projection_seq,updated_at) " +
+            "VALUES (s.stat_date,s.normalize_rule_id,s.max_log_time,1,SYSDATE)";
+
+    private static final String COLLECT_MERGE =
+            "MERGE INTO device_collect_stats t USING (SELECT ? device_collect_id," +
+            "? first_success_time,? last_success_time FROM DUAL) s " +
+            "ON (t.device_collect_id=s.device_collect_id) " +
+            "WHEN MATCHED THEN UPDATE SET t.first_success_time=CASE " +
+            "WHEN s.first_success_time IS NULL THEN t.first_success_time " +
+            "WHEN t.first_success_time IS NULL OR t.first_success_time>s.first_success_time " +
+            "THEN s.first_success_time ELSE t.first_success_time END," +
+            "t.last_success_time=CASE WHEN s.last_success_time IS NULL THEN t.last_success_time " +
+            "WHEN t.last_success_time IS NULL OR t.last_success_time<s.last_success_time " +
+            "THEN s.last_success_time ELSE t.last_success_time END," +
+            "t.projection_seq=t.projection_seq+1,t.updated_at=SYSDATE " +
+            "WHEN NOT MATCHED THEN INSERT (device_collect_id,first_success_time,last_success_time," +
+            "last_fail_time,projection_seq,updated_at) VALUES " +
+            "(s.device_collect_id,s.first_success_time,s.last_success_time,NULL,1,SYSDATE)";
+
+    private final JdbcTemplate jdbc;
+    private final PlatformTransactionManager transactionManager;
+    private final ObjectMapper objectMapper;
+    private final StatsMaterializationPublisher materializationPublisher;
+
+    @Value("${stats.projection-async.enabled:true}") private boolean enabled;
+    @Value("${stats.projection-async.applied-table:stats_projection_applied_event}") private String appliedTable;
+    @Value("${stats.projection-async.outbox-table:stats_projection_outbox}") private String outboxTable;
+    @Value("${stats.direct.device-table:device_daily_stats}") private String deviceTable;
+    @Value("${stats.direct.rule-table:normalize_rule_daily_stats}") private String ruleTable;
+    @Value("${stats.incremental.transaction-timeout-seconds:45}") private int transactionTimeoutSeconds;
+    @Value("${stats.projection-async.max-concurrency-retries:3}") private int maxConcurrencyRetries;
+    @Value("${stats.projection-async.concurrency-retry-backoff-ms:50}") private long retryBackoffMs;
+
+    private final AtomicLong consumed = new AtomicLong();
+    private final AtomicLong appliedEvents = new AtomicLong();
+    private final AtomicLong duplicates = new AtomicLong();
+    private final AtomicLong deviceGroups = new AtomicLong();
+    private final AtomicLong ruleGroups = new AtomicLong();
+    private final AtomicLong collectGroups = new AtomicLong();
+    private final AtomicLong successfulBatches = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+    private final AtomicLong retries = new AtomicLong();
+    private volatile LocalDateTime lastSuccessTime;
+    private volatile LocalDateTime lastFailureTime;
+
+    public StatsProjectionDeltaConsumer(JdbcTemplate jdbc,
+                                        PlatformTransactionManager transactionManager,
+                                        ObjectMapper objectMapper,
+                                        StatsMaterializationPublisher materializationPublisher) {
+        this.jdbc = jdbc;
+        this.transactionManager = transactionManager;
+        this.objectMapper = objectMapper.copy();
+        this.objectMapper.deactivateDefaultTyping();
+        this.materializationPublisher = materializationPublisher;
+    }
+
+    @KafkaListener(
+            id = "syslog-stats-projection",
+            groupId = "${stats.projection-async.group-id:${spring.kafka.consumer.group-id}-stats-projection}",
+            topics = "${stats.projection-async.topic:${spring.kafka.consumer.topic}.STATS.PROJECTION}",
+            autoStartup = "${stats.projection-async.enabled:true}",
+            containerFactory = "statsProjectionKafkaListenerContainerFactory"
+    )
+    public void consume(List<ConsumerRecord<String, String>> records, Acknowledgment acknowledgment) {
+        if (records == null || records.isEmpty()) {
+            acknowledgment.acknowledge();
+            return;
+        }
+        consumed.addAndGet(records.size());
+        List<StatsProjectionOutboxEvent> events = parse(records);
+        long started = System.nanoTime();
+        try {
+            ApplyResult result = applyWithRetry(events);
+            acknowledgment.acknowledge();
+            appliedEvents.addAndGet(result.appliedEvents);
+            duplicates.addAndGet(result.duplicates);
+            deviceGroups.addAndGet(result.deviceGroups);
+            ruleGroups.addAndGet(result.ruleGroups);
+            collectGroups.addAndGet(result.collectGroups);
+            successfulBatches.incrementAndGet();
+            lastSuccessTime = LocalDateTime.now();
+            log.info("统计投影批次完成: received={}, appliedEvents={}, duplicates={}, " +
+                            "deviceGroups={}, ruleGroups={}, collectGroups={}, durationMs={}",
+                    records.size(), result.appliedEvents, result.duplicates,
+                    result.deviceGroups, result.ruleGroups, result.collectGroups,
+                    (System.nanoTime() - started) / 1_000_000L);
+        } catch (RuntimeException failure) {
+            failures.incrementAndGet();
+            lastFailureTime = LocalDateTime.now();
+            throw failure;
+        }
+    }
+
+    private List<StatsProjectionOutboxEvent> parse(List<ConsumerRecord<String, String>> records) {
+        Map<String, StatsProjectionOutboxEvent> unique = new LinkedHashMap<>();
+        for (ConsumerRecord<String, String> record : records) {
+            try {
+                if (!StatsProjectionOutboxEvent.KAFKA_KEY.equals(record.key())) {
+                    throw new IllegalArgumentException("Unexpected stats projection Kafka key: " + record.key());
+                }
+                StatsProjectionOutboxEvent event = objectMapper.readValue(
+                        record.value(), StatsProjectionOutboxEvent.class);
+                event.validate();
+                unique.put(event.getEventId(), event);
+            } catch (Exception invalid) {
+                throw new IllegalStateException("Invalid internal stats projection event at " +
+                        record.topic() + "-" + record.partition() + "@" + record.offset(), invalid);
+            }
+        }
+        return new ArrayList<>(unique.values());
+    }
+
+    ApplyResult applyWithRetry(List<StatsProjectionOutboxEvent> events) {
+        int attempt = 0;
+        while (true) {
+            try {
+                return applyOnce(events);
+            } catch (RuntimeException failure) {
+                if (!DirectStatsProjectionWriter.isConcurrentFailure(failure)
+                        || attempt >= Math.max(0, maxConcurrencyRetries)) {
+                    throw failure;
+                }
+                attempt++;
+                retries.incrementAndGet();
+                sleepBeforeRetry(attempt);
+            }
+        }
+    }
+
+    private ApplyResult applyOnce(List<StatsProjectionOutboxEvent> events) {
+        if (!enabled) throw new IllegalStateException("Stats projection consumer is disabled");
+        TransactionTemplate transaction = new TransactionTemplate(transactionManager);
+        transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
+        ApplyResult result = transaction.execute(status -> {
+            Set<String> existing = existingEventIds(events);
+            List<StatsProjectionOutboxEvent> fresh = new ArrayList<>();
+            for (StatsProjectionOutboxEvent event : events) {
+                if (!existing.contains(event.getEventId())) fresh.add(event);
+            }
+            ProjectionAggregation aggregation = aggregate(fresh);
+            if (!aggregation.devices.isEmpty()) {
+                jdbc.batchUpdate(String.format(DEVICE_MERGE,
+                        StatsProjectionOutboxWriter.safeTable(deviceTable)), aggregation.deviceParams());
+            }
+            if (!aggregation.rules.isEmpty()) {
+                jdbc.batchUpdate(String.format(RULE_MERGE,
+                        StatsProjectionOutboxWriter.safeTable(ruleTable)), aggregation.ruleParams());
+            }
+            if (!aggregation.collects.isEmpty()) {
+                jdbc.batchUpdate(COLLECT_MERGE, aggregation.collectParams());
+            }
+            if (!fresh.isEmpty()) {
+                List<Object[]> markers = new ArrayList<>(fresh.size());
+                for (StatsProjectionOutboxEvent event : fresh) {
+                    markers.add(new Object[]{event.getEventId(), event.getSchema()});
+                }
+                jdbc.batchUpdate("INSERT INTO " + StatsProjectionOutboxWriter.safeTable(appliedTable) +
+                        " (event_id,event_schema,applied_at) VALUES (?,?,SYSDATE)", markers);
+            }
+            if (!events.isEmpty()) {
+                List<Object[]> acknowledged = new ArrayList<>(events.size());
+                for (StatsProjectionOutboxEvent event : events) {
+                    acknowledged.add(new Object[]{event.getEventId()});
+                }
+                jdbc.batchUpdate("UPDATE " + StatsProjectionOutboxWriter.safeTable(outboxTable) +
+                        " SET status='APPLIED',applied_at=SYSDATE,next_attempt_at=NULL," +
+                        "updated_at=SYSDATE WHERE event_id=?", acknowledged);
+            }
+            Collection<StatsMaterializationCommand> dirty = aggregation.materializationCommands();
+            if (!dirty.isEmpty()) materializationPublisher.enqueueAfterCommit(dirty);
+            return new ApplyResult(fresh.size(), events.size() - fresh.size(),
+                    aggregation.devices.size(), aggregation.rules.size(), aggregation.collects.size());
+        });
+        return result == null ? new ApplyResult(0, events.size(), 0, 0, 0) : result;
+    }
+
+    private Set<String> existingEventIds(List<StatsProjectionOutboxEvent> events) {
+        if (events.isEmpty()) return Collections.emptySet();
+        Set<String> result = new HashSet<>();
+        int chunkSize = 900;
+        for (int start = 0; start < events.size(); start += chunkSize) {
+            int end = Math.min(events.size(), start + chunkSize);
+            StringBuilder placeholders = new StringBuilder();
+            Object[] ids = new Object[end - start];
+            for (int i = start; i < end; i++) {
+                if (placeholders.length() > 0) placeholders.append(',');
+                placeholders.append('?');
+                ids[i - start] = events.get(i).getEventId();
+            }
+            result.addAll(jdbc.queryForList("SELECT event_id FROM " +
+                    StatsProjectionOutboxWriter.safeTable(appliedTable) + " WHERE event_id IN (" +
+                    placeholders + ")", String.class, ids));
+        }
+        return result;
+    }
+
+    static ProjectionAggregation aggregate(List<StatsProjectionOutboxEvent> events) {
+        ProjectionAggregation result = new ProjectionAggregation();
+        for (StatsProjectionOutboxEvent event : events) {
+            event.validate();
+            for (StatsProjectionOutboxEvent.DeviceDelta delta : event.getDevices()) {
+                DeviceKey key = new DeviceKey(delta.date(), delta.getDeviceId());
+                DeviceAggregate value = result.devices.computeIfAbsent(key,
+                        ignored -> new DeviceAggregate());
+                value.parseDelta += delta.getParseDelta();
+                value.nonLogDelta += delta.getNonLogDelta();
+                LocalDateTime receiveTime = delta.receiveTime();
+                if (receiveTime != null && (value.lastReceiveTime == null
+                        || receiveTime.isAfter(value.lastReceiveTime))) {
+                    value.lastReceiveTime = receiveTime;
+                }
+            }
+            for (StatsProjectionOutboxEvent.RuleDelta delta : event.getRules()) {
+                RuleKey key = new RuleKey(delta.date(), delta.getNormalizeRuleId());
+                LocalDateTime value = delta.time();
+                LocalDateTime current = result.rules.get(key);
+                if (current == null || value.isAfter(current)) result.rules.put(key, value);
+            }
+            for (StatsProjectionOutboxEvent.CollectDelta delta : event.getCollects()) {
+                CollectAggregate value = result.collects.computeIfAbsent(delta.getDeviceCollectId(),
+                        ignored -> new CollectAggregate());
+                value.include(delta.firstTime());
+                value.include(delta.lastTime());
+            }
+        }
+        return result;
+    }
+
+    private void sleepBeforeRetry(int attempt) {
+        try {
+            Thread.sleep(Math.max(0L, retryBackoffMs) * attempt);
+        } catch (InterruptedException interrupted) {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException("Stats projection retry interrupted", interrupted);
+        }
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("enabled", enabled);
+        result.put("consumed_total", consumed.get());
+        result.put("applied_events_total", appliedEvents.get());
+        result.put("duplicate_events_total", duplicates.get());
+        result.put("device_groups_total", deviceGroups.get());
+        result.put("rule_groups_total", ruleGroups.get());
+        result.put("collect_groups_total", collectGroups.get());
+        result.put("successful_batches", successfulBatches.get());
+        result.put("failures_total", failures.get());
+        result.put("concurrency_retries_total", retries.get());
+        result.put("last_success_time", lastSuccessTime);
+        result.put("last_failure_time", lastFailureTime);
+        return result;
+    }
+
+    static final class ProjectionAggregation {
+        final Map<DeviceKey, DeviceAggregate> devices = new TreeMap<>();
+        final Map<RuleKey, LocalDateTime> rules = new TreeMap<>();
+        final Map<Long, CollectAggregate> collects = new TreeMap<>();
+
+        List<Object[]> deviceParams() {
+            List<Object[]> result = new ArrayList<>(devices.size());
+            for (Map.Entry<DeviceKey, DeviceAggregate> entry : devices.entrySet()) {
+                DeviceAggregate value = entry.getValue();
+                result.add(new Object[]{Date.valueOf(entry.getKey().date), entry.getKey().id,
+                        value.parseDelta, value.nonLogDelta, value.lastReceiveTime});
+            }
+            return result;
+        }
+
+        List<Object[]> ruleParams() {
+            List<Object[]> result = new ArrayList<>(rules.size());
+            for (Map.Entry<RuleKey, LocalDateTime> entry : rules.entrySet()) {
+                result.add(new Object[]{Date.valueOf(entry.getKey().date), entry.getKey().id,
+                        entry.getValue()});
+            }
+            return result;
+        }
+
+        List<Object[]> collectParams() {
+            List<Object[]> result = new ArrayList<>(collects.size());
+            for (Map.Entry<Long, CollectAggregate> entry : collects.entrySet()) {
+                result.add(new Object[]{entry.getKey(), entry.getValue().min, entry.getValue().max});
+            }
+            return result;
+        }
+
+        Collection<StatsMaterializationCommand> materializationCommands() {
+            Set<StatsMaterializationCommand> result = new LinkedHashSet<>();
+            for (DeviceKey key : devices.keySet()) result.add(StatsMaterializationCommand.device(key.date, key.id));
+            for (RuleKey key : rules.keySet()) result.add(StatsMaterializationCommand.rule(key.date, key.id));
+            for (Long id : collects.keySet()) result.add(StatsMaterializationCommand.collect(id));
+            return result;
+        }
+    }
+
+    static class DimensionKey implements Comparable<DimensionKey> {
+        final LocalDate date;
+        final long id;
+        DimensionKey(LocalDate date, long id) { this.date = date; this.id = id; }
+        @Override public int compareTo(DimensionKey other) {
+            int dateOrder = date.compareTo(other.date);
+            return dateOrder != 0 ? dateOrder : Long.compare(id, other.id);
+        }
+        @Override public boolean equals(Object other) {
+            if (this == other) return true;
+            if (!(other instanceof DimensionKey)) return false;
+            DimensionKey key = (DimensionKey) other;
+            return id == key.id && date.equals(key.date);
+        }
+        @Override public int hashCode() { return 31 * date.hashCode() + Long.hashCode(id); }
+    }
+    static final class DeviceKey extends DimensionKey { DeviceKey(LocalDate date, long id) { super(date, id); } }
+    static final class RuleKey extends DimensionKey { RuleKey(LocalDate date, long id) { super(date, id); } }
+    static final class DeviceAggregate { long parseDelta; long nonLogDelta; LocalDateTime lastReceiveTime; }
+    static final class CollectAggregate {
+        LocalDateTime min; LocalDateTime max;
+        void include(LocalDateTime time) {
+            if (time == null) return;
+            if (min == null || time.isBefore(min)) min = time;
+            if (max == null || time.isAfter(max)) max = time;
+        }
+    }
+
+    static final class ApplyResult {
+        final int appliedEvents; final int duplicates;
+        final int deviceGroups; final int ruleGroups; final int collectGroups;
+        ApplyResult(int appliedEvents, int duplicates, int deviceGroups, int ruleGroups, int collectGroups) {
+            this.appliedEvents = appliedEvents; this.duplicates = duplicates;
+            this.deviceGroups = deviceGroups; this.ruleGroups = ruleGroups; this.collectGroups = collectGroups;
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxEvent.java

@@ -0,0 +1,221 @@
+package com.common.service;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * A durable, batch-aggregated statistics delta written beside the fact rows.
+ *
+ * <p>The event is additive/max-min only, so events from different application nodes may be
+ * published in a different order without changing the final projection. Kafka duplicates are
+ * eliminated by {@code eventId} in the projection consumer transaction.</p>
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class StatsProjectionOutboxEvent {
+    public static final String SCHEMA = "SYSLOG_STATS_PROJECTION_V1";
+    public static final String KAFKA_KEY = "SYSLOG_STATS_PROJECTION";
+
+    private String schema = SCHEMA;
+    private String eventId = UUID.randomUUID().toString();
+    private List<DeviceDelta> devices = new ArrayList<>();
+    private List<RuleDelta> rules = new ArrayList<>();
+    private List<CollectDelta> collects = new ArrayList<>();
+
+    public static StatsProjectionOutboxEvent fromAccumulator(DirectStatsAccumulator accumulator) {
+        StatsProjectionOutboxEvent event = new StatsProjectionOutboxEvent();
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry
+                : accumulator.getDevices().entrySet()) {
+            DirectStatsAccumulator.DeviceDelta value = entry.getValue();
+            if (value.getParseDelta() != 0L || value.getNonLogDelta() != 0L) {
+                event.devices.add(DeviceDelta.counts(entry.getKey().getDate(),
+                        entry.getKey().getDimensionId(), value.getParseDelta(), value.getNonLogDelta()));
+            }
+        }
+        for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry
+                : accumulator.getRules().entrySet()) {
+            if (entry.getValue().getMaxLogTime() != null) {
+                event.rules.add(RuleDelta.max(entry.getKey().getDate(),
+                        entry.getKey().getDimensionId(), entry.getValue().getMaxLogTime()));
+            }
+        }
+        return event;
+    }
+
+    static StatsProjectionOutboxEvent fromIngress(IngressStatsProjectionWriter.Aggregation aggregation) {
+        StatsProjectionOutboxEvent event = new StatsProjectionOutboxEvent();
+        for (Map.Entry<IngressStatsProjectionWriter.DeviceKey, LocalDateTime> entry
+                : aggregation.devices.entrySet()) {
+            event.devices.add(DeviceDelta.received(entry.getKey().date,
+                    entry.getKey().deviceId, entry.getValue()));
+        }
+        for (Map.Entry<Long, IngressStatsProjectionWriter.CollectTimes> entry
+                : aggregation.collects.entrySet()) {
+            event.collects.add(CollectDelta.range(entry.getKey(),
+                    entry.getValue().min, entry.getValue().max));
+        }
+        return event;
+    }
+
+    public void validate() {
+        if (!SCHEMA.equals(schema)) {
+            throw new IllegalArgumentException("Unsupported stats projection schema: " + schema);
+        }
+        if (eventId == null || eventId.trim().isEmpty()) {
+            throw new IllegalArgumentException("Stats projection event is missing eventId");
+        }
+        if (devices == null || rules == null || collects == null) {
+            throw new IllegalArgumentException("Stats projection event contains null collections");
+        }
+        if (devices.isEmpty() && rules.isEmpty() && collects.isEmpty()) {
+            throw new IllegalArgumentException("Stats projection event has no changes");
+        }
+        for (DeviceDelta delta : devices) delta.validate();
+        for (RuleDelta delta : rules) delta.validate();
+        for (CollectDelta delta : collects) delta.validate();
+    }
+
+    public boolean isEmpty() {
+        return devices.isEmpty() && rules.isEmpty() && collects.isEmpty();
+    }
+
+    public String getSchema() { return schema; }
+    public void setSchema(String schema) { this.schema = schema; }
+    public String getEventId() { return eventId; }
+    public void setEventId(String eventId) { this.eventId = eventId; }
+    public List<DeviceDelta> getDevices() { return devices; }
+    public void setDevices(List<DeviceDelta> devices) { this.devices = devices; }
+    public List<RuleDelta> getRules() { return rules; }
+    public void setRules(List<RuleDelta> rules) { this.rules = rules; }
+    public List<CollectDelta> getCollects() { return collects; }
+    public void setCollects(List<CollectDelta> collects) { this.collects = collects; }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class DeviceDelta {
+        private String statDate;
+        private long deviceId;
+        private long parseDelta;
+        private long nonLogDelta;
+        private String lastReceiveTime;
+
+        public DeviceDelta() {}
+
+        static DeviceDelta counts(LocalDate date, long deviceId, long parseDelta, long nonLogDelta) {
+            DeviceDelta result = new DeviceDelta();
+            result.statDate = date.toString();
+            result.deviceId = deviceId;
+            result.parseDelta = parseDelta;
+            result.nonLogDelta = nonLogDelta;
+            return result;
+        }
+
+        static DeviceDelta received(LocalDate date, long deviceId, LocalDateTime receiveTime) {
+            DeviceDelta result = counts(date, deviceId, 0L, 0L);
+            result.lastReceiveTime = receiveTime.toString();
+            return result;
+        }
+
+        void validate() {
+            date();
+            if (deviceId <= 0L) throw new IllegalArgumentException("Invalid device projection id");
+            if (parseDelta == 0L && nonLogDelta == 0L && receiveTime() == null) {
+                throw new IllegalArgumentException("Device projection delta is empty");
+            }
+        }
+
+        public LocalDate date() { return LocalDate.parse(statDate); }
+        public LocalDateTime receiveTime() {
+            return lastReceiveTime == null || lastReceiveTime.trim().isEmpty()
+                    ? null : LocalDateTime.parse(lastReceiveTime);
+        }
+        public String getStatDate() { return statDate; }
+        public void setStatDate(String statDate) { this.statDate = statDate; }
+        public long getDeviceId() { return deviceId; }
+        public void setDeviceId(long deviceId) { this.deviceId = deviceId; }
+        public long getParseDelta() { return parseDelta; }
+        public void setParseDelta(long parseDelta) { this.parseDelta = parseDelta; }
+        public long getNonLogDelta() { return nonLogDelta; }
+        public void setNonLogDelta(long nonLogDelta) { this.nonLogDelta = nonLogDelta; }
+        public String getLastReceiveTime() { return lastReceiveTime; }
+        public void setLastReceiveTime(String lastReceiveTime) { this.lastReceiveTime = lastReceiveTime; }
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class RuleDelta {
+        private String statDate;
+        private long normalizeRuleId;
+        private String maxLogTime;
+
+        public RuleDelta() {}
+
+        static RuleDelta max(LocalDate date, long ruleId, LocalDateTime maxLogTime) {
+            RuleDelta result = new RuleDelta();
+            result.statDate = date.toString();
+            result.normalizeRuleId = ruleId;
+            result.maxLogTime = maxLogTime.toString();
+            return result;
+        }
+
+        void validate() {
+            date();
+            if (normalizeRuleId <= 0L || time() == null) {
+                throw new IllegalArgumentException("Invalid rule projection delta");
+            }
+        }
+
+        public LocalDate date() { return LocalDate.parse(statDate); }
+        public LocalDateTime time() {
+            return maxLogTime == null || maxLogTime.trim().isEmpty()
+                    ? null : LocalDateTime.parse(maxLogTime);
+        }
+        public String getStatDate() { return statDate; }
+        public void setStatDate(String statDate) { this.statDate = statDate; }
+        public long getNormalizeRuleId() { return normalizeRuleId; }
+        public void setNormalizeRuleId(long normalizeRuleId) { this.normalizeRuleId = normalizeRuleId; }
+        public String getMaxLogTime() { return maxLogTime; }
+        public void setMaxLogTime(String maxLogTime) { this.maxLogTime = maxLogTime; }
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class CollectDelta {
+        private long deviceCollectId;
+        private String firstSuccessTime;
+        private String lastSuccessTime;
+
+        public CollectDelta() {}
+
+        static CollectDelta range(long collectId, LocalDateTime first, LocalDateTime last) {
+            CollectDelta result = new CollectDelta();
+            result.deviceCollectId = collectId;
+            result.firstSuccessTime = first == null ? null : first.toString();
+            result.lastSuccessTime = last == null ? null : last.toString();
+            return result;
+        }
+
+        void validate() {
+            if (deviceCollectId <= 0L || (firstTime() == null && lastTime() == null)) {
+                throw new IllegalArgumentException("Invalid collector projection delta");
+            }
+        }
+
+        public LocalDateTime firstTime() {
+            return firstSuccessTime == null || firstSuccessTime.trim().isEmpty()
+                    ? null : LocalDateTime.parse(firstSuccessTime);
+        }
+        public LocalDateTime lastTime() {
+            return lastSuccessTime == null || lastSuccessTime.trim().isEmpty()
+                    ? null : LocalDateTime.parse(lastSuccessTime);
+        }
+        public long getDeviceCollectId() { return deviceCollectId; }
+        public void setDeviceCollectId(long deviceCollectId) { this.deviceCollectId = deviceCollectId; }
+        public String getFirstSuccessTime() { return firstSuccessTime; }
+        public void setFirstSuccessTime(String firstSuccessTime) { this.firstSuccessTime = firstSuccessTime; }
+        public String getLastSuccessTime() { return lastSuccessTime; }
+        public void setLastSuccessTime(String lastSuccessTime) { this.lastSuccessTime = lastSuccessTime; }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxRelay.java

@@ -0,0 +1,151 @@
+package com.common.service;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.SendResult;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import org.springframework.util.concurrent.ListenableFuture;
+
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Relays durable outbox events to the dedicated projection topic.
+ *
+ * <p>Multiple application nodes may relay the same row during a race. The consumer's applied-event
+ * table makes that harmless, so the relay never needs a distributed lease that could stall the
+ * fact path.</p>
+ */
+@Slf4j
+@Component
+public class StatsProjectionOutboxRelay {
+    private final JdbcTemplate jdbc;
+    private final KafkaTemplate<String, String> kafkaTemplate;
+    private final AtomicBoolean running = new AtomicBoolean();
+    private final AtomicLong published = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+    private final AtomicLong cleaned = new AtomicLong();
+    private volatile long lastSuccessAt;
+    private volatile long lastFailureAt;
+
+    @Value("${stats.projection-async.enabled:true}") private boolean enabled;
+    @Value("${stats.projection-async.outbox-table:stats_projection_outbox}") private String outboxTable;
+    @Value("${stats.projection-async.applied-table:stats_projection_applied_event}") private String appliedTable;
+    @Value("${stats.projection-async.topic:${spring.kafka.consumer.topic}.STATS.PROJECTION}") private String topic;
+    @Value("${stats.projection-async.relay-batch-size:100}") private int batchSize;
+    @Value("${stats.projection-async.publish-timeout-ms:10000}") private long publishTimeoutMs;
+    @Value("${stats.projection-async.retry-backoff-ms:1000}") private long retryBackoffMs;
+    @Value("${stats.projection-async.redelivery-interval-ms:60000}") private long redeliveryIntervalMs;
+    @Value("${stats.projection-async.retention-days:14}") private int retentionDays;
+
+    public StatsProjectionOutboxRelay(JdbcTemplate jdbc, KafkaTemplate<String, String> kafkaTemplate) {
+        this.jdbc = jdbc;
+        this.kafkaTemplate = kafkaTemplate;
+    }
+
+    @Scheduled(fixedDelayString = "${stats.projection-async.relay-interval-ms:200}")
+    public void relay() {
+        if (!enabled || !running.compareAndSet(false, true)) return;
+        try {
+            for (OutboxRow row : loadDueRows()) {
+                publish(row);
+            }
+        } catch (RuntimeException failure) {
+            failures.incrementAndGet();
+            lastFailureAt = System.currentTimeMillis();
+            log.error("统计投影 Outbox 扫描失败，下轮自动重试", failure);
+        } finally {
+            running.set(false);
+        }
+    }
+
+    private List<OutboxRow> loadDueRows() {
+        String table = StatsProjectionOutboxWriter.safeTable(outboxTable);
+        String sql = "SELECT event_id,payload FROM (SELECT event_id,payload FROM " + table +
+                " WHERE status IN ('PENDING','FAILED','PUBLISHED') AND (next_attempt_at IS NULL " +
+                "OR next_attempt_at<=SYSDATE) ORDER BY created_at,event_id) WHERE ROWNUM<=?";
+        return jdbc.query(sql, ps -> ps.setInt(1, Math.max(1, batchSize)),
+                (rs, rowNum) -> new OutboxRow(rs.getString(1), rs.getString(2)));
+    }
+
+    private void publish(OutboxRow row) {
+        try {
+            ListenableFuture<SendResult<String, String>> future = kafkaTemplate.send(
+                    topic, StatsProjectionOutboxEvent.KAFKA_KEY, row.payload);
+            future.get(Math.max(1L, publishTimeoutMs), TimeUnit.MILLISECONDS);
+            jdbc.update("UPDATE " + StatsProjectionOutboxWriter.safeTable(outboxTable) +
+                            " SET status='PUBLISHED',published_at=SYSDATE,next_attempt_at=?,last_error=NULL," +
+                            "updated_at=SYSDATE WHERE event_id=? AND status<>'APPLIED'",
+                    new Timestamp(System.currentTimeMillis() + Math.max(1000L, redeliveryIntervalMs)),
+                    row.eventId);
+            published.incrementAndGet();
+            lastSuccessAt = System.currentTimeMillis();
+        } catch (InterruptedException interrupted) {
+            Thread.currentThread().interrupt();
+            markFailure(row, interrupted);
+        } catch (Exception failure) {
+            markFailure(row, failure);
+        }
+    }
+
+    private void markFailure(OutboxRow row, Exception failure) {
+        failures.incrementAndGet();
+        lastFailureAt = System.currentTimeMillis();
+        String message = String.valueOf(failure.getMessage());
+        if (message.length() > 1800) message = message.substring(0, 1800);
+        try {
+            jdbc.update("UPDATE " + StatsProjectionOutboxWriter.safeTable(outboxTable) +
+                            " SET status='FAILED',attempt_count=attempt_count+1,next_attempt_at=?," +
+                            "last_error=?,updated_at=SYSDATE WHERE event_id=? AND status<>'APPLIED'",
+                    new Timestamp(System.currentTimeMillis() + Math.max(0L, retryBackoffMs)),
+                    message, row.eventId);
+        } catch (RuntimeException markFailure) {
+            failure.addSuppressed(markFailure);
+        }
+        log.error("统计投影 Outbox 发布失败，将自动重试: eventId={}", row.eventId, failure);
+    }
+
+    @Scheduled(initialDelayString = "${stats.projection-async.cleanup-initial-delay-ms:3600000}",
+            fixedDelayString = "${stats.projection-async.cleanup-interval-ms:3600000}")
+    public void cleanup() {
+        if (!enabled) return;
+        long keepMillis = TimeUnit.DAYS.toMillis(Math.max(1, retentionDays));
+        Timestamp cutoff = new Timestamp(System.currentTimeMillis() - keepMillis);
+        int removed = jdbc.update("DELETE FROM " + StatsProjectionOutboxWriter.safeTable(outboxTable) +
+                " WHERE status='APPLIED' AND applied_at<?", cutoff);
+        removed += jdbc.update("DELETE FROM " + StatsProjectionOutboxWriter.safeTable(appliedTable) +
+                " WHERE applied_at<?", cutoff);
+        cleaned.addAndGet(removed);
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("enabled", enabled);
+        result.put("topic", topic);
+        result.put("running", running.get());
+        result.put("published_total", published.get());
+        result.put("failures_total", failures.get());
+        result.put("cleaned_rows_total", cleaned.get());
+        result.put("last_success_epoch_ms", lastSuccessAt == 0L ? null : lastSuccessAt);
+        result.put("last_failure_epoch_ms", lastFailureAt == 0L ? null : lastFailureAt);
+        return result;
+    }
+
+    private static final class OutboxRow {
+        private final String eventId;
+        private final String payload;
+        private OutboxRow(String eventId, String payload) {
+            this.eventId = eventId;
+            this.payload = payload;
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxWriter.java

@@ -0,0 +1,70 @@
+package com.common.service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** Writes one aggregated outbox row in the same transaction as the corresponding fact batch. */
+@Component
+public class StatsProjectionOutboxWriter {
+    private final JdbcTemplate jdbc;
+    private final ObjectMapper objectMapper;
+    private final AtomicLong events = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+
+    @Value("${stats.projection-async.enabled:true}") private boolean enabled;
+    @Value("${stats.projection-async.outbox-table:stats_projection_outbox}") private String outboxTable;
+
+    public StatsProjectionOutboxWriter(JdbcTemplate jdbc, ObjectMapper objectMapper) {
+        this.jdbc = jdbc;
+        this.objectMapper = objectMapper.copy();
+        this.objectMapper.deactivateDefaultTyping();
+    }
+
+    public boolean isEnabled() { return enabled; }
+
+    public int append(StatsProjectionOutboxEvent event) {
+        if (!enabled || event == null || event.isEmpty()) return 0;
+        if (!TransactionSynchronizationManager.isActualTransactionActive()) {
+            throw new IllegalStateException("Stats projection outbox must be written inside a transaction");
+        }
+        event.validate();
+        try {
+            int inserted = jdbc.update("INSERT INTO " + safeTable(outboxTable) +
+                            " (event_id,event_schema,payload,status,attempt_count,next_attempt_at," +
+                            "created_at,updated_at) VALUES (?,?,?,'PENDING',0,SYSDATE,SYSDATE,SYSDATE)",
+                    event.getEventId(), event.getSchema(), objectMapper.writeValueAsString(event));
+            events.incrementAndGet();
+            return inserted;
+        } catch (JsonProcessingException serializationFailure) {
+            failures.incrementAndGet();
+            throw new IllegalStateException("Failed to serialize stats projection outbox event", serializationFailure);
+        } catch (RuntimeException databaseFailure) {
+            failures.incrementAndGet();
+            throw databaseFailure;
+        }
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("enabled", enabled);
+        result.put("outbox_table", outboxTable);
+        result.put("events_written_total", events.get());
+        result.put("write_failures_total", failures.get());
+        return result;
+    }
+
+    static String safeTable(String table) {
+        if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
+            throw new IllegalStateException("Invalid stats projection table: " + table);
+        }
+        return table;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java

@@ -9,6 +9,7 @@ import com.common.mapper.SyslogNonNormalMessageMapper;
 import com.common.service.SyslogNonNormalMessageService;
 import com.common.service.StatsChangeCaptureService;
 import com.common.service.DirectStatsProjectionWriter;
+import com.common.service.DmLargeTextLimiter;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -40,6 +41,9 @@ public class SyslogNonNormalMessageServiceImpl
     @Autowired
     private DirectStatsProjectionWriter directStatsProjectionWriter;
 
+    @Autowired(required = false)
+    private DmLargeTextLimiter largeTextLimiter;
+
     @Value("${dm.non-normal-write.batch-size:500}")
     private int nonNormalBatchSize;
 
@@ -47,6 +51,7 @@ public class SyslogNonNormalMessageServiceImpl
     @Transactional(rollbackFor = Exception.class)
     public boolean saveMessage(SyslogNonNormalMessage message) {
         try {
+            truncateLargeFields(message);
             // 设置ID（如果未提供）
             if (message.getId() == null || message.getId().isEmpty()) {
                 message.setId(generateMessageId());
@@ -101,6 +106,7 @@ public class SyslogNonNormalMessageServiceImpl
         try {
             // 为每条记录设置默认值
             for (SyslogNonNormalMessage message : messages) {
+                truncateLargeFields(message);
                 if (message.getId() == null || message.getId().isEmpty()) {
                     message.setId(generateMessageId());
                 }
@@ -159,10 +165,15 @@ public class SyslogNonNormalMessageServiceImpl
         }
     }
 
+    private void truncateLargeFields(SyslogNonNormalMessage message) {
+        if (largeTextLimiter != null) largeTextLimiter.truncateNonNormal(message);
+    }
+
     @Override
     @Transactional(rollbackFor = Exception.class)
     public boolean updateMessage(SyslogNonNormalMessage message) {
         try {
+            truncateLargeFields(message);
             SyslogNonNormalMessage oldValue = getMessageById(message.getId());
             // 设置更新时间
             message.setUpdateTime(LocalDateTime.now());

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/Utf8TextTruncator.java

@@ -0,0 +1,88 @@
+package com.common.util;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+
+/** UTF-8 字节预算截断器；标记属于最终存储值的一部分，且永不切断 Unicode 字符。 */
+public final class Utf8TextTruncator {
+    private static final String MARKER_FORMAT = "...[HB_XDR_TRUNCATED original_utf8_bytes=%d]";
+
+    private Utf8TextTruncator() { }
+
+    public static Result truncate(String text, long maxUtf8Bytes) {
+        if (text == null) return new Result(null, false, 0L, 0L, null);
+        if (maxUtf8Bytes < 0L) throw new IllegalArgumentException("maxUtf8Bytes 不能小于 0");
+
+        byte[] original = text.getBytes(StandardCharsets.UTF_8);
+        long originalBytes = original.length;
+        if (maxUtf8Bytes == 0L || originalBytes <= maxUtf8Bytes) {
+            return new Result(text, false, originalBytes, originalBytes, null);
+        }
+
+        String marker = String.format(MARKER_FORMAT, originalBytes);
+        int markerBytes = marker.getBytes(StandardCharsets.UTF_8).length;
+        if (maxUtf8Bytes < markerBytes) {
+            throw new IllegalArgumentException("UTF-8 截断上限小于截断标记长度: limit="
+                    + maxUtf8Bytes + ", markerBytes=" + markerBytes);
+        }
+        long prefixBudget = maxUtf8Bytes - markerBytes;
+        int end = prefixEnd(text, prefixBudget);
+        String stored = text.substring(0, end) + marker;
+        return new Result(stored, true, originalBytes,
+                stored.getBytes(StandardCharsets.UTF_8).length, sha256(original));
+    }
+
+    private static int prefixEnd(String text, long byteBudget) {
+        long used = 0L;
+        int index = 0;
+        while (index < text.length()) {
+            int codePoint = text.codePointAt(index);
+            int width = utf8Width(codePoint);
+            if (used + width > byteBudget) break;
+            used += width;
+            index += Character.charCount(codePoint);
+        }
+        return index;
+    }
+
+    private static int utf8Width(int codePoint) {
+        if (codePoint <= 0x7f) return 1;
+        if (codePoint <= 0x7ff) return 2;
+        if (codePoint <= 0xffff) return 3;
+        return 4;
+    }
+
+    private static String sha256(byte[] bytes) {
+        try {
+            byte[] digest = MessageDigest.getInstance("SHA-256").digest(bytes);
+            StringBuilder hex = new StringBuilder(digest.length * 2);
+            for (byte item : digest) hex.append(String.format("%02x", item & 0xff));
+            return hex.toString();
+        } catch (Exception impossible) {
+            throw new IllegalStateException("JVM 不支持 SHA-256", impossible);
+        }
+    }
+
+    public static final class Result {
+        private final String value;
+        private final boolean truncated;
+        private final long originalUtf8Bytes;
+        private final long storedUtf8Bytes;
+        private final String originalSha256;
+
+        Result(String value, boolean truncated, long originalUtf8Bytes,
+               long storedUtf8Bytes, String originalSha256) {
+            this.value = value;
+            this.truncated = truncated;
+            this.originalUtf8Bytes = originalUtf8Bytes;
+            this.storedUtf8Bytes = storedUtf8Bytes;
+            this.originalSha256 = originalSha256;
+        }
+
+        public String getValue() { return value; }
+        public boolean isTruncated() { return truncated; }
+        public long getOriginalUtf8Bytes() { return originalUtf8Bytes; }
+        public long getStoredUtf8Bytes() { return storedUtf8Bytes; }
+        public String getOriginalSha256() { return originalSha256; }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/KafkaReliabilityConfig.java

@@ -1,15 +1,20 @@
 package com.config;
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.springframework.boot.autoconfigure.kafka.ConcurrentKafkaListenerContainerFactoryConfigurer;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
 import org.springframework.kafka.core.ConsumerFactory;
+import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
 import org.springframework.kafka.listener.DefaultErrorHandler;
 import org.springframework.util.backoff.FixedBackOff;
 
+import java.util.HashMap;
+import java.util.Map;
+
 /** Kafka 消费失败边界：业务未确认的消息绝不能被框架默认恢复器静默跳过。 */
 @Configuration
 public class KafkaReliabilityConfig {
@@ -50,6 +55,32 @@ public class KafkaReliabilityConfig {
         return factory;
     }
 
+    /** 大日志使用独立 ConsumerFactory，限制单次 poll 条数并显式放开单条拉取上限。 */
+    @Bean("largeRawKafkaListenerContainerFactory")
+    public ConcurrentKafkaListenerContainerFactory<Object, Object> largeRawKafkaListenerContainerFactory(
+            ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
+            ConsumerFactory<Object, Object> consumerFactory,
+            DefaultErrorHandler syslogDefaultErrorHandler,
+            @Value("${syslog.large-listener.concurrency:1}") int concurrency,
+            @Value("${syslog.large-listener.max-poll-records:5}") int maxPollRecords,
+            @Value("${syslog.large-listener.max-partition-fetch-bytes:33554432}") int maxPartitionFetchBytes,
+            @Value("${syslog.large-listener.fetch-max-bytes:67108864}") int fetchMaxBytes) {
+        if (concurrency < 1 || maxPollRecords < 1 || maxPartitionFetchBytes < 1
+                || fetchMaxBytes < maxPartitionFetchBytes) {
+            throw new IllegalArgumentException("大日志 Kafka 并发、poll 和 fetch 参数非法");
+        }
+        Map<String, Object> properties = new HashMap<>(consumerFactory.getConfigurationProperties());
+        properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
+        properties.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, maxPartitionFetchBytes);
+        properties.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, fetchMaxBytes);
+        DefaultKafkaConsumerFactory<Object, Object> largeConsumerFactory =
+                new DefaultKafkaConsumerFactory<>(properties);
+        ConcurrentKafkaListenerContainerFactory<Object, Object> factory =
+                configuredFactory(configurer, largeConsumerFactory, syslogDefaultErrorHandler);
+        factory.setConcurrency(concurrency);
+        return factory;
+    }
+
     /** 统计脏键与原始日志隔离；同一维度 key 由 Kafka 分区保证只被一个组成员处理。 */
     @Bean("statsMaterializationKafkaListenerContainerFactory")
     public ConcurrentKafkaListenerContainerFactory<Object, Object> statsMaterializationKafkaListenerContainerFactory(
@@ -66,6 +97,21 @@ public class KafkaReliabilityConfig {
         return factory;
     }
 
+    /**
+     * Projection deltas intentionally use one consumer per instance. All events use one Kafka key,
+     * so only one group member owns the active partition and daily projection rows cannot deadlock.
+     */
+    @Bean("statsProjectionKafkaListenerContainerFactory")
+    public ConcurrentKafkaListenerContainerFactory<Object, Object> statsProjectionKafkaListenerContainerFactory(
+            ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
+            ConsumerFactory<Object, Object> consumerFactory,
+            DefaultErrorHandler syslogDefaultErrorHandler) {
+        ConcurrentKafkaListenerContainerFactory<Object, Object> factory =
+                configuredFactory(configurer, consumerFactory, syslogDefaultErrorHandler);
+        factory.setConcurrency(1);
+        return factory;
+    }
+
     private ConcurrentKafkaListenerContainerFactory<Object, Object> configuredFactory(
             ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
             ConsumerFactory<Object, Object> consumerFactory,

haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java

@@ -25,6 +25,15 @@ public class ThreadPoolConfig {
     @Value("${app.processor.thread-pool.keep-alive-seconds:60}")
     private int keepAliveSeconds;
 
+    @Value("${syslog.large-processor.thread-pool.core-pool-size:1}")
+    private int largeCorePoolSize;
+
+    @Value("${syslog.large-processor.thread-pool.max-pool-size:2}")
+    private int largeMaxPoolSize;
+
+    @Value("${syslog.large-processor.thread-pool.queue-capacity:10}")
+    private int largeQueueCapacity;
+
     @Value("${dm.standardized-write.writer-concurrency:2}")
     private int dmWriterConcurrency;
 
@@ -54,6 +63,25 @@ public class ThreadPoolConfig {
         return executor;
     }
 
+    /** 大日志解析/规则计算独立线程池，避免占满普通日志的计算线程与任务队列。 */
+    @Bean("largeLogProcessorExecutor")
+    public Executor largeLogProcessorExecutor() {
+        if (largeCorePoolSize < 1 || largeMaxPoolSize < largeCorePoolSize || largeQueueCapacity < 1) {
+            throw new IllegalArgumentException("大日志处理线程池参数非法");
+        }
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+        executor.setCorePoolSize(largeCorePoolSize);
+        executor.setMaxPoolSize(largeMaxPoolSize);
+        executor.setQueueCapacity(largeQueueCapacity);
+        executor.setKeepAliveSeconds(keepAliveSeconds);
+        executor.setThreadNamePrefix("large-log-processor-");
+        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
+        executor.setWaitForTasksToCompleteOnShutdown(true);
+        executor.setAwaitTerminationSeconds(60);
+        executor.initialize();
+        return executor;
+    }
+
     /**
      * 达梦批写使用独立的有界线程池，不能复用规则计算线程池。
      *

haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/DirectStatsController.java

@@ -7,6 +7,9 @@ import com.common.service.DmStandardizedBatchExecutor;
 import com.common.service.StatsMaterializationConsumer;
 import com.common.service.StatsMaterializationPublisher;
 import com.common.service.StatsMaterializationReconciler;
+import com.common.service.StatsProjectionOutboxWriter;
+import com.common.service.StatsProjectionOutboxRelay;
+import com.common.service.StatsProjectionDeltaConsumer;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -25,6 +28,9 @@ public class DirectStatsController {
     private final StatsMaterializationPublisher materializationPublisher;
     private final StatsMaterializationConsumer materializationConsumer;
     private final StatsMaterializationReconciler materializationReconciler;
+    private final StatsProjectionOutboxWriter projectionOutboxWriter;
+    private final StatsProjectionOutboxRelay projectionOutboxRelay;
+    private final StatsProjectionDeltaConsumer projectionDeltaConsumer;
 
     public DirectStatsController(DirectStatsProjectionWriter writer,
                                  IngressStatsProjectionWriter ingressWriter,
@@ -32,7 +38,10 @@ public class DirectStatsController {
                                  DmStandardizedBatchExecutor batchExecutor,
                                  StatsMaterializationPublisher materializationPublisher,
                                  StatsMaterializationConsumer materializationConsumer,
-                                 StatsMaterializationReconciler materializationReconciler) {
+                                 StatsMaterializationReconciler materializationReconciler,
+                                 StatsProjectionOutboxWriter projectionOutboxWriter,
+                                 StatsProjectionOutboxRelay projectionOutboxRelay,
+                                 StatsProjectionDeltaConsumer projectionDeltaConsumer) {
         this.writer = writer;
         this.ingressWriter = ingressWriter;
         this.dayRolloverService = dayRolloverService;
@@ -40,6 +49,9 @@ public class DirectStatsController {
         this.materializationPublisher = materializationPublisher;
         this.materializationConsumer = materializationConsumer;
         this.materializationReconciler = materializationReconciler;
+        this.projectionOutboxWriter = projectionOutboxWriter;
+        this.projectionOutboxRelay = projectionOutboxRelay;
+        this.projectionDeltaConsumer = projectionDeltaConsumer;
     }
 
     @GetMapping("/status")
@@ -51,6 +63,9 @@ public class DirectStatsController {
         status.put("materialization_publisher", materializationPublisher.status());
         status.put("materialization_consumer", materializationConsumer.status());
         status.put("materialization_reconciler", materializationReconciler.status());
+        status.put("projection_outbox_writer", projectionOutboxWriter.status());
+        status.put("projection_outbox_relay", projectionOutboxRelay.status());
+        status.put("projection_delta_consumer", projectionDeltaConsumer.status());
         return ResponseEntity.ok(status);
     }
 }

haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/StorageLimitMetricsController.java

@@ -0,0 +1,26 @@
+package com.controllers;
+
+import com.common.service.DmLargeTextLimiter;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/** 大字段截断观测，指标只包含计数，不暴露设备原文。 */
+@RestController
+@RequestMapping("/api/syslog/storage-limits")
+public class StorageLimitMetricsController {
+    private final DmLargeTextLimiter limiter;
+
+    public StorageLimitMetricsController(DmLargeTextLimiter limiter) { this.limiter = limiter; }
+
+    @GetMapping("/metrics")
+    public Map<String, Long> metrics() {
+        Map<String, Long> values = new LinkedHashMap<>();
+        values.put("dm_normal_fields_truncated_total", limiter.getNormalFieldsTruncated());
+        values.put("dm_non_normal_fields_truncated_total", limiter.getNonNormalFieldsTruncated());
+        return values;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/InfluxRawPointFactory.java

@@ -1,15 +1,27 @@
 package com.influx.raw;
 
+import com.common.util.Utf8TextTruncator;
 import com.influxdb.client.domain.WritePrecision;
 import com.influxdb.client.write.Point;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 /** 原始 Point 的唯一构造器；V2 的 uuid 只作为 field。 */
 @Component
 public class InfluxRawPointFactory {
     private final RawStorageProperties properties;
+    private final RawInfluxMetrics metrics;
 
-    public InfluxRawPointFactory(RawStorageProperties properties) { this.properties = properties; }
+    @Autowired
+    public InfluxRawPointFactory(RawStorageProperties properties, RawInfluxMetrics metrics) {
+        this.properties = properties;
+        this.metrics = metrics;
+    }
+
+    /** 保留已有单元测试及非 Spring 工具入口。 */
+    public InfluxRawPointFactory(RawStorageProperties properties) {
+        this(properties, new RawInfluxMetrics());
+    }
 
     public RawPoint create(String eventId, String message, String receiveTime,
                            String deviceId, String deviceCollectId, String originalTopic,
@@ -17,21 +29,34 @@ public class InfluxRawPointFactory {
         require(eventId, "event_id"); require(receiveTime, "receive_time");
         require(deviceId, "device_id"); require(deviceCollectId, "device_collect_id");
         require(originalTopic, "originalTopic");
+        Utf8TextTruncator.Result storedMessage = Utf8TextTruncator.truncate(
+                message, properties.getMessageMaxUtf8Bytes());
+        if (storedMessage.isTruncated()) {
+            metrics.messageTruncated(storedMessage.getOriginalUtf8Bytes(), storedMessage.getStoredUtf8Bytes());
+        }
         String measurement = "syslog_security_" + parsed.getMeasurementDate();
         if (properties.getWriteSchema() == RawStorageProperties.WriteSchema.V1) {
             Point point = Point.measurement(measurement)
                     .addTag("device_id", deviceId).addTag("device_collect_id", deviceCollectId)
                     .addTag("uuid", eventId).addTag("topic", originalTopic)
-                    .addField("message", message).addField("receive_time", receiveTime)
+                    .addField("message", storedMessage.getValue()).addField("receive_time", receiveTime)
+                    .addField("message_truncated", storedMessage.isTruncated())
+                    .addField("message_original_bytes", storedMessage.getOriginalUtf8Bytes())
+                    .addField("message_stored_bytes", storedMessage.getStoredUtf8Bytes())
                     .addField("uuid", eventId).time(parsed.getEpochMillis(), WritePrecision.MS);
+            if (storedMessage.isTruncated()) point.addField("message_original_sha256", storedMessage.getOriginalSha256());
             return new RawPoint(point, properties.getV1Bucket(), false);
         }
         if (pointTimeNs == null) throw new IllegalArgumentException("V2 Point 缺少 point_time_ns");
         Point point = Point.measurement(measurement)
                 .addTag("device_id", deviceId).addTag("device_collect_id", deviceCollectId)
                 .addTag("topic", originalTopic).addTag("event_bucket", InfluxEventBucket.bucketOf(eventId))
-                .addField("uuid", eventId).addField("message", message).addField("receive_time", receiveTime)
+                .addField("uuid", eventId).addField("message", storedMessage.getValue()).addField("receive_time", receiveTime)
+                .addField("message_truncated", storedMessage.isTruncated())
+                .addField("message_original_bytes", storedMessage.getOriginalUtf8Bytes())
+                .addField("message_stored_bytes", storedMessage.getStoredUtf8Bytes())
                 .time(pointTimeNs, WritePrecision.NS);
+        if (storedMessage.isTruncated()) point.addField("message_original_sha256", storedMessage.getOriginalSha256());
         return new RawPoint(point, properties.getV2Bucket(), true);
     }
 

haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawInfluxMetrics.java

@@ -16,6 +16,9 @@ public class RawInfluxMetrics {
     private final LongAdder v2FailedBatches = new LongAdder();
     private final LongAdder dualReadDuplicate = new LongAdder();
     private final LongAdder dualReadConflict = new LongAdder();
+    private final LongAdder messagesTruncated = new LongAdder();
+    private final LongAdder originalBytesTruncated = new LongAdder();
+    private final LongAdder storedBytesAfterTruncation = new LongAdder();
 
     public void legacyMissing() { legacyMissing.increment(); }
     public void legacyAssigned() { legacyAssigned.increment(); }
@@ -24,6 +27,11 @@ public class RawInfluxMetrics {
     public void v2FailedBatch() { v2FailedBatches.increment(); }
     public void dualReadDuplicate() { dualReadDuplicate.increment(); }
     public void dualReadConflict() { dualReadConflict.increment(); }
+    public void messageTruncated(long originalBytes, long storedBytes) {
+        messagesTruncated.increment();
+        originalBytesTruncated.add(originalBytes);
+        storedBytesAfterTruncation.add(storedBytes);
+    }
 
     public Map<String, Object> snapshot() {
         Map<String, Object> values = new LinkedHashMap<>();
@@ -36,6 +44,9 @@ public class RawInfluxMetrics {
         values.put("raw_write_v2_failed_batches_total", v2FailedBatches.sum());
         values.put("influx_dual_read_duplicate_total", dualReadDuplicate.sum());
         values.put("influx_dual_read_conflict_total", dualReadConflict.sum());
+        values.put("raw_message_truncated_total", messagesTruncated.sum());
+        values.put("raw_message_truncated_original_bytes_total", originalBytesTruncated.sum());
+        values.put("raw_message_truncated_stored_bytes_total", storedBytesAfterTruncation.sum());
         return values;
     }
 }

haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawStorageProperties.java

@@ -19,6 +19,10 @@ public class RawStorageProperties {
     @Value("${influxdb.raw.legacy-point-time-fallback-enabled:true}") private boolean legacyFallbackEnabled;
     @Value("${influxdb.raw.query-max-rows:10000}") private int queryMaxRows;
     @Value("${influxdb.raw.query-max-response-bytes:10485760}") private long queryMaxResponseBytes;
+    /** 原始 message 单字段的 UTF-8 存储上限；0 表示不限制。 */
+    @Value("${influxdb.raw.message-max-utf8-bytes:0}") private long messageMaxUtf8Bytes;
+    /** 一次同步写请求的估算 Line Protocol 上限；按字节切批，不能只按条数。 */
+    @Value("${influxdb.raw.write-batch-max-bytes:8388608}") private long writeBatchMaxBytes;
 
     private WriteSchema writeSchema;
     private ReadMode readMode;
@@ -35,6 +39,12 @@ public class RawStorageProperties {
         if (queryMaxResponseBytes <= 0L) {
             throw new IllegalArgumentException("influxdb.raw.query-max-response-bytes 必须大于 0");
         }
+        if (messageMaxUtf8Bytes < 0L || (messageMaxUtf8Bytes > 0L && messageMaxUtf8Bytes < 64L)) {
+            throw new IllegalArgumentException("influxdb.raw.message-max-utf8-bytes 必须为 0 或至少 64");
+        }
+        if (writeBatchMaxBytes <= 0L) {
+            throw new IllegalArgumentException("influxdb.raw.write-batch-max-bytes 必须大于 0");
+        }
     }
 
     private static String normalize(String value) {
@@ -49,6 +59,8 @@ public class RawStorageProperties {
     public boolean isLegacyFallbackEnabled() { return legacyFallbackEnabled; }
     public int getQueryMaxRows() { return queryMaxRows; }
     public long getQueryMaxResponseBytes() { return queryMaxResponseBytes; }
+    public long getMessageMaxUtf8Bytes() { return messageMaxUtf8Bytes; }
+    public long getWriteBatchMaxBytes() { return writeBatchMaxBytes; }
 
     public enum WriteSchema { V1, V2 }
     public enum ReadMode { V1_ONLY, V1_V2, V2_ONLY }

haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties

@@ -34,6 +34,10 @@ influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
 influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
 influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
 influxdb.raw.query-max-response-bytes=${INFLUXDB_RAW_QUERY_MAX_RESPONSE_BYTES:10485760}
+# 0 表示不限制；启用后最终 message（含截断标记）不超过该 UTF-8 字节数。
+influxdb.raw.message-max-utf8-bytes=${INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES:1048576}
+# Influx 同步写按 Line Protocol 估算字节切批，避免少量大日志形成超大 HTTP 请求。
+influxdb.raw.write-batch-max-bytes=${INFLUXDB_RAW_WRITE_BATCH_MAX_BYTES:8388608}
 # InfluxDB 2.7 连接超时配置
 influxdb.connection.timeout=30s
 influxdb.connection.read-timeout=30s
@@ -70,16 +74,31 @@ spring.kafka.consumer.auto-offset-reset=latest
 spring.kafka.consumer.enable-auto-commit=false
 spring.kafka.consumer.auto-commit-interval=1000
 spring.kafka.consumer.topic=test-topic
+spring.kafka.consumer.large-topic=${KAFKA_CONSUMER_LARGE_TOPIC:${spring.kafka.consumer.topic}.large}
 # consumer-rule 是当前活跃的 Kafka 原始日志标准化节点，默认启动监听。
 syslog.raw-consumer.enabled=true
 # Raw Topic 当前有 3 个分区，使用独立工厂显式创建 3 个 consumer；可通过环境变量灰度调整。
 syslog.raw-listener.concurrency=${SYSLOG_RAW_LISTENER_CONCURRENCY:3}
+# 大日志使用独立 Topic、消费组和 ConsumerFactory；建议生产环境独立 JVM 部署。
+syslog.large-listener.enabled=${SYSLOG_LARGE_LISTENER_ENABLED:true}
+syslog.large-listener.group-id=${SYSLOG_LARGE_LISTENER_GROUP_ID:${spring.kafka.consumer.group-id}-large}
+syslog.large-listener.concurrency=${SYSLOG_LARGE_LISTENER_CONCURRENCY:1}
+syslog.large-listener.max-poll-records=${SYSLOG_LARGE_LISTENER_MAX_POLL_RECORDS:5}
+syslog.large-listener.max-partition-fetch-bytes=${SYSLOG_LARGE_LISTENER_MAX_PARTITION_FETCH_BYTES:33554432}
+syslog.large-listener.fetch-max-bytes=${SYSLOG_LARGE_LISTENER_FETCH_MAX_BYTES:67108864}
+syslog.large-listener.processing-batch-max-bytes=${SYSLOG_LARGE_PROCESSING_BATCH_MAX_BYTES:8388608}
+# 大日志规则计算不复用普通日志线程池，过载时由 Kafka offset 背压。
+syslog.large-processor.thread-pool.core-pool-size=${SYSLOG_LARGE_PROCESSOR_CORE_POOL_SIZE:1}
+syslog.large-processor.thread-pool.max-pool-size=${SYSLOG_LARGE_PROCESSOR_MAX_POOL_SIZE:2}
+syslog.large-processor.thread-pool.queue-capacity=${SYSLOG_LARGE_PROCESSOR_QUEUE_CAPACITY:10}
 # P0：Retry/DLT 发布与原始消费使用同一 Kafka 集群；Topic 名称全部可由环境变量覆盖。
 spring.kafka.bootstrap-servers=${spring.kafka.consumer.bootstrap-servers}
 spring.kafka.producer.acks=all
 spring.kafka.producer.properties.enable.idempotence=true
 spring.kafka.producer.properties.delivery.timeout.ms=120000
 spring.kafka.producer.properties.request.timeout.ms=30000
+# Retry/DLT 信封也可能携带大日志密文，不能保留 Kafka 默认 1MiB。
+spring.kafka.producer.properties.max.request.size=${KAFKA_RELIABILITY_MAX_REQUEST_SIZE:33554432}
 syslog.reliability.retry-1m-topic=${spring.kafka.consumer.topic}.RETRY.1M
 syslog.reliability.retry-10m-topic=${spring.kafka.consumer.topic}.RETRY.10M
 syslog.reliability.retry-1h-topic=${spring.kafka.consumer.topic}.RETRY.1H
@@ -91,6 +110,14 @@ syslog.reliability.consumer-version=${APP_VERSION:1.0.0}
 syslog.reliability.consumer-group-id=${spring.kafka.consumer.group-id}-retry
 syslog.reliability.consumer.enabled=true
 syslog.reliability.listener.concurrency=1
+# 大日志 Retry/DLT 不回流普通重试通道，避免再次形成队头阻塞。
+syslog.large-reliability.retry-1m-topic=${spring.kafka.consumer.large-topic}.RETRY.1M
+syslog.large-reliability.retry-10m-topic=${spring.kafka.consumer.large-topic}.RETRY.10M
+syslog.large-reliability.retry-1h-topic=${spring.kafka.consumer.large-topic}.RETRY.1H
+syslog.large-reliability.dlt-topic=${spring.kafka.consumer.large-topic}.DLT
+syslog.large-reliability.replay-topic=${spring.kafka.consumer.large-topic}.REPLAY
+syslog.large-reliability.consumer-group-id=${spring.kafka.consumer.group-id}-large-retry
+syslog.large-reliability.consumer.enabled=true
 
 
 spring.kafka.consumer.max-poll-records=500
@@ -152,6 +179,10 @@ dm.standardized-write.mode=BATCH
 dm.standardized-write.batch-size=${DM_STANDARDIZED_WRITE_BATCH_SIZE:500}
 dm.standardized-write.writer-concurrency=2
 dm.standardized-write.writer-queue-capacity=100
+# 标准化大字段和非标原文均在达梦持久化边界截断；0 表示不限制。
+dm.normal.large-field-max-utf8-bytes=${DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES:65536}
+dm.normal.large-fields=${DM_NORMAL_LARGE_FIELDS:http_req_body,http_resp_body,http_req_header,http_resp_header,detail,description,cmdline,shell_cmdline,parent_cmdline,mail_message}
+dm.non-normal.large-field-max-utf8-bytes=${DM_NON_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES:262144}
 dm.standardized-write.transaction-timeout-seconds=45
 dm.standardized-write.kafka-batch-deadline-ms=90000
 dm.standardized-write.max-split-depth=10
@@ -181,7 +212,7 @@ spring.datasource.hikari.validation-timeout=5000
 spring.datasource.hikari.leak-detection-threshold=30000
 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule
 spring.datasource.hikari.auto-commit=false
-# 统计写入采用事实同事务直写，不再为每条结果写 change_log + pending。
+# DIRECT 不再写逐事实 change_log；默认只在事实事务追加一条批次 Outbox，日统计异步投影。
 # 首次部署 DIRECT 前必须先执行 08/09/10/11 脚本并按设计文档排空旧 pending。
 stats.write-mode=DIRECT
 stats.incremental.capture-enabled=false
@@ -197,11 +228,11 @@ stats.direct.control-table=stats_direct_control
 stats.direct.today-stats-date-enabled=true
 stats.direct.max-concurrency-retries=${STATS_DIRECT_MAX_CONCURRENCY_RETRIES:3}
 stats.direct.concurrency-retry-backoff-ms=${STATS_DIRECT_CONCURRENCY_RETRY_BACKOFF_MS:50}
-stats.direct.transaction-log-enabled=false
+stats.direct.transaction-log-enabled=${STATS_DIRECT_TRANSACTION_LOG_ENABLED:false}
 stats.direct.day-rollover-check-cron=5 * * * * ?
 stats.business-zone=Asia/Shanghai
 stats.fact-only.confirm-rebuild-required=false
-# 事实事务只维护小型精确统计表；提交后按维度脏键异步刷新业务展示字段。
+# 日统计投影完成后按维度脏键异步刷新业务展示字段。
 # 三个实例必须配置相同 topic/group-id，Topic 分区数建议与原始 Topic 相同。
 stats.materialization.enabled=${STATS_MATERIALIZATION_ENABLED:true}
 stats.materialization.topic=${STATS_MATERIALIZATION_TOPIC:${spring.kafka.consumer.topic}.STATS.MATERIALIZE}
@@ -216,6 +247,25 @@ stats.materialization.concurrency-retry-backoff-ms=${STATS_MATERIALIZATION_CONCU
 stats.materialization.reconcile-initial-delay-ms=${STATS_MATERIALIZATION_RECONCILE_INITIAL_DELAY_MS:30000}
 stats.materialization.reconcile-interval-ms=${STATS_MATERIALIZATION_RECONCILE_INTERVAL_MS:60000}
 
+# Fact transactions append one aggregated durable event instead of locking the daily projection rows.
+# A dedicated single-lane Kafka consumer applies device/rule/collector projection deltas and then
+# emits the existing idempotent business-materialization dirty keys.
+stats.projection-async.enabled=${STATS_PROJECTION_ASYNC_ENABLED:true}
+stats.projection-async.outbox-table=${STATS_PROJECTION_OUTBOX_TABLE:stats_projection_outbox}
+stats.projection-async.applied-table=${STATS_PROJECTION_APPLIED_TABLE:stats_projection_applied_event}
+stats.projection-async.topic=${STATS_PROJECTION_TOPIC:${spring.kafka.consumer.topic}.STATS.PROJECTION}
+stats.projection-async.group-id=${STATS_PROJECTION_GROUP_ID:${spring.kafka.consumer.group-id}-stats-projection}
+stats.projection-async.relay-interval-ms=${STATS_PROJECTION_RELAY_INTERVAL_MS:200}
+stats.projection-async.relay-batch-size=${STATS_PROJECTION_RELAY_BATCH_SIZE:100}
+stats.projection-async.publish-timeout-ms=${STATS_PROJECTION_PUBLISH_TIMEOUT_MS:10000}
+stats.projection-async.retry-backoff-ms=${STATS_PROJECTION_RETRY_BACKOFF_MS:1000}
+stats.projection-async.redelivery-interval-ms=${STATS_PROJECTION_REDELIVERY_INTERVAL_MS:60000}
+stats.projection-async.max-concurrency-retries=${STATS_PROJECTION_MAX_CONCURRENCY_RETRIES:3}
+stats.projection-async.concurrency-retry-backoff-ms=${STATS_PROJECTION_CONCURRENCY_RETRY_BACKOFF_MS:50}
+stats.projection-async.retention-days=${STATS_PROJECTION_RETENTION_DAYS:14}
+stats.projection-async.cleanup-initial-delay-ms=${STATS_PROJECTION_CLEANUP_INITIAL_DELAY_MS:3600000}
+stats.projection-async.cleanup-interval-ms=${STATS_PROJECTION_CLEANUP_INTERVAL_MS:3600000}
+
 # 标准化规则编译与批次缓存灰度开关
 normalization.jetcache.enabled=true
 normalization.rule-source-cache.enabled=true

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SyslogFailureRouterReliabilityTest.java

@@ -42,6 +42,11 @@ class SyslogFailureRouterReliabilityTest {
         ReflectionTestUtils.setField(router, "retry10mTopic", "raw.RETRY.10M");
         ReflectionTestUtils.setField(router, "retry1hTopic", "raw.RETRY.1H");
         ReflectionTestUtils.setField(router, "dltTopic", "raw.DLT");
+        ReflectionTestUtils.setField(router, "largeOriginalTopic", "raw.large");
+        ReflectionTestUtils.setField(router, "largeRetry1mTopic", "raw.large.RETRY.1M");
+        ReflectionTestUtils.setField(router, "largeRetry10mTopic", "raw.large.RETRY.10M");
+        ReflectionTestUtils.setField(router, "largeRetry1hTopic", "raw.large.RETRY.1H");
+        ReflectionTestUtils.setField(router, "largeDltTopic", "raw.large.DLT");
         ReflectionTestUtils.setField(router, "publishTimeoutMs", 1000L);
         ReflectionTestUtils.setField(router, "consumerVersion", "test");
         record = new ConsumerRecord<>("raw", 2, 42L, "device-1", "ciphertext");
@@ -86,6 +91,22 @@ class SyslogFailureRouterReliabilityTest {
         assertFalse(body.getValue().contains("schema_version"));
     }
 
+    @Test
+    void largeMessageFailureNeverFallsBackIntoNormalRetryTopic() {
+        ConsumerRecord<String, String> largeRecord = new ConsumerRecord<>(
+                "raw.large", 1, 7L, "event", "large-ciphertext");
+        SettableListenableFuture<SendResult<String, String>> acknowledged = new SettableListenableFuture<>();
+        acknowledged.set(null);
+        when(template.send(eq("raw.large.RETRY.1M"), anyString(), anyString()))
+                .thenReturn(acknowledged);
+
+        SyslogFailureRouter.TerminalState state = router.route(
+                largeRecord, router.resolve(largeRecord), new SocketException("reset"));
+
+        assertEquals(SyslogFailureRouter.TerminalState.RETRY_PUBLISHED, state);
+        verify(template).send(eq("raw.large.RETRY.1M"), anyString(), anyString());
+    }
+
     @Test
     void publishFailureNeverBecomesTerminal() {
         SettableListenableFuture<SendResult<String, String>> failed = new SettableListenableFuture<>();

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SyslogKafkaTopologyTest.java

@@ -4,12 +4,17 @@ import org.junit.jupiter.api.Test;
 import org.springframework.kafka.annotation.KafkaListener;
 
 import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.List;
 
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import com.common.service.StatsMaterializationConsumer;
+import com.common.service.StatsProjectionDeltaConsumer;
 
 /** 防止多节点 consumer-rule 的监听器 ID 或显式消费组在后续合并中回退。 */
 class SyslogKafkaTopologyTest {
@@ -50,6 +55,56 @@ class SyslogKafkaTopologyTest {
         }, listener.topics());
     }
 
+    @Test
+    void largeRawListenerUsesDedicatedTopicGroupAndFactory() throws Exception {
+        Method method = SysLogProcessor.class.getMethod(
+                "consumeLargeMessages", List.class,
+                org.springframework.kafka.support.Acknowledgment.class);
+        KafkaListener listener = method.getAnnotation(KafkaListener.class);
+
+        assertNotNull(listener);
+        assertEquals("syslog-consumer-rule-raw-large", listener.id());
+        assertEquals("${syslog.large-listener.group-id:${spring.kafka.consumer.group-id}-large}",
+                listener.groupId());
+        assertEquals("largeRawKafkaListenerContainerFactory", listener.containerFactory());
+        assertArrayEquals(new String[]{"${spring.kafka.consumer.large-topic:agent-syslog-topic.large}"},
+                listener.topics());
+    }
+
+    @Test
+    void largeRetryReplayListenerIsAlsoIsolated() throws Exception {
+        Method method = SysLogProcessor.class.getMethod(
+                "consumeLargeRetryAndReplay", List.class,
+                org.springframework.kafka.support.Acknowledgment.class);
+        KafkaListener listener = method.getAnnotation(KafkaListener.class);
+
+        assertNotNull(listener);
+        assertEquals("syslog-consumer-rule-large-retry-replay", listener.id());
+        assertEquals("largeRawKafkaListenerContainerFactory", listener.containerFactory());
+        assertArrayEquals(new String[]{
+                "${syslog.large-reliability.retry-1m-topic:${spring.kafka.consumer.large-topic}.RETRY.1M}",
+                "${syslog.large-reliability.retry-10m-topic:${spring.kafka.consumer.large-topic}.RETRY.10M}",
+                "${syslog.large-reliability.retry-1h-topic:${spring.kafka.consumer.large-topic}.RETRY.1H}",
+                "${syslog.large-reliability.replay-topic:${spring.kafka.consumer.large-topic}.REPLAY}"
+        }, listener.topics());
+    }
+
+    @Test
+    void largePollIsPartitionedByUtf8BytesInsteadOfOnlyRecordCount() {
+        List<ConsumerRecord<String, String>> records = Arrays.asList(
+                new ConsumerRecord<>("large", 0, 0L, "k1", "中文"),
+                new ConsumerRecord<>("large", 0, 1L, "k2", "12"),
+                new ConsumerRecord<>("large", 0, 2L, "k3", "🙂"));
+
+        List<List<ConsumerRecord<String, String>>> chunks =
+                SysLogProcessor.partitionRecordsByBytes(records, 7L);
+
+        assertEquals(2, chunks.size());
+        assertEquals(1, chunks.get(0).size());
+        assertEquals(2, chunks.get(1).size());
+        assertEquals(6, chunks.get(0).get(0).value().getBytes(StandardCharsets.UTF_8).length);
+    }
+
     @Test
     void statsMaterializerUsesDedicatedTopicAndGroup() throws Exception {
         Method method = StatsMaterializationConsumer.class.getMethod(
@@ -65,4 +120,20 @@ class SyslogKafkaTopologyTest {
                 "${stats.materialization.topic:${spring.kafka.consumer.topic}.STATS.MATERIALIZE}"
         }, listener.topics());
     }
+
+    @Test
+    void statsProjectionUsesSeparateSingleLaneTopicAndGroup() throws Exception {
+        Method method = StatsProjectionDeltaConsumer.class.getMethod(
+                "consume", List.class, org.springframework.kafka.support.Acknowledgment.class);
+        KafkaListener listener = method.getAnnotation(KafkaListener.class);
+
+        assertNotNull(listener);
+        assertEquals("syslog-stats-projection", listener.id());
+        assertEquals("${stats.projection-async.group-id:${spring.kafka.consumer.group-id}-stats-projection}",
+                listener.groupId());
+        assertEquals("statsProjectionKafkaListenerContainerFactory", listener.containerFactory());
+        assertArrayEquals(new String[]{
+                "${stats.projection-async.topic:${spring.kafka.consumer.topic}.STATS.PROJECTION}"
+        }, listener.topics());
+    }
 }

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DirectStatsProjectionWriterTest.java

@@ -119,12 +119,14 @@ class DirectStatsProjectionWriterTest {
     }
 
     @Test
-    void kafkaMaterializationRemovesBusinessTableUpdatesFromFactTransaction() {
+    void durableOutboxRemovesDailyAndBusinessProjectionUpdatesFromFactTransaction() {
         JdbcTemplate jdbc = mock(JdbcTemplate.class);
         StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
         when(publisher.isEnabled()).thenReturn(true);
+        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
+        when(outbox.isEnabled()).thenReturn(true);
         DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(
-                jdbc, new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), publisher);
+                jdbc, new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), publisher, outbox);
         configure(writer, true);
         TransactionSynchronizationManager.setActualTransactionActive(true);
 
@@ -135,20 +137,19 @@ class DirectStatsProjectionWriterTest {
         row.put("normalize_rule_id", 11L);
         writer.writeStandardizedRow(row);
 
-        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
-        verify(jdbc, times(2)).batchUpdate(sql.capture(), anyList());
-        assertTrue(sql.getAllValues().get(0).contains("MERGE INTO device_daily_stats"));
-        assertTrue(sql.getAllValues().get(1).contains("MERGE INTO normalize_rule_daily_stats"));
-        verify(publisher).enqueueAfterCommit(anyCollection());
-        assertTrue("KAFKA_DIRTY_KEY".equals(writer.status().get("business_materialization_mode")));
+        verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class));
+        verifyNoInteractions(jdbc);
+        assertTrue("DURABLE_OUTBOX".equals(writer.status().get("daily_projection_write_mode")));
     }
 
     private DirectStatsProjectionWriter writer(JdbcTemplate jdbc, String mode, boolean readEnabled) {
         StatsWriteModeProvider modeProvider = new StatsWriteModeProvider(mode);
         StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
         when(publisher.isEnabled()).thenReturn(false);
+        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
+        when(outbox.isEnabled()).thenReturn(false);
         DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(
-                jdbc, modeProvider, new DirectStatsWriteMetrics(), publisher);
+                jdbc, modeProvider, new DirectStatsWriteMetrics(), publisher, outbox);
         configure(writer, readEnabled);
         return writer;
     }

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmLargeTextLimiterTest.java

@@ -0,0 +1,62 @@
+package com.common.service;
+
+import com.common.entity.SyslogNonNormalMessage;
+import org.junit.jupiter.api.Test;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class DmLargeTextLimiterTest {
+
+    @Test
+    void normalConfiguredFieldsAreTruncatedButOtherFieldsRemainUntouched() {
+        DmLargeTextLimiter limiter = limiter(96L, "http_req_body,description", 0L);
+        Map<String, Object> row = new HashMap<>();
+        row.put("http_req_body", repeat("中文🙂", 100));
+        row.put("other", repeat("x", 200));
+
+        limiter.truncateNormalRow(row);
+
+        String body = (String) row.get("http_req_body");
+        assertTrue(body.contains("HB_XDR_TRUNCATED"));
+        assertTrue(body.getBytes(StandardCharsets.UTF_8).length <= 96);
+        assertEquals(repeat("x", 200), row.get("other"));
+        assertEquals(1L, limiter.getNormalFieldsTruncated());
+    }
+
+    @Test
+    void allNonNormalLargeColumnsUseTheSameVisibleMarker() {
+        DmLargeTextLimiter limiter = limiter(0L, "", 96L);
+        SyslogNonNormalMessage message = new SyslogNonNormalMessage();
+        message.setSyslogMessage(repeat("原文", 100));
+        message.setHeaderMessage(repeat("头", 100));
+        message.setReasonDetail(repeat("原因", 100));
+
+        limiter.truncateNonNormal(message);
+
+        assertTrue(message.getSyslogMessage().contains("HB_XDR_TRUNCATED"));
+        assertTrue(message.getHeaderMessage().contains("HB_XDR_TRUNCATED"));
+        assertTrue(message.getReasonDetail().contains("HB_XDR_TRUNCATED"));
+        assertEquals(3L, limiter.getNonNormalFieldsTruncated());
+    }
+
+    private DmLargeTextLimiter limiter(long normal, String fields, long nonNormal) {
+        DmLargeTextLimiter limiter = new DmLargeTextLimiter();
+        ReflectionTestUtils.setField(limiter, "normalMaxBytes", normal);
+        ReflectionTestUtils.setField(limiter, "normalFieldsValue", fields);
+        ReflectionTestUtils.setField(limiter, "nonNormalMaxBytes", nonNormal);
+        limiter.validate();
+        return limiter;
+    }
+
+    private static String repeat(String value, int count) {
+        StringBuilder result = new StringBuilder(value.length() * count);
+        for (int i = 0; i < count; i++) result.append(value);
+        return result.toString();
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsProjectionWriterTest.java

@@ -2,6 +2,11 @@ package com.common.service;
 
 import com.common.entity.IngressStatsEvent;
 import org.junit.jupiter.api.Test;
+import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.support.SimpleTransactionStatus;
+import org.springframework.jdbc.core.JdbcTemplate;
 
 import java.time.LocalDateTime;
 import java.util.Arrays;
@@ -9,6 +14,11 @@ import java.util.ArrayList;
 import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
 
 class IngressStatsProjectionWriterTest {
     @Test
@@ -41,4 +51,28 @@ class IngressStatsProjectionWriterTest {
         assertEquals(Arrays.asList(8L, 2L, 9L), devices);
         assertEquals(Arrays.asList(3L, 4L, 7L), new ArrayList<>(result.collects.keySet()));
     }
+
+    @Test
+    void directAsyncModeWritesOnlyOneOutboxEvent() {
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        PlatformTransactionManager manager = mock(PlatformTransactionManager.class);
+        when(manager.getTransaction(any(TransactionDefinition.class)))
+                .thenReturn(new SimpleTransactionStatus());
+        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
+        when(outbox.isEnabled()).thenReturn(true);
+        IngressStatsProjectionWriter writer = new IngressStatsProjectionWriter(
+                jdbc, manager, new StatsWriteModeProvider("DIRECT"),
+                mock(StatsMaterializationPublisher.class), outbox);
+        ReflectionTestUtils.setField(writer, "enabled", true);
+        ReflectionTestUtils.setField(writer, "transactionTimeoutSeconds", 45);
+
+        int groups = writer.writeBatch(Arrays.asList(
+                new IngressStatsEvent("a", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 0)),
+                new IngressStatsEvent("b", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 1))));
+
+        assertEquals(2, groups);
+        verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class));
+        verifyNoInteractions(jdbc);
+        verify(manager).commit(any());
+    }
 }

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncTest.java

@@ -0,0 +1,151 @@
+package com.common.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.support.SimpleTransactionStatus;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class StatsProjectionAsyncTest {
+    @AfterEach
+    void clearTransactionState() {
+        TransactionSynchronizationManager.clear();
+    }
+
+    @Test
+    void outboxEventAggregatesOneFactBatchByDimension() {
+        LocalDateTime time = LocalDateTime.of(2026, 9, 12, 8, 0);
+        List<Map<String, Object>> rows = new ArrayList<>();
+        for (long ruleId : Arrays.asList(11L, 12L, 11L)) {
+            Map<String, Object> row = new HashMap<>();
+            row.put("created_at", time);
+            row.put("log_time", time.plusMinutes(ruleId));
+            row.put("device_id", 7L);
+            row.put("normalize_rule_id", ruleId);
+            rows.add(row);
+        }
+
+        StatsProjectionOutboxEvent event = StatsProjectionOutboxEvent.fromAccumulator(
+                new DirectStatsAccumulator().addStandardizedRows(rows));
+
+        event.validate();
+        assertEquals(1, event.getDevices().size());
+        assertEquals(3L, event.getDevices().get(0).getParseDelta());
+        assertEquals(2, event.getRules().size());
+    }
+
+    @Test
+    void outboxWriterRequiresAndUsesFactTransaction() {
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        StatsProjectionOutboxWriter writer = new StatsProjectionOutboxWriter(jdbc, new ObjectMapper());
+        ReflectionTestUtils.setField(writer, "enabled", true);
+        ReflectionTestUtils.setField(writer, "outboxTable", "stats_projection_outbox");
+        StatsProjectionOutboxEvent event = oneDeviceEvent("event-1");
+
+        assertThrows(IllegalStateException.class, () -> writer.append(event));
+
+        TransactionSynchronizationManager.setActualTransactionActive(true);
+        writer.append(event);
+        verify(jdbc).update(org.mockito.ArgumentMatchers.contains("INSERT INTO stats_projection_outbox"),
+                org.mockito.ArgumentMatchers.eq("event-1"),
+                org.mockito.ArgumentMatchers.eq(StatsProjectionOutboxEvent.SCHEMA),
+                org.mockito.ArgumentMatchers.anyString());
+    }
+
+    @Test
+    void projectionConsumerAppliesFreshEventAndRecordsIdempotencyMarker() {
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        when(jdbc.queryForList(org.mockito.ArgumentMatchers.anyString(),
+                org.mockito.ArgumentMatchers.eq(String.class), org.mockito.ArgumentMatchers.<Object>any()))
+                .thenReturn(Collections.emptyList());
+        PlatformTransactionManager manager = mock(PlatformTransactionManager.class);
+        when(manager.getTransaction(any(TransactionDefinition.class))).thenReturn(new SimpleTransactionStatus());
+        StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
+        StatsProjectionDeltaConsumer consumer = new StatsProjectionDeltaConsumer(
+                jdbc, manager, new ObjectMapper(), publisher);
+        configure(consumer);
+
+        StatsProjectionDeltaConsumer.ApplyResult result = consumer.applyWithRetry(
+                Collections.singletonList(oneDeviceEvent("event-1")));
+
+        assertEquals(1, result.appliedEvents);
+        assertEquals(1, result.deviceGroups);
+        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
+        verify(jdbc, times(3)).batchUpdate(sql.capture(), anyList());
+        assertTrue(sql.getAllValues().get(0).contains("MERGE INTO device_daily_stats"));
+        assertTrue(sql.getAllValues().get(1).contains("stats_projection_applied_event"));
+        assertTrue(sql.getAllValues().get(2).contains("SET status='APPLIED'"));
+        verify(publisher).enqueueAfterCommit(org.mockito.ArgumentMatchers.anyCollection());
+        verify(manager).commit(any());
+    }
+
+    @Test
+    void projectionConsumerDoesNotApplyDuplicateDeltaTwice() {
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        when(jdbc.queryForList(org.mockito.ArgumentMatchers.anyString(),
+                org.mockito.ArgumentMatchers.eq(String.class), org.mockito.ArgumentMatchers.<Object>any()))
+                .thenReturn(Collections.singletonList("event-1"));
+        PlatformTransactionManager manager = mock(PlatformTransactionManager.class);
+        when(manager.getTransaction(any(TransactionDefinition.class))).thenReturn(new SimpleTransactionStatus());
+        StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
+        StatsProjectionDeltaConsumer consumer = new StatsProjectionDeltaConsumer(
+                jdbc, manager, new ObjectMapper(), publisher);
+        configure(consumer);
+
+        StatsProjectionDeltaConsumer.ApplyResult result = consumer.applyWithRetry(
+                Collections.singletonList(oneDeviceEvent("event-1")));
+
+        assertEquals(0, result.appliedEvents);
+        assertEquals(1, result.duplicates);
+        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
+        verify(jdbc).batchUpdate(sql.capture(), anyList());
+        assertTrue(sql.getValue().contains("SET status='APPLIED'"));
+    }
+
+    private StatsProjectionOutboxEvent oneDeviceEvent(String eventId) {
+        DirectStatsAccumulator accumulator = new DirectStatsAccumulator();
+        Map<String, Object> row = new HashMap<>();
+        row.put("created_at", LocalDate.of(2026, 9, 12));
+        row.put("log_time", LocalDateTime.of(2026, 9, 12, 8, 0));
+        row.put("device_id", 7L);
+        StatsProjectionOutboxEvent event = StatsProjectionOutboxEvent.fromAccumulator(
+                accumulator.addStandardizedRow(row));
+        event.setEventId(eventId);
+        return event;
+    }
+
+    private void configure(StatsProjectionDeltaConsumer consumer) {
+        ReflectionTestUtils.setField(consumer, "enabled", true);
+        ReflectionTestUtils.setField(consumer, "appliedTable", "stats_projection_applied_event");
+        ReflectionTestUtils.setField(consumer, "outboxTable", "stats_projection_outbox");
+        ReflectionTestUtils.setField(consumer, "deviceTable", "device_daily_stats");
+        ReflectionTestUtils.setField(consumer, "ruleTable", "normalize_rule_daily_stats");
+        ReflectionTestUtils.setField(consumer, "transactionTimeoutSeconds", 45);
+        ReflectionTestUtils.setField(consumer, "maxConcurrencyRetries", 3);
+        ReflectionTestUtils.setField(consumer, "retryBackoffMs", 0L);
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/util/Utf8TextTruncatorTest.java

@@ -0,0 +1,43 @@
+package com.common.util;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class Utf8TextTruncatorTest {
+    @Test
+    void truncatesWithinBudgetWithoutSplittingChineseOrEmoji() {
+        String source = repeat("中文🙂", 100);
+
+        Utf8TextTruncator.Result result = Utf8TextTruncator.truncate(source, 96L);
+
+        assertTrue(result.isTruncated());
+        assertTrue(result.getValue().endsWith("]"));
+        assertTrue(result.getValue().contains("HB_XDR_TRUNCATED"));
+        assertTrue(result.getValue().getBytes(StandardCharsets.UTF_8).length <= 96);
+        assertEquals(source.getBytes(StandardCharsets.UTF_8).length, result.getOriginalUtf8Bytes());
+        assertNotNull(result.getOriginalSha256());
+        assertFalse(result.getValue().contains("�"));
+    }
+
+    @Test
+    void zeroMeansUnlimitedAndSmallEnabledLimitFailsClearly() {
+        Utf8TextTruncator.Result result = Utf8TextTruncator.truncate("中文🙂", 0L);
+        assertFalse(result.isTruncated());
+        assertEquals("中文🙂", result.getValue());
+        assertThrows(IllegalArgumentException.class,
+                () -> Utf8TextTruncator.truncate(repeat("x", 100), 8L));
+    }
+
+    private static String repeat(String value, int count) {
+        StringBuilder result = new StringBuilder(value.length() * count);
+        for (int i = 0; i < count; i++) result.append(value);
+        return result.toString();
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java

@@ -67,6 +67,25 @@ class InfluxRawV2ContractTest {
         String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket_v2",raw.getBucket());
     }
 
+    @Test void oversizedRawMessageIsMarkedAndCarriesStructuredTruncationMetadata() {
+        RawStorageProperties properties=mock(RawStorageProperties.class);
+        when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2);
+        when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
+        when(properties.getMessageMaxUtf8Bytes()).thenReturn(128L);
+        RawInfluxMetrics metrics=new RawInfluxMetrics();
+        InfluxRawPointFactory factory=new InfluxRawPointFactory(properties,metrics);
+
+        String line=factory.create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5",
+                repeat("中文🙂",100),"20260814000000000","126","1","raw.large",
+                ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L)
+                .getPoint().toLineProtocol();
+
+        assertTrue(line.contains("HB_XDR_TRUNCATED"));
+        assertTrue(line.contains("message_truncated=true"));
+        assertTrue(line.contains("message_original_sha256="));
+        assertEquals(1L,metrics.snapshot().get("raw_message_truncated_total"));
+    }
+
     @Test void onlyMissingPointTimeFallsBack() {
         RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); long millis=ReceiveTimeParser.parse("20260814000000000").getEpochMillis();
         assertNotNull(resolver.resolve(false,null,millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,"-1",millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,"not-number",millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,Long.toString(millis*1_000_000L+1_000_000L),millis,true)); assertEquals(1L,metrics.snapshot().get("legacy_missing_point_time_total"));
@@ -108,4 +127,10 @@ class InfluxRawV2ContractTest {
         assertTrue(query.indexOf("|> group()") < query.indexOf("|> sort("));
         assertTrue(query.indexOf("|> sort(") < query.indexOf("|> limit(n: 2)"));
     }
+
+    private static String repeat(String value, int count) {
+        StringBuilder result = new StringBuilder(value.length() * count);
+        for (int i=0;i<count;i++) result.append(value);
+        return result.toString();
+    }
 }

haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-prod.yaml

@@ -20,8 +20,15 @@ services:
     environment:
       # 环境配置
       - SPRING_PROFILES_ACTIVE=prod
+      - SYSLOG_SERVER_ENABLED=true
       # 设备上送日志为 UTF-8；显式声明，Netty 与 WAL 共用 Spring 配置。
       - SYSLOG_CHARSET=UTF-8
+      # 256KiB 以上走大日志 Topic，单条最多接纳 6MiB；WAL/Dispatcher 均按字节背压。
+      - SYSLOG_MAX_FRAME_LENGTH=6291456
+      - SYSLOG_MESSAGE_MAX_BYTES=6291456
+      - SYSLOG_LARGE_MESSAGE_THRESHOLD_BYTES=262144
+      - SYSLOG_WAL_MAX_PENDING_BYTES=268435456
+      - SYSLOG_DISPATCH_MAX_INFLIGHT_BYTES=268435456
       - TZ=Asia/Shanghai
       # 探针标识ID
       - APP_SERVICE_DEVICE_COLLECT_ID=1
@@ -43,6 +50,8 @@ services:
       # Kafka配置
       - SPRING_KAFKA_PRODUCER_BOOTSTRAP_SERVERS=192.168.4.99:9092
       - SPRING_KAFKA_PRODUCER_TOPIC=agent-01-syslog-topic
+      - KAFKA_PRODUCER_LARGE_TOPIC=agent-01-syslog-topic.large
+      - KAFKA_PRODUCER_MAX_REQUEST_SIZE=33554432
       # 仅兼容仍使用独立原文归档的旧版 JAR；当前 WAL 版本已删除该 Java 实现。
       - SYSLOG_RAW_ARCHIVE_ENABLED=false
       - SYSLOG_RAW_ARCHIVE_DIRECTORY=/data/syslog-archive

haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/ApiPullRunner.java

@@ -415,6 +415,8 @@ public class ApiPullRunner {
     while (true) {
       AppendAttempt attempt = ingress.tryAppend(event);
       if (attempt.getStatus() == AdmissionStatus.ACCEPTED) return attempt.getDurableStage();
+      if (attempt.getStatus() == AdmissionStatus.OVERSIZED)
+        throw new AdapterException("MESSAGE_TOO_LARGE", "设备事件超过单条消息硬上限");
       if (attempt.getStatus() == AdmissionStatus.UNAVAILABLE)
         throw new AdapterException("WAL_UNAVAILABLE", "WAL 当前不可用");
       if (System.nanoTime() >= deadline)

haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java

@@ -19,9 +19,17 @@ public class AppProperties {
     @Value("${syslog.udp.port:515}")
     private int syslogUdpPort;
 
-    @Value("${syslog.max.frame.length:65536}")
+    @Value("${syslog.max.frame.length:6291456}")
     private int syslogMaxFrameLength;
 
+    /** 所有接入方式共用的单条原文字节硬上限；0 表示不限制。 */
+    @Value("${syslog.message.max-bytes:6291456}")
+    private long syslogMessageMaxBytes;
+
+    /** 超过该值投递大日志 Topic；0 表示关闭分流。 */
+    @Value("${syslog.large-message.threshold-bytes:262144}")
+    private long syslogLargeMessageThresholdBytes;
+
     @Value("${syslog.buffer.size:1024}")
     private int syslogBufferSize;
 
@@ -50,6 +58,10 @@ public class AppProperties {
     @Value("${syslog.wal.queue-capacity:10000}")
     private int syslogWalQueueCapacity;
 
+    /** 队列与待 force 记录共用的内存字节预算；0 表示不限制。 */
+    @Value("${syslog.wal.max-pending-bytes:268435456}")
+    private long syslogWalMaxPendingBytes;
+
     @Value("${syslog.wal.high-watermark-percent:70}")
     private int syslogWalHighWatermarkPercent;
 
@@ -114,6 +126,10 @@ public class AppProperties {
     @Value("${syslog.dispatch.max-inflight:5000}")
     private int syslogDispatchMaxInflight;
 
+    /** Dispatcher 正文在途字节预算；0 表示不限制。 */
+    @Value("${syslog.dispatch.max-inflight-bytes:268435456}")
+    private long syslogDispatchMaxInflightBytes;
+
     @Value("${syslog.dispatch.retry-backoff-ms:1000}")
     private long syslogDispatchRetryBackoffMs;
 
@@ -166,6 +182,9 @@ public class AppProperties {
     @Value("${spring.kafka.producer.topic:agent-syslog-topic}")
     private String kafkaProducerTopic;
 
+    @Value("${spring.kafka.producer.large-topic:agent-syslog-topic.large}")
+    private String kafkaProducerLargeTopic;
+
     @Value("${spring.kafka.producer.ack-timeout-ms:130000}")
     private long kafkaProducerAckTimeoutMs;
 

haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/SyslogWalDispatcher.java

@@ -48,9 +48,11 @@ public class SyslogWalDispatcher {
     private final LongAdder kafkaAcked = new LongAdder();
     private final LongAdder terminalFiltered = new LongAdder();
     private final LongAdder retryCount = new LongAdder();
+    private final LongAdder largeDispatched = new LongAdder();
     private final AtomicLong inflight = new AtomicLong();
 
     private Semaphore permits;
+    private ByteBudget byteBudget;
     private ExecutorService workers;
     private ScheduledExecutorService scheduler;
     private Thread readerThread;
@@ -72,6 +74,7 @@ public class SyslogWalDispatcher {
     public void start() throws Exception {
         validateConfiguration();
         permits = new Semaphore(properties.getSyslogDispatchMaxInflight());
+        byteBudget = new ByteBudget(properties.getSyslogDispatchMaxInflightBytes());
         workers = Executors.newFixedThreadPool(properties.getSyslogDispatchWorkerThreads(),
                 namedFactory("syslog-dispatch-worker"));
         scheduler = Executors.newScheduledThreadPool(2, namedFactory("syslog-dispatch-scheduler"));
@@ -85,8 +88,9 @@ public class SyslogWalDispatcher {
         scheduler.scheduleWithFixedDelay(this::logStatus,
                 properties.getSyslogStatusLogIntervalMs(), properties.getSyslogStatusLogIntervalMs(),
                 TimeUnit.MILLISECONDS);
-        log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, checkpointSequence={}",
+        log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, maxInflightBytes={}, checkpointSequence={}",
                 properties.getSyslogDispatchWorkerThreads(), properties.getSyslogDispatchMaxInflight(),
+                properties.getSyslogDispatchMaxInflightBytes(),
                 walService.getCheckpointSequence());
     }
 
@@ -99,7 +103,11 @@ public class SyslogWalDispatcher {
                     permits.release();
                     continue;
                 }
+                record.setDispatchPayloadBytes(Math.max(1L,
+                        Math.max(record.getRawLength(), utf8Length(record.getRawMessage()))));
+                byteBudget.acquire(record.getDispatchPayloadBytes());
                 if (record.getSequence() != nextReadSequence) {
+                    byteBudget.release(record.getDispatchPayloadBytes());
                     permits.release();
                     throw new IllegalStateException("WAL sequence 不连续，拒绝越过: expected="
                             + nextReadSequence + ", actual=" + record.getSequence());
@@ -158,8 +166,13 @@ public class SyslogWalDispatcher {
             eventTime = LocalDateTime.parse(walEventTime, WAL_TIME).format(HEADER_TIME);
 
             // v2/v3 继续调用旧重载，确保升级前已落盘记录产生的 Kafka 消息完全不变。
-            java.util.concurrent.CompletableFuture<kafkaProducer.DeliveryResult> deliveryFuture =
-                    record.getSchemaVersion() >= 4
+            boolean large = properties.getSyslogLargeMessageThresholdBytes() > 0L
+                    && record.getDispatchPayloadBytes() > properties.getSyslogLargeMessageThresholdBytes();
+            java.util.concurrent.CompletableFuture<kafkaProducer.DeliveryResult> deliveryFuture = large
+                    ? producer.messagePushLarge(
+                            resolution.getDevice(), record.getRawMessage(), receiveTime, eventTime,
+                            record.getEventId(), record.getPointTimeNs())
+                    : record.getSchemaVersion() >= 4
                             ? producer.messagePush(
                                     resolution.getDevice(),
                                     record.getRawMessage(),
@@ -177,6 +190,7 @@ public class SyslogWalDispatcher {
                     .whenComplete((delivery, failure) -> {
                         if (failure == null) {
                             kafkaAcked.increment();
+                            if (large) largeDispatched.increment();
                             markComplete(record);
                         } else {
                             scheduleRetry(record, attempt, "KAFKA_ACK", unwrap(failure));
@@ -206,7 +220,9 @@ public class SyslogWalDispatcher {
     private void markComplete(SyslogWalRecord record) {
         synchronized (completionMonitor) {
             if (record.getSequence() < nextCommitSequence) return;
-            completedOutOfOrder.put(record.getSequence(), record);
+            if (completedOutOfOrder.putIfAbsent(record.getSequence(), checkpointOnly(record)) != null) return;
+            // 已 ACK 的正文不再占字节预算；条数许可仍要等连续 checkpoint，限制乱序完成表大小。
+            byteBudget.release(record.getDispatchPayloadBytes());
             SyslogWalRecord latest = null;
             int released = 0;
             while (true) {
@@ -224,16 +240,26 @@ public class SyslogWalDispatcher {
         }
     }
 
+    private SyslogWalRecord checkpointOnly(SyslogWalRecord source) {
+        SyslogWalRecord value = new SyslogWalRecord();
+        value.setSequence(source.getSequence());
+        value.setSegment(source.getSegment());
+        value.setStartOffset(source.getStartOffset());
+        value.setEndOffset(source.getEndOffset());
+        return value;
+    }
+
     private void logStatus() {
         if (!running.get()) return;
-        log.info("Syslog WAL状态: networkReceived={}, appended={}, walPendingApprox={}, walQueue={}/{}, durableSequence={}, "
-                        + "checkpointSequence={}, kafkaInflight={}, kafkaAcked={}, terminalFiltered={}, "
+        log.info("Syslog WAL状态: networkReceived={}, appended={}, admissionOversized={}, walPendingApprox={}, walPendingBytes={}, walQueue={}/{}, durableSequence={}, "
+                        + "checkpointSequence={}, kafkaInflight={}, kafkaInflightBytes={}, kafkaAcked={}, largeDispatched={}, terminalFiltered={}, "
                         + "retry={}, tcpPauseTotal={}, udpDropped={}, deviceCacheSize={}, deviceCacheHits={}, "
                         + "deviceCacheMisses={}",
                 SyslogMessageHandler.getNetworkReceivedCount(), walService.getAppendedCount(),
-                walService.getPendingRecordsApprox(),
+                walService.getOversizedCount(), walService.getPendingRecordsApprox(), walService.getPendingAppendBytes(),
                 walService.getQueueSize(), walService.getQueueCapacity(), walService.getDurableSequence(),
-                walService.getCheckpointSequence(), inflight.get(), kafkaAcked.sum(), terminalFiltered.sum(),
+                walService.getCheckpointSequence(), inflight.get(), byteBudget == null ? 0L : byteBudget.used(),
+                kafkaAcked.sum(), largeDispatched.sum(), terminalFiltered.sum(),
                 retryCount.sum(), SyslogMessageHandler.getTcpPauseCount(),
                 SyslogMessageHandler.getUdpDroppedCount(), deviceResolver.getCacheSize(),
                 deviceResolver.getHitCount(), deviceResolver.getMissCount());
@@ -246,6 +272,23 @@ public class SyslogWalDispatcher {
         return failure;
     }
 
+    /** 不分配临时 byte[] 的 UTF-8 长度估算；非法孤立代理项按 3 字节保守计算。 */
+    static long utf8Length(String value) {
+        if (value == null || value.isEmpty()) return 0L;
+        long bytes = 0L;
+        for (int index = 0; index < value.length(); index++) {
+            char current = value.charAt(index);
+            if (current <= 0x7f) bytes++;
+            else if (current <= 0x7ff) bytes += 2L;
+            else if (Character.isHighSurrogate(current) && index + 1 < value.length()
+                    && Character.isLowSurrogate(value.charAt(index + 1))) {
+                bytes += 4L;
+                index++;
+            } else bytes += 3L;
+        }
+        return bytes;
+    }
+
     private ThreadFactory namedFactory(String prefix) {
         AtomicLong sequence = new AtomicLong();
         return runnable -> {
@@ -258,18 +301,58 @@ public class SyslogWalDispatcher {
     private void validateConfiguration() {
         if (properties.getSyslogDispatchWorkerThreads() <= 0
                 || properties.getSyslogDispatchMaxInflight() <= 0
+                || properties.getSyslogDispatchMaxInflightBytes() < 0L
                 || properties.getSyslogDispatchRetryBackoffMs() <= 0
                 || properties.getSyslogDispatchShutdownTimeoutMs() <= 0
                 || properties.getSyslogDeviceContextCacheTtlMs() <= 0
                 || properties.getSyslogStatusLogIntervalMs() <= 0) {
             throw new IllegalArgumentException("Dispatcher 线程、在途、重试、缓存和状态日志参数必须大于 0");
         }
+        if (properties.getSyslogMessageMaxBytes() > 0L
+                && properties.getSyslogDispatchMaxInflightBytes() > 0L
+                && properties.getSyslogMessageMaxBytes() > properties.getSyslogDispatchMaxInflightBytes()) {
+            throw new IllegalArgumentException("单条消息上限不能大于 Dispatcher 在途字节预算");
+        }
     }
 
     public long getInflight() { return inflight.get(); }
     public long getKafkaAcked() { return kafkaAcked.sum(); }
     public long getTerminalFiltered() { return terminalFiltered.sum(); }
     public long getRetryCount() { return retryCount.sum(); }
+    public long getLargeDispatched() { return largeDispatched.sum(); }
+    public long getInflightBytes() { return byteBudget == null ? 0L : byteBudget.used(); }
+
+    /** 同时保留条数 Semaphore 和字节预算，避免少量大消息占满堆。 */
+    static final class ByteBudget {
+        private final long capacity;
+        private long used;
+
+        ByteBudget(long capacity) {
+            if (capacity < 0L) throw new IllegalArgumentException("capacity 不能小于 0");
+            this.capacity = capacity;
+        }
+
+        synchronized void acquire(long bytes) throws InterruptedException {
+            if (bytes <= 0L) throw new IllegalArgumentException("bytes 必须大于 0");
+            if (capacity == 0L) { used += bytes; return; }
+            if (bytes > capacity) {
+                // 硬上限关闭时仍保证进展：超预算单条只能独占，不允许与其他正文并存。
+                while (used != 0L) wait();
+                used = bytes;
+                return;
+            }
+            while (used > capacity - bytes) wait();
+            used += bytes;
+        }
+
+        synchronized void release(long bytes) {
+            used -= bytes;
+            if (used < 0L) throw new IllegalStateException("Dispatcher 字节预算释放越界");
+            notifyAll();
+        }
+
+        synchronized long used() { return used; }
+    }
 
     @PreDestroy
     public void shutdown() {

haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AdmissionStatus.java

@@ -4,5 +4,6 @@ package com.haobang.wal;
 public enum AdmissionStatus {
   ACCEPTED,
   BACKPRESSURE,
+  OVERSIZED,
   UNAVAILABLE
 }

haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AppendAttempt.java

@@ -24,7 +24,7 @@ public final class AppendAttempt {
 
   public static AppendAttempt rejected(AdmissionStatus status) {
     if (status == null || status == AdmissionStatus.ACCEPTED) {
-      throw new IllegalArgumentException("rejected 只能使用 BACKPRESSURE 或 UNAVAILABLE");
+      throw new IllegalArgumentException("rejected 不能使用 ACCEPTED");
     }
     return new AppendAttempt(status, null);
   }

haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/PendingWalAppend.java

@@ -5,18 +5,22 @@ import java.util.concurrent.CompletableFuture;
 /** WAL 写线程队列中的事件及其待完成 durable future。 sequence/schemaVersion 由唯一写线程赋值，生产者线程不得提前推断 WAL 位置。 */
 final class PendingWalAppend {
   private final IngressEvent event;
+  private final int rawBytes;
   private final CompletableFuture<DurableReceipt> durableFuture = new CompletableFuture<>();
   private long sequence;
   private int schemaVersion;
 
-  PendingWalAppend(IngressEvent event) {
+  PendingWalAppend(IngressEvent event, int rawBytes) {
     this.event = event;
+    this.rawBytes = rawBytes;
   }
 
   IngressEvent getEvent() {
     return event;
   }
 
+  int getRawBytes() { return rawBytes; }
+
   CompletableFuture<DurableReceipt> getDurableFuture() {
     return durableFuture;
   }

haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogLegacyArchiveImporter.java

@@ -112,6 +112,10 @@ public class SyslogLegacyArchiveImporter {
         while (true) {
             SyslogWalService.AdmissionResult result = walService.tryAppend(event);
             if (result == SyslogWalService.AdmissionResult.ACCEPTED) return;
+            if (result == SyslogWalService.AdmissionResult.OVERSIZED) {
+                throw new IllegalStateException("旧归档事件超过当前单条消息硬上限，拒绝永久重试: eventId="
+                        + event.getEventId());
+            }
             if (result == SyslogWalService.AdmissionResult.UNAVAILABLE) {
                 throw new IllegalStateException("WAL 不可用，无法导入旧归档 eventId=" + event.getEventId(),
                         walService.getFatalFailure());

haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecord.java

@@ -87,6 +87,8 @@ public class SyslogWalRecord {
   @JsonIgnore private long endOffset;
   @JsonIgnore private boolean quarantinedInvalid;
   @JsonIgnore private String validationError;
+  /** Dispatcher 对正文使用的 UTF-8/源编码较大值；只存在内存中，不改变 WAL 协议。 */
+  @JsonIgnore private long dispatchPayloadBytes;
 
   public int getSchemaVersion() {
     return schemaVersion;
@@ -232,6 +234,14 @@ public class SyslogWalRecord {
     this.rawLength = rawLength;
   }
 
+  public long getDispatchPayloadBytes() {
+    return dispatchPayloadBytes;
+  }
+
+  public void setDispatchPayloadBytes(long dispatchPayloadBytes) {
+    this.dispatchPayloadBytes = dispatchPayloadBytes;
+  }
+
   public String getRawSha256() {
     return rawSha256;
   }

haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalService.java

@@ -64,7 +64,7 @@ public class SyslogWalService {
     private static final String SEGMENT_GLOB = "syslog-wal-*.jsonl";
     static final long COMPLETED_CLEANUP_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(30L);
 
-    public enum AdmissionResult { ACCEPTED, BACKPRESSURE, UNAVAILABLE }
+    public enum AdmissionResult { ACCEPTED, BACKPRESSURE, OVERSIZED, UNAVAILABLE }
 
     private final AppProperties properties;
     private final ObjectMapper objectMapper;
@@ -76,6 +76,8 @@ public class SyslogWalService {
     private final AtomicLong durableSequence = new AtomicLong();
     private final LongAdder appended = new LongAdder();
     private final LongAdder admissionBackpressure = new LongAdder();
+    private final LongAdder admissionOversized = new LongAdder();
+    private final AtomicLong pendingAppendBytes = new AtomicLong();
     private final LongAdder quarantined = new LongAdder();
     private final LongAdder terminalRecorded = new LongAdder();
     private final Object durableMonitor = new Object();
@@ -211,6 +213,7 @@ public class SyslogWalService {
         AppendAttempt attempt = tryAppendUnified(toUnified(event));
         if (attempt.getStatus() == AdmissionStatus.ACCEPTED) return AdmissionResult.ACCEPTED;
         if (attempt.getStatus() == AdmissionStatus.BACKPRESSURE) return AdmissionResult.BACKPRESSURE;
+        if (attempt.getStatus() == AdmissionStatus.OVERSIZED) return AdmissionResult.OVERSIZED;
         return AdmissionResult.UNAVAILABLE;
     }
 
@@ -231,8 +234,19 @@ public class SyslogWalService {
         if (!running || fatalFailure != null || diskState == DiskState.CRITICAL) {
             return AppendAttempt.rejected(AdmissionStatus.UNAVAILABLE);
         }
-        PendingWalAppend pending = new PendingWalAppend(event);
+        int rawBytes = event.getRawMessage().getBytes(event.getCharset()).length;
+        if (properties.getSyslogMessageMaxBytes() > 0L
+                && rawBytes > properties.getSyslogMessageMaxBytes()) {
+            admissionOversized.increment();
+            return AppendAttempt.rejected(AdmissionStatus.OVERSIZED);
+        }
+        if (!reservePendingBytes(rawBytes)) {
+            admissionBackpressure.increment();
+            return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE);
+        }
+        PendingWalAppend pending = new PendingWalAppend(event, rawBytes);
         if (!appendQueue.offer(pending)) {
+            releasePendingBytes(rawBytes);
             admissionBackpressure.increment();
             return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE);
         }
@@ -243,14 +257,18 @@ public class SyslogWalService {
         refreshDiskState(false);
         if (fatalFailure != null || diskState != DiskState.NORMAL) return true;
         int high = percentageCount(properties.getSyslogWalHighWatermarkPercent());
-        return appendQueue != null && appendQueue.size() >= high;
+        long byteHigh = percentageBytes(properties.getSyslogWalHighWatermarkPercent());
+        return appendQueue != null && (appendQueue.size() >= high
+                || (byteHigh > 0L && pendingAppendBytes.get() >= byteHigh));
     }
 
     public boolean canResumeTcp() {
         refreshDiskState(false);
         if (fatalFailure != null || diskState != DiskState.NORMAL) return false;
         int low = percentageCount(properties.getSyslogWalLowWatermarkPercent());
-        return appendQueue != null && appendQueue.size() <= low;
+        long byteLow = percentageBytes(properties.getSyslogWalLowWatermarkPercent());
+        return appendQueue != null && appendQueue.size() <= low
+                && (byteLow == 0L || pendingAppendBytes.get() <= byteLow);
     }
 
     public long getRetryIntervalMs() {
@@ -263,6 +281,8 @@ public class SyslogWalService {
     public int getQueueCapacity() { return properties.getSyslogWalQueueCapacity(); }
     public long getAppendedCount() { return appended.sum(); }
     public long getBackpressureCount() { return admissionBackpressure.sum(); }
+    public long getOversizedCount() { return admissionOversized.sum(); }
+    public long getPendingAppendBytes() { return pendingAppendBytes.get(); }
     public long getQuarantinedCount() { return quarantined.sum(); }
     public long getTerminalRecordedCount() { return terminalRecorded.sum(); }
     public long getIngressPointTimeAssignedCount() { return pointTimeAllocator.getAssignedCount(); }
@@ -495,6 +515,7 @@ public class SyslogWalService {
                     pending.getSequence(),
                     pending.getSchemaVersion(),
                     durableAt);
+            releasePendingBytes(pending.getRawBytes());
             pending.getDurableFuture().complete(receipt);
         }
         pendingForce.clear();
@@ -538,12 +559,14 @@ public class SyslogWalService {
 
     private void completePendingExceptionally(Throwable failure) {
         for (PendingWalAppend pending : pendingForce) {
+            releasePendingBytes(pending.getRawBytes());
             pending.getDurableFuture().completeExceptionally(failure);
         }
         pendingForce.clear();
         if (appendQueue == null) return;
         PendingWalAppend pending;
         while ((pending = appendQueue.poll()) != null) {
+            releasePendingBytes(pending.getRawBytes());
             pending.getDurableFuture().completeExceptionally(failure);
         }
     }
@@ -552,6 +575,38 @@ public class SyslogWalService {
         return Math.max(1, (int) Math.floor(properties.getSyslogWalQueueCapacity() * percent / 100.0));
     }
 
+    private long percentageBytes(int percent) {
+        long capacity = properties.getSyslogWalMaxPendingBytes();
+        return capacity <= 0L ? 0L : Math.max(1L, capacity * percent / 100L);
+    }
+
+    private boolean reservePendingBytes(long bytes) {
+        long capacity = properties.getSyslogWalMaxPendingBytes();
+        if (capacity <= 0L) {
+            pendingAppendBytes.addAndGet(bytes);
+            return true;
+        }
+        while (true) {
+            long current = pendingAppendBytes.get();
+            // 单条硬上限显式关闭时，允许一个超过预算的事件独占通道，避免永久 BACKPRESSURE。
+            if (bytes > capacity) {
+                if (current != 0L) return false;
+                if (pendingAppendBytes.compareAndSet(0L, bytes)) return true;
+                continue;
+            }
+            if (bytes > capacity - current) return false;
+            if (pendingAppendBytes.compareAndSet(current, current + bytes)) return true;
+        }
+    }
+
+    private void releasePendingBytes(long bytes) {
+        long remaining = pendingAppendBytes.addAndGet(-bytes);
+        if (remaining < 0L) {
+            pendingAppendBytes.set(0L);
+            throw new IllegalStateException("WAL pending 字节计数出现负值");
+        }
+    }
+
     private void validateConfiguration() {
         if (!properties.isSyslogWalEnabled()) {
             throw new IllegalArgumentException("syslog.wal.enabled=false 会破坏可靠接入边界，正式链路禁止关闭 WAL");
@@ -563,6 +618,20 @@ public class SyslogWalService {
                 || properties.getSyslogWalCheckpointFlushIntervalMs() <= 0) {
             throw new IllegalArgumentException("WAL 队列、刷盘、分段、游标和停机参数必须大于 0");
         }
+        if (properties.getSyslogMessageMaxBytes() < 0L
+                || properties.getSyslogLargeMessageThresholdBytes() < 0L
+                || properties.getSyslogWalMaxPendingBytes() < 0L) {
+            throw new IllegalArgumentException("消息上限、大日志阈值和 WAL 字节预算不能小于 0");
+        }
+        if (properties.getSyslogMessageMaxBytes() > 0L
+                && properties.getSyslogWalMaxPendingBytes() > 0L
+                && properties.getSyslogMessageMaxBytes() > properties.getSyslogWalMaxPendingBytes()) {
+            throw new IllegalArgumentException("单条消息上限不能大于 WAL pending 字节预算");
+        }
+        if (properties.getSyslogMessageMaxBytes() > 0L
+                && properties.getSyslogLargeMessageThresholdBytes() > properties.getSyslogMessageMaxBytes()) {
+            throw new IllegalArgumentException("大日志分流阈值不能大于单条消息硬上限");
+        }
         if (properties.getSyslogWalRetryIntervalMs() <= 0
                 || properties.getSyslogWalDiskRetryIntervalMs() <= 0) {
             throw new IllegalArgumentException("WAL 队列和磁盘重试间隔必须大于 0");

haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalStatusController.java

@@ -29,13 +29,16 @@ public class SyslogWalStatusController {
     public Map<String, Object> status() {
         Map<String, Object> value = new LinkedHashMap<>();
         value.put("network_received", SyslogMessageHandler.getNetworkReceivedCount());
+        value.put("network_oversized_dropped", SyslogMessageHandler.getOversizedDroppedCount());
         value.put("wal_appended", wal.getAppendedCount());
         value.put("wal_queue_size", wal.getQueueSize());
         value.put("wal_queue_capacity", wal.getQueueCapacity());
+        value.put("wal_pending_bytes", wal.getPendingAppendBytes());
         value.put("wal_durable_sequence", wal.getDurableSequence());
         value.put("wal_checkpoint_sequence", wal.getCheckpointSequence());
         value.put("wal_pending_approx", wal.getPendingRecordsApprox());
         value.put("wal_admission_backpressure", wal.getBackpressureCount());
+        value.put("wal_admission_oversized", wal.getOversizedCount());
         value.put("wal_terminal_recorded", wal.getTerminalRecordedCount());
         value.put("wal_quarantined", wal.getQuarantinedCount());
         Map<String, Long> pointTimeAssigned = new LinkedHashMap<>();
@@ -49,7 +52,9 @@ public class SyslogWalStatusController {
         value.put("wal_fatal_error", wal.getFatalFailure() == null
                 ? null : wal.getFatalFailure().toString());
         value.put("dispatch_inflight", dispatcher.getInflight());
+        value.put("dispatch_inflight_bytes", dispatcher.getInflightBytes());
         value.put("kafka_acked", dispatcher.getKafkaAcked());
+        value.put("kafka_large_dispatched", dispatcher.getLargeDispatched());
         value.put("dispatch_terminal_filtered", dispatcher.getTerminalFiltered());
         value.put("dispatch_retry", dispatcher.getRetryCount());
         value.put("tcp_pause_total", SyslogMessageHandler.getTcpPauseCount());

haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java

@@ -53,7 +53,7 @@ public class kafkaProducer {
                          @Value("${spring.kafka.producer.compression-type:lz4}") String compressionType,
                          @Value("${spring.kafka.producer.properties.max.block.ms:10000}") int maxBlockMs,
                          @Value("${spring.kafka.producer.properties.buffer.memory:134217728}") long bufferMemory,
-                         @Value("${spring.kafka.producer.properties.max.request.size:8388608}") int maxRequestSize,
+                         @Value("${spring.kafka.producer.properties.max.request.size:33554432}") int maxRequestSize,
                          @Value("${syslog.kafka.partition-key-strategy:event_id}") String partitionKeyStrategy) {
         this.appProperties = appProperties;
         this.partitionKeyStrategy = PartitionKeyStrategy.parse(partitionKeyStrategy);
@@ -130,12 +130,34 @@ public class kafkaProducer {
                                                           String eventTime,
                                                           String eventId,
                                                           Long pointTimeNs) {
+        return messagePushToTopic(appProperties.getKafkaProducerTopic(), deviceDevice, syslog,
+                receiveTime, eventTime, eventId, pointTimeNs);
+    }
+
+    /** 大日志只改变目标 Topic，载荷协议、event_id 和 ACK 语义与普通通道完全一致。 */
+    public CompletableFuture<DeliveryResult> messagePushLarge(DeviceDevice deviceDevice,
+                                                               String syslog,
+                                                               String receiveTime,
+                                                               String eventTime,
+                                                               String eventId,
+                                                               Long pointTimeNs) {
+        return messagePushToTopic(appProperties.getKafkaProducerLargeTopic(), deviceDevice, syslog,
+                receiveTime, eventTime, eventId, pointTimeNs);
+    }
+
+    private CompletableFuture<DeliveryResult> messagePushToTopic(String topic,
+                                                                  DeviceDevice deviceDevice,
+                                                                  String syslog,
+                                                                  String receiveTime,
+                                                                  String eventTime,
+                                                                  String eventId,
+                                                                  Long pointTimeNs) {
         // [流程节点3.1] event_id 在接入边界生成并写入加密载荷，Retry/Replay 不再重新生成。
         String fullMessage = DeviceInfoUtil.getFullLogString(
                 deviceDevice, syslog, receiveTime, eventTime, eventId, pointTimeNs, appProperties);
         // [流程节点3.2] Kafka 中保存携带 event_id 的完整密文，原始正文不写普通应用日志。
         String encrypted = Sm4Util.encryptCbc(fullMessage, appProperties.getSm4Key());
-        ProducerRecord<String, String> record = buildRecord(deviceDevice.getId(), eventId, encrypted);
+        ProducerRecord<String, String> record = buildRecord(topic, deviceDevice.getId(), eventId, encrypted);
 
         CompletableFuture<DeliveryResult> result = new CompletableFuture<>();
         // [SYSLOG-ALARM][KPROD][已修复] 回调成功才表示 Broker 接管；send() 返回本身不再记录 PRODUCED。
@@ -150,8 +172,13 @@ public class kafkaProducer {
     }
 
     ProducerRecord<String, String> buildRecord(Integer deviceId, String eventId, String encrypted) {
+        return buildRecord(appProperties.getKafkaProducerTopic(), deviceId, eventId, encrypted);
+    }
+
+    ProducerRecord<String, String> buildRecord(String topic, Integer deviceId, String eventId, String encrypted) {
+        if (topic == null || topic.trim().isEmpty()) throw new IllegalArgumentException("Kafka Topic 不能为空");
         String partitionKey = partitionKeyStrategy.partitionKey(deviceId, eventId);
-        return new ProducerRecord<>(appProperties.getKafkaProducerTopic(), partitionKey, encrypted);
+        return new ProducerRecord<>(topic, partitionKey, encrypted);
     }
 
     public static String newEventId() {

haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java

@@ -33,6 +33,7 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
     private static final LongAdder NETWORK_RECEIVED = new LongAdder();
     private static final LongAdder TCP_PAUSE_COUNT = new LongAdder();
     private static final LongAdder UDP_DROPPED = new LongAdder();
+    private static final LongAdder OVERSIZED_DROPPED = new LongAdder();
 
     /** 每个 TCP Channel 独立 Handler，pending 只保存关闭 AUTO_READ 前已进入 Pipeline 的帧。 */
     private final Deque<SyslogIngressEvent> pendingTcp = new ArrayDeque<>();
@@ -93,6 +94,10 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
     private void acceptUdp(SyslogWalService wal, SyslogIngressEvent event) {
         SyslogWalService.AdmissionResult result = wal.tryAppend(event);
         if (result == SyslogWalService.AdmissionResult.ACCEPTED) return;
+        if (result == SyslogWalService.AdmissionResult.OVERSIZED) {
+            recordOversized(event, "UDP");
+            return;
+        }
         UDP_DROPPED.increment();
         long dropped = UDP_DROPPED.sum();
         if (dropped == 1 || dropped % 100 == 0) {
@@ -110,6 +115,11 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
             return;
         }
         SyslogWalService.AdmissionResult result = wal.tryAppend(event);
+        if (result == SyslogWalService.AdmissionResult.OVERSIZED) {
+            recordOversized(event, "TCP");
+            ctx.close();
+            return;
+        }
         if (result != SyslogWalService.AdmissionResult.ACCEPTED) {
             pendingTcp.addLast(event);
             pauseAndSchedule(ctx, wal);
@@ -144,6 +154,11 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
         while (!pendingTcp.isEmpty()) {
             SyslogIngressEvent event = pendingTcp.peekFirst();
             SyslogWalService.AdmissionResult result = wal.tryAppend(event);
+            if (result == SyslogWalService.AdmissionResult.OVERSIZED) {
+                pendingTcp.removeFirst();
+                recordOversized(event, "TCP");
+                continue;
+            }
             if (result != SyslogWalService.AdmissionResult.ACCEPTED) {
                 scheduleDrain(ctx, wal);
                 return;
@@ -172,4 +187,12 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
     public static long getNetworkReceivedCount() { return NETWORK_RECEIVED.sum(); }
     public static long getTcpPauseCount() { return TCP_PAUSE_COUNT.sum(); }
     public static long getUdpDroppedCount() { return UDP_DROPPED.sum(); }
+    public static long getOversizedDroppedCount() { return OVERSIZED_DROPPED.sum(); }
+
+    private void recordOversized(SyslogIngressEvent event, String protocol) {
+        OVERSIZED_DROPPED.increment();
+        log.error("Syslog 超过单条消息硬上限，已拒绝且不记录原文: eventId={}, protocol={}, sourceIp={}, chars={}, rejectedTotal={}",
+                event.getEventId(), protocol, event.getSourceIp(), event.getRawMessage().length(),
+                OVERSIZED_DROPPED.sum());
+    }
 }

haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogServerBoth.java

@@ -47,6 +47,10 @@ public class SyslogServerBoth {
         this.syslogCharset = java.util.Objects.requireNonNull(syslogCharset, "syslogCharset");
         this.tcpPort = properties.getSyslogTcpPort();
         this.udpPort = properties.getSyslogUdpPort();
+        if (properties.getSyslogMessageMaxBytes() > 0L
+                && properties.getSyslogMaxFrameLength() < properties.getSyslogMessageMaxBytes()) {
+            throw new IllegalArgumentException("syslog.max.frame.length 不能小于 syslog.message.max-bytes");
+        }
     }
 
     public void start() throws InterruptedException {

haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties

@@ -9,7 +9,7 @@ server.error.include-binding-errors=always
 # Syslog Server Configuration
 syslog.tcp.port=514
 syslog.udp.port=514
-syslog.max.frame.length=262144
+syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456}
 syslog.buffer.size=1000
 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8
 syslog.charset=GBK

haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties

@@ -9,7 +9,7 @@ server.error.include-binding-errors=always
 # Syslog Server Configuration
 syslog.tcp.port=514
 syslog.udp.port=514
-syslog.max.frame.length=262144
+syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456}
 syslog.buffer.size=1000
 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8
 syslog.charset=GBK
@@ -56,4 +56,4 @@ spring.redis.lettuce.pool.max-idle=10
 spring.redis.lettuce.pool.min-idle=5
 
 # 生产环境缓存时间较长
-spring.cache.redis.time-to-live=3600000
+spring.cache.redis.time-to-live=3600000

haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties

@@ -9,7 +9,7 @@ server.error.include-binding-errors=always
 # Syslog Server Configuration
 syslog.tcp.port=514
 syslog.udp.port=514
-syslog.max.frame.length=2621440
+syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456}
 syslog.buffer.size=10000
 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8
 syslog.charset=UTF-8

haobang-security-dm/syslog-serve/src/main/resources/application-test.properties

@@ -9,7 +9,7 @@ server.error.include-binding-errors=always
 # Syslog Server Configuration
 syslog.tcp.port=514
 syslog.udp.port=514
-syslog.max.frame.length=262144
+syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456}
 syslog.buffer.size=1000
 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8
 syslog.charset=GBK

haobang-security-dm/syslog-serve/src/main/resources/application.properties

@@ -20,7 +20,11 @@ spring.kafka.producer.properties.linger.ms=5
 spring.kafka.producer.properties.batch.size=65536
 spring.kafka.producer.compression-type=lz4
 spring.kafka.producer.ack-timeout-ms=130000
-syslog.max.frame.length=262144
+syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456}
+# 单条原文硬上限；0 表示不限制。超过阈值的消息在进入 WAL 前被明确拒绝。
+syslog.message.max-bytes=${SYSLOG_MESSAGE_MAX_BYTES:6291456}
+# 超过 256KiB 的已接纳消息投递独立大日志 Topic；0 表示关闭分流。
+syslog.large-message.threshold-bytes=${SYSLOG_LARGE_MESSAGE_THRESHOLD_BYTES:262144}
 syslog.buffer.size=1000
 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8
 syslog.charset=GBK
@@ -31,6 +35,8 @@ syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a
 syslog.wal.enabled=${SYSLOG_WAL_ENABLED:true}
 syslog.wal.directory=${SYSLOG_WAL_DIRECTORY:${SYSLOG_RAW_ARCHIVE_DIRECTORY:/data/syslog-wal}}
 syslog.wal.queue-capacity=${SYSLOG_WAL_QUEUE_CAPACITY:10000}
+# 队列和待 force 记录共享的内存字节预算，防止大日志把“有界条数”放大成无界堆占用。
+syslog.wal.max-pending-bytes=${SYSLOG_WAL_MAX_PENDING_BYTES:268435456}
 syslog.wal.high-watermark-percent=${SYSLOG_WAL_HIGH_WATERMARK_PERCENT:70}
 syslog.wal.low-watermark-percent=${SYSLOG_WAL_LOW_WATERMARK_PERCENT:40}
 syslog.wal.retry-interval-ms=${SYSLOG_WAL_RETRY_INTERVAL_MS:10}
@@ -52,6 +58,7 @@ syslog.point-time.write-enabled=${SYSLOG_POINT_TIME_WRITE_ENABLED:true}
 # WAL 后台投递 Kafka：内存只保留有限在途消息，失败记录留在 WAL 自动重试。
 syslog.dispatch.worker-threads=${SYSLOG_DISPATCH_WORKER_THREADS:4}
 syslog.dispatch.max-inflight=${SYSLOG_DISPATCH_MAX_INFLIGHT:5000}
+syslog.dispatch.max-inflight-bytes=${SYSLOG_DISPATCH_MAX_INFLIGHT_BYTES:268435456}
 syslog.dispatch.retry-backoff-ms=${SYSLOG_DISPATCH_RETRY_BACKOFF_MS:1000}
 syslog.dispatch.shutdown-timeout-ms=${SYSLOG_DISPATCH_SHUTDOWN_TIMEOUT_MS:30000}
 syslog.device-context.cache-ttl-ms=${SYSLOG_DEVICE_CONTEXT_CACHE_TTL_MS:300000}
@@ -77,6 +84,7 @@ app.service.data_type=json
 # kafka Configuration
 spring.kafka.producer.bootstrap-servers=192.168.4.99:9092
 spring.kafka.producer.topic =${KAFKA_PRODUCER_TOPIC:test-topic}
+spring.kafka.producer.large-topic=${KAFKA_PRODUCER_LARGE_TOPIC:${spring.kafka.producer.topic}.large}
 # kafka  Configuration 新增优化配置
 spring.kafka.producer.properties.retries=10
 spring.kafka.producer.properties.retry.backoff.ms=500
@@ -87,7 +95,8 @@ spring.kafka.producer.properties.delivery.timeout.ms=120000
 spring.kafka.producer.properties.max.block.ms=10000
 spring.kafka.producer.properties.buffer.memory=134217728
 # 允许投递加密后超过 Kafka 默认 1 MiB 的 Syslog；Broker/Topic 上限必须不小于此值。
-spring.kafka.producer.properties.max.request.size=${KAFKA_PRODUCER_MAX_REQUEST_SIZE:8388608}
+# 6MiB GBK 原文转 UTF-8 后再做 Base64 加密可能明显膨胀，按 32MiB 留出确定余量。
+spring.kafka.producer.properties.max.request.size=${KAFKA_PRODUCER_MAX_REQUEST_SIZE:33554432}
 # event_id 使高流量设备的独立事件均匀分布到多个分区；device_id 仅用于兼容回滚。
 syslog.kafka.partition-key-strategy=${SYSLOG_KAFKA_PARTITION_KEY_STRATEGY:event_id}
 

haobang-security-dm/syslog-serve/src/test/java/com/haobang/dispatch/SyslogWalDispatcherTest.java

@@ -121,6 +121,56 @@ class SyslogWalDispatcherTest {
         verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
     }
 
+    @Test
+    void largeRecordUsesDedicatedKafkaLaneAndReleasesByteBudgetAfterAck() throws Exception {
+        AppProperties properties = properties(tempDirectory);
+        properties.setSyslogLargeMessageThresholdBytes(4L);
+        properties.setSyslogMessageMaxBytes(32L);
+        properties.setSyslogWalMaxPendingBytes(64L);
+        properties.setSyslogDispatchMaxInflightBytes(64L);
+        wal = new SyslogWalService(properties, new ObjectMapper());
+        wal.initialize();
+        wal.tryAppend(event("20260811-large", "12345"));
+        waitUntil(() -> wal.getAppendedCount() == 1L);
+
+        DeviceDevice device = new DeviceDevice();
+        device.setId(126);
+        DeviceContextResolver resolver = mock(DeviceContextResolver.class);
+        when(resolver.resolve(anyString(), anyString()))
+                .thenReturn(DeviceContextResolver.Resolution.registered(device));
+        kafkaProducer producer = mock(kafkaProducer.class);
+        when(producer.messagePushLarge(any(DeviceDevice.class), anyString(), anyString(),
+                anyString(), anyString(), any()))
+                .thenReturn(CompletableFuture.completedFuture(null));
+
+        dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
+        dispatcher.start();
+        waitUntil(() -> wal.getCheckpointSequence() == 1L);
+
+        verify(producer).messagePushLarge(eq(device), eq("12345"),
+                eq("20260811200000000"), eq("20260811200000000"),
+                eq("20260811-large"), eq(null));
+        verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
+        assertEquals(1L, dispatcher.getLargeDispatched());
+        assertEquals(0L, dispatcher.getInflightBytes());
+    }
+
+    @Test
+    void singleRecordLargerThanByteBudgetCanRunExclusivelyWhenHardLimitIsDisabled() throws Exception {
+        SyslogWalDispatcher.ByteBudget budget = new SyslogWalDispatcher.ByteBudget(4L);
+
+        budget.acquire(5L);
+
+        assertEquals(5L, budget.used());
+        budget.release(5L);
+        assertEquals(0L, budget.used());
+    }
+
+    @Test
+    void utf8BudgetCountsChineseAndEmojiWithoutAllocatingEncodedPayload() {
+        assertEquals(11L, SyslogWalDispatcher.utf8Length("中文🙂x"));
+    }
+
     private SyslogIngressEvent event(String id, String raw) {
         return new SyslogIngressEvent(id, LocalDateTime.of(2026, 8, 11, 20, 0),
                 "192.168.4.19", 514, "TCP", raw);
@@ -147,6 +197,7 @@ class SyslogWalDispatcherTest {
         value.setSyslogCharset("UTF-8");
         value.setSyslogDispatchWorkerThreads(2);
         value.setSyslogDispatchMaxInflight(8);
+        value.setSyslogDispatchMaxInflightBytes(0L);
         value.setSyslogDispatchRetryBackoffMs(10);
         value.setSyslogDispatchShutdownTimeoutMs(2000);
         value.setSyslogDeviceContextCacheTtlMs(60000);

haobang-security-dm/syslog-serve/src/test/java/com/haobang/wal/SyslogWalServiceTest.java

@@ -98,6 +98,58 @@ class SyslogWalServiceTest {
         assertEquals(null, attempt.getDurableStage());
     }
 
+    @Test
+    void oversizedMessageIsRejectedBeforeEnteringWal() {
+        AppProperties properties = properties(tempDirectory);
+        properties.setSyslogMessageMaxBytes(4L);
+        properties.setSyslogWalMaxPendingBytes(16L);
+        service = new SyslogWalService(properties, new ObjectMapper());
+        service.initialize();
+
+        SyslogWalService.AdmissionResult result = service.tryAppend(
+                event("20260811-too-large", "12345"));
+
+        assertEquals(SyslogWalService.AdmissionResult.OVERSIZED, result);
+        assertEquals(1L, service.getOversizedCount());
+        assertEquals(0, service.getQueueSize());
+        assertEquals(0L, service.getPendingAppendBytes());
+    }
+
+    @Test
+    void pendingByteBudgetBackpressuresEvenWhenRecordQueueHasCapacity() {
+        AppProperties properties = properties(tempDirectory);
+        properties.setSyslogMessageMaxBytes(5L);
+        properties.setSyslogWalMaxPendingBytes(5L);
+        properties.setSyslogWalFlushRecords(100);
+        properties.setSyslogWalFlushIntervalMs(10000L);
+        service = new SyslogWalService(properties, new ObjectMapper());
+        service.initialize();
+
+        assertEquals(SyslogWalService.AdmissionResult.ACCEPTED,
+                service.tryAppend(event("20260811-first", "12345")));
+        assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE,
+                service.tryAppend(event("20260811-second", "x")));
+        assertEquals(5L, service.getPendingAppendBytes());
+        assertEquals(1L, service.getBackpressureCount());
+    }
+
+    @Test
+    void disabledHardLimitLetsOneOverBudgetRecordMakeProgressExclusively() {
+        AppProperties properties = properties(tempDirectory);
+        properties.setSyslogMessageMaxBytes(0L);
+        properties.setSyslogWalMaxPendingBytes(4L);
+        properties.setSyslogWalFlushRecords(100);
+        properties.setSyslogWalFlushIntervalMs(10000L);
+        service = new SyslogWalService(properties, new ObjectMapper());
+        service.initialize();
+
+        assertEquals(SyslogWalService.AdmissionResult.ACCEPTED,
+                service.tryAppend(event("20260811-over-budget", "12345")));
+        assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE,
+                service.tryAppend(event("20260811-waits", "x")));
+        assertEquals(5L, service.getPendingAppendBytes());
+    }
+
     @Test
     void forceBeforeCursorAndPersistSchemaV2() throws Exception {
         service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());

haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java

@@ -63,6 +63,20 @@ class KafkaProducerEventIdTest {
         assertEquals("126", record.key());
     }
 
+    @Test
+    void largeLaneUsesDedicatedTopicWithoutChangingPartitionKey() {
+        AppProperties properties = new AppProperties();
+        properties.setKafkaProducerTopic("raw");
+        properties.setKafkaProducerLargeTopic("raw.large");
+        kafkaProducer producer = new kafkaProducer(properties, mock(Producer.class), "event_id");
+
+        ProducerRecord<String, String> record = producer.buildRecord(
+                properties.getKafkaProducerLargeTopic(), 126, "20260811-event", "ciphertext");
+
+        assertEquals("raw.large", record.topic());
+        assertEquals("20260811-event", record.key());
+    }
+
     @Test
     void invalidPartitionKeyStrategyFailsFast() {
         assertThrows(IllegalArgumentException.class, () -> producer("random"));

haobang-security-dm/upgrade-doc/README.md

@@ -6,9 +6,12 @@
 | --- | --- |
 | `v1/` | 原有升级资料 |
 | `v2/` | 从提交 `b61ec2e` 升级到当前源码所需的增量资料 |
+| `v3/` | 日统计异步投影，以及大日志独立 Kafka 通道、字节背压和存储截断升级资料 |
 
-现场如果已经执行过 v1，只执行 v2；新环境按 v1、v2 的顺序执行。
+新环境按 v1、v2、v3 的顺序执行；已经完成 v2 的现场只执行 v3。已经执行过早期 v3 日统计升级的
+现场，只需补做 v3 中的大日志 Topic、应用升级和验证步骤，不要重复执行数据库升级脚本。
 
 数据库脚本不要重复执行。执行过程中有报错时，先确认已经落库的对象，再决定从哪里继续。
 
 v2 的执行步骤见 [v2/README.md](./v2/README.md)。
+v3 的执行步骤见 [v3/README.md](./v3/README.md)。

haobang-security-dm/upgrade-doc/replace-syslog-consumer-rule.md

@@ -1,4 +1,4 @@
-
+先执行这条查看启动目录和启动命令：
 
 docker inspect syslog-consumer-rule-dm \
   --format 'WorkingDir={{.Config.WorkingDir}}{{println}}Entrypoint={{json .Config.Entrypoint}}{{println}}Cmd={{json .Config.Cmd}}'
@@ -7,4 +7,9 @@ docker inspect syslog-consumer-rule-dm \
 再确认文件确实存在：
 
   docker exec syslog-consumer-rule-dm \
-  sh -c 'ls -lh /app/*.jar 2>/dev/null'
+  sh -c 'ls -lh /app/*.jar 2>/dev/null'
+
+
+  最后执行 
+
+  ./upgrade-doc/replace-syslog-consumer-rule-jar.sh /实际路径/syslog-consumer-rule-1.0.0.jar

haobang-security-dm/upgrade-doc/v3/README.md

@@ -0,0 +1,193 @@
+# v3 日统计异步投影与大日志通道升级说明
+
+## 升级内容
+
+v3 将以下统计表的热点更新移出 `syslog-consumer-rule` 事实事务：
+
+- `device_daily_stats`
+- `normalize_rule_daily_stats`
+- `device_collect_stats`
+
+事实事务只批量写事实表并追加一条聚合 Outbox。独立的 Kafka 投影消费者串行更新日统计，之后继续通过
+已有的 `STATS.MATERIALIZE` Topic 刷新 `device_device`、`dm_normalize_rule` 和
+`device_collect_task`。这样三台 `syslog-consumer-rule` 可以并行写事实，不再同时争抢相同日统计行锁。
+
+v3 同时增加大日志独立处理链路：
+
+- `syslog-serve` 在进入 WAL 前执行单条硬上限检查；超过上限明确拒绝，不把原文写入应用日志；
+- 接纳后的消息仍先进入 `syslog-serve` 本地 WAL，WAL 入队和 Kafka 投递同时受条数、字节数约束；
+- 超过分流阈值的消息进入独立 `.large` Topic，其 Retry、DLT、Replay 也使用独立 Topic；
+- `syslog-consumer-rule` 使用独立的大日志 listener、消费组、ConsumerFactory、poll 批量和规则计算线程池；
+- 规则解析使用完整内容，写 InfluxDB 和达梦前才按 UTF-8 字节数截断，结尾带
+  `HB_XDR_TRUNCATED` 标记；InfluxDB 另存原长度、保存长度和原文 SHA-256。
+
+## 文件与执行顺序
+
+| 顺序 | 文件 | 作用 |
+| --- | --- | --- |
+| 1 | `database/01-stats-async-projection-precheck.sql` | 只读预检 |
+| 2 | `database/02-stats-async-projection-upgrade.sql` | 创建 Outbox 和消费幂等表 |
+| 3 | `database/03-stats-async-projection-verify.sql` | 只读验证表、约束和索引 |
+| 4 | `kafka/01-create-stats-projection-topic.sh` | 创建独立统计投影 Topic |
+| 5 | `kafka/02-create-large-log-topics.sh` | 创建大日志主 Topic 及独立 Retry、DLT、Replay Topic |
+| 6 | `config/01-new-environment-variables.md` | 升级应用前配置新增环境变量，并核对 Kafka Broker 参数 |
+
+数据库脚本必须使用 `syslog-consumer-rule` 实际连接的达梦用户执行。
+
+如果现场已经执行过早期 v3 日统计异步投影升级，只执行 `kafka/02-create-large-log-topics.sh` 和本页的
+大日志应用配置、验证步骤；不要重复执行数据库升级脚本。
+
+## 第一步：达梦
+
+依次执行：
+
+```text
+01-stats-async-projection-precheck.sql
+02-stats-async-projection-upgrade.sql
+03-stats-async-projection-verify.sql
+```
+
+预检判断：
+
+- 第一组应返回 `DEVICE_DAILY_STATS`、`NORMALIZE_RULE_DAILY_STATS`、`DEVICE_COLLECT_STATS` 三行；
+- 后面三组必须为空；
+- 如果目标表、约束或索引已经存在，不要执行 `02`，先执行 `03` 判断是否已经升级完成。
+
+验证判断：
+
+- 返回两张新表；
+- 三个约束状态为 `ENABLED`；
+- 两个索引状态为 `VALID`；
+- 首次建表时两张表的行数都是 0。
+
+## 第二步：Kafka
+
+### 统计投影 Topic
+
+在 Kafka 主机的 `kafka_2.13-3.6.1/bin` 目录执行：
+
+```bash
+bash /脚本所在路径/01-create-stats-projection-topic.sh
+```
+
+默认创建：
+
+```text
+agent-01-syslog-topic.STATS.PROJECTION
+```
+
+新建时为 1 个分区，副本数沿用 `agent-01-syslog-topic`。如果现场此前已经建成 6 个分区，脚本只告警，
+不会修改或删除；应用使用固定 Kafka key，投影事件仍然只落入其中一个稳定分区。
+
+必须看到：
+
+```text
+PASS: agent-01-syslog-topic.STATS.PROJECTION 已创建/确认。
+```
+
+`STATS.PROJECTION` 不能与已有的 `STATS.MATERIALIZE` 配成同一个 Topic。
+
+### 大日志 Topic
+
+先确认所有 Kafka Broker 的 `replica.fetch.max.bytes` 不小于 `33554432`，并按现场 Kafka 版本要求使
+配置生效。然后执行：
+
+```bash
+bash /脚本所在路径/02-create-large-log-topics.sh
+```
+
+脚本默认从 `agent-01-syslog-topic` 读取分区数和副本数，并创建：
+
+```text
+agent-01-syslog-topic.large
+agent-01-syslog-topic.large.RETRY.1M
+agent-01-syslog-topic.large.RETRY.10M
+agent-01-syslog-topic.large.RETRY.1H
+agent-01-syslog-topic.large.DLT
+agent-01-syslog-topic.large.REPLAY
+```
+
+必须看到：
+
+```text
+PASS: 大日志 Topic 已创建并统一配置。
+```
+
+## 第三步：升级应用
+
+升级 `syslog-serve` 和三台 `syslog-consumer-rule`；`syslog-consumer` 不需要因为本次改造升级。
+
+升级前先按 [新增变量配置清单](./config/01-new-environment-variables.md) 修改现场实际使用的 Compose 文件。
+三台 `syslog-consumer-rule` 的 Topic、消费组和截断上限必须保持一致。
+
+统计投影默认已经启用。确认现场没有显式配置：
+
+```text
+STATS_PROJECTION_ASYNC_ENABLED=false
+```
+
+先逐台升级 `syslog-consumer-rule`，可以启动一台观察后再升级另外两台；确认大日志 listener 正常后再
+升级 `syslog-serve`。新旧版本滚动期间，同一条原始 Kafka 消息只会被消费组中的一个实例处理，不会因为
+滚动升级重复统计。
+
+## 升级后验证
+
+查看处理日志：
+
+```bash
+docker logs --since 2m syslog-consumer-rule-dm 2>&1 \
+| grep -E 'Syslog批次完成|统计投影批次完成|统计物化批次完成|Outbox|Deadlock' \
+| tail -100
+```
+
+查看 Outbox 状态：
+
+```sql
+SELECT status, COUNT(*)
+FROM stats_projection_outbox
+GROUP BY status
+ORDER BY status;
+```
+
+正常表现：
+
+- 日志持续出现 `统计投影批次完成`；
+- `APPLIED` 持续增长；
+- `PENDING`、`PUBLISHED` 只短暂停留，不持续增长；
+- `FAILED` 不持续增长；
+- `Syslog批次完成` 中 `dmWriteMs` 明显下降；
+- 不再出现 daily stats 相关死锁。
+
+查看新消费组积压：
+
+```bash
+./kafka-consumer-groups.sh \
+  --bootstrap-server 10.141.121.35:9092,10.141.121.50:9092,10.141.121.59:9092 \
+  --group agent-01-syslog-group-dm-stats-projection \
+  --describe
+```
+
+验证大日志链路：
+
+- `GET /syslogserve/api/syslog/wal/status`：检查 `wal_admission_oversized`、`wal_pending_bytes`、
+  `dispatch_inflight_bytes`、`kafka_large_dispatched`；
+- `GET /xdrrule/api/syslog/raw-storage/metrics`：检查 `raw_message_truncated_total` 及截断前后字节累计值；
+- `GET /xdrrule/api/syslog/storage-limits/metrics`：检查达梦正常/非标字段截断计数；
+- 发送一条略大于 256 KiB、但小于 6 MiB 的测试消息，确认 `.large` Topic 有数据且普通 Topic 无该事件；
+- 查询 InfluxDB/达梦，确认被截断值以 `...[HB_XDR_TRUNCATED original_utf8_bytes=...]` 结尾。
+
+## 回滚
+
+回滚最后一台新版本前，先确认 Outbox 没有未应用事件：
+
+```sql
+SELECT status, COUNT(*)
+FROM stats_projection_outbox
+WHERE status <> 'APPLIED'
+GROUP BY status;
+```
+
+查询为空后再回滚最后一台。两张新表、Kafka Topic 和消费组 offset 全部保留，不要删除或重置。
+
+回滚旧版本前，还要先停止新生产者并排空 `.large` 主 Topic；大日志 Topic 和消费组 offset 同样保留，
+不要删除或重置。

haobang-security-dm/upgrade-doc/v3/config/01-new-environment-variables.md

@@ -0,0 +1,147 @@
+# v3 新增变量配置
+
+应用侧只需要修改两个 Docker Compose 文件。变量放到对应服务的 `environment:` 下，不要修改 JAR 中的
+`application.properties`；Kafka Broker 参数单独按第 4 节核对。
+
+## 1. syslog-serve
+
+配置文件：
+
+```text
+syslog-serve/docker-compose/docker-compose-serve-prod.yaml
+```
+
+在 `services.syslog-serve.environment` 中增加：
+
+```yaml
+      # 开启 Syslog TCP/UDP 监听，字符集按现场设备实际编码调整。
+      - SYSLOG_SERVER_ENABLED=true
+      - SYSLOG_CHARSET=UTF-8
+
+      # 单条最多接收 6 MiB，超过 256 KiB 时发送到大日志 Topic。
+      - SYSLOG_MAX_FRAME_LENGTH=6291456
+      - SYSLOG_MESSAGE_MAX_BYTES=6291456
+      - SYSLOG_LARGE_MESSAGE_THRESHOLD_BYTES=262144
+
+      # WAL 和 Kafka 投递最多各占用 256 MiB 在途内存。
+      - SYSLOG_WAL_MAX_PENDING_BYTES=268435456
+      - SYSLOG_DISPATCH_MAX_INFLIGHT_BYTES=268435456
+
+      # 大日志 Kafka Topic 及 Producer 请求上限。
+      - KAFKA_PRODUCER_LARGE_TOPIC=agent-01-syslog-topic.large
+      - KAFKA_PRODUCER_MAX_REQUEST_SIZE=33554432
+```
+
+其中：
+
+- `SYSLOG_MAX_FRAME_LENGTH`：TCP 单帧最大长度；
+- `SYSLOG_MESSAGE_MAX_BYTES`：所有来源的单条日志硬上限，超过后拒绝；
+- `SYSLOG_LARGE_MESSAGE_THRESHOLD_BYTES`：超过该值后走大日志 Topic；
+- `SYSLOG_WAL_MAX_PENDING_BYTES`、`SYSLOG_DISPATCH_MAX_INFLIGHT_BYTES`：限制 JVM 在途数据量。
+
+`SYSLOG_MAX_FRAME_LENGTH` 必须大于或等于 `SYSLOG_MESSAGE_MAX_BYTES`。如果设备发送 GBK，应将
+`SYSLOG_CHARSET` 改为 `GBK`。
+
+## 2. syslog-consumer-rule
+
+配置文件：
+
+```text
+syslog-consumer-rule/docker-compose/docker-compose-consumer-rule-prod.yaml
+```
+
+在 `services.syslog-consumer-rule.environment` 中增加，三台实例保持一致：
+
+```yaml
+      # 大日志独立消费通道。
+      - KAFKA_CONSUMER_LARGE_TOPIC=agent-01-syslog-topic.large
+      - SYSLOG_LARGE_LISTENER_ENABLED=true
+      - SYSLOG_LARGE_LISTENER_CONCURRENCY=1
+      - SYSLOG_LARGE_LISTENER_MAX_POLL_RECORDS=5
+      - SYSLOG_LARGE_LISTENER_MAX_PARTITION_FETCH_BYTES=33554432
+      - SYSLOG_LARGE_LISTENER_FETCH_MAX_BYTES=67108864
+
+      # 大日志规则处理批次和独立线程池。
+      - SYSLOG_LARGE_PROCESSING_BATCH_MAX_BYTES=8388608
+      - SYSLOG_LARGE_PROCESSOR_CORE_POOL_SIZE=1
+      - SYSLOG_LARGE_PROCESSOR_MAX_POOL_SIZE=2
+      - SYSLOG_LARGE_PROCESSOR_QUEUE_CAPACITY=10
+
+      # Retry、DLT、Replay 也允许传输大日志。
+      - KAFKA_RELIABILITY_MAX_REQUEST_SIZE=33554432
+
+      # InfluxDB 原文最多保存 1 MiB，单批写入最多 8 MiB。
+      - INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES=1048576
+      - INFLUXDB_RAW_WRITE_BATCH_MAX_BYTES=8388608
+
+      # 达梦正常日志大字段最多保存 64 KiB，非标日志原文最多保存 256 KiB。
+      - DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES=65536
+      - DM_NON_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES=262144
+```
+
+被截断的数据会以以下标记结尾：
+
+```text
+...[HB_XDR_TRUNCATED original_utf8_bytes=原始字节数]
+```
+
+以下三个存储上限配置为 `0` 时表示不截断：
+
+```text
+INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES
+DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES
+DM_NON_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES
+```
+
+规则解析始终使用完整原文，只在写入 InfluxDB 或达梦时截断。
+
+## 3. 日统计投影
+
+日统计投影默认开启，一般不需要新增配置。只需确认现场没有配置：
+
+```text
+STATS_PROJECTION_ASYNC_ENABLED=false
+```
+
+如果现场需要显式写明，可以在 `syslog-consumer-rule` 的 `environment` 中配置：
+
+```yaml
+      - STATS_PROJECTION_ASYNC_ENABLED=true
+      - STATS_PROJECTION_TOPIC=agent-01-syslog-topic.STATS.PROJECTION
+      - STATS_PROJECTION_GROUP_ID=agent-01-syslog-group-dm-stats-projection
+```
+
+## 4. Kafka 配置
+
+每个 Kafka Broker 的 `replica.fetch.max.bytes` 必须不小于 32 MiB：
+
+```properties
+replica.fetch.max.bytes=33554432
+```
+
+然后执行：
+
+```bash
+bash upgrade-doc/v3/kafka/02-create-large-log-topics.sh
+```
+
+生产者和消费者的大日志 Topic 名称必须完全相同：
+
+```text
+KAFKA_PRODUCER_LARGE_TOPIC=agent-01-syslog-topic.large
+KAFKA_CONSUMER_LARGE_TOPIC=agent-01-syslog-topic.large
+```
+
+## 5. 使配置生效
+
+在现场对应的部署目录先检查 Compose，再重建容器：
+
+```bash
+docker compose -f docker-compose-serve-prod.yaml config
+docker compose -f docker-compose-serve-prod.yaml up -d --force-recreate syslog-serve
+
+docker compose -f docker-compose-consumer-rule-prod.yaml config
+docker compose -f docker-compose-consumer-rule-prod.yaml up -d --force-recreate syslog-consumer-rule
+```
+
+建议先升级三台 `syslog-consumer-rule`，确认大日志消费者正常后，再升级 `syslog-serve`。

haobang-security-dm/upgrade-doc/v3/database/01-stats-async-projection-precheck.sql

@@ -0,0 +1,40 @@
+-- v3 达梦升级预检：只读，不修改任何数据。
+-- 第一组应返回 3 行；第二、三、四组应全部为空。
+
+-- v1 的三张 DIRECT 统计表必须已经存在。
+SELECT table_name
+FROM user_tables
+WHERE table_name IN (
+  'DEVICE_DAILY_STATS',
+  'NORMALIZE_RULE_DAILY_STATS',
+  'DEVICE_COLLECT_STATS'
+)
+ORDER BY table_name;
+
+-- v3 目标表必须尚未创建。若有返回，停止执行 02，先确认是否执行过 v3 或曾中途失败。
+SELECT table_name
+FROM user_tables
+WHERE table_name IN (
+  'STATS_PROJECTION_OUTBOX',
+  'STATS_PROJECTION_APPLIED_EVENT'
+)
+ORDER BY table_name;
+
+-- 约束名不得与现有对象冲突。
+SELECT constraint_name, table_name
+FROM user_constraints
+WHERE constraint_name IN (
+  'PK_STATS_PROJECTION_OUTBOX',
+  'CK_STATS_PROJECTION_OUTBOX_STATUS',
+  'PK_STATS_PROJECTION_APPLIED'
+)
+ORDER BY constraint_name;
+
+-- 索引名不得与现有对象冲突。
+SELECT index_name, table_name
+FROM user_indexes
+WHERE index_name IN (
+  'IDX_STATS_PROJECTION_OUTBOX_DUE',
+  'IDX_STATS_PROJECTION_APPLIED_TIME'
+)
+ORDER BY index_name;

haobang-security-dm/upgrade-doc/v3/database/02-stats-async-projection-upgrade.sql

@@ -0,0 +1,41 @@
+-- v3 达梦升级：为事实事务外的可靠日统计投影建立 Outbox 和消费幂等表。
+-- 仅在 01 预检通过后执行一次；不要重复执行。
+
+CREATE TABLE stats_projection_outbox (
+  event_id VARCHAR2(64) NOT NULL,
+  event_schema VARCHAR2(64) NOT NULL,
+  payload CLOB NOT NULL,
+  status VARCHAR2(16) DEFAULT 'PENDING' NOT NULL,
+  attempt_count INT DEFAULT 0 NOT NULL,
+  next_attempt_at TIMESTAMP DEFAULT SYSDATE,
+  published_at TIMESTAMP,
+  applied_at TIMESTAMP,
+  last_error VARCHAR2(2000),
+  created_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
+  updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
+  CONSTRAINT pk_stats_projection_outbox PRIMARY KEY(event_id),
+  CONSTRAINT ck_stats_projection_outbox_status
+    CHECK(status IN ('PENDING','FAILED','PUBLISHED','APPLIED'))
+);
+
+CREATE INDEX idx_stats_projection_outbox_due
+ON stats_projection_outbox(status,next_attempt_at,created_at);
+
+CREATE TABLE stats_projection_applied_event (
+  event_id VARCHAR2(64) NOT NULL,
+  event_schema VARCHAR2(64) NOT NULL,
+  applied_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
+  CONSTRAINT pk_stats_projection_applied PRIMARY KEY(event_id)
+);
+
+CREATE INDEX idx_stats_projection_applied_time
+ON stats_projection_applied_event(applied_at);
+
+COMMENT ON TABLE stats_projection_outbox IS
+'事实批次统计 Outbox；Kafka 发布或消费失败时可持续恢复';
+COMMENT ON COLUMN stats_projection_outbox.status IS
+'PENDING/FAILED/PUBLISHED/APPLIED；只有 APPLIED 才允许到期清理';
+COMMENT ON TABLE stats_projection_applied_event IS
+'统计投影消费幂等标记；保留时间必须长于 STATS.PROJECTION Topic';
+
+COMMIT;

haobang-security-dm/upgrade-doc/v3/database/03-stats-async-projection-verify.sql

@@ -0,0 +1,44 @@
+-- v3 达梦升级验证：只读。
+
+-- 应返回 2 行。
+SELECT table_name
+FROM user_tables
+WHERE table_name IN (
+  'STATS_PROJECTION_OUTBOX',
+  'STATS_PROJECTION_APPLIED_EVENT'
+)
+ORDER BY table_name;
+
+-- 核对关键字段；OUTBOX 应包含 11 列，APPLIED_EVENT 应包含 3 列。
+SELECT table_name, column_id, column_name, data_type, nullable
+FROM user_tab_columns
+WHERE table_name IN (
+  'STATS_PROJECTION_OUTBOX',
+  'STATS_PROJECTION_APPLIED_EVENT'
+)
+ORDER BY table_name, column_id;
+
+-- 三个约束都应为 ENABLED。
+SELECT table_name, constraint_name, constraint_type, status
+FROM user_constraints
+WHERE constraint_name IN (
+  'PK_STATS_PROJECTION_OUTBOX',
+  'CK_STATS_PROJECTION_OUTBOX_STATUS',
+  'PK_STATS_PROJECTION_APPLIED'
+)
+ORDER BY constraint_name;
+
+-- 两个索引都应为 VALID。
+SELECT table_name, index_name, uniqueness, status
+FROM user_indexes
+WHERE index_name IN (
+  'IDX_STATS_PROJECTION_OUTBOX_DUE',
+  'IDX_STATS_PROJECTION_APPLIED_TIME'
+)
+ORDER BY index_name;
+
+-- 刚建表时应返回 0、0；应用启动后数字会正常增长。
+SELECT
+  (SELECT COUNT(*) FROM stats_projection_outbox) AS outbox_rows,
+  (SELECT COUNT(*) FROM stats_projection_applied_event) AS applied_rows
+FROM dual;

haobang-security-dm/upgrade-doc/v3/kafka/01-create-stats-projection-topic-docker.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Docker 单文件版：直接在 Kafka 容器内创建并校验统计投影 Topic。
+# 默认值适配测试环境；如容器名、Kafka 地址或安装目录不同，可通过环境变量覆盖。
+
+DOCKER_BIN="${DOCKER_BIN:-docker}"
+KAFKA_CONTAINER="${KAFKA_CONTAINER:-kafka_ycgy-bt-kafka-1}"
+BOOTSTRAP="${BOOTSTRAP:-localhost:9092}"
+KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}"
+RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
+PROJECTION_TOPIC="${PROJECTION_TOPIC:-${RAW_TOPIC}.STATS.PROJECTION}"
+RETENTION_MS="${RETENTION_MS:-604800000}"
+COMMAND_CONFIG="${COMMAND_CONFIG:-}"
+
+die() {
+  echo "ERROR: $*" >&2
+  exit 1
+}
+
+command -v "${DOCKER_BIN}" >/dev/null 2>&1 || die "未找到 Docker CLI: ${DOCKER_BIN}"
+
+RUNNING="$("${DOCKER_BIN}" inspect -f '{{.State.Running}}' "${KAFKA_CONTAINER}" 2>/dev/null || true)"
+[[ "${RUNNING}" == "true" ]] || die "Kafka 容器不存在或未运行: ${KAFKA_CONTAINER}"
+
+"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-topics.sh" \
+  || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-topics.sh"
+"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-configs.sh" \
+  || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-configs.sh"
+"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" bash -c 'exit 0' \
+  || die "容器内缺少 bash，无法执行原始升级脚本"
+
+DOCKER_ENV=(
+  -e "BOOTSTRAP=${BOOTSTRAP}"
+  -e "KAFKA_BIN=${KAFKA_BIN}"
+  -e "RAW_TOPIC=${RAW_TOPIC}"
+  -e "PROJECTION_TOPIC=${PROJECTION_TOPIC}"
+  -e "RETENTION_MS=${RETENTION_MS}"
+)
+KAFKA_COMMON=(--bootstrap-server "${BOOTSTRAP}")
+
+if [[ -n "${COMMAND_CONFIG}" ]]; then
+  "${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -r "${COMMAND_CONFIG}" \
+    || die "容器内无法读取 COMMAND_CONFIG: ${COMMAND_CONFIG}"
+  DOCKER_ENV+=(-e "COMMAND_CONFIG=${COMMAND_CONFIG}")
+  KAFKA_COMMON+=(--command-config "${COMMAND_CONFIG}")
+fi
+
+echo "检查 Kafka 连接及原始 topic ..."
+"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" \
+  "${KAFKA_BIN}/kafka-topics.sh" \
+  "${KAFKA_COMMON[@]}" \
+  --describe \
+  --topic "${RAW_TOPIC}" >/dev/null \
+  || die "无法连接 Kafka 或原始 topic 不存在: ${RAW_TOPIC}"
+
+echo "Kafka 容器: ${KAFKA_CONTAINER}"
+echo "Bootstrap: ${BOOTSTRAP}"
+echo "原始 topic: ${RAW_TOPIC}"
+echo "投影 topic: ${PROJECTION_TOPIC}"
+echo "开始执行 Docker 版升级 ..."
+
+exec "${DOCKER_BIN}" exec -i "${DOCKER_ENV[@]}" "${KAFKA_CONTAINER}" \
+  bash -s <<'KAFKA_UPGRADE_SCRIPT'
+set -euo pipefail
+
+if [[ "${PROJECTION_TOPIC}" == "${RAW_TOPIC}.STATS.MATERIALIZE" ]]; then
+  echo "ERROR: STATS.PROJECTION 不能与 STATS.MATERIALIZE 使用同一个 Topic" >&2
+  exit 1
+fi
+
+TOPICS="${KAFKA_BIN}/kafka-topics.sh"
+CONFIGS="${KAFKA_BIN}/kafka-configs.sh"
+COMMON=(--bootstrap-server "${BOOTSTRAP}")
+if [[ -n "${COMMAND_CONFIG:-}" ]]; then
+  [[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; }
+  COMMON+=(--command-config "${COMMAND_CONFIG}")
+fi
+
+echo "Kafka=${BOOTSTRAP}"
+echo "原始 Topic=${RAW_TOPIC}"
+echo "统计投影 Topic=${PROJECTION_TOPIC}"
+
+RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}")
+REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
+  /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}}
+')
+if [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
+  echo "ERROR: 无法从 ${RAW_TOPIC} 读取副本数" >&2
+  exit 1
+fi
+
+echo "[1/3] 创建 Topic（已经存在时不重复创建）"
+"${TOPICS}" "${COMMON[@]}" --create --if-not-exists \
+  --topic "${PROJECTION_TOPIC}" \
+  --partitions 1 \
+  --replication-factor "${REPLICAS}" \
+  --config cleanup.policy=delete \
+  --config "retention.ms=${RETENTION_MS}"
+
+echo "[2/3] 更新保留策略"
+if ! "${CONFIGS}" "${COMMON[@]}" --alter \
+  --entity-type topics --entity-name "${PROJECTION_TOPIC}" \
+  --add-config "cleanup.policy=delete,retention.ms=${RETENTION_MS}"; then
+  echo "WARN: Kafka 配置更新超时；新建命令已携带相同策略，继续验证 Topic" >&2
+fi
+
+echo "[3/3] 验证 Topic"
+INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${PROJECTION_TOPIC}")
+printf '%s\n' "${INFO}"
+PARTITIONS=$(printf '%s\n' "${INFO}" | awk '
+  /PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}}
+')
+ACTUAL_REPLICAS=$(printf '%s\n' "${INFO}" | awk '
+  /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}}
+')
+if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]]; then
+  echo "ERROR: ${PROJECTION_TOPIC} 没有有效分区" >&2
+  exit 1
+fi
+if [[ "${PARTITIONS}" != "1" ]]; then
+  echo "WARN: Topic 已有 ${PARTITIONS} 个分区；不会修改分区数，固定 key 仍只使用其中一个分区" >&2
+fi
+if [[ "${ACTUAL_REPLICAS}" != "${REPLICAS}" ]]; then
+  echo "WARN: Topic 副本数=${ACTUAL_REPLICAS}，原始 Topic 副本数=${REPLICAS}" >&2
+fi
+
+echo "PASS: ${PROJECTION_TOPIC} 已创建/确认。分区数=${PARTITIONS}，保留时间=${RETENTION_MS}ms。"
+echo "脚本未删除消息，也未修改任何消费组 offset。"
+KAFKA_UPGRADE_SCRIPT

haobang-security-dm/upgrade-doc/v3/kafka/01-create-stats-projection-topic.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# v3 统计投影 Topic。新建时使用 1 个分区；如果现场已经建成多个分区则保留现状，
+# 应用使用固定 Kafka key，投影消息仍只会落入其中一个稳定分区。
+BOOTSTRAP="${BOOTSTRAP:-10.141.121.35:9092,10.141.121.50:9092,10.141.121.59:9092}"
+RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
+PROJECTION_TOPIC="${PROJECTION_TOPIC:-${RAW_TOPIC}.STATS.PROJECTION}"
+RETENTION_MS="${RETENTION_MS:-604800000}"
+
+if [[ "${PROJECTION_TOPIC}" == "${RAW_TOPIC}.STATS.MATERIALIZE" ]]; then
+  echo "ERROR: STATS.PROJECTION 不能与 STATS.MATERIALIZE 使用同一个 Topic" >&2
+  exit 1
+fi
+
+if [[ -z "${KAFKA_BIN:-}" ]]; then
+  for candidate in \
+    "$(pwd)" \
+    "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" \
+    "/opt/kafka/kafka_2.13-3.6.1/bin" \
+    "/opt/kafka/bin" \
+    "/usr/local/kafka/bin"; do
+    if [[ -x "${candidate}/kafka-topics.sh" ]] \
+        && [[ -x "${candidate}/kafka-configs.sh" ]]; then
+      KAFKA_BIN="${candidate}"
+      break
+    fi
+  done
+fi
+
+if [[ -z "${KAFKA_BIN:-}" ]] && command -v kafka-topics.sh >/dev/null 2>&1; then
+  KAFKA_BIN=$(dirname -- "$(command -v kafka-topics.sh)")
+fi
+if [[ -z "${KAFKA_BIN:-}" ]]; then
+  echo "ERROR: 未找到 kafka-topics.sh；请在 Kafka bin 目录执行，或设置 KAFKA_BIN" >&2
+  exit 1
+fi
+
+TOPICS="${KAFKA_BIN}/kafka-topics.sh"
+CONFIGS="${KAFKA_BIN}/kafka-configs.sh"
+COMMON=(--bootstrap-server "${BOOTSTRAP}")
+if [[ -n "${COMMAND_CONFIG:-}" ]]; then
+  [[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; }
+  COMMON+=(--command-config "${COMMAND_CONFIG}")
+fi
+
+echo "Kafka=${BOOTSTRAP}"
+echo "原始 Topic=${RAW_TOPIC}"
+echo "统计投影 Topic=${PROJECTION_TOPIC}"
+
+RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}")
+REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
+  /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}}
+')
+if [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
+  echo "ERROR: 无法从 ${RAW_TOPIC} 读取副本数" >&2
+  exit 1
+fi
+
+echo "[1/3] 创建 Topic（已经存在时不重复创建）"
+"${TOPICS}" "${COMMON[@]}" --create --if-not-exists \
+  --topic "${PROJECTION_TOPIC}" \
+  --partitions 1 \
+  --replication-factor "${REPLICAS}" \
+  --config cleanup.policy=delete \
+  --config "retention.ms=${RETENTION_MS}"
+
+echo "[2/3] 更新保留策略"
+if ! "${CONFIGS}" "${COMMON[@]}" --alter \
+  --entity-type topics --entity-name "${PROJECTION_TOPIC}" \
+  --add-config "cleanup.policy=delete,retention.ms=${RETENTION_MS}"; then
+  echo "WARN: Kafka 配置更新超时；新建命令已携带相同策略，继续验证 Topic" >&2
+fi
+
+echo "[3/3] 验证 Topic"
+INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${PROJECTION_TOPIC}")
+printf '%s\n' "${INFO}"
+PARTITIONS=$(printf '%s\n' "${INFO}" | awk '
+  /PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}}
+')
+ACTUAL_REPLICAS=$(printf '%s\n' "${INFO}" | awk '
+  /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}}
+')
+if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]]; then
+  echo "ERROR: ${PROJECTION_TOPIC} 没有有效分区" >&2
+  exit 1
+fi
+if [[ "${PARTITIONS}" != "1" ]]; then
+  echo "WARN: Topic 已有 ${PARTITIONS} 个分区；不会修改分区数，固定 key 仍只使用其中一个分区" >&2
+fi
+if [[ "${ACTUAL_REPLICAS}" != "${REPLICAS}" ]]; then
+  echo "WARN: Topic 副本数=${ACTUAL_REPLICAS}，原始 Topic 副本数=${REPLICAS}" >&2
+fi
+
+echo "PASS: ${PROJECTION_TOPIC} 已创建/确认。分区数=${PARTITIONS}，保留时间=${RETENTION_MS}ms。"
+echo "脚本未删除消息，也未修改任何消费组 offset。"

haobang-security-dm/upgrade-doc/v3/kafka/02-create-large-log-topics-docker.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Docker 单文件版：直接在 Kafka 容器内创建并校验大日志相关 Topic。
+# 默认值适配测试环境；如容器名、Kafka 地址或安装目录不同，可通过环境变量覆盖。
+
+DOCKER_BIN="${DOCKER_BIN:-docker}"
+KAFKA_CONTAINER="${KAFKA_CONTAINER:-kafka_ycgy-bt-kafka-1}"
+BOOTSTRAP="${BOOTSTRAP:-localhost:9092}"
+KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}"
+RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
+LARGE_TOPIC="${LARGE_TOPIC:-${RAW_TOPIC}.large}"
+MAX_MESSAGE_BYTES="${MAX_MESSAGE_BYTES:-33554432}"
+RETENTION_MS="${RETENTION_MS:-604800000}"
+DLT_RETENTION_MS="${DLT_RETENTION_MS:-2592000000}"
+COMMAND_CONFIG="${COMMAND_CONFIG:-}"
+
+die() {
+  echo "ERROR: $*" >&2
+  exit 1
+}
+
+command -v "${DOCKER_BIN}" >/dev/null 2>&1 || die "未找到 Docker CLI: ${DOCKER_BIN}"
+
+RUNNING="$("${DOCKER_BIN}" inspect -f '{{.State.Running}}' "${KAFKA_CONTAINER}" 2>/dev/null || true)"
+[[ "${RUNNING}" == "true" ]] || die "Kafka 容器不存在或未运行: ${KAFKA_CONTAINER}"
+
+"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-topics.sh" \
+  || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-topics.sh"
+"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-configs.sh" \
+  || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-configs.sh"
+"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" bash -c 'exit 0' \
+  || die "容器内缺少 bash，无法执行原始升级脚本"
+
+DOCKER_ENV=(
+  -e "BOOTSTRAP=${BOOTSTRAP}"
+  -e "KAFKA_BIN=${KAFKA_BIN}"
+  -e "RAW_TOPIC=${RAW_TOPIC}"
+  -e "LARGE_TOPIC=${LARGE_TOPIC}"
+  -e "MAX_MESSAGE_BYTES=${MAX_MESSAGE_BYTES}"
+  -e "RETENTION_MS=${RETENTION_MS}"
+  -e "DLT_RETENTION_MS=${DLT_RETENTION_MS}"
+)
+KAFKA_COMMON=(--bootstrap-server "${BOOTSTRAP}")
+
+if [[ -n "${COMMAND_CONFIG}" ]]; then
+  "${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -r "${COMMAND_CONFIG}" \
+    || die "容器内无法读取 COMMAND_CONFIG: ${COMMAND_CONFIG}"
+  DOCKER_ENV+=(-e "COMMAND_CONFIG=${COMMAND_CONFIG}")
+  KAFKA_COMMON+=(--command-config "${COMMAND_CONFIG}")
+fi
+
+echo "检查 Kafka 连接及原始 topic ..."
+"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" \
+  "${KAFKA_BIN}/kafka-topics.sh" \
+  "${KAFKA_COMMON[@]}" \
+  --describe \
+  --topic "${RAW_TOPIC}" >/dev/null \
+  || die "无法连接 Kafka 或原始 topic 不存在: ${RAW_TOPIC}"
+
+echo "Kafka 容器: ${KAFKA_CONTAINER}"
+echo "Bootstrap: ${BOOTSTRAP}"
+echo "原始 topic: ${RAW_TOPIC}"
+echo "大日志 topic: ${LARGE_TOPIC}"
+echo "开始执行 Docker 版升级 ..."
+
+exec "${DOCKER_BIN}" exec -i "${DOCKER_ENV[@]}" "${KAFKA_CONTAINER}" \
+  bash -s <<'KAFKA_UPGRADE_SCRIPT'
+set -euo pipefail
+
+TOPICS="${KAFKA_BIN}/kafka-topics.sh"
+CONFIGS="${KAFKA_BIN}/kafka-configs.sh"
+COMMON=(--bootstrap-server "${BOOTSTRAP}")
+if [[ -n "${COMMAND_CONFIG:-}" ]]; then
+  [[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; }
+  COMMON+=(--command-config "${COMMAND_CONFIG}")
+fi
+
+RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}")
+PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk '
+  /PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}}
+')
+REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
+  /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}}
+')
+if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
+  echo "ERROR: 无法读取 ${RAW_TOPIC} 的分区数或副本数" >&2
+  exit 1
+fi
+
+declare -a TOPIC_NAMES=(
+  "${LARGE_TOPIC}"
+  "${LARGE_TOPIC}.RETRY.1M"
+  "${LARGE_TOPIC}.RETRY.10M"
+  "${LARGE_TOPIC}.RETRY.1H"
+  "${LARGE_TOPIC}.DLT"
+  "${LARGE_TOPIC}.REPLAY"
+)
+
+echo "Kafka=${BOOTSTRAP}，分区=${PARTITIONS}，副本=${REPLICAS}，单条上限=${MAX_MESSAGE_BYTES}"
+for topic in "${TOPIC_NAMES[@]}"; do
+  topic_retention="${RETENTION_MS}"
+  [[ "${topic}" == "${LARGE_TOPIC}.DLT" ]] && topic_retention="${DLT_RETENTION_MS}"
+  "${TOPICS}" "${COMMON[@]}" --create --if-not-exists \
+    --topic "${topic}" --partitions "${PARTITIONS}" --replication-factor "${REPLICAS}" \
+    --config cleanup.policy=delete \
+    --config "retention.ms=${topic_retention}" \
+    --config "max.message.bytes=${MAX_MESSAGE_BYTES}"
+  "${CONFIGS}" "${COMMON[@]}" --alter --entity-type topics --entity-name "${topic}" \
+    --add-config "cleanup.policy=delete,retention.ms=${topic_retention},max.message.bytes=${MAX_MESSAGE_BYTES}"
+  "${TOPICS}" "${COMMON[@]}" --describe --topic "${topic}"
+done
+
+echo "PASS: 大日志 Topic 已创建并统一配置。"
+echo "注意：Broker 的 replica.fetch.max.bytes 也必须不小于 ${MAX_MESSAGE_BYTES}。"
+echo "脚本未删除消息，也未修改任何消费组 offset。"
+KAFKA_UPGRADE_SCRIPT

haobang-security-dm/upgrade-doc/v3/kafka/02-create-large-log-topics.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# 创建大日志主 Topic 及独立 Retry/DLT/Replay Topic。不删除消息、不修改消费组 offset。
+BOOTSTRAP="${BOOTSTRAP:-10.141.121.35:9092,10.141.121.50:9092,10.141.121.59:9092}"
+RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
+LARGE_TOPIC="${LARGE_TOPIC:-${RAW_TOPIC}.large}"
+MAX_MESSAGE_BYTES="${MAX_MESSAGE_BYTES:-33554432}"
+RETENTION_MS="${RETENTION_MS:-604800000}"
+DLT_RETENTION_MS="${DLT_RETENTION_MS:-2592000000}"
+
+if [[ -z "${KAFKA_BIN:-}" ]]; then
+  for candidate in \
+    "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" \
+    "/opt/kafka/kafka_2.13-3.6.1/bin" \
+    "/opt/kafka/bin" \
+    "/usr/local/kafka/bin"; do
+    if [[ -x "${candidate}/kafka-topics.sh" ]] && [[ -x "${candidate}/kafka-configs.sh" ]]; then
+      KAFKA_BIN="${candidate}"
+      break
+    fi
+  done
+fi
+if [[ -z "${KAFKA_BIN:-}" ]] && command -v kafka-topics.sh >/dev/null 2>&1; then
+  KAFKA_BIN=$(dirname -- "$(command -v kafka-topics.sh)")
+fi
+if [[ -z "${KAFKA_BIN:-}" ]]; then
+  echo "ERROR: 未找到 Kafka CLI；请设置 KAFKA_BIN" >&2
+  exit 1
+fi
+
+TOPICS="${KAFKA_BIN}/kafka-topics.sh"
+CONFIGS="${KAFKA_BIN}/kafka-configs.sh"
+COMMON=(--bootstrap-server "${BOOTSTRAP}")
+if [[ -n "${COMMAND_CONFIG:-}" ]]; then
+  [[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; }
+  COMMON+=(--command-config "${COMMAND_CONFIG}")
+fi
+
+RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}")
+PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk '
+  /PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}}
+')
+REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
+  /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}}
+')
+if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
+  echo "ERROR: 无法读取 ${RAW_TOPIC} 的分区数或副本数" >&2
+  exit 1
+fi
+
+declare -a TOPIC_NAMES=(
+  "${LARGE_TOPIC}"
+  "${LARGE_TOPIC}.RETRY.1M"
+  "${LARGE_TOPIC}.RETRY.10M"
+  "${LARGE_TOPIC}.RETRY.1H"
+  "${LARGE_TOPIC}.DLT"
+  "${LARGE_TOPIC}.REPLAY"
+)
+
+echo "Kafka=${BOOTSTRAP}，分区=${PARTITIONS}，副本=${REPLICAS}，单条上限=${MAX_MESSAGE_BYTES}"
+for topic in "${TOPIC_NAMES[@]}"; do
+  topic_retention="${RETENTION_MS}"
+  [[ "${topic}" == "${LARGE_TOPIC}.DLT" ]] && topic_retention="${DLT_RETENTION_MS}"
+  "${TOPICS}" "${COMMON[@]}" --create --if-not-exists \
+    --topic "${topic}" --partitions "${PARTITIONS}" --replication-factor "${REPLICAS}" \
+    --config cleanup.policy=delete \
+    --config "retention.ms=${topic_retention}" \
+    --config "max.message.bytes=${MAX_MESSAGE_BYTES}"
+  "${CONFIGS}" "${COMMON[@]}" --alter --entity-type topics --entity-name "${topic}" \
+    --add-config "cleanup.policy=delete,retention.ms=${topic_retention},max.message.bytes=${MAX_MESSAGE_BYTES}"
+  "${TOPICS}" "${COMMON[@]}" --describe --topic "${topic}"
+done
+
+echo "PASS: 大日志 Topic 已创建并统一配置。"
+echo "注意：Broker 的 replica.fetch.max.bytes 也必须不小于 ${MAX_MESSAGE_BYTES}。"
+echo "脚本未删除消息，也未修改任何消费组 offset。"