perf(syslog): 统计异步化并按采样保留率自适应调整业务批量

- 入口统计改用独立 Kafka 消费组，移除主消费同步统计写库
- DIRECT 模式入库计数改为事务提交后异步投递，取消事实事务内统计 Outbox 写入
- 采样审计、告警分钟桶通知改为后台处理，异常不阻断主消费或回滚告警
- RAW 基础业务批量默认 500，按保留率动态扩大，上限 3000
- 限制业务块字节数及规则任务数量，保留告警全量处理和整次 poll 完成后提交
- 细化 SLOW_SYSLOG_BATCH，补充批次关联、线程排队、数据库执行及提交耗时
- 更新 v4 升级说明，明确异步统计损失风险及现场验证方法

验证：JDK 8 构建成功，232 项测试通过。
升级：复用现有表和 Kafka Topic，无新增建表或建 Topic 脚本要求。

haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/AdaptiveRawBatchSizer.java

@@ -0,0 +1,35 @@
+package com.Modules.NormalData;
+
+/** 普通/非标采用较高保留率，倍率由当前策略自动计算；不改变实际采样或告警保留规则。 */
+@org.springframework.stereotype.Component
+public final class AdaptiveRawBatchSizer {
+    private final int baseRecords;
+    private final int maxRecords;
+
+    public AdaptiveRawBatchSizer(
+            @org.springframework.beans.factory.annotation.Value("${syslog.raw-listener.max-poll-records:500}") int baseRecords,
+            @org.springframework.beans.factory.annotation.Value("${syslog.raw-listener.adaptive-max-records:3000}") int maxRecords) {
+        if (baseRecords < 1 || maxRecords < baseRecords) {
+            throw new IllegalArgumentException("普通自适应批量要求 1 <= 基础条数 <= 上限条数");
+        }
+        this.baseRecords = baseRecords;
+        this.maxRecords = maxRecords;
+    }
+
+    public int targetRecords(int normalRetainBps, int nonNormalRetainBps) {
+        validateRate(normalRetainBps);
+        validateRate(nonNormalRetainBps);
+        int retainBps = Math.max(normalRetainBps, nonNormalRetainBps);
+        if (retainBps == 0) return maxRecords;
+        long numerator = (long) baseRecords * 10000L;
+        long calculated = (numerator + retainBps - 1L) / retainBps;
+        return (int) Math.min(maxRecords, calculated);
+    }
+
+    public int getBaseRecords() { return baseRecords; }
+    public int getMaxRecords() { return maxRecords; }
+
+    private static void validateRate(int rate) {
+        if (rate < 0 || rate > 10000) throw new IllegalArgumentException("保留万分比必须在0~10000之间");
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java

@@ -525,10 +525,8 @@ public class LogNormalProcessor {
                 if (compiledEnabled) {
                     try {
                         CompiledRule compiled = compiledRuleCache.get(source, ctx.executionMetrics);
-                        long parseStarted = System.nanoTime();
                         ParsedMessage parsed = ctx.messageParseContext.parse(
                                 compiled, ctx.strLogMsg, ctx.executionMetrics);
-                        ctx.executionMetrics.addMessageParseNanos(System.nanoTime() - parseStarted);
                         if (parsed == null) {
                             if (isJsonRule(source)) {
                                 markInvalidJson(ctx, source);

haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java

@@ -8,13 +8,11 @@ import com.common.util.SyslogParser;
 import com.config.AppProperties;
 import com.config.KafkaConsumerProperties;
 import com.common.service.DmStandardizedBatchWriter;
-import com.common.service.IngressStatsProjectionWriter;
 import com.common.service.DirectStatsProjectionWriter;
 import com.common.service.SyslogNonNormalMessageService;
 import com.common.service.SyslogNormalDataService;
 import com.common.service.BacklogRetentionDelta;
 import com.common.service.BacklogSamplingAuditService;
-import com.common.entity.IngressStatsEvent;
 import com.common.entity.SyslogNonNormalMessage;
 import com.influxdb.client.write.Point;
 import com.influx.raw.InfluxRawPointFactory;
@@ -46,6 +44,8 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.Arrays;
 import java.time.LocalDateTime;
 import java.nio.charset.StandardCharsets;
 
@@ -68,6 +68,17 @@ public class SysLogProcessor {
     @Value("${app.processor.process-timeout-ms:120000}")
     private long processTimeoutMs;
 
+    @Value("${syslog.normalization.workers-per-batch:8}")
+    private int normalizationWorkersPerBatch = 8;
+
+    @Value("${syslog.raw-listener.processing-batch-max-bytes:16777216}")
+    private long rawProcessingBatchMaxBytes = 16777216L;
+
+    @Autowired
+    private AdaptiveRawBatchSizer adaptiveRawBatchSizer = new AdaptiveRawBatchSizer(500, 3000);
+    // 每个普通listener线程独立学习上一个业务块的阶段，避免不同节点/通道互相放大批量。
+    private final ThreadLocal<RawAdaptiveState> rawAdaptiveState = ThreadLocal.withInitial(RawAdaptiveState::new);
+
     @Value("${syslog.large-listener.processing-batch-max-bytes:8388608}")
     private long largeProcessingBatchMaxBytes;
 
@@ -79,6 +90,7 @@ public class SysLogProcessor {
 
     /** listener、业务处理和 Kafka commit 均在同一个 consumer 线程，ThreadLocal 用于拼接端到端慢批次。 */
     private final ThreadLocal<SlowBatchTrace> slowBatchTrace = new ThreadLocal<>();
+    private final ThreadLocal<Long> lastListenerFinished = new ThreadLocal<>();
 
     @Autowired
     private AppProperties appProperties;
@@ -108,9 +120,6 @@ public class SysLogProcessor {
     @Value("${dm.non-normal-write.concurrency-retry-backoff-ms:50}")
     private long nonNormalConcurrencyRetryBackoffMs;
 
-    @Autowired
-    private IngressStatsProjectionWriter ingressStatsProjectionWriter;
-
     @Autowired
     private com.influx.InfluxDBClient influxClient;
 
@@ -201,8 +210,11 @@ public class SysLogProcessor {
             }
             long processingMs = elapsedMillis(processingStartedAt);
             acknowledgeAndCompleteSlowBatchTrace(ack, processingMs);
+        } catch (RuntimeException aborted) {
+            completeSlowBatchTrace(elapsedMillis(processingStartedAt), 0L, aborted, "NOT_ATTEMPTED");
+            throw aborted;
         } finally {
-            slowBatchTrace.remove();
+            clearSlowBatchTrace();
         }
     }
 
@@ -236,14 +248,69 @@ public class SysLogProcessor {
         beginSlowBatchTrace(lane, records);
         long processingStartedAt = System.nanoTime();
         try {
-            processBatchWithContinuity(records, enforceRetryDue, lane, normalizationExecutor);
+            if ("RAW".equals(lane)) {
+                processAdaptiveRawChunks(records, normalizationExecutor);
+            } else {
+                for (List<ConsumerRecord<String, String>> chunk : partitionRecordsByBytes(
+                        records, Math.max(1L, rawProcessingBatchMaxBytes))) {
+                    processBatchWithContinuity(chunk, enforceRetryDue, lane, normalizationExecutor);
+                }
+            }
             long processingMs = elapsedMillis(processingStartedAt);
             acknowledgeAndCompleteSlowBatchTrace(ack, processingMs);
+        } catch (RuntimeException aborted) {
+            if ("RAW".equals(lane)) observeRawRetention(10000);
+            completeSlowBatchTrace(elapsedMillis(processingStartedAt), 0L, aborted, "NOT_ATTEMPTED");
+            throw aborted;
         } finally {
-            slowBatchTrace.remove();
+            clearSlowBatchTrace();
         }
     }
 
+    private void processAdaptiveRawChunks(List<ConsumerRecord<String, String>> records, Executor executor) {
+        RawAdaptiveState state = rawAdaptiveState.get();
+        // 首次启动、长时间无反馈后从基础批量探测，避免沿用过时的积压阶段。
+        if (state.observedAt == 0L || System.nanoTime() - state.observedAt > TimeUnit.MINUTES.toNanos(1)) {
+            state.retainBps = 10000;
+        }
+        for (int start = 0; start < records.size();) {
+            enterStage("ADAPTIVE_CHUNK_SELECTION");
+            int target = adaptiveRawBatchSizer.targetRecords(state.retainBps, state.retainBps);
+            int end = adaptiveChunkEnd(records, start, target, Math.max(1L, rawProcessingBatchMaxBytes));
+            SlowBatchTrace trace = slowBatchTrace.get();
+            if (trace != null) {
+                trace.adaptiveTargets.merge(target, 1, Integer::sum);
+                trace.actualChunkSizes.merge(end - start, 1, Integer::sum);
+            }
+            processBatchWithContinuity(new ArrayList<>(records.subList(start, end)), false, "RAW", executor);
+            start = end;
+        }
+    }
+
+    static int adaptiveChunkEnd(List<ConsumerRecord<String, String>> records, int start, int target, long maxBytes) {
+        if (target < 1 || maxBytes < 1 || start < 0 || start >= records.size()) {
+            throw new IllegalArgumentException("自适应分块参数非法");
+        }
+        long bytes = 0L;
+        int end = start;
+        while (end < records.size() && end - start < target) {
+            ConsumerRecord<String, String> record = records.get(end);
+            long size = record.serializedValueSize();
+            if (size < 0L) size = record.value() == null ? 0L : record.value().getBytes(StandardCharsets.UTF_8).length;
+            if (end > start && size > maxBytes - bytes) break;
+            bytes += size;
+            end++;
+            if (bytes >= maxBytes) break; // 单条超限仍独立处理，不切断、不丢弃。
+        }
+        return end;
+    }
+
+    void observeRawRetention(int retainBps) {
+        RawAdaptiveState state = rawAdaptiveState.get();
+        state.retainBps = retainBps;
+        state.observedAt = System.nanoTime();
+    }
+
     private void processBatchWithContinuity(List<ConsumerRecord<String, String>> records,
                                             boolean enforceRetryDue, String lane,
                                             Executor normalizationExecutor) {
@@ -253,16 +320,27 @@ public class SysLogProcessor {
             // Retry Topic 的计划时间尚未到达，不属于处理异常，仍按原延时语义保留 offset。
             throw notDue;
         } catch (RuntimeException failure) {
+            if ("RAW".equals(lane)) observeRawRetention(10000);
+            SlowBatchTrace trace = slowBatchTrace.get();
+            if (trace != null) {
+                trace.failedChunks++;
+                trace.failureStage = trace.activeStage;
+                trace.businessError = failure.getClass().getName();
+            }
             if (!failureContinuityEnabled) throw failure;
+            enterStage("CONTINUITY_DLT");
             routeContinuityBatch(records, failure, "BATCH_" + lane);
             log.error("[SYSLOG_CONTINUITY] 批次特殊异常已记录，继续消费后续消息: lane={}, "
                             + "records={}, errorType={}",
                     lane, records.size(), failure.getClass().getName(), failure);
+        } finally {
+            endStage();
         }
     }
 
     void processBatch(List<ConsumerRecord<String, String>> records,
                       boolean enforceRetryDue, String lane, Executor normalizationExecutor) {
+        enterStage("PREPARE");
         long batchStartedAt = System.currentTimeMillis();
         long prepareStartedAt = batchStartedAt;
         BatchMetrics metrics = new BatchMetrics();
@@ -282,39 +360,48 @@ public class SysLogProcessor {
         }
         long prepareMs = System.currentTimeMillis() - prepareStartedAt;
 
-        // 入口状态是 Kafka 已可靠接收的事实。批内按设备/探针聚合，失败时禁止 ACK；重投只重复 MIN/MAX。
-        long ingressStartedAt = System.nanoTime();
-        int ingressGroups = ingressStatsProjectionWriter.writeBatch(ingressEvents(preparedMessages));
-        long ingressMs = elapsedMillis(ingressStartedAt);
+        // 入口统计由独立 Kafka 消费组处理；主链路不再调用统计写库或 Outbox。
+        int ingressGroups = 0;
+        long ingressMs = 0L;
 
         // 必须先完成规则计算，才能确保告警全量保留，并让未保留消息同时跳过 InfluxDB 和达梦。
         long normalizeStartedAt = System.currentTimeMillis();
         List<PreparedNormalization> computed = computePreparedNormalizations(
                 preparedMessages, metrics, normalizationExecutor);
+        enterStage("SAMPLING_DECISION");
         SamplingPlan samplingPlan = buildSamplingPlan(computed);
         if (!samplingPlan.auditDeltas.isEmpty()) {
-            boolean audited = backlogSamplingAuditService.recordBeforeDropping(samplingPlan.auditDeltas);
-            if (!audited) {
-                samplingPlan = allRetainedPlan(computed);
-                log.error("[BACKLOG_SAMPLING_FALLBACK] 采样审计不可用，当前批次已回退全量保留: "
-                        + "lane={}, records={}", lane, records.size());
-            } else {
-                metrics.recordSampling(samplingPlan);
+            enterStage("AUDIT_ENQUEUE");
+            boolean accepted = backlogSamplingAuditService.recordAsync(samplingPlan.auditDeltas);
+            SlowBatchTrace trace = slowBatchTrace.get();
+            if (trace != null) {
+                if (accepted) trace.auditAcceptedChunks++;
+                else trace.auditLostChunks++;
             }
+            // 审计失败不回退全量保存，避免进一步放大业务写入压力。
+            metrics.recordSampling(samplingPlan);
         }
         long normalizeMs = System.currentTimeMillis() - normalizeStartedAt;
 
         // 告警是最高优先级：先独立提交告警表，不能让 InfluxDB 或普通表的慢写阻塞告警入库。
         long persistencePlanStartedAt = System.nanoTime();
+        enterStage("PERSISTENCE_PLAN");
         StandardizedPersistencePlan standardizedPlan = prepareStandardizedPersistence(
                 samplingPlan.normalizations, metrics);
         long persistencePlanMs = elapsedMillis(persistencePlanStartedAt);
         long alarmWriteStartedAt = System.nanoTime();
+        enterStage("ALARM_DM");
+        SlowBatchTrace rowTrace = slowBatchTrace.get();
+        if (rowTrace != null) rowTrace.alarmRequested += standardizedPlan.alarmWriteRequests.size();
         DmStandardizedBatchWriter.WriteReport<PreparedMessage> alarmReport =
                 standardizedBatchWriter.write(standardizedPlan.alarmWriteRequests);
+        if (rowTrace != null) {
+            rowTrace.alarmSaved += alarmReport.getSuccessfulRows();
+        }
         long alarmWriteMs = elapsedMillis(alarmWriteStartedAt);
 
         long influxStartedAt = System.currentTimeMillis();
+        enterStage("INFLUX_AND_FAILURE_ROUTE");
         List<PreparedMessage> persistedMessages = persistRawWithoutBlockingMainFlow(
                 samplingPlan.rawMessages, metrics, lane);
         long influxMs = System.currentTimeMillis() - influxStartedAt;
@@ -322,6 +409,12 @@ public class SysLogProcessor {
         long remainingDmStartedAt = System.nanoTime();
         persistRemainingStandardizedData(standardizedPlan, alarmReport, alarmWriteMs, metrics);
         long remainingDmMs = elapsedMillis(remainingDmStartedAt);
+        enterStage("BATCH_FINISH");
+
+        if ("RAW".equals(lane)) {
+            // 任意准备/规则失败采用基础批量；多个阶段混合时取更高保留率保守放大。
+            observeRawRetention(preparedMessages.size() == records.size() ? samplingPlan.maxRetainBps : 10000);
+        }
 
         totalProcessed.addAndGet(records.size());
         long businessTotalMs = System.currentTimeMillis() - batchStartedAt;
@@ -369,8 +462,46 @@ public class SysLogProcessor {
             polledMessageBytes += messageBytes;
             maxMessageBytes = Math.max(maxMessageBytes, messageBytes);
         }
-        slowBatchTrace.set(new SlowBatchTrace(lane, topicPartitions(records), records.size(),
-                polledMessageBytes, maxMessageBytes, System.nanoTime()));
+        long now = System.nanoTime();
+        SlowBatchTrace trace = new SlowBatchTrace(lane, topicPartitions(records), records.size(),
+                polledMessageBytes, maxMessageBytes, now);
+        Long last = lastListenerFinished.get();
+        trace.listenerGapMs = last == null ? -1 : TimeUnit.NANOSECONDS.toMillis(now - last);
+        trace.previousMdc = org.slf4j.MDC.get("syslogBatchId");
+        org.slf4j.MDC.put("syslogBatchId", trace.batchId);
+        Map<String, long[]> ranges = new LinkedHashMap<>();
+        for (ConsumerRecord<String, String> record : records) {
+            String key = record.topic() + ":" + record.partition();
+            long[] range = ranges.computeIfAbsent(key, ignored -> new long[]{record.offset(), record.offset()});
+            range[0] = Math.min(range[0], record.offset());
+            range[1] = Math.max(range[1], record.offset());
+        }
+        for (Map.Entry<String, long[]> range : ranges.entrySet()) {
+            trace.offsetRanges.put(range.getKey(), range.getValue()[0] + ".." + range.getValue()[1]);
+        }
+        slowBatchTrace.set(trace);
+    }
+
+    private void clearSlowBatchTrace() {
+        SlowBatchTrace trace = slowBatchTrace.get();
+        if (trace == null || trace.previousMdc == null) org.slf4j.MDC.remove("syslogBatchId");
+        else org.slf4j.MDC.put("syslogBatchId", trace.previousMdc);
+        slowBatchTrace.remove();
+        lastListenerFinished.set(System.nanoTime());
+    }
+
+    private void enterStage(String name) {
+        endStage();
+        SlowBatchTrace trace = slowBatchTrace.get();
+        if (trace != null) { trace.activeStage = name; trace.stageStarted = System.nanoTime(); }
+    }
+
+    private void endStage() {
+        SlowBatchTrace trace = slowBatchTrace.get();
+        if (trace != null && trace.activeStage != null) {
+            trace.stageNanos.merge(trace.activeStage, System.nanoTime() - trace.stageStarted, Long::sum);
+            trace.activeStage = null;
+        }
     }
 
     private void recordSlowBatchStages(int records, int prepared, int persisted, int normalizedRows,
@@ -394,6 +525,11 @@ public class SysLogProcessor {
         trace.ruleComputeMs += metrics.ruleComputeMs();
         trace.messageParseMs += metrics.messageParseMs();
         trace.fieldMappingMs += metrics.fieldMappingMs();
+        trace.ruleQueueMaxMs = Math.max(trace.ruleQueueMaxMs, metrics.ruleQueueMaxMs.get());
+        trace.ruleWorkerBusySumMs += metrics.ruleWorkerBusySumMs.get();
+        trace.rawSampledOut += metrics.rawSampledOut.get();
+        trace.normalSampledOut += metrics.normalRowsSampledOut.get();
+        trace.nonNormalSampledOut += metrics.nonNormalRowsSampledOut.get();
     }
 
     private void acknowledgeAndCompleteSlowBatchTrace(Acknowledgment ack, long businessProcessMs) {
@@ -409,25 +545,53 @@ public class SysLogProcessor {
 
     private void completeSlowBatchTrace(long businessProcessMs, long kafkaCommitMs,
                                         RuntimeException commitFailure) {
+        completeSlowBatchTrace(businessProcessMs, kafkaCommitMs, commitFailure,
+                commitFailure == null ? "SUCCESS" : "FAILED");
+    }
+
+    private void completeSlowBatchTrace(long businessProcessMs, long kafkaCommitMs,
+                                        RuntimeException commitFailure, String commitStatus) {
         SlowBatchTrace trace = slowBatchTrace.get();
-        if (trace == null) return;
+        if (trace == null || trace.completed) return;
+        if ("NOT_ATTEMPTED".equals(commitStatus) && trace.failureStage == null) trace.failureStage = trace.activeStage;
+        endStage();
+        trace.completed = true;
         long endToEndMs = elapsedMillis(trace.startedAtNanos);
-        if (slowBatchThresholdMs <= 0L || endToEndMs < slowBatchThresholdMs) return;
-        log.warn("[SLOW_SYSLOG_BATCH] lane={}, topicPartitions={}, polledRecords={}, "
+        if (commitFailure == null && trace.failedChunks == 0
+                && (slowBatchThresholdMs <= 0L || endToEndMs < slowBatchThresholdMs)) return;
+        Map<String, Long> stages = new LinkedHashMap<>();
+        trace.stageNanos.forEach((key, value) -> stages.put(key, TimeUnit.NANOSECONDS.toMillis(value)));
+        log.warn("[SLOW_SYSLOG_BATCH] batchId={}, lane={}, topicPartitions={}, polledRecords={}, "
                         + "polledMessageBytes={}, maxMessageBytes={}, processedRecords={}, chunks={}, "
                         + "prepared={}, persisted={}, normalizedRows={}, prepareMs={}, ingressMs={}, "
                         + "normalizeMs={}, ruleComputeWorkerMs={}, messageParseWorkerMs={}, "
                         + "fieldMappingWorkerMs={}, persistencePlanMs={}, alarmDmWriteMs={}, influxMs={}, "
                         + "remainingDmWriteMs={}, businessProcessMs={}, kafkaCommitMs={}, "
-                        + "kafkaCommitStatus={}, kafkaCommitErrorType={}, endToEndMs={}",
-                trace.lane, trace.topicPartitions, trace.polledRecords,
+                        + "kafkaCommitStatus={}, kafkaCommitErrorType={}, endToEndMs={}, "
+                        + "offsetRanges={}, listenerGapMs={}, stageMs={}, failedChunks={}, failureStage={}, businessError={}, "
+                        + "alarmRequested={}, alarmSaved={}, normalRequested={}, normalSaved={}, nonNormalQueued={}, nonNormalRejected={}, "
+                        + "auditAcceptedChunks={}, auditLostChunks={}, rawSampledOut={}, normalSampledOut={}, nonNormalSampledOut={}, "
+                        + "ruleQueueMaxMs={}, ruleWorkerBusySumMs={}, adaptiveRaw={}, "
+                        + "baseProcessingRecords={}, adaptiveMaxRecords={}, adaptiveTargetHistogram={}, actualChunkHistogram={}, "
+                        + "nextTargetRecords={}, observedRetainBps={}",
+                trace.batchId, trace.lane, trace.topicPartitions, trace.polledRecords,
                 trace.polledMessageBytes, trace.maxMessageBytes, trace.recordsProcessed,
                 trace.chunks, trace.prepared, trace.persisted, trace.normalizedRows,
                 trace.prepareMs, trace.ingressMs, trace.normalizeMs, trace.ruleComputeMs,
                 trace.messageParseMs, trace.fieldMappingMs, trace.persistencePlanMs,
                 trace.alarmDmMs, trace.influxMs, trace.remainingDmMs,
-                businessProcessMs, kafkaCommitMs, commitFailure == null ? "SUCCESS" : "FAILED",
-                commitFailure == null ? "none" : commitFailure.getClass().getName(), endToEndMs);
+                businessProcessMs, kafkaCommitMs, commitStatus,
+                "FAILED".equals(commitStatus) ? commitFailure.getClass().getName() : "none", endToEndMs,
+                trace.offsetRanges, trace.listenerGapMs, stages, trace.failedChunks, trace.failureStage, trace.businessError,
+                trace.alarmRequested, trace.alarmSaved, trace.normalRequested, trace.normalSaved,
+                trace.nonNormalQueued, trace.nonNormalRejected, trace.auditAcceptedChunks, trace.auditLostChunks,
+                trace.rawSampledOut, trace.normalSampledOut, trace.nonNormalSampledOut,
+                trace.ruleQueueMaxMs, trace.ruleWorkerBusySumMs, "RAW".equals(trace.lane),
+                adaptiveRawBatchSizer.getBaseRecords(), adaptiveRawBatchSizer.getMaxRecords(),
+                trace.adaptiveTargets, trace.actualChunkSizes,
+                "RAW".equals(trace.lane) ? adaptiveRawBatchSizer.targetRecords(rawAdaptiveState.get().retainBps,
+                        rawAdaptiveState.get().retainBps) : 0,
+                "RAW".equals(trace.lane) ? rawAdaptiveState.get().retainBps : 10000);
     }
 
     private void routeContinuityBatch(List<ConsumerRecord<String, String>> records,
@@ -504,8 +668,11 @@ public class SysLogProcessor {
         List<ConsumerRecord<String, String>> current = new ArrayList<>();
         long currentBytes = 0L;
         for (ConsumerRecord<String, String> record : records) {
-            String value = record.value();
-            long bytes = value == null ? 0L : value.getBytes(StandardCharsets.UTF_8).length;
+            long bytes = record.serializedValueSize();
+            if (bytes < 0L) {
+                String value = record.value();
+                bytes = value == null ? 0L : value.getBytes(StandardCharsets.UTF_8).length;
+            }
             if (!current.isEmpty() && bytes > maxBytes - currentBytes) {
                 result.add(current);
                 current = new ArrayList<>();
@@ -859,6 +1026,7 @@ public class SysLogProcessor {
     private List<PreparedNormalization> computePreparedNormalizations(
             List<PreparedMessage> messages, BatchMetrics metrics, Executor normalizationExecutor) {
         if (messages.isEmpty()) return Collections.emptyList();
+        enterStage("RULE_SNAPSHOT");
         long snapshotStartedAt = System.nanoTime();
         Set<Long> deviceIds = new HashSet<>();
         for (PreparedMessage message : messages) {
@@ -869,17 +1037,34 @@ public class SysLogProcessor {
         RuleExecutionMetrics executionMetrics = new RuleExecutionMetrics();
         metrics.recordRuleSnapshotMs(elapsedMillis(snapshotStartedAt));
         long ruleComputeStartedAt = System.nanoTime();
-        List<CompletableFuture<PreparedNormalization>> futures = new ArrayList<>(messages.size());
+        enterStage("RULE_EXECUTION_WAIT");
+        // 每批固定数量的 worker，避免多个 consumer 按日志条数同时向线程池提交任务。
+        int workers = Math.min(messages.size(), Math.max(1, normalizationWorkersPerBatch));
+        List<CompletableFuture<Void>> futures = new ArrayList<>(workers);
+        PreparedNormalization[] results = new PreparedNormalization[messages.size()];
+        AtomicInteger next = new AtomicInteger();
+        AtomicBoolean stopped = new AtomicBoolean();
         try {
-            for (PreparedMessage message : messages) {
-                futures.add(CompletableFuture.supplyAsync(() -> {
+            for (int worker = 0; worker < workers; worker++) {
+                long queuedAt = System.nanoTime();
+                futures.add(CompletableFuture.runAsync(() -> {
+                    long workerBegin = System.nanoTime();
+                    metrics.ruleQueueMaxMs.accumulateAndGet(elapsedMillis(queuedAt), Math::max);
                     try {
-                        return prepareNormalization(message, ruleSnapshot, enrichmentContext, executionMetrics);
-                    } catch (Exception failure) {
-                        // 解析/规则计算失败仍按原消息可靠路由。只有取得 Retry/DLT Broker ACK，
-                        // 这个消息才会携带 terminalState 返回并允许原批次继续。
-                        return new PreparedNormalization(message, null,
-                                routeFailure(message.record, message.context, failure));
+                    while (!stopped.get() && !Thread.currentThread().isInterrupted()) {
+                        int index = next.getAndIncrement();
+                        if (index >= messages.size()) return;
+                        PreparedMessage message = messages.get(index);
+                        try {
+                            results[index] = prepareNormalization(message, ruleSnapshot,
+                                    enrichmentContext, executionMetrics);
+                        } catch (Exception failure) {
+                            results[index] = new PreparedNormalization(message, null,
+                                    routeFailure(message.record, message.context, failure));
+                        }
+                    }
+                    } finally {
+                        metrics.ruleWorkerBusySumMs.addAndGet(elapsedMillis(workerBegin));
                     }
                 }, normalizationExecutor));
             }
@@ -887,11 +1072,10 @@ public class SysLogProcessor {
                     .get(processTimeoutMs, TimeUnit.MILLISECONDS);
             metrics.recordRuleComputeMs(elapsedMillis(ruleComputeStartedAt));
             metrics.recordRuleExecution(executionMetrics, enrichmentContext);
-            List<PreparedNormalization> computed = new ArrayList<>(futures.size());
-            for (CompletableFuture<PreparedNormalization> future : futures) {
-                computed.add(future.join());
+            for (PreparedNormalization result : results) {
+                if (result == null) throw new IllegalStateException("标准化工作任务提前结束");
             }
-            return computed;
+            return new ArrayList<>(Arrays.asList(results));
         } catch (TimeoutException timeout) {
             cancelOutstanding(futures);
             throw new IllegalStateException("标准化批次处理超时，禁止提交 offset", timeout);
@@ -905,10 +1089,12 @@ public class SysLogProcessor {
         } catch (RuntimeException submitFailure) {
             cancelOutstanding(futures);
             throw submitFailure;
+        } finally {
+            stopped.set(true);
         }
     }
 
-    /** 根据已审计的保留决策写两张标准化表和非标表。 */
+    /** 根据保留决策写两张标准化表和非标表，不等待后台采样审计。 */
     private void persistComputedNormalizations(List<RetentionPreparedNormalization> normalizations,
                                                BatchMetrics metrics) {
         StandardizedPersistencePlan plan = prepareStandardizedPersistence(normalizations, metrics);
@@ -963,8 +1149,14 @@ public class SysLogProcessor {
             DmStandardizedBatchWriter.WriteReport<PreparedMessage> alarmReport,
             long alarmWriteMs, BatchMetrics metrics) {
         long normalWriteStartedAt = System.nanoTime();
+        enterStage("NORMAL_DM");
+        SlowBatchTrace rowTrace = slowBatchTrace.get();
+        if (rowTrace != null) rowTrace.normalRequested += plan.normalWriteRequests.size();
         DmStandardizedBatchWriter.WriteReport<PreparedMessage> normalReport =
                 standardizedBatchWriter.write(plan.normalWriteRequests);
+        if (rowTrace != null) {
+            rowTrace.normalSaved += normalReport.getSuccessfulRows();
+        }
         metrics.recordDmWriteMs(alarmWriteMs + elapsedMillis(normalWriteStartedAt));
         Map<PreparedMessage, RuntimeException> failures = new LinkedHashMap<>();
         failures.putAll(alarmReport.getFailures());
@@ -976,6 +1168,7 @@ public class SysLogProcessor {
         int successfulRows = alarmReport.getSuccessfulRows() + normalReport.getSuccessfulRows();
         metrics.recordDmWrite(requestedRows, successfulRows, failedMessages.size());
 
+        enterStage("DM_FAILURE_ROUTE");
         for (Map.Entry<PreparedMessage, RuntimeException> failure : failures.entrySet()) {
             PreparedMessage message = failure.getKey();
             metrics.record(routeFailure(message.record, message.context, failure.getValue()));
@@ -990,15 +1183,22 @@ public class SysLogProcessor {
         }
 
         // 非标日志仍使用现有独立线程；采样决策已经在进入队列前完成。
+        enterStage("NON_NORMAL_ENQUEUE");
         submitNonNormalMessages(plan.nonNormalMessages, metrics);
+        if (rowTrace != null) {
+            rowTrace.nonNormalQueued += metrics.nonNormalQueued();
+            rowTrace.nonNormalRejected += metrics.nonNormalRejected();
+        }
     }
 
     SamplingPlan buildSamplingPlan(List<PreparedNormalization> computed) {
         List<RetentionPreparedNormalization> retained = new ArrayList<>(computed.size());
         List<PreparedMessage> rawMessages = new ArrayList<>(computed.size());
         List<BacklogRetentionDelta> auditDeltas = new ArrayList<>();
+        int maxRetainBps = computed.isEmpty() ? 10000 : 0;
         for (PreparedNormalization prepared : computed) {
             if (prepared.terminalState != null) {
+                maxRetainBps = 10000;
                 retained.add(new RetentionPreparedNormalization(prepared, false, false));
                 // 规则异常时无法证明该消息不是告警，保留已构造的 InfluxDB 原文点。
                 rawMessages.add(prepared.message);
@@ -1014,6 +1214,8 @@ public class SysLogProcessor {
             BacklogSamplingDecision decision = backlogSamplingDecider.decide(
                     prepared.message.eventId, prepared.message.receivedAt,
                     alarmRows > 0L, normalRows > 0L, nonNormalRows > 0L);
+            maxRetainBps = Math.max(maxRetainBps,
+                    Math.max(decision.getNormalRetainBps(), decision.getNonNormalRetainBps()));
             retained.add(new RetentionPreparedNormalization(
                     prepared, decision.isKeepNormal(), decision.isKeepNonNormal()));
             if (decision.isKeepRaw()) rawMessages.add(prepared.message);
@@ -1032,7 +1234,7 @@ public class SysLogProcessor {
                         decision.getLagMs(), decision.getLagMs()));
             }
         }
-        return new SamplingPlan(retained, rawMessages, auditDeltas);
+        return new SamplingPlan(retained, rawMessages, auditDeltas, maxRetainBps);
     }
 
     private SamplingPlan allRetainedPlan(List<PreparedNormalization> computed) {
@@ -1053,9 +1255,12 @@ public class SysLogProcessor {
             return;
         }
         List<SyslogNonNormalMessage> batch = new ArrayList<>(messages);
+        final String batchId = org.slf4j.MDC.get("syslogBatchId");
+        final long queuedAt = System.nanoTime();
         try {
             nonNormalWriterExecutor.execute(() -> {
                 long startedAt = System.nanoTime();
+                long queueMs = elapsedMillis(queuedAt);
                 try {
                     int inserted = saveNonNormalBatchWithConcurrencyRetry(batch);
                     if (inserted != batch.size()) {
@@ -1067,13 +1272,17 @@ public class SysLogProcessor {
                         return;
                     }
                     nonNormalAsyncCompletedTotal.addAndGet(inserted);
+                    if (slowBatchThresholdMs > 0 && queueMs + elapsedMillis(startedAt) >= slowBatchThresholdMs) {
+                        log.warn("[SLOW_NON_NORMAL_WRITE] batchId={}, rows={}, inserted={}, queueMs={}, writeMs={}, status=SUCCESS",
+                                batchId, batch.size(), inserted, queueMs, elapsedMillis(startedAt));
+                    }
                     log.debug("非标日志异步批写完成: rows={}, durationMs={}, completedTotal={}",
                             inserted, elapsedMillis(startedAt), nonNormalAsyncCompletedTotal.get());
                 } catch (Exception failure) {
                     nonNormalAsyncFailedTotal.addAndGet(batch.size());
-                    log.error("非标日志异步批写失败，Kafka offset 不回退，当前非标数据可能丢失: "
+                    log.error("[SLOW_NON_NORMAL_WRITE] batchId={}, queueMs={}, status=FAILED, 非标日志异步批写失败，Kafka offset 不回退，当前非标数据可能丢失: "
                                     + "rows={}, durationMs={}, failedTotal={}",
-                            batch.size(), elapsedMillis(startedAt),
+                            batchId, queueMs, batch.size(), elapsedMillis(startedAt),
                             nonNormalAsyncFailedTotal.get(), failure);
                 }
             });
@@ -1209,15 +1418,6 @@ public class SysLogProcessor {
         }
     }
 
-    private List<IngressStatsEvent> ingressEvents(List<PreparedMessage> messages) {
-        List<IngressStatsEvent> events = new ArrayList<>(messages.size());
-        for (PreparedMessage message : messages) {
-            events.add(new IngressStatsEvent(message.eventId, message.deviceId,
-                    message.deviceCollectId, message.receivedAt));
-        }
-        return events;
-    }
-
     private static Long optionalLong(String value) {
         if (value == null || value.trim().isEmpty()) return null;
         try { return Long.valueOf(value.trim()); }
@@ -1256,6 +1456,7 @@ public class SysLogProcessor {
     }
 
     static final class SamplingPlan {
+        private final int maxRetainBps;
         private final List<RetentionPreparedNormalization> normalizations;
         private final List<PreparedMessage> rawMessages;
         private final List<BacklogRetentionDelta> auditDeltas;
@@ -1263,14 +1464,22 @@ public class SysLogProcessor {
         private SamplingPlan(List<RetentionPreparedNormalization> normalizations,
                              List<PreparedMessage> rawMessages,
                              List<BacklogRetentionDelta> auditDeltas) {
+            this(normalizations, rawMessages, auditDeltas, 10000);
+        }
+
+        private SamplingPlan(List<RetentionPreparedNormalization> normalizations,
+                             List<PreparedMessage> rawMessages, List<BacklogRetentionDelta> auditDeltas,
+                             int maxRetainBps) {
             this.normalizations = normalizations;
             this.rawMessages = rawMessages;
             this.auditDeltas = auditDeltas;
+            this.maxRetainBps = maxRetainBps;
         }
 
         int rawMessageCount() { return rawMessages.size(); }
         int auditMessageCount() { return auditDeltas.size(); }
         List<BacklogRetentionDelta> auditDeltas() { return auditDeltas; }
+        int maxRetainBps() { return maxRetainBps; }
     }
 
     private static final class StandardizedPersistencePlan {
@@ -1293,7 +1502,38 @@ public class SysLogProcessor {
         }
     }
 
+    private static final class RawAdaptiveState {
+        private int retainBps = 10000;
+        private long observedAt;
+    }
+
     private static final class SlowBatchTrace {
+        private final Map<Integer, Integer> adaptiveTargets = new LinkedHashMap<>();
+        private final Map<Integer, Integer> actualChunkSizes = new LinkedHashMap<>();
+        private final String batchId = java.util.UUID.randomUUID().toString();
+        private String previousMdc;
+        private String activeStage;
+        private long stageStarted;
+        private final Map<String, Long> stageNanos = new LinkedHashMap<>();
+        private final Map<String, String> offsetRanges = new LinkedHashMap<>();
+        private long listenerGapMs;
+        private boolean completed;
+        private int failedChunks;
+        private String failureStage;
+        private String businessError;
+        private int auditAcceptedChunks;
+        private int auditLostChunks;
+        private int alarmRequested;
+        private int alarmSaved;
+        private int normalRequested;
+        private int normalSaved;
+        private int nonNormalQueued;
+        private int nonNormalRejected;
+        private long ruleQueueMaxMs;
+        private long ruleWorkerBusySumMs;
+        private int rawSampledOut;
+        private int normalSampledOut;
+        private int nonNormalSampledOut;
         private final String lane;
         private final String topicPartitions;
         private final int polledRecords;
@@ -1343,6 +1583,8 @@ public class SysLogProcessor {
         private final AtomicInteger normalRowsSampledOut = new AtomicInteger();
         private final AtomicInteger nonNormalRowsSampledOut = new AtomicInteger();
         private final AtomicLong ruleComputeMs = new AtomicLong();
+        private final AtomicLong ruleQueueMaxMs = new AtomicLong();
+        private final AtomicLong ruleWorkerBusySumMs = new AtomicLong();
         private final AtomicLong ruleSnapshotMs = new AtomicLong();
         private final AtomicLong dmWriteMs = new AtomicLong();
         private final AtomicInteger nonNormalQueued = new AtomicInteger();

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BacklogSamplingAuditService.java

@@ -12,7 +12,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
-/** 先提交小时审计，再允许调用方执行有损采样；审计失败时调用方必须回退全量保留。 */
+/** 消费优先：小时采样审计后台尽力保存，失败不改变采样决策，不等待主消费。 */
 @Slf4j
 @Service
 public class BacklogSamplingAuditService {
@@ -21,6 +21,11 @@ public class BacklogSamplingAuditService {
     private final BacklogSamplingMetrics metrics;
     private final PlatformTransactionManager transactionManager;
     private volatile long retryAfterMs;
+    private final BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("SAMPLING_AUDIT", 256, 50000, false);
+
+    @javax.annotation.PostConstruct public void start() { queue.start(); }
+    @javax.annotation.PreDestroy public void stop() { queue.stop(); }
+    public Map<String, Object> status() { return queue.status(); }
 
     public BacklogSamplingAuditService(BacklogRetentionAuditMapper mapper,
                                        BacklogSamplingProperties properties,
@@ -32,34 +37,46 @@ public class BacklogSamplingAuditService {
         this.transactionManager = transactionManager;
     }
 
-    /** 返回 false 表示不能安全采样，调用方应保留当前批次全部数据。 */
-    public boolean recordBeforeDropping(List<BacklogRetentionDelta> deltas) {
+    /** 返回值仅代表入队是否成功，不影响调用方原采样决策。 */
+    public boolean recordAsync(List<BacklogRetentionDelta> deltas) {
         if (deltas == null || deltas.isEmpty()) return true;
-        long affectedMessages = messageCount(deltas);
-        long now = System.currentTimeMillis();
-        if (now < retryAfterMs) {
-            metrics.auditFailure(affectedMessages);
+        long affectedMessages = 0L;
+        try {
+            affectedMessages = messageCount(deltas);
+            final long taskMessages = affectedMessages;
+            List<BacklogRetentionDelta> aggregated = aggregate(deltas);
+            metrics.record(aggregated); // 决策指标，不再假称这些数字已写入审计数据库。
+            if (System.currentTimeMillis() < retryAfterMs) {
+                queue.loss("DATABASE_BACKOFF", null);
+                metrics.auditLoss(affectedMessages);
+                return false;
+            }
+            boolean offered = queue.offer(aggregated.size(), () -> writeAudit(aggregated, taskMessages));
+            if (!offered) metrics.auditLoss(affectedMessages);
+            return offered;
+        } catch (RuntimeException failure) {
+            queue.loss("PREPARE_FAILED", failure);
+            metrics.auditLoss(affectedMessages);
             return false;
         }
+    }
 
-        List<BacklogRetentionDelta> aggregated = aggregate(deltas);
+    private void writeAudit(List<BacklogRetentionDelta> aggregated, long affectedMessages) {
         try {
             TransactionTemplate transaction = new TransactionTemplate(transactionManager);
+            transaction.setPropagationBehavior(org.springframework.transaction.TransactionDefinition.PROPAGATION_REQUIRES_NEW);
             transaction.setTimeout(Math.max(1, properties.getAuditTimeoutSeconds()));
             transaction.execute(status -> {
                 for (BacklogRetentionDelta delta : aggregated) mapper.upsert(delta);
                 return null;
             });
             retryAfterMs = 0L;
-            metrics.record(aggregated);
-            return true;
         } catch (RuntimeException failure) {
             retryAfterMs = System.currentTimeMillis() + Math.max(1L, properties.getAuditRetryBackoffMs());
-            metrics.auditFailure(affectedMessages);
-            log.error("[BACKLOG_SAMPLING_AUDIT_FAILURE] 小时采样审计写入失败，当前批次自动回退全量保留: "
-                            + "messages={}, groups={}, retryBackoffMs={}",
-                    affectedMessages, aggregated.size(), properties.getAuditRetryBackoffMs(), failure);
-            return false;
+            queue.pause(properties.getAuditRetryBackoffMs());
+            metrics.auditWriteFailed(affectedMessages);
+            // 审计为累加UPSERT；提交结果不确定时自动重放可能双计数，当前选择记录损失而不重放。
+            throw failure;
         }
     }
 

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BacklogSamplingMetrics.java

@@ -20,6 +20,10 @@ public class BacklogSamplingMetrics {
     private final LongAdder rawDropped = new LongAdder();
     private final LongAdder auditWriteFailures = new LongAdder();
     private final LongAdder auditFallbackMessages = new LongAdder();
+    private final LongAdder auditLostMessages = new LongAdder();
+
+    public void auditLoss(long messages) { auditLostMessages.add(Math.max(0L, messages)); }
+    public void auditWriteFailed(long messages) { auditWriteFailures.increment(); auditLoss(messages); }
 
     public void record(List<BacklogRetentionDelta> deltas) {
         for (BacklogRetentionDelta delta : deltas) {
@@ -51,6 +55,8 @@ public class BacklogSamplingMetrics {
         values.put("raw_dropped_total", rawDropped.sum());
         values.put("audit_write_failures_total", auditWriteFailures.sum());
         values.put("audit_fallback_messages_total", auditFallbackMessages.sum());
+        values.put("audit_lost_or_unconfirmed_messages_total", auditLostMessages.sum());
+        values.put("counter_semantics", "SAMPLING_DECISIONS_NOT_DB_CONFIRMATIONS");
         return values;
     }
 }

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BestEffortAuxiliaryQueue.java

@@ -0,0 +1,147 @@
+package com.common.service;
+
+import lombok.extern.slf4j.Slf4j;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** 辅助写库专用有界队列。入队不执行任务、不等待数据库、不使用CallerRuns。 */
+@Slf4j
+final class BestEffortAuxiliaryQueue {
+    private final String kind;
+    private final ArrayBlockingQueue<Task> queue;
+    private final long maxGroups;
+    private final boolean retry;
+    private final AtomicLong groups = new AtomicLong();
+    private final AtomicLong accepted = new AtomicLong();
+    private final AtomicLong completed = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+    private final AtomicLong lost = new AtomicLong();
+    private final AtomicLong logAfter = new AtomicLong();
+    private volatile Task current;
+    private volatile boolean closed;
+    private volatile long resumeAt;
+    private ScheduledExecutorService executor;
+
+    BestEffortAuxiliaryQueue(String kind, int capacity, long maxGroups, boolean retry) {
+        this.kind = kind;
+        this.queue = new ArrayBlockingQueue<>(capacity);
+        this.maxGroups = maxGroups;
+        this.retry = retry;
+    }
+
+    void start() {
+        executor = Executors.newSingleThreadScheduledExecutor(r -> {
+            Thread t = new Thread(r, "syslog-aux-" + kind.toLowerCase(java.util.Locale.ROOT));
+            t.setDaemon(true);
+            return t;
+        });
+        executor.scheduleWithFixedDelay(this::drain, 0, 200, TimeUnit.MILLISECONDS);
+    }
+
+    boolean offer(int weight, Runnable work) {
+        boolean reserved = false;
+        try {
+            if (closed || weight < 1) { loss("CLOSED_OR_INVALID", null); return false; }
+            long value;
+            do {
+                value = groups.get();
+                if (weight > maxGroups - value) { loss("GROUP_LIMIT", null); return false; }
+            } while (!groups.compareAndSet(value, value + weight));
+            reserved = true;
+            if (!queue.offer(new Task(weight, work))) { loss("QUEUE_FULL", null); return false; }
+            reserved = false;
+            accepted.incrementAndGet();
+            return true;
+        } catch (RuntimeException failure) {
+            loss("ENQUEUE_FAILED", failure);
+            return false;
+        } finally {
+            if (reserved) groups.addAndGet(-weight);
+        }
+    }
+
+    // 只有后台线程/测试调用；入队和状态查询不获取该锁。
+    synchronized void drain() {
+        if (resumeAt != 0L && System.nanoTime() < resumeAt) return;
+        for (int n = 0; n < 32 && !closed; n++) {
+            if (current == null) current = queue.poll();
+            Task task = current;
+            if (task == null || (task.retryAt != 0L && System.nanoTime() < task.retryAt)) return;
+            long begin = System.nanoTime();
+            long queueMs = TimeUnit.NANOSECONDS.toMillis(begin - task.createdAt);
+            try {
+                task.work.run();
+                completed.incrementAndGet();
+                groups.addAndGet(-task.weight);
+                current = null;
+                long workMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - begin);
+                if (workMs >= 500 || queueMs >= 500) log.warn(
+                        "[SLOW_SYSLOG_AUX] kind={}, groups={}, queueAgeMs={}, executeMs={}, status=SUCCESS",
+                        kind, task.weight, queueMs, workMs);
+            } catch (RuntimeException failure) {
+                failures.incrementAndGet();
+                if (retry) {
+                    task.retryAt = System.nanoTime() + TimeUnit.SECONDS.toNanos(1);
+                } else {
+                    groups.addAndGet(-task.weight);
+                    current = null;
+                    lost.incrementAndGet();
+                }
+                if (mayLog()) log.error("[SYSLOG_AUX_FAILURE] kind={}, groups={}, queueAgeMs={}, executeMs={}, "
+                                + "action={}, failuresTotal={}", kind, task.weight, queueMs,
+                        TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - begin),
+                        retry ? "BACKGROUND_RETRY" : "DROP_AUX_ONLY", failures.get(), failure);
+                return;
+            }
+        }
+    }
+
+    void loss(String reason, RuntimeException failure) {
+        lost.incrementAndGet();
+        if (mayLog()) log.error("[SYSLOG_AUX_LOSS] kind={}, reason={}, lostTasksTotal={}, action=CONTINUE_FACT_CONSUMPTION",
+                kind, reason, lost.get(), failure);
+    }
+
+    void pause(long millis) {
+        resumeAt = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(Math.max(1L, millis));
+    }
+
+    private boolean mayLog() {
+        long now = System.nanoTime();
+        long after = logAfter.get();
+        return (after == 0 || now >= after) && logAfter.compareAndSet(after, now + TimeUnit.SECONDS.toNanos(10));
+    }
+
+    Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("mode", "BEST_EFFORT_BACKGROUND");
+        result.put("pending_tasks", queue.size() + (current == null ? 0 : 1));
+        result.put("pending_groups", groups.get());
+        result.put("max_groups", maxGroups);
+        result.put("enqueued_total", accepted.get());
+        result.put("completed_total", completed.get());
+        result.put("failures_total", failures.get());
+        result.put("lost_tasks_total", lost.get());
+        return result;
+    }
+
+    void stop() {
+        closed = true;
+        if (executor != null) executor.shutdownNow();
+        if (groups.get() > 0) log.warn("[SYSLOG_AUX_SHUTDOWN_PENDING] kind={}, pendingGroups={}, action=AUX_DATA_MAY_BE_LOST",
+                kind, groups.get());
+    }
+
+    private static final class Task {
+        final int weight;
+        final Runnable work;
+        final long createdAt = System.nanoTime();
+        long retryAt;
+        Task(int weight, Runnable work) { this.weight = weight; this.work = work; }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java

@@ -21,12 +21,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.function.Supplier;
 
 /**
- * 把当前事实事务的内存聚合结果写入同步投影或可靠异步投影入口。
+ * 把当前事实事务的内存聚合结果交给提交后的有界异步统计队列。
  *
- * <p>DIRECT 默认只追加一条批次 Outbox；SHADOW/显式回退仍同步更新统计表。调用方必须
- * 已经开启事实事务，本类不会吞掉任何数据库异常。</p>
+ * <p>DIRECT 连续消费优先，不写 Outbox 或统计表，统计异常记录后放行。
+ * 历史 SHADOW 模式仍同步写影子投影，保留原有校验语义。</p>
  */
 @Slf4j
 @Service
@@ -64,7 +65,7 @@ public class DirectStatsProjectionWriter {
     private final StatsWriteModeProvider modeProvider;
     private final DirectStatsWriteMetrics metrics;
     private final StatsMaterializationPublisher materializationPublisher;
-    private final StatsProjectionOutboxWriter projectionOutboxWriter;
+    private final StatsProjectionAsyncPublisher asyncPublisher;
 
     @Value("${stats.incremental.read-enabled:false}")
     private boolean readEnabled;
@@ -94,62 +95,51 @@ public class DirectStatsProjectionWriter {
                                        StatsWriteModeProvider modeProvider,
                                        DirectStatsWriteMetrics metrics,
                                        StatsMaterializationPublisher materializationPublisher,
-                                       StatsProjectionOutboxWriter projectionOutboxWriter) {
+                                       StatsProjectionAsyncPublisher asyncPublisher) {
         this.jdbc = jdbc;
         this.modeProvider = modeProvider;
         this.metrics = metrics;
         this.materializationPublisher = materializationPublisher;
-        this.projectionOutboxWriter = projectionOutboxWriter;
+        this.asyncPublisher = asyncPublisher;
     }
 
     public WriteResult writeStandardizedRows(List<Map<String, Object>> rows) {
-        if (!shouldBuildAggregation()) {
-            handleFactOnlyMode();
-            return WriteResult.inactive(rows == null ? 0 : rows.size());
-        }
-        long aggregateStarted = System.nanoTime();
-        DirectStatsAccumulator accumulator = accumulator().addStandardizedRows(rows);
-        return write(accumulator, elapsedMs(aggregateStarted));
+        return project(rows == null ? 0 : rows.size(), () -> accumulator().addStandardizedRows(rows), false);
     }
 
     public WriteResult writeStandardizedRow(Map<String, Object> row) {
-        if (!shouldBuildAggregation()) {
-            handleFactOnlyMode();
-            return WriteResult.inactive(row == null ? 0 : 1);
-        }
-        long aggregateStarted = System.nanoTime();
-        DirectStatsAccumulator accumulator = accumulator().addStandardizedRow(row);
-        return write(accumulator, elapsedMs(aggregateStarted));
+        return project(row == null ? 0 : 1, () -> accumulator().addStandardizedRow(row), false);
     }
 
     public WriteResult writeNonNormalRows(List<SyslogNonNormalMessage> messages) {
-        if (!shouldBuildAggregation()) {
-            handleFactOnlyMode();
-            return WriteResult.inactive(messages == null ? 0 : messages.size());
-        }
-        long aggregateStarted = System.nanoTime();
-        DirectStatsAccumulator accumulator = accumulator().addNonNormalRows(messages);
-        return write(accumulator, elapsedMs(aggregateStarted));
+        return project(messages == null ? 0 : messages.size(), () -> accumulator().addNonNormalRows(messages), false);
     }
 
     public WriteResult writeNonNormalInsert(SyslogNonNormalMessage message) {
-        if (!shouldBuildAggregation()) {
-            handleFactOnlyMode();
-            return WriteResult.inactive(message == null ? 0 : 1);
-        }
-        long aggregateStarted = System.nanoTime();
-        DirectStatsAccumulator accumulator = accumulator().addNonNormalInsert(message);
-        return write(accumulator, elapsedMs(aggregateStarted));
+        return project(message == null ? 0 : 1, () -> accumulator().addNonNormalInsert(message), false);
     }
 
     public WriteResult writeNonNormalUpdate(SyslogNonNormalMessage oldValue, SyslogNonNormalMessage patch) {
-        if (!shouldBuildAggregation()) {
-            handleFactOnlyMode();
-            return WriteResult.inactive(oldValue == null || patch == null ? 0 : 1);
+        return project(oldValue == null || patch == null ? 0 : 1,
+                () -> accumulator().addNonNormalUpdate(oldValue, patch), true);
+    }
+
+    private WriteResult project(int rows, Supplier<DirectStatsAccumulator> build, boolean exact) {
+        try {
+            if (!shouldBuildAggregation()) {
+                handleFactOnlyMode();
+                return WriteResult.inactive(rows);
+            }
+            long started = System.nanoTime();
+            DirectStatsAccumulator aggregate = build.get();
+            return exact ? writeExactDeviceGroups(aggregate, elapsedMs(started))
+                    : write(aggregate, elapsedMs(started));
+        } catch (RuntimeException failure) {
+            if (!asynchronousProjection()) throw failure;
+            metrics.recordFailure();
+            asyncPublisher.recordLoss(rows, "AGGREGATION_OR_REGISTRATION_FAILED", failure);
+            return WriteResult.inactive(rows);
         }
-        long aggregateStarted = System.nanoTime();
-        DirectStatsAccumulator accumulator = accumulator().addNonNormalUpdate(oldValue, patch);
-        return writeExactDeviceGroups(accumulator, elapsedMs(aggregateStarted));
     }
 
     public void recordTransactionRetry() {
@@ -165,7 +155,8 @@ public class DirectStatsProjectionWriter {
         status.put("business_materialization_mode", asynchronousMaterialization()
                 ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS");
         status.put("daily_projection_write_mode", asynchronousProjection()
-                ? "DURABLE_OUTBOX" : "SYNCHRONOUS_MERGE");
+                ? "BEST_EFFORT_AFTER_COMMIT_KAFKA" : "SYNCHRONOUS_MERGE");
+        status.put("async_publisher", asyncPublisher.status());
         return status;
     }
 
@@ -201,21 +192,16 @@ public class DirectStatsProjectionWriter {
         requireTransaction();
 
         if (asynchronousProjection()) {
-            try {
-                long outboxStarted = System.nanoTime();
-                projectionOutboxWriter.append(StatsProjectionOutboxEvent.fromAccumulator(accumulator));
-                long outboxMs = elapsedMs(outboxStarted);
-                int deviceGroups = effectiveDeviceEntries(accumulator).size();
-                int ruleGroups = effectiveRuleEntries(accumulator).size();
-                metrics.recordSuccess(accumulator.getInputRows(), deviceGroups, ruleGroups,
-                        aggregateMs, outboxMs, 0L);
-                return WriteResult.direct(accumulator.getInputRows(), deviceGroups, ruleGroups,
-                        aggregateMs, outboxMs, 0L);
-            } catch (RuntimeException failure) {
-                metrics.recordFailure();
-                throw new DirectStatsWriteException(
-                        "统计投影 Outbox 写入失败，事实事务必须整体回滚", failure);
-            }
+            long enqueueStarted = System.nanoTime();
+            asyncPublisher.enqueueAfterCommit(StatsProjectionOutboxEvent.fromAccumulator(accumulator),
+                    accumulator.getInputRows());
+            long enqueueMs = elapsedMs(enqueueStarted);
+            int deviceGroups = effectiveDeviceEntries(accumulator).size();
+            int ruleGroups = effectiveRuleEntries(accumulator).size();
+            metrics.recordSuccess(accumulator.getInputRows(), deviceGroups, ruleGroups,
+                    aggregateMs, enqueueMs, 0L);
+            return WriteResult.direct(accumulator.getInputRows(), deviceGroups, ruleGroups,
+                    aggregateMs, enqueueMs, 0L);
         }
 
         List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> devices =
@@ -438,8 +424,8 @@ public class DirectStatsProjectionWriter {
     }
 
     private boolean asynchronousProjection() {
-        return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT
-                && projectionOutboxWriter != null && projectionOutboxWriter.isEnabled();
+        // 关闭统计发布也绝不回退同步统计SQL，以免再次阻塞事实消费。
+        return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT;
     }
 
     private List<StatsMaterializationCommand> materializationCommands(

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java

@@ -36,6 +36,9 @@ public class DmStandardizedBatchExecutor {
     @org.springframework.beans.factory.annotation.Value("${stats.direct.transaction-log-enabled:false}")
     private boolean transactionLogEnabled;
 
+    @org.springframework.beans.factory.annotation.Value("${syslog.slow-batch.threshold-ms:500}")
+    private long slowTransactionThresholdMs;
+
     @org.springframework.beans.factory.annotation.Value("${stats.direct.max-concurrency-retries:1}")
     private int maxConcurrencyRetries;
 
@@ -96,11 +99,14 @@ public class DmStandardizedBatchExecutor {
         transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
         long transactionStarted = System.nanoTime();
         final TransactionTiming timing = new TransactionTiming();
+        try {
         transaction.execute(status -> {
+            timing.beginMs = elapsedMs(transactionStarted);
             TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() {
                 @Override
                 public void beforeCommit(boolean readOnly) {
                     timing.commitStarted = System.nanoTime();
+                    timing.phase("FACT_COMMIT");
                 }
 
                 @Override
@@ -110,31 +116,56 @@ public class DmStandardizedBatchExecutor {
             });
             long flushStarted = System.nanoTime();
             // 事实表使用一个固定 PreparedStatement 直接 addBatch，彻底绕开逐行 OGNL 动态 SQL。
+            timing.phase("JDBC_FACT_WRITE");
             timing.batchExecution = preparedBatchInserter.insert(
                     alarmTable, rows, transactionTimeoutSeconds);
 
             // 兼容期变更日志仍与事实行共用当前事务；默认关闭时完全不创建 MyBatis BatchSession。
             if (statsChangeCaptureService.isEnabled()) {
+                timing.phase("LEGACY_STATS_CAPTURE");
                 statsChangeCaptureService.captureStandardizedBatch(batchSqlSession, rows, alarmTable);
                 batchSqlSession.flushStatements();
             }
             timing.factFlushMs = elapsedMs(flushStarted);
 
-            // 与标准化告警事实共用同一事务写入事件时间桶。事实成功而通知失败时必须整体回滚，
-            // 否则迟到数据可能永久失去重放机会。
+            // 只注册提交后入队；分钟桶在独立后台事务写入，失败不回滚当前告警。
             if (alarmTable) {
+                timing.phase("LATE_BUCKET_REGISTER");
+                long bucketStarted = System.nanoTime();
                 lateAlarmBucketWriter.record(rows);
+                timing.lateBucketMs = elapsedMs(bucketStarted);
             }
 
-            // DIRECT 异步投影模式只在当前事实事务追加一条聚合 Outbox；独立 Kafka consumer
-            // 再更新日统计。SHADOW/回退模式仍保留原同步 MERGE。
+            // DIRECT 只聚合并注册afterCommit入队；此处不写Outbox/统计表、不发送Kafka。
+            // 独立后台线程发送统计事件，失败不影响事实提交；历史SHADOW模式保留同步MERGE。
+            timing.phase("STATS_REGISTER");
             timing.statsResult = directStatsProjectionWriter.writeStandardizedRows(rows);
             return null;
         });
+        } catch (RuntimeException failure) {
+            log.error("[SLOW_DM_TRANSACTION] batchId={}, table={}, factRows={}, status=FAILED, "
+                            + "failureStage={}, phaseElapsedMs={}, transactionMs={}, connectionBeginMs={}, "
+                            + "jdbcExecuteMs={}, lateAlarmBucketMs={}, retry={}, errorType={}",
+                    org.slf4j.MDC.get("syslogBatchId"), alarmTable ? "syslog_normal_alarm" : "syslog_normal_data",
+                    rows.size(), timing.phase, elapsedMs(timing.phaseStarted), elapsedMs(transactionStarted),
+                    timing.beginMs, timing.batchExecution.getExecuteMs(), timing.lateBucketMs, retries,
+                    failure.getClass().getName());
+            throw failure;
+        }
         long transactionMs = elapsedMs(transactionStarted);
         transactionMetrics.recordSuccess(rows.size(), timing.batchExecution.getPlanMs(),
                 timing.batchExecution.getBindMs(), timing.batchExecution.getExecuteMs(),
                 timing.factFlushMs, timing.commitMs, transactionMs);
+        if (slowTransactionThresholdMs > 0 && transactionMs >= slowTransactionThresholdMs) {
+            log.warn("[SLOW_DM_TRANSACTION] batchId={}, table={}, factRows={}, status=SUCCESS, connectionBeginMs={}, jdbcPlanMs={}, "
+                            + "jdbcBindMs={}, jdbcExecuteMs={}, lateAlarmBucketMs={}, statsProjectionStageMs={}, "
+                            + "commitMs={}, transactionMs={}, retry={}",
+                    org.slf4j.MDC.get("syslogBatchId"), alarmTable ? "syslog_normal_alarm" : "syslog_normal_data", rows.size(), timing.beginMs,
+                    timing.batchExecution.getPlanMs(), timing.batchExecution.getBindMs(),
+                    timing.batchExecution.getExecuteMs(), timing.lateBucketMs,
+                    timing.statsResult == null ? 0L : timing.statsResult.getMergeMs(),
+                    timing.commitMs, transactionMs, retries);
+        }
         if (transactionLogEnabled && timing.statsResult != null && timing.statsResult.isDirect()) {
             log.info("DIRECT 达梦事务完成: factRows={}, deviceGroups={}, ruleGroups={}, aggregateMs={}, " +
                             "jdbcPlanMs={}, jdbcBindMs={}, jdbcExecuteMs={}, factFlushMs={}, "
@@ -148,7 +179,9 @@ public class DmStandardizedBatchExecutor {
     }
 
     public Map<String, Object> status() {
-        return transactionMetrics.snapshot();
+        Map<String, Object> result = transactionMetrics.snapshot();
+        result.put("late_alarm_bucket_async", lateAlarmBucketWriter.status());
+        return result;
     }
 
     private static long elapsedMs(long startedNanos) {
@@ -156,6 +189,11 @@ public class DmStandardizedBatchExecutor {
     }
 
     private static final class TransactionTiming {
+        private String phase = "CONNECTION_BEGIN";
+        private long phaseStarted = System.nanoTime();
+        private void phase(String value) { phase = value; phaseStarted = System.nanoTime(); }
+        private long beginMs;
+        private long lateBucketMs;
         private long factFlushMs;
         private long commitStarted;
         private long commitMs;

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java

@@ -69,6 +69,9 @@ public class DmStandardizedBatchWriter {
     @Value("${dm.standardized-write.batch-size:500}")
     private int batchSize;
 
+    @Value("${syslog.slow-batch.threshold-ms:500}")
+    private long slowQueueThresholdMs;
+
     @Value("${dm.standardized-write.transaction-timeout-seconds:45}")
     private int transactionTimeoutSeconds;
 
@@ -190,10 +193,25 @@ public class DmStandardizedBatchWriter {
         List<CompletableFuture<WriteReport<T>>> futures = new ArrayList<>(groups.size());
         try {
             for (Map.Entry<BatchKey, List<WriteRequest<T>>> entry : groups.entrySet()) {
+                long queuedAt = System.nanoTime();
+                String batchId = org.slf4j.MDC.get("syslogBatchId");
                 futures.add(CompletableFuture.supplyAsync(() -> {
+                    String previous = org.slf4j.MDC.get("syslogBatchId");
+                    if (batchId != null) org.slf4j.MDC.put("syslogBatchId", batchId);
+                    else org.slf4j.MDC.remove("syslogBatchId");
+                    try {
+                    long queueMs = (System.nanoTime() - queuedAt) / 1_000_000L;
+                    if (slowQueueThresholdMs > 0 && queueMs >= slowQueueThresholdMs) {
+                        log.warn("[SLOW_DM_QUEUE] batchId={}, alarmTable={}, rows={}, queueMs={}",
+                                batchId, entry.getKey().alarmTable, entry.getValue().size(), queueMs);
+                    }
                     WriteReport<T> groupReport = new WriteReport<>();
                     writeGroup(entry.getKey(), entry.getValue(), groupReport, budget);
                     return groupReport;
+                    } finally {
+                        if (previous == null) org.slf4j.MDC.remove("syslogBatchId");
+                        else org.slf4j.MDC.put("syslogBatchId", previous);
+                    }
                 }, writerExecutor));
             }
 

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsConsumer.java

@@ -0,0 +1,140 @@
+package com.common.service;
+
+import com.Modules.NormalData.SyslogFailureRouter;
+import com.common.entity.IngressStatsEvent;
+import com.common.util.Sm4Util;
+import com.common.util.SyslogParser;
+import com.config.AppProperties;
+import com.influx.raw.ReceiveTimeParser;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.kafka.annotation.KafkaListener;
+import org.springframework.kafka.support.Acknowledgment;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** 独立消费原始Topic，只维护接收时间；不执行规则、采样或事实表写入。 */
+@Component
+@Slf4j
+public class IngressStatsConsumer {
+    private final IngressStatsProjectionWriter writer;
+    private final SyslogFailureRouter failureRouter;
+    private final AppProperties appProperties;
+    private final AtomicLong batches = new AtomicLong();
+    private final AtomicLong invalidAttempts = new AtomicLong();
+    private final AtomicLong writeFailures = new AtomicLong();
+
+    @Value("${spring.kafka.consumer.group-id}") private String rawGroup;
+    @Value("${syslog.large-listener.group-id:${spring.kafka.consumer.group-id}-large}")
+    private String largeRawGroup;
+    @Value("${stats.ingress-metadata.group-id:${spring.kafka.consumer.group-id}-ingress-v1}")
+    private String group;
+    @Value("${stats.ingress-metadata.large-group-id:${stats.ingress-metadata.group-id}-large}")
+    private String largeGroup;
+
+    public IngressStatsConsumer(IngressStatsProjectionWriter writer,
+                                SyslogFailureRouter failureRouter, AppProperties appProperties) {
+        this.writer = writer;
+        this.failureRouter = failureRouter;
+        this.appProperties = appProperties;
+    }
+
+    @PostConstruct
+    public void validateGroups() {
+        if (group == null || group.trim().isEmpty() || largeGroup == null || largeGroup.trim().isEmpty()
+                || group.equals(largeGroup) || group.equals(rawGroup) || group.equals(largeRawGroup)
+                || largeGroup.equals(rawGroup) || largeGroup.equals(largeRawGroup)) {
+            throw new IllegalStateException("入口统计必须使用与主消费隔离的非空 Kafka group-id");
+        }
+        log.info("Ingress stats consumer ready: group={}, largeGroup={}, mode=INDEPENDENT_KAFKA_DIRECT_MERGE",
+                group, largeGroup);
+    }
+
+    @KafkaListener(id = "syslog-ingress-stats", groupId = "${stats.ingress-metadata.group-id}",
+            topics = "${spring.kafka.consumer.topic}",
+            autoStartup = "${stats.ingress-metadata.enabled:true}",
+            containerFactory = "ingressStatsKafkaListenerContainerFactory")
+    public void consume(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
+        consumeBatch(records, ack, "RAW");
+    }
+
+    @KafkaListener(id = "syslog-ingress-stats-large", groupId = "${stats.ingress-metadata.large-group-id}",
+            topics = "${spring.kafka.consumer.large-topic}",
+            autoStartup = "#{${stats.ingress-metadata.enabled:true} && ${syslog.large-listener.enabled:true}}",
+            containerFactory = "largeIngressStatsKafkaListenerContainerFactory")
+    public void consumeLarge(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
+        consumeBatch(records, ack, "RAW_LARGE");
+    }
+
+    private void consumeBatch(List<ConsumerRecord<String, String>> records, Acknowledgment ack, String lane) {
+        long started = System.nanoTime();
+        List<IngressStatsEvent> events = new ArrayList<>(records.size());
+        int invalid = 0;
+        String firstInvalid = null;
+        for (ConsumerRecord<String, String> record : records) {
+            try {
+                events.add(decode(record));
+            } catch (RuntimeException malformed) {
+                invalid++;
+                if (firstInvalid == null) firstInvalid = record.topic() + "-" + record.partition()
+                        + "@" + record.offset() + ":" + malformed.getClass().getSimpleName();
+            }
+        }
+        if (invalid > 0) {
+            invalidAttempts.addAndGet(invalid);
+            // 原始消息仍留在Kafka；坏头不会永久挡住其它设备接收状态，也不向业务Retry重复投递。
+            log.error("[INGRESS_STATS_INVALID] lane={}, invalidRecords={}, firstInvalid={}, "
+                    + "action=SKIP_INVALID_METADATA", lane, invalid, firstInvalid);
+        }
+        int groups;
+        try {
+            groups = writer.writeBatch(events);
+        } catch (RuntimeException failure) {
+            writeFailures.incrementAndGet();
+            log.error("[INGRESS_STATS_WRITE_FAILED] lane={}, records={}, action=RETRY_STATS_GROUP",
+                    lane, records.size(), failure);
+            throw failure;
+        }
+        // MIN/MAX更新和达梦提交均已完成，才推进独立统计组；重复消费不重复增加事实数量。
+        ack.acknowledge();
+        batches.incrementAndGet();
+        log.info("[INGRESS_STATS_BATCH] lane={}, records={}, valid={}, invalid={}, groups={}, totalMs={}",
+                lane, records.size(), events.size(), invalid, groups, (System.nanoTime() - started) / 1_000_000L);
+    }
+
+    IngressStatsEvent decode(ConsumerRecord<String, String> record) {
+        SyslogFailureRouter.MessageContext context = failureRouter.resolve(record);
+        String decrypted = Sm4Util.decryptCbc(context.getPayload(), appProperties.getSm4Key());
+        if (decrypted.indexOf(']') < 0) throw new IllegalArgumentException("缺少设备元数据头");
+        Map<String, String> header = SyslogParser.parseKeyValuePairs(
+                SyslogParser.substringBeforeFirstChar(decrypted, ']'));
+        Long device = positiveId(header.get("device_id"));
+        Long collector = positiveId(header.get("device_collect_id"));
+        if (device == null && collector == null) throw new IllegalArgumentException("缺少有效设备/采集任务ID");
+        return new IngressStatsEvent(context.getEventId(), device, collector,
+                ReceiveTimeParser.parse(header.get("receive_time")).getLocalDateTime());
+    }
+
+    private static Long positiveId(String value) {
+        if (value == null || value.trim().isEmpty()) return null;
+        long parsed = Long.parseLong(value);
+        return parsed > 0L ? parsed : null;
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("group_id", group);
+        result.put("large_group_id", largeGroup);
+        result.put("successful_batches", batches.get());
+        result.put("invalid_record_attempts", invalidAttempts.get());
+        result.put("write_failures", writeFailures.get());
+        return result;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java

@@ -29,7 +29,6 @@ public class IngressStatsProjectionWriter {
     private final PlatformTransactionManager transactionManager;
     private final StatsWriteModeProvider modeProvider;
     private final StatsMaterializationPublisher materializationPublisher;
-    private final StatsProjectionOutboxWriter projectionOutboxWriter;
 
     @Value("${stats.ingress-metadata.enabled:false}")
     private boolean enabled;
@@ -51,13 +50,11 @@ public class IngressStatsProjectionWriter {
     public IngressStatsProjectionWriter(JdbcTemplate jdbc,
                                         PlatformTransactionManager transactionManager,
                                         StatsWriteModeProvider modeProvider,
-                                        StatsMaterializationPublisher materializationPublisher,
-                                        StatsProjectionOutboxWriter projectionOutboxWriter) {
+                                        StatsMaterializationPublisher materializationPublisher) {
         this.jdbc = jdbc;
         this.transactionManager = transactionManager;
         this.modeProvider = modeProvider;
         this.materializationPublisher = materializationPublisher;
-        this.projectionOutboxWriter = projectionOutboxWriter;
     }
 
     public int writeBatch(List<IngressStatsEvent> events) {
@@ -95,8 +92,7 @@ public class IngressStatsProjectionWriter {
                 ? "device_daily_stats+" + shadowDeviceTable : "device_daily_stats");
         status.put("business_materialization_mode", asynchronousMaterialization()
                 ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS");
-        status.put("daily_projection_write_mode", asynchronousProjection()
-                ? "DURABLE_OUTBOX" : "SYNCHRONOUS_MERGE");
+        status.put("daily_projection_write_mode", "INDEPENDENT_KAFKA_DIRECT_MERGE");
         status.put("successful_batches", successfulBatches.get());
         status.put("received_events", receivedEvents.get());
         status.put("updated_groups", updatedGroups.get());
@@ -114,10 +110,6 @@ public class IngressStatsProjectionWriter {
         TransactionTemplate transaction = new TransactionTemplate(transactionManager);
         transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
         Integer updated = transaction.execute(status -> {
-            if (asynchronousProjection()) {
-                projectionOutboxWriter.append(StatsProjectionOutboxEvent.fromIngress(aggregation));
-                return aggregation.devices.size() + aggregation.collects.size();
-            }
             int groups = 0;
             boolean asyncMaterialization = asynchronousMaterialization();
             List<StatsMaterializationCommand> commands = asyncMaterialization
@@ -246,11 +238,6 @@ public class IngressStatsProjectionWriter {
                 && materializationPublisher.isEnabled();
     }
 
-    private boolean asynchronousProjection() {
-        return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT
-                && projectionOutboxWriter != null && projectionOutboxWriter.isEnabled();
-    }
-
     private static String safeTable(String table) {
         if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
             throw new IllegalStateException("非法入口设备统计表名: " + table);

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/LateAlarmBucketWriter.java

@@ -3,7 +3,8 @@ package com.common.service;
 import java.sql.Timestamp;
 import java.time.LocalDateTime;
 import java.time.temporal.ChronoUnit;
-import java.util.LinkedHashSet;
+import java.util.TreeSet;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -12,11 +13,15 @@ import java.util.UUID;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionSynchronization;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+import org.springframework.transaction.support.TransactionTemplate;
 
 /**
- * 在标准化告警事实事务内登记事件时间桶。
+ * 告警提交后登记事件时间桶，后台独立事务失败只重试通知，不回滚告警。
  *
- * <p>该通知不是“告警一定迟到”的结论。通知与事实在同一事务提交；下游能领取通知时，事实必然
+ * <p>该通知不是“告警一定迟到”的结论。事实提交后才入队；下游能领取通知时，事实必然
  * 已经可见，再根据领取时已经完成的规则窗口决定是否重放。这里刻意不使用事实的
  * {@code created_at} 判断提交先后，因为 Kafka 积压时接收时间可能早于实际入库时间。
  */
@@ -38,12 +43,22 @@ public class LateAlarmBucketWriter {
           + "VALUES (?,?,?,?,SYSDATE,1,0,0,'PENDING',0,0,SYSDATE,SYSDATE)";
 
   private final JdbcTemplate jdbcTemplate;
+  private final PlatformTransactionManager transactionManager;
+  private final BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("LATE_ALARM_BUCKET", 256, 50000, true);
+
+  @Value("${analysis.late-replay.notify-timeout-seconds:5}")
+  private int transactionTimeoutSeconds = 5;
+
+  @javax.annotation.PostConstruct public void start() { queue.start(); }
+  @javax.annotation.PreDestroy public void stop() { queue.stop(); }
+  public Map<String, Object> status() { return queue.status(); }
 
   @Value("${analysis.late-replay.outbox-enabled:true}")
   private boolean enabled;
 
-  public LateAlarmBucketWriter(JdbcTemplate jdbcTemplate) {
+  public LateAlarmBucketWriter(JdbcTemplate jdbcTemplate, PlatformTransactionManager transactionManager) {
     this.jdbcTemplate = jdbcTemplate;
+    this.transactionManager = transactionManager;
   }
 
   /**
@@ -56,7 +71,27 @@ public class LateAlarmBucketWriter {
       return;
     }
 
-    Set<BucketKey> buckets = new LinkedHashSet<>();
+    try {
+      // 只保留设备/分钟的小键，不把原始日志和事实行滞留在后台队列。
+      Set<BucketKey> buckets = buckets(rows);
+      if (!TransactionSynchronizationManager.isActualTransactionActive()
+          || !TransactionSynchronizationManager.isSynchronizationActive()) {
+        throw new IllegalStateException("迟到告警通知必须在事实提交后入队");
+      }
+      TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
+        @Override public void afterCommit() {
+          queue.offer(buckets.size(), () -> writeBuckets(buckets));
+        }
+      });
+    } catch (RuntimeException failure) {
+      queue.loss("PREPARE_OR_REGISTER_FAILED", failure);
+    }
+  }
+
+  private Set<BucketKey> buckets(List<Map<String, Object>> rows) {
+    // 不同Kafka批次可能含有相反顺序的设备/分钟，统一加锁顺序避免交叉等待。
+    Set<BucketKey> buckets = new TreeSet<>(Comparator
+        .comparing((BucketKey key) -> key.deviceId).thenComparing(key -> key.bucketStart));
     for (Map<String, Object> row : rows) {
       Integer deviceId = integerValue(row.get("device_id"));
       LocalDateTime eventTime = timeValue(row.get("log_time"));
@@ -65,7 +100,14 @@ public class LateAlarmBucketWriter {
       }
       buckets.add(new BucketKey(deviceId, eventTime.truncatedTo(ChronoUnit.MINUTES)));
     }
+    return buckets;
+  }
 
+  private void writeBuckets(Set<BucketKey> buckets) {
+    TransactionTemplate transaction = new TransactionTemplate(transactionManager);
+    transaction.setPropagationBehavior(org.springframework.transaction.TransactionDefinition.PROPAGATION_REQUIRES_NEW);
+    transaction.setTimeout(Math.max(1, transactionTimeoutSeconds));
+    transaction.execute(status -> {
     for (BucketKey key : buckets) {
       LocalDateTime bucketEnd = key.bucketStart.plusMinutes(1);
       try {
@@ -79,14 +121,16 @@ public class LateAlarmBucketWriter {
             Timestamp.valueOf(bucketEnd));
       } catch (RuntimeException failure) {
         // 多个 Kafka 分区可能同时首次写同一设备分钟，MERGE 的并发插入会竞争唯一键。
-        // 交给外层事实事务整体回滚并有限重试，不能只吞掉 outbox 失败。
+        // 仅后台通知事务回滚重试，绝不返回事实事务。
         if (isConcurrentWrite(failure)) {
           throw new DirectStatsProjectionWriter.ConcurrentStatsWriteException(
-              "迟到告警分钟桶并发写入失败，可安全回滚后有限重试", failure);
+              "迟到告警分钟桶并发写入失败，仅后台通知事务回滚重试", failure);
         }
         throw failure;
       }
     }
+      return null;
+    });
   }
 
   private Integer integerValue(Object value) {

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionAsyncPublisher.java

@@ -0,0 +1,217 @@
+package com.common.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
+import org.springframework.kafka.core.DefaultKafkaProducerFactory;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.support.TransactionSynchronization;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** 连续消费优先：事实提交后只做有界内存offer，Kafka发送与重试在独立线程执行。 */
+@Slf4j
+@Component
+public class StatsProjectionAsyncPublisher {
+    private final KafkaTemplate<String, String> kafka;
+    private final ObjectMapper mapper;
+    private final ArrayBlockingQueue<Pending> queue;
+    private final long maxPendingGroups;
+    private final AtomicLong pendingGroups = new AtomicLong();
+    private final AtomicLong enqueued = new AtomicLong();
+    private final AtomicLong published = new AtomicLong();
+    private final AtomicLong failures = new AtomicLong();
+    private final AtomicLong droppedEvents = new AtomicLong();
+    private final AtomicLong droppedFactRows = new AtomicLong();
+    private final AtomicLong nextLossLogAt = new AtomicLong();
+    private volatile boolean closed;
+    private volatile Pending inFlight;
+    private ScheduledExecutorService executor;
+    private DefaultKafkaProducerFactory<String, String> ownedProducerFactory;
+
+    // 与旧统计路径使用相同的 mapper；显式区分 WebConfig 的 httpObjectMapper。
+    // 下方构造器复制后关闭类型信息，不修改共享缓存 mapper。
+    @Autowired
+    public StatsProjectionAsyncPublisher(KafkaProperties properties,
+            @Qualifier("objectMapper") ObjectMapper mapper,
+            @Value("${stats.projection-async.memory-queue-capacity:1000}") int capacity,
+            @Value("${stats.projection-async.memory-max-pending-groups:50000}") long maxPendingGroups) {
+        this(dedicatedFactory(properties), mapper, capacity, maxPendingGroups);
+    }
+
+    private StatsProjectionAsyncPublisher(DefaultKafkaProducerFactory<String, String> factory,
+            ObjectMapper mapper, int capacity, long maxPendingGroups) {
+        this(new KafkaTemplate<>(factory), mapper, capacity, maxPendingGroups);
+        this.ownedProducerFactory = factory;
+    }
+
+    private static DefaultKafkaProducerFactory<String, String> dedicatedFactory(KafkaProperties properties) {
+        Map<String, Object> config = new LinkedHashMap<>(properties.buildProducerProperties());
+        config.put(ProducerConfig.CLIENT_ID_CONFIG, "syslog-stats-projection-memory");
+        // 不与主业务Retry/DLT/告警发送共用KafkaProducer缓冲区，统计断链只影响后台发送。
+        return new DefaultKafkaProducerFactory<>(config);
+    }
+
+    @Value("${stats.projection-async.enabled:true}") private boolean enabled;
+    @Value("${stats.projection-async.topic:${spring.kafka.consumer.topic}.STATS.PROJECTION}") private String topic;
+    @Value("${stats.projection-async.publish-timeout-ms:10000}") private long timeoutMs;
+    @Value("${stats.projection-async.retry-backoff-ms:1000}") private long retryBackoffMs;
+
+    StatsProjectionAsyncPublisher(KafkaTemplate<String, String> kafka, ObjectMapper mapper,
+            int capacity, long maxPendingGroups) {
+        if (capacity < 1 || maxPendingGroups < 1) throw new IllegalArgumentException("统计内存队列容量必须大于0");
+        this.kafka = kafka;
+        this.mapper = mapper.copy();
+        this.mapper.deactivateDefaultTyping();
+        this.queue = new ArrayBlockingQueue<>(capacity);
+        this.maxPendingGroups = maxPendingGroups;
+    }
+
+    @PostConstruct
+    public void start() {
+        executor = Executors.newSingleThreadScheduledExecutor(task -> {
+            Thread thread = new Thread(task, "stats-projection-publisher");
+            thread.setDaemon(true);
+            return thread;
+        });
+        executor.scheduleWithFixedDelay(this::flush, 0, 200, TimeUnit.MILLISECONDS);
+        log.info("Stats projection publisher ready: mode=BEST_EFFORT_AFTER_COMMIT_KAFKA, enabled={}, "
+                        + "queueCapacity={}, maxPendingGroups={}, topic={}",
+                enabled, queue.remainingCapacity(), maxPendingGroups, topic);
+    }
+
+    public void enqueueAfterCommit(StatsProjectionOutboxEvent event, int factRows) {
+        try {
+            event.validate();
+            event.setOutboxBacked(false);
+            if (!TransactionSynchronizationManager.isActualTransactionActive()
+                    || !TransactionSynchronizationManager.isSynchronizationActive()) {
+                throw new IllegalStateException("统计事件必须在事实事务提交后入队");
+            }
+            TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
+                @Override public void afterCommit() {
+                    // 不做序列化、网络发送或数据库写入，任何运行时失败均不得影响已提交事实。
+                    offer(event, factRows);
+                }
+            });
+        } catch (RuntimeException failure) {
+            recordLoss(factRows, "PREPARE_OR_REGISTER_FAILED", failure);
+        }
+    }
+
+    private void offer(StatsProjectionOutboxEvent event, int factRows) {
+        long groups = (long) event.getDevices().size() + event.getRules().size() + event.getCollects().size();
+        boolean reserved = false;
+        try {
+            if (!enabled || closed) {
+                recordLoss(factRows, closed ? "SHUTTING_DOWN" : "PROJECTION_DISABLED", null);
+                return;
+            }
+            long current;
+            do {
+                current = pendingGroups.get();
+                if (groups > maxPendingGroups - current) {
+                    recordLoss(factRows, "GROUP_CAPACITY", null);
+                    return;
+                }
+            } while (!pendingGroups.compareAndSet(current, current + groups));
+            reserved = true;
+            if (queue.offer(new Pending(event, groups))) {
+                reserved = false; // 预算归队列/发送线程所有，发送成功后释放。
+                enqueued.incrementAndGet();
+            } else {
+                recordLoss(factRows, "QUEUE_FULL", null);
+            }
+        } catch (RuntimeException failure) {
+            recordLoss(factRows, "ENQUEUE_FAILED", failure);
+        } finally {
+            if (reserved) pendingGroups.addAndGet(-groups);
+        }
+    }
+
+    /** 独立线程每轮最多发送100个聚合事件；失败保留原eventId，下游可去重。 */
+    synchronized void flush() {
+        if (!enabled || closed) return;
+        for (int i = 0; i < 100 && !closed; i++) {
+            if (inFlight == null) inFlight = queue.poll();
+            Pending pending = inFlight;
+            if (pending == null || pending.retryAt > System.currentTimeMillis()) return;
+            try {
+                if (pending.payload == null) pending.payload = mapper.writeValueAsString(pending.event);
+                kafka.send(topic, StatsProjectionOutboxEvent.KAFKA_KEY, pending.payload)
+                        .get(Math.max(1L, timeoutMs), TimeUnit.MILLISECONDS);
+                published.incrementAndGet();
+                pendingGroups.addAndGet(-pending.groups);
+                inFlight = null;
+            } catch (Exception failure) {
+                failures.incrementAndGet();
+                pending.retryAt = System.currentTimeMillis() + Math.max(1000L, retryBackoffMs);
+                log.warn("[STATS_ASYNC_PUBLISH_FAILED] eventId={}, pendingEvents={}, action=BACKGROUND_RETRY, errorType={}",
+                        pending.event.getEventId(), queue.size() + 1, failure.getClass().getName());
+                if (failure instanceof InterruptedException) Thread.currentThread().interrupt();
+                return;
+            }
+        }
+    }
+
+    public void recordLoss(int factRows, String reason, RuntimeException failure) {
+        droppedEvents.incrementAndGet();
+        droppedFactRows.addAndGet(Math.max(0, factRows));
+        long now = System.currentTimeMillis();
+        long next = nextLossLogAt.get();
+        if (now >= next && nextLossLogAt.compareAndSet(next, now + 10000L)) {
+            log.error("[STATS_ASYNC_LOSS] reason={}, droppedEventsTotal={}, droppedFactRowsTotal={}, "
+                            + "action=CONTINUE_FACT_CONSUMPTION, errorType={}", reason, droppedEvents.get(),
+                    droppedFactRows.get(), failure == null ? "none" : failure.getClass().getName());
+        }
+    }
+
+    public Map<String, Object> status() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("mode", "BEST_EFFORT_AFTER_COMMIT_KAFKA");
+        result.put("enabled", enabled);
+        result.put("topic", topic);
+        result.put("pending_events", queue.size() + (inFlight == null ? 0 : 1));
+        result.put("queue_capacity", queue.size() + queue.remainingCapacity());
+        result.put("pending_groups", pendingGroups.get());
+        result.put("max_pending_groups", maxPendingGroups);
+        result.put("enqueued_total", enqueued.get());
+        result.put("published_total", published.get());
+        result.put("publish_failures_total", failures.get());
+        result.put("dropped_events_total", droppedEvents.get());
+        result.put("dropped_fact_rows_total", droppedFactRows.get());
+        return result;
+    }
+
+    @PreDestroy
+    public void stop() {
+        closed = true;
+        if (executor != null) executor.shutdownNow();
+        int pending = queue.size() + (inFlight == null ? 0 : 1);
+        if (pending > 0) log.warn("[STATS_ASYNC_SHUTDOWN_PENDING] pendingEvents={}, "
+                + "pendingGroups={}, action=STATISTICS_MAY_BE_LOST", pending, pendingGroups.get());
+        if (ownedProducerFactory != null) ownedProducerFactory.destroy();
+    }
+
+    private static final class Pending {
+        final StatsProjectionOutboxEvent event;
+        final long groups;
+        String payload;
+        long retryAt;
+        Pending(StatsProjectionOutboxEvent event, long groups) { this.event = event; this.groups = groups; }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionDeltaConsumer.java

@@ -211,11 +211,11 @@ public class StatsProjectionDeltaConsumer {
                 jdbc.batchUpdate("INSERT INTO " + StatsProjectionOutboxWriter.safeTable(appliedTable) +
                         " (event_id,event_schema,applied_at) VALUES (?,?,SYSDATE)", markers);
             }
-            if (!events.isEmpty()) {
-                List<Object[]> acknowledged = new ArrayList<>(events.size());
-                for (StatsProjectionOutboxEvent event : events) {
-                    acknowledged.add(new Object[]{event.getEventId()});
-                }
+            List<Object[]> acknowledged = new ArrayList<>();
+            for (StatsProjectionOutboxEvent event : events) {
+                if (event.isOutboxBacked()) acknowledged.add(new Object[]{event.getEventId()});
+            }
+            if (!acknowledged.isEmpty()) {
                 jdbc.batchUpdate("UPDATE " + StatsProjectionOutboxWriter.safeTable(outboxTable) +
                         " SET status='APPLIED',applied_at=SYSDATE,next_attempt_at=NULL," +
                         "updated_at=SYSDATE WHERE event_id=?", acknowledged);

haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxEvent.java

@@ -10,7 +10,7 @@ import java.util.Map;
 import java.util.UUID;
 
 /**
- * A durable, batch-aggregated statistics delta written beside the fact rows.
+ * Batch-aggregated delta shared by the after-commit memory publisher and legacy durable Outbox.
  *
  * <p>The event is additive/max-min only, so events from different application nodes may be
  * published in a different order without changing the final projection. Kafka duplicates are
@@ -23,6 +23,11 @@ public class StatsProjectionOutboxEvent {
 
     private String schema = SCHEMA;
     private String eventId = UUID.randomUUID().toString();
+    // 老版消息没有此字段，默认true，兼容清理升级前的Outbox。
+    private boolean outboxBacked = true;
+
+    public boolean isOutboxBacked() { return outboxBacked; }
+    public void setOutboxBacked(boolean outboxBacked) { this.outboxBacked = outboxBacked; }
     private List<DeviceDelta> devices = new ArrayList<>();
     private List<RuleDelta> rules = new ArrayList<>();
     private List<CollectDelta> collects = new ArrayList<>();

haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/KafkaReliabilityConfig.java

@@ -15,9 +15,11 @@ import org.springframework.util.backoff.FixedBackOff;
 
 import java.util.HashMap;
 import java.util.Map;
+import lombok.extern.slf4j.Slf4j;
 
 /** Kafka 消费失败边界：业务未确认的消息绝不能被框架默认恢复器静默跳过。 */
 @Configuration
+@Slf4j
 public class KafkaReliabilityConfig {
 
     @Bean
@@ -50,16 +52,26 @@ public class KafkaReliabilityConfig {
             ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
             ConsumerFactory<Object, Object> consumerFactory,
             DefaultErrorHandler syslogDefaultErrorHandler,
-            @Value("${syslog.raw-listener.concurrency:3}") int concurrency) {
+            @Value("${syslog.raw-listener.concurrency:3}") int concurrency,
+            @Value("${syslog.raw-listener.max-poll-records:500}") int baseRecords,
+            @Value("${syslog.raw-listener.adaptive-max-records:3000}") int maxPollRecords) {
         if (concurrency < 1) {
             throw new IllegalArgumentException("syslog.raw-listener.concurrency 必须大于 0");
         }
-        ConcurrentKafkaListenerContainerFactory<Object, Object> factory =
-                configuredFactory(configurer, consumerFactory, syslogDefaultErrorHandler);
+        new com.Modules.NormalData.AdaptiveRawBatchSizer(baseRecords, maxPollRecords);
+        Map<String, Object> properties = new HashMap<>(consumerFactory.getConfigurationProperties());
+        properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
+        // 只在所有业务子块完成后手动提交本次poll，禁止自动提交尚未处理的尾部记录。
+        properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
+        ConcurrentKafkaListenerContainerFactory<Object, Object> factory = configuredFactory(
+                configurer, new DefaultKafkaConsumerFactory<>(properties), syslogDefaultErrorHandler);
         factory.setConcurrency(concurrency);
         // 在 listener 线程调用 acknowledge() 时立即同步提交，使慢批次日志能够准确覆盖
         // “业务处理完成 -> Kafka offset 提交完成”的全部时间；可靠性语义仍是处理成功后提交。
         factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
+        log.info("Raw Kafka batch configuration: maxPollRecords={}, baseProcessingRecords={}, "
+                        + "adaptiveMaxRecords={}, concurrency={}",
+                maxPollRecords, baseRecords, maxPollRecords, concurrency);
         return factory;
     }
 
@@ -90,6 +102,47 @@ public class KafkaReliabilityConfig {
         return factory;
     }
 
+    /** 统计脏键与原始日志隔离；同一维度 key 由 Kafka 分区保证只被一个组成员处理。 */
+    @Bean("ingressStatsKafkaListenerContainerFactory")
+    public ConcurrentKafkaListenerContainerFactory<Object, Object> ingressStatsKafkaListenerContainerFactory(
+            ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
+            ConsumerFactory<Object, Object> consumerFactory,
+            DefaultErrorHandler syslogDefaultErrorHandler,
+            @Value("${stats.ingress-metadata.max-poll-records:2000}") int maxPollRecords,
+            @Value("${stats.ingress-metadata.auto-offset-reset:earliest}") String reset) {
+        return ingressFactory(configurer, consumerFactory, syslogDefaultErrorHandler, maxPollRecords, reset);
+    }
+
+    @Bean("largeIngressStatsKafkaListenerContainerFactory")
+    public ConcurrentKafkaListenerContainerFactory<Object, Object> largeIngressStatsKafkaListenerContainerFactory(
+            ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
+            ConsumerFactory<Object, Object> consumerFactory,
+            DefaultErrorHandler syslogDefaultErrorHandler,
+            @Value("${stats.ingress-metadata.auto-offset-reset:earliest}") String reset) {
+        return ingressFactory(configurer, consumerFactory, syslogDefaultErrorHandler, 5, reset);
+    }
+
+    private ConcurrentKafkaListenerContainerFactory<Object, Object> ingressFactory(
+            ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
+            ConsumerFactory<Object, Object> source, DefaultErrorHandler errorHandler,
+            int maxPollRecords, String reset) {
+        if (maxPollRecords < 1 || !("earliest".equals(reset) || "latest".equals(reset))) {
+            throw new IllegalArgumentException("入口统计 poll/reset 配置非法");
+        }
+        Map<String, Object> properties = new HashMap<>(source.getConfigurationProperties());
+        properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
+        properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
+        properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, reset);
+        properties.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, 33554432);
+        properties.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, 67108864);
+        ConcurrentKafkaListenerContainerFactory<Object, Object> factory = configuredFactory(
+                configurer, new DefaultKafkaConsumerFactory<>(properties), errorHandler);
+        factory.setBatchListener(true);
+        factory.setConcurrency(1);
+        factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
+        return factory;
+    }
+
     /** 统计脏键与原始日志隔离；同一维度 key 由 Kafka 分区保证只被一个组成员处理。 */
     @Bean("statsMaterializationKafkaListenerContainerFactory")
     public ConcurrentKafkaListenerContainerFactory<Object, Object> statsMaterializationKafkaListenerContainerFactory(

haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/SyslogBuildStartupLogger.java

@@ -0,0 +1,46 @@
+package com.config;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Component;
+
+/** 标识运行代码的功能修订；不依赖构建插件、镜像标签或可覆盖的配置版本号。 */
+@Slf4j
+@Component
+public class SyslogBuildStartupLogger {
+
+    // 确认包含此功能修订，不代表唯一构建；后续功能修订时同步更新此标识。
+    static final String CODE_REVISION = "ASYNC_STATS_ADAPTIVE_RAW_V20260913_56";
+    private final Environment environment;
+
+    public SyslogBuildStartupLogger(Environment environment) {
+        this.environment = environment;
+    }
+
+    @EventListener(ApplicationReadyEvent.class)
+    public void onApplicationReady() {
+        log.info("[SYSLOG_BUILD_READY] {}", startupSummary());
+    }
+
+    String startupSummary() {
+        return "codeRevision=" + CODE_REVISION
+                + ", application=syslog-consumer-rule"
+                + ", profiles=" + String.join(",", environment.getActiveProfiles())
+                + ", rawEnabled=" + value("syslog.raw-consumer.enabled", "true")
+                + ", rawConcurrency=" + value("syslog.raw-listener.concurrency", "3")
+                + ", baseProcessingRecords=" + value("syslog.raw-listener.max-poll-records", "500")
+                + ", adaptiveMaxRecords=" + value("syslog.raw-listener.adaptive-max-records", "3000")
+                + ", samplingEnabled=" + value("syslog.backlog-sampling.enabled", "true")
+                + ", statsWriteMode=" + value("stats.write-mode", "DIRECT")
+                + ", ingressStatsEnabled=" + value("stats.ingress-metadata.enabled", "true")
+                + ", statsPublisherEnabled=" + value("stats.projection-async.enabled", "true")
+                + ", lateBucketEnabled=" + value("analysis.late-replay.outbox-enabled", "true")
+                + ", slowBatchThresholdMs=" + value("syslog.slow-batch.threshold-ms", "500");
+    }
+
+    private String value(String key, String fallback) {
+        return environment.getProperty(key, fallback);
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/BacklogSamplingController.java

@@ -14,11 +14,14 @@ import java.util.Map;
 public class BacklogSamplingController {
     private final BacklogSamplingProperties properties;
     private final BacklogSamplingMetrics metrics;
+    private final com.common.service.BacklogSamplingAuditService auditService;
 
     public BacklogSamplingController(BacklogSamplingProperties properties,
-                                     BacklogSamplingMetrics metrics) {
+                                     BacklogSamplingMetrics metrics,
+                                     com.common.service.BacklogSamplingAuditService auditService) {
         this.properties = properties;
         this.metrics = metrics;
+        this.auditService = auditService;
     }
 
     @GetMapping("/status")
@@ -32,6 +35,7 @@ public class BacklogSamplingController {
         result.put("degrade_steps", properties.getDegradeSteps());
         result.put("policy_version", properties.getPolicyVersion());
         result.putAll(metrics.snapshot());
+        result.put("audit_async", auditService.status());
         return result;
     }
 }

haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/IngressStatsController.java

@@ -1,6 +1,7 @@
 package com.controllers;
 
 import com.common.service.IngressStatsProjectionWriter;
+import com.common.service.IngressStatsConsumer;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -13,13 +14,17 @@ import java.util.Map;
 @RequestMapping("/api/syslog/stats/ingress")
 public class IngressStatsController {
     private final IngressStatsProjectionWriter writer;
+    private final IngressStatsConsumer consumer;
 
-    public IngressStatsController(IngressStatsProjectionWriter writer) {
+    public IngressStatsController(IngressStatsProjectionWriter writer, IngressStatsConsumer consumer) {
         this.writer = writer;
+        this.consumer = consumer;
     }
 
     @GetMapping("/status")
     public ResponseEntity<Map<String, Object>> status() {
-        return ResponseEntity.ok(writer.status());
+        Map<String, Object> result = writer.status();
+        result.put("consumer", consumer.status());
+        return ResponseEntity.ok(result);
     }
 }

haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties

@@ -63,7 +63,7 @@ spring.kafka.consumer.topic=agent-01-syslog-topic
 syslog.raw-consumer.enabled=true
 
 
-# 调试修改：prod 环境普通 Kafka Consumer 单次最多拉取条数；如需改为2000，将 500 改为 2000。
+# 通用/Retry消费者上限。RAW底层上限由adaptive-max-records决定（3000），业务基础批量由max-poll-records决定（500）。
 spring.kafka.consumer.max-poll-records=500
 spring.kafka.consumer.properties.max.poll.interval.ms=900000
 spring.kafka.consumer.properties.session.timeout.ms=45000

haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties

@@ -84,6 +84,16 @@ syslog.raw-consumer.enabled=true
 syslog.raw-listener.concurrency=${SYSLOG_RAW_LISTENER_CONCURRENCY:3}
 # 端到端批次耗时达到该阈值时，打印业务各阶段及 Kafka offset 同步提交耗时；0 表示关闭。
 syslog.slow-batch.threshold-ms=${SYSLOG_SLOW_BATCH_THRESHOLD_MS:500}
+# 普通业务基础批量500（保留旧配置名）；不再表示底层poll上限。已有同名环境变量仍覆盖基础值。
+# 目标业务批量=ceil(基础条数*10000/max(普通保留万分比,非标保留万分比))，并受下面3000上限限制。
+syslog.raw-listener.max-poll-records=${SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS:500}
+# 底层RAW max.poll.records固定为此值；无积压时也可能poll到3000，再按500分块，全部处理后才ACK。
+# 基础值必须大于0且不超过上限；Retry/大日志/独立统计不采用此自适应策略。
+syslog.raw-listener.adaptive-max-records=${SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS:3000}
+# 大批次按 Kafka value 序列化字节拆成16MiB业务块，全部完成后提交本次poll；单条不会切断。
+syslog.raw-listener.processing-batch-max-bytes=${SYSLOG_RAW_PROCESSING_BATCH_MAX_BYTES:16777216}
+# 每个业务块只向共享规则线程池提交8个worker，避免2000条对应2000个排队任务。
+syslog.normalization.workers-per-batch=${SYSLOG_NORMALIZATION_WORKERS_PER_BATCH:8}
 # 连续性优先：特殊/系统性异常优先写 DLT；即使 DLT 失败也记录告警并推进原 offset，避免阻塞分区。
 syslog.failure-continuity.enabled=${SYSLOG_FAILURE_CONTINUITY_ENABLED:true}
 # DLT 故障时最多快速等待 1 秒；首次失败后整批直接记录并放行。
@@ -146,7 +156,7 @@ syslog.large-reliability.consumer-group-id=${spring.kafka.consumer.group-id}-lar
 syslog.large-reliability.consumer.enabled=true
 
 
-# 调试修改：普通 Kafka Consumer 单次最多拉取条数。如需改为2000，将 500 改为 2000。
+# 通用消费者（如Retry）的上限；RAW底层上限用adaptive-max-records，业务基础值用syslog.raw-listener.max-poll-records。
 # 使用 prod profile 时 application-prod.properties 中存在同名配置，必须同步修改，否则 prod 值会覆盖本项。
 spring.kafka.consumer.max-poll-records=500
 spring.kafka.consumer.properties.max.poll.interval.ms=900000
@@ -163,6 +173,8 @@ spring.task.scheduling.pool.size=10
 
 # 标准化告警事实与迟到重放通知使用同一个达梦事务。
 analysis.late-replay.outbox-enabled=${ANALYSIS_LATE_REPLAY_OUTBOX_ENABLED:true}
+# 告警提交后后台写分钟桶的事务超时；不在主消费线程等待。JDBC/连接池自身超时仍独立生效。
+analysis.late-replay.notify-timeout-seconds=${ANALYSIS_LATE_REPLAY_NOTIFY_TIMEOUT_SECONDS:5}
 
 # 日志配置
 logging.level.com.common.schedule=INFO
@@ -240,11 +252,18 @@ spring.datasource.hikari.validation-timeout=5000
 spring.datasource.hikari.leak-detection-threshold=30000
 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule
 spring.datasource.hikari.auto-commit=false
-# DIRECT 不再写逐事实 change_log；默认只在事实事务追加一条批次 Outbox，日统计异步投影。
+# DIRECT 连续消费优先：事实事务不再写统计Outbox/日统计表，提交后通过有界内存队列异步发布统计。
 # 首次部署 DIRECT 前必须先执行 08/09/10/11 脚本并按设计文档排空旧 pending。
 stats.write-mode=DIRECT
 stats.incremental.capture-enabled=false
 stats.ingress-metadata.enabled=true
+# 入口统计独立消费原始Topic，直接聚合写统计表；主消费不再写入口Outbox。
+# 各节点必须使用同一组名，且不得与业务主消费组相同。
+stats.ingress-metadata.group-id=${STATS_INGRESS_GROUP_ID:${spring.kafka.consumer.group-id}-ingress-v1}
+stats.ingress-metadata.large-group-id=${STATS_INGRESS_LARGE_GROUP_ID:${stats.ingress-metadata.group-id}-large}
+stats.ingress-metadata.max-poll-records=${STATS_INGRESS_MAX_POLL_RECORDS:2000}
+# 新组默认从Kafka仍保留的最早消息开始，确保不跳过历史接收时间；已有offset继续原位置。
+stats.ingress-metadata.auto-offset-reset=${STATS_INGRESS_AUTO_OFFSET_RESET:earliest}
 stats.incremental.read-enabled=true
 stats.incremental.transaction-timeout-seconds=45
 stats.device-receive.change-capture-enabled=false
@@ -275,10 +294,13 @@ stats.materialization.concurrency-retry-backoff-ms=${STATS_MATERIALIZATION_CONCU
 stats.materialization.reconcile-initial-delay-ms=${STATS_MATERIALIZATION_RECONCILE_INITIAL_DELAY_MS:30000}
 stats.materialization.reconcile-interval-ms=${STATS_MATERIALIZATION_RECONCILE_INTERVAL_MS:60000}
 
-# Fact transactions append one aggregated durable event instead of locking the daily projection rows.
-# A dedicated single-lane Kafka consumer applies device/rule/collector projection deltas and then
-# emits the existing idempotent business-materialization dirty keys.
+# 事实提交后只做内存offer，独立线程发送到统计Kafka；满队列/进程退出可能少统计，不阻碍事实消费。
+# enabled=false 会跳过统计并记录损失，绝不回退为同步统计写库。正常保持true。
+# 旧Outbox继续后台排空，applied表仍用于统计事件去重，不要删表。
 stats.projection-async.enabled=${STATS_PROJECTION_ASYNC_ENABLED:true}
+# 队列最多1000个批次聚合事件；含正在重试事件的聚合分组总数最多50000，限制内存占用。
+stats.projection-async.memory-queue-capacity=${STATS_PROJECTION_MEMORY_QUEUE_CAPACITY:1000}
+stats.projection-async.memory-max-pending-groups=${STATS_PROJECTION_MEMORY_MAX_PENDING_GROUPS:50000}
 stats.projection-async.outbox-table=${STATS_PROJECTION_OUTBOX_TABLE:stats_projection_outbox}
 stats.projection-async.applied-table=${STATS_PROJECTION_APPLIED_TABLE:stats_projection_applied_event}
 stats.projection-async.topic=${STATS_PROJECTION_TOPIC:${spring.kafka.consumer.topic}.STATS.PROJECTION}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/AdaptiveRawBatchSizerTest.java

@@ -0,0 +1,33 @@
+package com.Modules.NormalData;
+
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.*;
+
+class AdaptiveRawBatchSizerTest {
+    @Test void derivesMultiplierAndCapsWithoutPerStageConfiguration() {
+        AdaptiveRawBatchSizer sizer = new AdaptiveRawBatchSizer(500, 3000);
+        assertEquals(500, sizer.targetRecords(10000, 10000));
+        assertEquals(1000, sizer.targetRecords(5000, 5000));
+        assertEquals(2000, sizer.targetRecords(2500, 2500));
+        assertEquals(2500, sizer.targetRecords(1000, 2000));
+        assertEquals(3000, sizer.targetRecords(1000, 1000));
+        assertEquals(3000, sizer.targetRecords(0, 0));
+        assertEquals(500, sizer.targetRecords(0, 10000));
+        assertEquals(500, sizer.targetRecords(10000, 0));
+        assertEquals(834, sizer.targetRecords(5000, 6000));
+    }
+
+    @Test void supportsOtherBaseSizesWithoutOverflow() {
+        assertEquals(3000, new AdaptiveRawBatchSizer(1000, 3000).targetRecords(1000, 2000));
+        assertEquals(Integer.MAX_VALUE,
+                new AdaptiveRawBatchSizer(Integer.MAX_VALUE, Integer.MAX_VALUE).targetRecords(1, 1));
+    }
+
+    @Test void rejectsInvalidConfigurationAndRates() {
+        assertThrows(IllegalArgumentException.class, () -> new AdaptiveRawBatchSizer(0, 3000));
+        assertThrows(IllegalArgumentException.class, () -> new AdaptiveRawBatchSizer(500, 499));
+        AdaptiveRawBatchSizer sizer = new AdaptiveRawBatchSizer(500, 3000);
+        assertThrows(IllegalArgumentException.class, () -> sizer.targetRecords(-1, 2000));
+        assertThrows(IllegalArgumentException.class, () -> sizer.targetRecords(1000, 10001));
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogAdaptiveRawBatchTest.java

@@ -0,0 +1,155 @@
+package com.Modules.NormalData;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.kafka.support.Acknowledgment;
+import org.springframework.test.util.ReflectionTestUtils;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+class SysLogAdaptiveRawBatchTest {
+    private SysLogProcessor processor;
+    private Acknowledgment ack;
+
+    @BeforeEach void setup() {
+        processor = spy(new SysLogProcessor());
+        ReflectionTestUtils.setField(processor, "taskExecutor", (Executor) Runnable::run);
+        ReflectionTestUtils.setField(processor, "largeTaskExecutor", (Executor) Runnable::run);
+        ReflectionTestUtils.setField(processor, "largeProcessingBatchMaxBytes", 16777216L);
+        ReflectionTestUtils.setField(processor, "failureContinuityEnabled", false);
+        ack = mock(Acknowledgment.class);
+    }
+
+    @Test void livePollSplitsInto500WithoutEarlyAckOrRecordLoss() {
+        List<Integer> sizes = new ArrayList<>();
+        List<Long> offsets = new ArrayList<>();
+        doAnswer(call -> {
+            List<ConsumerRecord<String, String>> chunk = call.getArgument(0);
+            sizes.add(chunk.size());
+            chunk.forEach(record -> offsets.add(record.offset()));
+            verifyNoInteractions(ack);
+            processor.observeRawRetention(10000);
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(3000), ack);
+        assertEquals(Arrays.asList(500, 500, 500, 500, 500, 500), sizes);
+        for (int i = 0; i < 3000; i++) assertEquals((long) i, offsets.get(i).longValue());
+        verify(ack).acknowledge();
+    }
+
+    @Test void backlogExpandsNextChunkAndRecoveryShrinksFollowingChunk() {
+        List<Integer> sizes = new ArrayList<>();
+        doAnswer(call -> {
+            sizes.add(((List<?>) call.getArgument(0)).size());
+            processor.observeRawRetention(2000);
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(3000), ack);
+        assertEquals(Arrays.asList(500, 2500), sizes);
+        sizes.clear();
+        doAnswer(call -> {
+            sizes.add(((List<?>) call.getArgument(0)).size());
+            processor.observeRawRetention(10000);
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(3000), ack);
+        assertEquals(Arrays.asList(2500, 500), sizes);
+        sizes.clear();
+        processor.consumeAndParallelProcess(records(3000), ack);
+        assertEquals(Arrays.asList(500, 500, 500, 500, 500, 500), sizes);
+        verify(ack, times(3)).acknowledge();
+    }
+
+    @Test void tenPercentHits3000CapAndIncompleteTailIsNotHeldWaitingForMore() {
+        processor.observeRawRetention(1000);
+        List<Integer> sizes = new ArrayList<>();
+        doAnswer(call -> { sizes.add(((List<?>) call.getArgument(0)).size()); return null; })
+                .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(3000), ack);
+        processor.consumeAndParallelProcess(records(17), ack);
+        assertEquals(Arrays.asList(3000, 17), sizes);
+    }
+
+    @Test void bytesLimitStillWinsAndOversizedSingleRecordIsNotLost() {
+        List<ConsumerRecord<String, String>> records = Arrays.asList(
+                new ConsumerRecord<>("raw", 0, 0, "k", "中"),
+                new ConsumerRecord<>("raw", 0, 1, "k", "中"),
+                new ConsumerRecord<>("raw", 0, 2, "k", "oversized"),
+                new ConsumerRecord<>("raw", 0, 3, "k", "x"));
+        ReflectionTestUtils.setField(processor, "rawProcessingBatchMaxBytes", 3L);
+        processor.observeRawRetention(1000);
+        List<Integer> sizes = new ArrayList<>();
+        List<Long> offsets = new ArrayList<>();
+        doAnswer(call -> {
+            List<ConsumerRecord<String, String>> chunk = call.getArgument(0);
+            sizes.add(chunk.size());
+            chunk.forEach(record -> offsets.add(record.offset()));
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records, ack);
+        assertEquals(Arrays.asList(1, 1, 1, 1), sizes);
+        assertEquals(Arrays.asList(0L, 1L, 2L, 3L), offsets);
+        verify(ack).acknowledge();
+    }
+
+    @Test void failingLaterChunkDoesNotCommitUnfinishedPollAndResetsToBase() {
+        List<Integer> sizes = new ArrayList<>();
+        doAnswer(call -> {
+            sizes.add(((List<?>) call.getArgument(0)).size());
+            processor.observeRawRetention(2000);
+            if (sizes.size() == 2) throw new IllegalStateException("DM unavailable");
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        assertThrows(IllegalStateException.class, () -> processor.consumeAndParallelProcess(records(3000), ack));
+        verifyNoInteractions(ack);
+        assertEquals(Arrays.asList(500, 2500), sizes);
+        sizes.clear();
+        doAnswer(call -> { sizes.add(((List<?>) call.getArgument(0)).size()); return null; })
+                .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(1000), ack);
+        assertEquals(Arrays.asList(500, 500), sizes);
+    }
+
+    @Test void feedbackDoesNotLeakAcrossConsumerThreadsOrChangeRetryAndLargeLanes() throws Exception {
+        processor.observeRawRetention(1000);
+        List<Integer> sizes = Collections.synchronizedList(new ArrayList<>());
+        doAnswer(call -> { sizes.add(((List<?>) call.getArgument(0)).size()); return null; })
+                .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        ExecutorService otherConsumer = Executors.newSingleThreadExecutor();
+        try {
+            otherConsumer.submit(() -> processor.consumeAndParallelProcess(records(1000), ack)).get(3, TimeUnit.SECONDS);
+        } finally { otherConsumer.shutdownNow(); }
+        assertEquals(Arrays.asList(500, 500), sizes);
+        sizes.clear();
+        processor.consumeRetryAndReplay(records(1000), ack);
+        processor.consumeLargeMessages(records(1000), ack);
+        assertEquals(Arrays.asList(1000, 1000), sizes);
+    }
+
+    @Test void staleFeedbackResetsAfterOneMinuteWithoutObservation() {
+        processor.observeRawRetention(1000);
+        Object state = ((ThreadLocal<?>) ReflectionTestUtils.getField(processor, "rawAdaptiveState")).get();
+        ReflectionTestUtils.setField(state, "observedAt", System.nanoTime() - TimeUnit.MINUTES.toNanos(2));
+        List<Integer> sizes = new ArrayList<>();
+        doAnswer(call -> { sizes.add(((List<?>) call.getArgument(0)).size()); return null; })
+                .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(1000), ack);
+        assertEquals(Arrays.asList(500, 500), sizes);
+    }
+
+    private List<ConsumerRecord<String, String>> records(int count) {
+        List<ConsumerRecord<String, String>> result = new ArrayList<>();
+        for (int i = 0; i < count; i++) result.add(new ConsumerRecord<>("raw", 0, i, "k", "value"));
+        return result;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java

@@ -137,6 +137,32 @@ class SysLogProcessorParallelTest {
                 () -> processor.normalizePreparedMessages(messages(2), metrics));
     }
 
+    @Test
+    void concurrentLargePollsSubmitOnlyBoundedWorkersAndProcessEveryRecord() throws Exception {
+        AtomicInteger submissions = new AtomicInteger();
+        Executor countedExecutor = task -> {
+            submissions.incrementAndGet();
+            executor.execute(task);
+        };
+        ReflectionTestUtils.setField(processor, "taskExecutor", countedExecutor);
+        when(normalProcessor.prepare(any(PreparedNormalizationInput.class), any(), any(), any()))
+                .thenReturn(LogNormalProcessor.NormalizationResult.success(
+                        Collections.singletonList(normalizedRow())));
+        ExecutorService callers = Executors.newFixedThreadPool(3);
+        try {
+            List<java.util.concurrent.Future<?>> runs = new ArrayList<>();
+            for (int i = 0; i < 3; i++) {
+                runs.add(callers.submit(() -> processor.normalizePreparedMessages(messages(3000),
+                        new SysLogProcessor.BatchMetrics())));
+            }
+            for (java.util.concurrent.Future<?> run : runs) run.get(10, TimeUnit.SECONDS);
+            assertEquals(24, submissions.get(), "3个并发批次应提交24个worker，而非9000个任务");
+            verify(normalProcessor, times(9000)).prepare(any(PreparedNormalizationInput.class), any(), any(), any());
+        } finally {
+            callers.shutdownNow();
+        }
+    }
+
     @Test
     void recordsRuleComputeAndDmWriteTimingsSeparately() {
         when(normalProcessor.prepare(any(PreparedNormalizationInput.class), any(), any(), any()))
@@ -382,6 +408,31 @@ class SysLogProcessorParallelTest {
         assertEquals(1L, normalDropped);
         assertEquals(1L, nonNormalDropped);
         assertEquals(2L, rawDropped);
+        assertEquals(0, plan.maxRetainBps());
+        ReflectionTestUtils.setField(properties, "normalRetainBps", 1000);
+        ReflectionTestUtils.setField(properties, "nonNormalRetainBps", 2000);
+        assertEquals(2000, processor.buildSamplingPlan(computed).maxRetainBps(),
+                "放大倍率必须使用两类中较高保留率");
+        ReflectionTestUtils.setField(properties, "enabled", false);
+        assertEquals(10000, processor.buildSamplingPlan(computed).maxRetainBps());
+        ReflectionTestUtils.setField(properties, "enabled", true);
+        computed.add(new SysLogProcessor.PreparedNormalization(source.get(0), null,
+                SyslogFailureRouter.TerminalState.DLT_PUBLISHED));
+        assertEquals(10000, processor.buildSamplingPlan(computed).maxRetainBps(),
+                "规则结果未知时保守回到基础批量");
+        computed.remove(computed.size() - 1);
+        BacklogSamplingDecider mixed = mock(BacklogSamplingDecider.class);
+        LocalDateTime hour = LocalDateTime.of(2026, 9, 13, 0, 0);
+        BacklogSamplingDecision degraded = new BacklogSamplingDecision(true, true, true, true,
+                3600000L, hour, "test", 1000, 2000);
+        BacklogSamplingDecision recent = new BacklogSamplingDecision(false, true, true, true,
+                0L, hour, "test", 10000, 10000);
+        when(mixed.decide(any(), any(), org.mockito.ArgumentMatchers.anyBoolean(),
+                org.mockito.ArgumentMatchers.anyBoolean(), org.mockito.ArgumentMatchers.anyBoolean()))
+                .thenReturn(degraded, recent, degraded);
+        ReflectionTestUtils.setField(processor, "backlogSamplingDecider", mixed);
+        assertEquals(10000, processor.buildSamplingPlan(computed).maxRetainBps(),
+                "同块包含新旧不同阶段消息时取最高保留率");
     }
 
     private List<ConsumerRecord<String, String>> records(int count) {

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogSlowBatchTraceTest.java

@@ -0,0 +1,110 @@
+package com.Modules.NormalData;
+
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.kafka.support.Acknowledgment;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.stream.Collectors;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+import static org.mockito.ArgumentMatchers.*;
+
+class SysLogSlowBatchTraceTest {
+    private final Logger logger = (Logger) LoggerFactory.getLogger(SysLogProcessor.class);
+    private ListAppender<ILoggingEvent> appender;
+    private SysLogProcessor processor;
+
+    @BeforeEach void setup() {
+        appender = new ListAppender<>();
+        appender.start();
+        logger.addAppender(appender);
+        processor = spy(new SysLogProcessor());
+        ReflectionTestUtils.setField(processor, "taskExecutor", (Executor) Runnable::run);
+        ReflectionTestUtils.setField(processor, "slowBatchThresholdMs", 1L);
+        ReflectionTestUtils.setField(processor, "failureContinuityEnabled", false);
+        MDC.put("syslogBatchId", "previous-context");
+    }
+
+    @AfterEach void cleanup() {
+        logger.detachAppender(appender);
+        appender.stop();
+        MDC.remove("syslogBatchId");
+    }
+
+    @Test void successfulTraceContainsStageBreakdownOffsetsAndRestoresContext() {
+        doAnswer(call -> {
+            ReflectionTestUtils.invokeMethod(processor, "enterStage", "PREPARE");
+            Object trace = trace();
+            ReflectionTestUtils.setField(trace, "startedAtNanos", System.nanoTime() - 10_000_000L);
+            ReflectionTestUtils.setField(trace, "stageStarted", System.nanoTime() - 2_000_000L);
+            ReflectionTestUtils.invokeMethod(processor, "enterStage", "ALARM_DM");
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        Acknowledgment ack = mock(Acknowledgment.class);
+        processor.consumeAndParallelProcess(records(), ack);
+        verify(ack).acknowledge();
+        String log = onlyTrace();
+        assertTrue(log.contains("PREPARE="));
+        assertTrue(log.contains("ADAPTIVE_CHUNK_SELECTION="));
+        assertTrue(log.contains("adaptiveTargetHistogram={500=1}"));
+        assertTrue(log.contains("actualChunkHistogram={2=1}"));
+        assertTrue(log.contains("ALARM_DM="));
+        assertTrue(log.contains("raw:0=12..14"));
+        assertTrue(log.contains("kafkaCommitStatus=SUCCESS"));
+        assertTrue(log.contains("listenerGapMs=-1"));
+        assertFalse(log.contains("batchId=previous-context"));
+        assertEquals("previous-context", MDC.get("syslogBatchId"));
+        assertNull(trace());
+    }
+
+    @Test void businessAbortReportsFailingStageAndNoCommitInsteadOfFalseSuccess() {
+        doAnswer(call -> {
+            ReflectionTestUtils.invokeMethod(processor, "enterStage", "NORMAL_DM");
+            throw new IllegalStateException("DB stalled");
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        Acknowledgment ack = mock(Acknowledgment.class);
+        assertThrows(IllegalStateException.class, () -> processor.consumeAndParallelProcess(records(), ack));
+        verifyNoInteractions(ack);
+        String log = onlyTrace();
+        assertTrue(log.contains("failureStage=NORMAL_DM"));
+        assertTrue(log.contains("kafkaCommitStatus=NOT_ATTEMPTED"));
+        assertTrue(log.contains("failedChunks=1"));
+        assertEquals("previous-context", MDC.get("syslogBatchId"));
+    }
+
+    @Test void commitFailureIsLoggedExactlyOnce() {
+        doNothing().when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        Acknowledgment ack = mock(Acknowledgment.class);
+        doThrow(new IllegalStateException("commit rejected")).when(ack).acknowledge();
+        assertThrows(IllegalStateException.class, () -> processor.consumeAndParallelProcess(records(), ack));
+        String log = onlyTrace();
+        assertTrue(log.contains("kafkaCommitStatus=FAILED"));
+        assertTrue(log.contains("kafkaCommitErrorType=java.lang.IllegalStateException"));
+        assertEquals("previous-context", MDC.get("syslogBatchId"));
+    }
+
+    private Object trace() {
+        return ((ThreadLocal<?>) ReflectionTestUtils.getField(processor, "slowBatchTrace")).get();
+    }
+    private String onlyTrace() {
+        List<String> logs = appender.list.stream().map(ILoggingEvent::getFormattedMessage)
+                .filter(line -> line.contains("[SLOW_SYSLOG_BATCH]")).collect(Collectors.toList());
+        assertEquals(1, logs.size());
+        assertFalse(logs.get(0).contains("batchId={}"));
+        return logs.get(0);
+    }
+    private List<ConsumerRecord<String, String>> records() {
+        return Arrays.asList(new ConsumerRecord<>("raw", 0, 12L, "k", "first"),
+                new ConsumerRecord<>("raw", 0, 14L, "k", "second"));
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BacklogSamplingAuditServiceTest.java

@@ -41,7 +41,7 @@ class BacklogSamplingAuditServiceTest {
     }
 
     @Test
-    void auditFailureReturnsFalseAndRecordsFullRetentionFallback() {
+    void auditFailureIsBackgroundOnlyAndNeverTriggersFullRetentionFallback() {
         BacklogRetentionAuditMapper mapper = mock(BacklogRetentionAuditMapper.class);
         when(mapper.upsert(any())).thenThrow(new IllegalStateException("table unavailable"));
         PlatformTransactionManager transactionManager = transactionManager();
@@ -49,13 +49,18 @@ class BacklogSamplingAuditServiceTest {
         BacklogSamplingAuditService service = new BacklogSamplingAuditService(
                 mapper, properties(), metrics, transactionManager);
 
-        boolean audited = service.recordBeforeDropping(Arrays.asList(
+        boolean audited = service.recordAsync(Arrays.asList(
                 delta(2, 1, 1, 1, 1)));
 
-        assertFalse(audited);
+        assertTrue(audited);
+        org.mockito.Mockito.verifyNoInteractions(mapper, transactionManager);
+        ((BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(service, "queue")).drain();
         Map<String, Object> snapshot = metrics.snapshot();
         assertEquals(1L, snapshot.get("audit_write_failures_total"));
-        assertEquals(2L, snapshot.get("audit_fallback_messages_total"));
+        assertEquals(0L, snapshot.get("audit_fallback_messages_total"));
+        assertEquals(2L, snapshot.get("audit_lost_or_unconfirmed_messages_total"));
+        assertFalse(service.recordAsync(Arrays.asList(delta(1, 0, 1, 0, 1))));
+        assertEquals(3L, metrics.snapshot().get("degraded_messages_total"));
     }
 
     @Test
@@ -67,10 +72,13 @@ class BacklogSamplingAuditServiceTest {
         BacklogSamplingAuditService service = new BacklogSamplingAuditService(
                 mapper, properties(), metrics, transactionManager);
 
-        assertTrue(service.recordBeforeDropping(Arrays.asList(
+        assertTrue(service.recordAsync(Arrays.asList(
                 delta(1, 0, 1, 0, 1))));
 
+        org.mockito.Mockito.verifyNoInteractions(mapper, transactionManager);
+        ((BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(service, "queue")).drain();
         verify(mapper).upsert(any());
+        verify(transactionManager).commit(any());
         assertEquals(1L, metrics.snapshot().get("degraded_messages_total"));
         assertEquals(1L, metrics.snapshot().get("normal_rows_dropped_total"));
     }

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BestEffortAuxiliaryQueueTest.java

@@ -0,0 +1,64 @@
+package com.common.service;
+
+import org.junit.jupiter.api.Test;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import static org.junit.jupiter.api.Assertions.*;
+
+class BestEffortAuxiliaryQueueTest {
+    @Test
+    void hungAuxiliaryDatabaseDoesNotBlockMainEnqueueAndCapacityIsBounded() throws Exception {
+        BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("TEST", 1, 2, true);
+        CountDownLatch entered = new CountDownLatch(1);
+        CountDownLatch release = new CountDownLatch(1);
+        assertTrue(queue.offer(1, () -> {
+            entered.countDown();
+            try { release.await(5, TimeUnit.SECONDS); }
+            catch (InterruptedException e) { Thread.currentThread().interrupt(); }
+        }));
+        CompletableFuture<Void> writer = CompletableFuture.runAsync(queue::drain);
+        try {
+            assertTrue(entered.await(2, TimeUnit.SECONDS));
+            CompletableFuture.runAsync(() -> {
+                assertTrue(queue.offer(1, () -> { }));
+                assertFalse(queue.offer(1, () -> fail("overflow must never run")));
+            }).get(2, TimeUnit.SECONDS);
+            assertEquals(2L, queue.status().get("pending_groups"));
+            assertEquals(1L, queue.status().get("lost_tasks_total"));
+        } finally {
+            release.countDown();
+            writer.get(3, TimeUnit.SECONDS);
+            queue.stop();
+        }
+        assertEquals(2L, queue.status().get("completed_total"));
+        assertEquals(0L, queue.status().get("pending_groups"));
+    }
+
+    @Test
+    void auditFailureIsNotAutomaticallyReplayedAndCapacityIsReleased() {
+        BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("AUDIT_TEST", 1, 1, false);
+        AtomicInteger attempts = new AtomicInteger();
+        assertTrue(queue.offer(1, () -> { attempts.incrementAndGet(); throw new IllegalStateException("unknown commit"); }));
+        assertDoesNotThrow(queue::drain);
+        queue.drain();
+        assertEquals(1, attempts.get());
+        assertEquals(0L, queue.status().get("pending_groups"));
+        assertTrue(queue.offer(1, () -> { }));
+        queue.drain();
+        assertEquals(1L, queue.status().get("completed_total"));
+        queue.stop();
+        assertFalse(queue.offer(1, () -> fail("closed")));
+    }
+
+    @Test
+    void queueCountLimitAlsoReleasesReservedGroups() {
+        BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("TEST", 1, 100, false);
+        assertTrue(queue.offer(1, () -> { }));
+        assertFalse(queue.offer(1, () -> fail("overflow")));
+        assertEquals(1L, queue.status().get("pending_groups"));
+        queue.drain();
+        assertEquals(0L, queue.status().get("pending_groups"));
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DirectStatsProjectionWriterTest.java

@@ -36,7 +36,7 @@ class DirectStatsProjectionWriterTest {
     }
 
     @Test
-    void directModeWritesOnlyDimensionBatchesAndMaterializesAffectedRows() {
+    void directModeAggregatesWithoutAnyStatisticsSql() {
         JdbcTemplate jdbc = mock(JdbcTemplate.class);
         DirectStatsProjectionWriter writer = writer(jdbc, "DIRECT", true);
         TransactionSynchronizationManager.setActualTransactionActive(true);
@@ -53,13 +53,7 @@ class DirectStatsProjectionWriterTest {
 
         writer.writeStandardizedRows(rows);
 
-        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
-        verify(jdbc, times(4)).batchUpdate(sql.capture(), anyList());
-        String combined = String.join("\n", sql.getAllValues());
-        assertTrue(combined.contains("MERGE INTO device_daily_stats"));
-        assertTrue(combined.contains("MERGE INTO normalize_rule_daily_stats"));
-        assertTrue(combined.contains("UPDATE device_device"));
-        assertTrue(combined.contains("UPDATE dm_normalize_rule"));
+        verifyNoInteractions(jdbc);
         assertTrue(((Number) writer.status().get("input_rows")).longValue() == 100L);
         assertTrue(((Number) writer.status().get("device_groups")).longValue() == 1L);
         assertTrue(((Number) writer.status().get("rule_groups")).longValue() == 2L);
@@ -76,7 +70,7 @@ class DirectStatsProjectionWriterTest {
     }
 
     @Test
-    void nonNormalUpdateRecomputesOnlyAffectedFactGroupsExactly() {
+    void shadowNonNormalUpdateRetainsExactRecomputation() {
         AtomicInteger exactQueries = new AtomicInteger();
         JdbcTemplate jdbc = mock(JdbcTemplate.class, invocation -> {
             if ("queryForMap".equals(invocation.getMethod().getName())) {
@@ -88,7 +82,7 @@ class DirectStatsProjectionWriterTest {
             }
             return Answers.RETURNS_DEFAULTS.answer(invocation);
         });
-        DirectStatsProjectionWriter writer = writer(jdbc, "DIRECT", true);
+        DirectStatsProjectionWriter writer = writer(jdbc, "DIRECT_SHADOW", true);
         TransactionSynchronizationManager.setActualTransactionActive(true);
         LocalDate today = LocalDate.now(ZoneId.of("Asia/Shanghai"));
         com.common.entity.SyslogNonNormalMessage oldValue =
@@ -104,9 +98,8 @@ class DirectStatsProjectionWriterTest {
 
         assertTrue(exactQueries.get() == 2);
         ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
-        verify(jdbc, times(2)).batchUpdate(sql.capture(), anyList());
+        verify(jdbc).batchUpdate(sql.capture(), anyList());
         assertTrue(sql.getAllValues().get(0).contains("parse_count=s.parse_count"));
-        assertTrue(sql.getAllValues().get(1).contains("UPDATE device_device"));
     }
 
     @Test
@@ -119,12 +112,11 @@ class DirectStatsProjectionWriterTest {
     }
 
     @Test
-    void durableOutboxRemovesDailyAndBusinessProjectionUpdatesFromFactTransaction() {
+    void afterCommitPublisherRemovesAllStatisticsSqlFromFactTransaction() {
         JdbcTemplate jdbc = mock(JdbcTemplate.class);
         StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
         when(publisher.isEnabled()).thenReturn(true);
-        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
-        when(outbox.isEnabled()).thenReturn(true);
+        StatsProjectionAsyncPublisher outbox = mock(StatsProjectionAsyncPublisher.class);
         DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(
                 jdbc, new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), publisher, outbox);
         configure(writer, true);
@@ -137,17 +129,17 @@ class DirectStatsProjectionWriterTest {
         row.put("normalize_rule_id", 11L);
         writer.writeStandardizedRow(row);
 
-        verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class));
+        verify(outbox).enqueueAfterCommit(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class),
+                org.mockito.ArgumentMatchers.eq(1));
         verifyNoInteractions(jdbc);
-        assertTrue("DURABLE_OUTBOX".equals(writer.status().get("daily_projection_write_mode")));
+        assertTrue("BEST_EFFORT_AFTER_COMMIT_KAFKA".equals(writer.status().get("daily_projection_write_mode")));
     }
 
     private DirectStatsProjectionWriter writer(JdbcTemplate jdbc, String mode, boolean readEnabled) {
         StatsWriteModeProvider modeProvider = new StatsWriteModeProvider(mode);
         StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
         when(publisher.isEnabled()).thenReturn(false);
-        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
-        when(outbox.isEnabled()).thenReturn(false);
+        StatsProjectionAsyncPublisher outbox = mock(StatsProjectionAsyncPublisher.class);
         DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(
                 jdbc, modeProvider, new DirectStatsWriteMetrics(), publisher, outbox);
         configure(writer, readEnabled);

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsConsumerTest.java

@@ -0,0 +1,90 @@
+package com.common.service;
+
+import com.Modules.NormalData.SyslogFailureRouter;
+import com.common.entity.IngressStatsEvent;
+import com.common.util.Sm4Util;
+import com.config.AppProperties;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.Acknowledgment;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+import static org.mockito.ArgumentMatchers.anyList;
+
+class IngressStatsConsumerTest {
+    private static final String KEY = "0123456789abcdeffedcba9876543210";
+    private final IngressStatsProjectionWriter writer = mock(IngressStatsProjectionWriter.class);
+    private final Acknowledgment ack = mock(Acknowledgment.class);
+
+    private IngressStatsConsumer consumer() {
+        AppProperties properties = mock(AppProperties.class);
+        when(properties.getSm4Key()).thenReturn(KEY);
+        return new IngressStatsConsumer(writer,
+                new SyslogFailureRouter(mock(KafkaTemplate.class), new ObjectMapper()), properties);
+    }
+
+    private ConsumerRecord<String, String> record(String topic, long offset, String time) {
+        String value = Sm4Util.encryptCbc("[device_id=126 device_collect_id=3 receive_time="
+                + time + " event_time=20000101000000000]正文不参与规则或时间统计", KEY);
+        return new ConsumerRecord<>(topic, 0, offset, "key", value);
+    }
+
+    @Test
+    void decodesActualEncryptedHeaderUsingReceiveTimeNotEventTime() {
+        IngressStatsEvent event = consumer().decode(record("raw", 0, "20260913044500000"));
+        assertEquals(Long.valueOf(126), event.getDeviceId());
+        assertEquals(Long.valueOf(3), event.getDeviceCollectId());
+        assertEquals(LocalDateTime.of(2026, 9, 13, 4, 45), event.getReceiveTime());
+    }
+
+    @Test
+    void commitsOnlyAfterStatsDatabaseReturnsAndSupportsLargeTopic() {
+        consumer().consumeLarge(Collections.singletonList(record("raw.large", 1, "20260913044500000")), ack);
+        InOrder order = inOrder(writer, ack);
+        order.verify(writer).writeBatch(anyList());
+        order.verify(ack).acknowledge();
+    }
+
+    @Test
+    void databaseFailureRetainsStatsOffsetForRetry() {
+        when(writer.writeBatch(anyList())).thenThrow(new IllegalStateException("database down"));
+        assertThrows(IllegalStateException.class, () -> consumer().consume(
+                Collections.singletonList(record("raw", 0, "20260913044500000")), ack));
+        verifyNoInteractions(ack);
+    }
+
+    @Test
+    void invalidHeaderIsCountedWithoutDiscardingOtherRecords() {
+        IngressStatsConsumer consumer = consumer();
+        consumer.consume(Arrays.asList(new ConsumerRecord<>("raw", 0, 0L, "k", "bad ciphertext"),
+                record("raw", 1L, "20260913044500000")), ack);
+        ArgumentCaptor<List<IngressStatsEvent>> events = ArgumentCaptor.forClass(List.class);
+        verify(writer).writeBatch(events.capture());
+        assertEquals(1, events.getValue().size());
+        assertEquals(1L, consumer.status().get("invalid_record_attempts"));
+        verify(ack).acknowledge();
+    }
+
+    @Test
+    void rejectsSharingBusinessConsumerGroup() {
+        IngressStatsConsumer consumer = consumer();
+        ReflectionTestUtils.setField(consumer, "rawGroup", "business");
+        ReflectionTestUtils.setField(consumer, "largeRawGroup", "business-large");
+        ReflectionTestUtils.setField(consumer, "group", "business");
+        ReflectionTestUtils.setField(consumer, "largeGroup", "stats-large");
+        assertThrows(IllegalStateException.class, consumer::validateGroups);
+        ReflectionTestUtils.setField(consumer, "group", "stats");
+        assertDoesNotThrow(consumer::validateGroups);
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsProjectionWriterTest.java

@@ -53,26 +53,27 @@ class IngressStatsProjectionWriterTest {
     }
 
     @Test
-    void directAsyncModeWritesOnlyOneOutboxEvent() {
+    void independentIngressWritesAggregatedMinMaxDirectlyAndCommits() {
         JdbcTemplate jdbc = mock(JdbcTemplate.class);
         PlatformTransactionManager manager = mock(PlatformTransactionManager.class);
         when(manager.getTransaction(any(TransactionDefinition.class)))
                 .thenReturn(new SimpleTransactionStatus());
-        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
-        when(outbox.isEnabled()).thenReturn(true);
         IngressStatsProjectionWriter writer = new IngressStatsProjectionWriter(
                 jdbc, manager, new StatsWriteModeProvider("DIRECT"),
-                mock(StatsMaterializationPublisher.class), outbox);
+                mock(StatsMaterializationPublisher.class));
         ReflectionTestUtils.setField(writer, "enabled", true);
         ReflectionTestUtils.setField(writer, "transactionTimeoutSeconds", 45);
+        ReflectionTestUtils.setField(writer, "businessZone", "Asia/Shanghai");
 
         int groups = writer.writeBatch(Arrays.asList(
                 new IngressStatsEvent("a", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 0)),
                 new IngressStatsEvent("b", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 1))));
 
         assertEquals(2, groups);
-        verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class));
-        verifyNoInteractions(jdbc);
+        verify(jdbc).update(org.mockito.ArgumentMatchers.contains("MERGE INTO device_daily_stats"),
+                any(), any(), any());
+        verify(jdbc).update(org.mockito.ArgumentMatchers.contains("MERGE INTO device_collect_stats"),
+                any(), any(), any());
         verify(manager).commit(any());
     }
 }

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/LateAlarmBucketWriterTest.java

@@ -6,6 +6,8 @@ import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.doAnswer;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import java.time.LocalDateTime;
 import java.util.Arrays;
@@ -16,14 +18,60 @@ import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.test.util.ReflectionTestUtils;
 
 class LateAlarmBucketWriterTest {
+  private final org.springframework.transaction.support.TransactionTemplate transaction =
+      new org.springframework.transaction.support.TransactionTemplate(new TestTransactionManager());
+
+  @org.junit.jupiter.api.AfterEach void clean() {
+    org.springframework.transaction.support.TransactionSynchronizationManager.clear();
+  }
+
+  private LateAlarmBucketWriter writer(JdbcTemplate jdbc) {
+    LateAlarmBucketWriter writer = new LateAlarmBucketWriter(jdbc, new TestTransactionManager());
+    ReflectionTestUtils.setField(writer, "enabled", true);
+    return writer;
+  }
+
+  private void committed(LateAlarmBucketWriter writer, java.util.List<Map<String, Object>> rows) {
+    transaction.execute(status -> { writer.record(rows); return null; });
+    ((BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(writer, "queue")).drain();
+  }
+
+  private static class TestTransactionManager extends org.springframework.transaction.support.AbstractPlatformTransactionManager {
+    protected Object doGetTransaction() { return new Object(); }
+    protected void doBegin(Object tx, org.springframework.transaction.TransactionDefinition def) { }
+    protected void doCommit(org.springframework.transaction.support.DefaultTransactionStatus status) { }
+    protected void doRollback(org.springframework.transaction.support.DefaultTransactionStatus status) { }
+  }
+
+  @Test
+  void takesBucketLocksInSameOrderForReversedBatches() {
+    JdbcTemplate jdbc = mock(JdbcTemplate.class);
+    LateAlarmBucketWriter writer = writer(jdbc);
+    ReflectionTestUtils.setField(writer, "enabled", true);
+    java.util.List<String> locks = new java.util.ArrayList<>();
+    doAnswer(call -> {
+      locks.add(call.getArgument(1) + ":" + call.getArgument(2));
+      return 1;
+    }).when(jdbc).update(anyString(), any(), any(), any(), any(), any(), any());
+    LocalDateTime time = LocalDateTime.of(2026, 9, 13, 4, 0);
+    java.util.List<Map<String, Object>> rows = new java.util.ArrayList<>(Arrays.asList(
+        row(126, time.plusMinutes(1), 1), row(125, time, 2), row(126, time, 3)));
+    committed(writer, rows);
+    java.util.List<String> first = new java.util.ArrayList<>(locks);
+    locks.clear();
+    java.util.Collections.reverse(rows);
+    committed(writer, rows);
+    assertEquals(first, locks);
+    assertEquals(3, locks.size());
+  }
 
   @Test
   void groupsRowsByDeviceAndEventMinute() {
     JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class);
-    LateAlarmBucketWriter writer = new LateAlarmBucketWriter(jdbcTemplate);
+    LateAlarmBucketWriter writer = writer(jdbcTemplate);
     ReflectionTestUtils.setField(writer, "enabled", true);
 
-    writer.record(
+    committed(writer,
         Arrays.asList(
             row(126, LocalDateTime.of(2026, 9, 4, 10, 1, 2), 3),
             row(126, LocalDateTime.of(2026, 9, 4, 10, 1, 59), 4),
@@ -41,13 +89,37 @@ class LateAlarmBucketWriterTest {
   }
 
   @Test
-  void rejectsAlarmRowsWithoutStableTimes() {
-    LateAlarmBucketWriter writer = new LateAlarmBucketWriter(mock(JdbcTemplate.class));
+  void invalidNotificationDoesNotAbortAlarmTransaction() {
+    LateAlarmBucketWriter writer = writer(mock(JdbcTemplate.class));
     ReflectionTestUtils.setField(writer, "enabled", true);
     Map<String, Object> invalid = new HashMap<>();
     invalid.put("device_id", 126);
 
-    assertThrows(IllegalArgumentException.class, () -> writer.record(Arrays.asList(invalid)));
+    org.junit.jupiter.api.Assertions.assertDoesNotThrow(() -> committed(writer, Arrays.asList(invalid)));
+    assertEquals(1L, writer.status().get("lost_tasks_total"));
+  }
+
+  @Test
+  void commitOnlyEnqueuesRollbackNeverEnqueuesAndDatabaseFailureOnlyRetriesBackground() {
+    JdbcTemplate jdbc = mock(JdbcTemplate.class);
+    LateAlarmBucketWriter writer = writer(jdbc);
+    java.util.List<Map<String, Object>> rows = Arrays.asList(row(126, LocalDateTime.now(), 1));
+    transaction.execute(status -> { writer.record(rows); status.setRollbackOnly(); return null; });
+    assertEquals(0L, writer.status().get("enqueued_total"));
+    transaction.execute(status -> { writer.record(rows); return null; });
+    org.mockito.Mockito.verifyNoInteractions(jdbc);
+    org.mockito.Mockito.doThrow(new IllegalStateException("database unavailable"))
+        .when(jdbc).update(anyString(), any(), any(), any(), any(), any(), any());
+    BestEffortAuxiliaryQueue queue = (BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(writer, "queue");
+    org.junit.jupiter.api.Assertions.assertDoesNotThrow(queue::drain);
+    assertEquals(1L, writer.status().get("failures_total"));
+    assertEquals(1, writer.status().get("pending_tasks"));
+    assertEquals(0L, writer.status().get("lost_tasks_total"));
+    Object current = ReflectionTestUtils.getField(queue, "current");
+    ReflectionTestUtils.setField(current, "retryAt", 0L);
+    org.mockito.Mockito.doReturn(1).when(jdbc).update(anyString(), any(), any(), any(), any(), any(), any());
+    queue.drain();
+    assertEquals(1L, writer.status().get("completed_total"));
   }
 
   private Map<String, Object> row(int deviceId, LocalDateTime eventTime, int receiveSecond) {

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncPublisherTest.java

@@ -0,0 +1,238 @@
+package com.common.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.SendResult;
+import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.support.AbstractPlatformTransactionManager;
+import org.springframework.transaction.support.DefaultTransactionStatus;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+import org.springframework.transaction.support.TransactionTemplate;
+import org.springframework.util.concurrent.SettableListenableFuture;
+
+import java.time.LocalDateTime;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+class StatsProjectionAsyncPublisherTest {
+    private final KafkaTemplate<String, String> kafka = mock(KafkaTemplate.class);
+    private final TransactionTemplate transaction = new TransactionTemplate(new TestTransactionManager());
+
+    @AfterEach
+    void cleanup() { TransactionSynchronizationManager.clear(); }
+
+    @Test
+    void productionConstructorUsesDedicatedProducerWithExistingKafkaSettings() {
+        org.springframework.boot.autoconfigure.kafka.KafkaProperties properties =
+                new org.springframework.boot.autoconfigure.kafka.KafkaProperties();
+        properties.setBootstrapServers(java.util.Collections.singletonList("broker.example:9092"));
+        properties.getProperties().put("security.protocol", "SASL_PLAINTEXT");
+        properties.getProducer().setAcks("all");
+        StatsProjectionAsyncPublisher publisher = new StatsProjectionAsyncPublisher(properties, new ObjectMapper(), 10, 10);
+        org.springframework.kafka.core.DefaultKafkaProducerFactory<?, ?> factory =
+                (org.springframework.kafka.core.DefaultKafkaProducerFactory<?, ?>)
+                        ReflectionTestUtils.getField(publisher, "ownedProducerFactory");
+        assertNotNull(factory);
+        assertEquals("SASL_PLAINTEXT", factory.getConfigurationProperties().get("security.protocol"));
+        assertEquals("all", factory.getConfigurationProperties().get("acks"));
+        assertEquals("syslog-stats-projection-memory", factory.getConfigurationProperties().get("client.id"));
+        assertEquals(java.util.Collections.singletonList("broker.example:9092"),
+                factory.getConfigurationProperties().get("bootstrap.servers"));
+        publisher.stop(); // 不创建连接即可构造/销毁，未依赖真实Kafka。
+    }
+
+    @Test
+    void commitOnlyOffersAndRollbackNeverEnqueues() throws Exception {
+        StatsProjectionAsyncPublisher publisher = publisher(10, 10);
+        StatsProjectionOutboxEvent event = event();
+        transaction.execute(status -> {
+            publisher.enqueueAfterCommit(event, 1);
+            assertEquals(0, metric(publisher, "pending_events"));
+            verifyNoInteractions(kafka);
+            return null;
+        });
+        assertEquals(1, metric(publisher, "pending_events"));
+        verifyNoInteractions(kafka); // 即使Kafka发送会阻塞，事实提交也不调用它。
+        transaction.execute(status -> {
+            publisher.enqueueAfterCommit(event(), 1);
+            status.setRollbackOnly();
+            return null;
+        });
+        assertEquals(1, metric(publisher, "enqueued_total"));
+        SettableListenableFuture<SendResult<String, String>> success = new SettableListenableFuture<>();
+        success.set(null);
+        when(kafka.send(anyString(), anyString(), anyString())).thenReturn(success);
+        publisher.flush();
+        assertEquals(1, metric(publisher, "published_total"));
+        assertEquals(0, metric(publisher, "pending_groups"));
+        ArgumentCaptor<String> payload = ArgumentCaptor.forClass(String.class);
+        verify(kafka).send(eq("stats-topic"), eq(StatsProjectionOutboxEvent.KAFKA_KEY), payload.capture());
+        StatsProjectionOutboxEvent sent = new ObjectMapper().readValue(payload.getValue(), StatsProjectionOutboxEvent.class);
+        assertFalse(sent.isOutboxBacked());
+        assertEquals(event.getEventId(), sent.getEventId());
+    }
+
+    @Test
+    void kafkaFailureRetriesSameEventWhileQueueOverflowLetsFactCommit() {
+        StatsProjectionAsyncPublisher publisher = publisher(1, 10);
+        SettableListenableFuture<SendResult<String, String>> failure = new SettableListenableFuture<>();
+        failure.setException(new IllegalStateException("Kafka unavailable"));
+        SettableListenableFuture<SendResult<String, String>> success = new SettableListenableFuture<>();
+        success.set(null);
+        when(kafka.send(anyString(), anyString(), anyString())).thenReturn(failure, success, success);
+        commit(publisher);
+        assertDoesNotThrow(publisher::flush);
+        assertEquals(1, metric(publisher, "publish_failures_total"));
+        commit(publisher);
+        commit(publisher); // queue=1，另有一个正在重试；第三条只损失统计。
+        assertEquals(2, metric(publisher, "pending_events"));
+        assertEquals(1, metric(publisher, "dropped_events_total"));
+        assertEquals(1, metric(publisher, "dropped_fact_rows_total"));
+        Object pending = ReflectionTestUtils.getField(publisher, "inFlight");
+        ReflectionTestUtils.setField(pending, "retryAt", 0L);
+        publisher.flush();
+        assertEquals(2, metric(publisher, "published_total"));
+        assertEquals(0, metric(publisher, "pending_groups"));
+        ArgumentCaptor<String> payload = ArgumentCaptor.forClass(String.class);
+        verify(kafka, times(3)).send(anyString(), anyString(), payload.capture());
+        assertEquals(payload.getAllValues().get(0), payload.getAllValues().get(1));
+    }
+
+    @Test
+    void blockedBackgroundSendDoesNotBlockNextCommit() throws Exception {
+        StatsProjectionAsyncPublisher publisher = publisher(1, 10);
+        SettableListenableFuture<SendResult<String, String>> blocked = new SettableListenableFuture<>();
+        java.util.concurrent.CountDownLatch entered = new java.util.concurrent.CountDownLatch(1);
+        when(kafka.send(anyString(), anyString(), anyString())).thenAnswer(call -> {
+            entered.countDown();
+            return blocked;
+        });
+        commit(publisher);
+        CompletableFuture<Void> worker = CompletableFuture.runAsync(publisher::flush);
+        try {
+            assertTrue(entered.await(3, TimeUnit.SECONDS));
+            // 使用另一线程+有界等待，防止错误地共享flush锁使测试永久挂住。
+            CompletableFuture.runAsync(() -> { commit(publisher); commit(publisher); }).get(2, TimeUnit.SECONDS);
+            assertEquals(1, metric(publisher, "dropped_events_total"));
+        } finally {
+            blocked.set(null);
+            worker.get(3, TimeUnit.SECONDS);
+        }
+    }
+
+    @Test
+    void groupBudgetIsBoundedAndReleasedAfterSend() {
+        StatsProjectionAsyncPublisher publisher = publisher(10, 1);
+        commit(publisher);
+        commit(publisher);
+        assertEquals(1, metric(publisher, "pending_groups"));
+        assertEquals(1, metric(publisher, "dropped_events_total"));
+        SettableListenableFuture<SendResult<String, String>> success = new SettableListenableFuture<>();
+        success.set(null);
+        when(kafka.send(anyString(), anyString(), anyString())).thenReturn(success);
+        publisher.flush();
+        commit(publisher);
+        assertEquals(2, metric(publisher, "enqueued_total"));
+    }
+
+    @Test
+    void missingTransactionDisabledPublisherAndShutdownFailOpen() {
+        StatsProjectionAsyncPublisher publisher = publisher(10, 10);
+        assertDoesNotThrow(() -> publisher.enqueueAfterCommit(event(), 1));
+        assertEquals(1, metric(publisher, "dropped_events_total"));
+        ReflectionTestUtils.setField(publisher, "enabled", false);
+        commit(publisher);
+        ReflectionTestUtils.setField(publisher, "enabled", true);
+        publisher.stop();
+        commit(publisher);
+        assertEquals(3, metric(publisher, "dropped_events_total"));
+        assertEquals(0, metric(publisher, "pending_events"));
+        verifyNoInteractions(kafka);
+    }
+
+    @Test
+    void directStatisticsAggregationFailureDoesNotRollbackFacts() {
+        StatsProjectionAsyncPublisher publisher = publisher(10, 10);
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(jdbc,
+                new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(),
+                mock(StatsMaterializationPublisher.class), publisher);
+        ReflectionTestUtils.setField(writer, "businessZone", "invalid-zone");
+        assertDoesNotThrow(() -> transaction.execute(status -> {
+            writer.writeStandardizedRow(row());
+            assertFalse(status.isRollbackOnly());
+            return null;
+        }));
+        assertEquals(1, metric(publisher, "dropped_events_total"));
+        verifyNoInteractions(jdbc, kafka);
+    }
+
+    @Test
+    void directNonNormalUpdateAndDisabledPublishingNeverUseStatisticsSql() {
+        StatsProjectionAsyncPublisher publisher = publisher(10, 10);
+        ReflectionTestUtils.setField(publisher, "enabled", false);
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(jdbc,
+                new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(),
+                mock(StatsMaterializationPublisher.class), publisher);
+        ReflectionTestUtils.setField(writer, "businessZone", "Asia/Shanghai");
+        com.common.entity.SyslogNonNormalMessage old = new com.common.entity.SyslogNonNormalMessage();
+        old.setDeviceId(7);
+        old.setCreatedAt(java.time.OffsetDateTime.now());
+        com.common.entity.SyslogNonNormalMessage patch = new com.common.entity.SyslogNonNormalMessage();
+        patch.setDeviceId(8);
+        transaction.execute(status -> {
+            writer.writeStandardizedRow(row());
+            writer.writeNonNormalInsert(old);
+            writer.writeNonNormalUpdate(old, patch);
+            return null;
+        });
+        assertEquals(3, metric(publisher, "dropped_events_total"));
+        verifyNoInteractions(jdbc, kafka);
+    }
+
+    private StatsProjectionAsyncPublisher publisher(int capacity, long groups) {
+        StatsProjectionAsyncPublisher publisher = new StatsProjectionAsyncPublisher(kafka, new ObjectMapper(), capacity, groups);
+        ReflectionTestUtils.setField(publisher, "enabled", true);
+        ReflectionTestUtils.setField(publisher, "topic", "stats-topic");
+        ReflectionTestUtils.setField(publisher, "timeoutMs", 10000L);
+        return publisher;
+    }
+
+    private void commit(StatsProjectionAsyncPublisher publisher) {
+        transaction.execute(status -> { publisher.enqueueAfterCommit(event(), 1); return null; });
+    }
+
+    private static Map<String, Object> row() {
+        Map<String, Object> row = new HashMap<>();
+        row.put("created_at", LocalDateTime.of(2026, 9, 13, 8, 0));
+        row.put("device_id", 7L);
+        return row;
+    }
+
+    private static StatsProjectionOutboxEvent event() {
+        return StatsProjectionOutboxEvent.fromAccumulator(new DirectStatsAccumulator().addStandardizedRow(row()));
+    }
+
+    private static long metric(StatsProjectionAsyncPublisher publisher, String key) {
+        return ((Number) publisher.status().get(key)).longValue();
+    }
+
+    private static class TestTransactionManager extends AbstractPlatformTransactionManager {
+        protected Object doGetTransaction() { return new Object(); }
+        protected void doBegin(Object transaction, TransactionDefinition definition) { }
+        protected void doCommit(DefaultTransactionStatus status) { }
+        protected void doRollback(DefaultTransactionStatus status) { }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncTest.java

@@ -138,6 +138,32 @@ class StatsProjectionAsyncTest {
         return event;
     }
 
+    @Test
+    void memoryEventUpdatesProjectionAndDedupMarkerWithoutTouchingOutbox() throws Exception {
+        StatsProjectionOutboxEvent event = oneDeviceEvent("memory-1");
+        event.setOutboxBacked(false);
+        ObjectMapper mapper = new ObjectMapper();
+        event = mapper.readValue(mapper.writeValueAsString(event), StatsProjectionOutboxEvent.class);
+        org.junit.jupiter.api.Assertions.assertFalse(event.isOutboxBacked());
+        assertTrue(mapper.readValue("{\"eventId\":\"legacy\"}", StatsProjectionOutboxEvent.class).isOutboxBacked());
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        PlatformTransactionManager manager = mock(PlatformTransactionManager.class);
+        when(manager.getTransaction(any(TransactionDefinition.class))).thenReturn(new SimpleTransactionStatus());
+        StatsProjectionDeltaConsumer consumer = new StatsProjectionDeltaConsumer(
+                jdbc, manager, mapper, mock(StatsMaterializationPublisher.class));
+        configure(consumer);
+        consumer.applyWithRetry(Collections.singletonList(event));
+        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
+        verify(jdbc, times(2)).batchUpdate(sql.capture(), anyList());
+        assertTrue(sql.getAllValues().get(0).contains("MERGE INTO device_daily_stats"));
+        assertTrue(sql.getAllValues().get(1).contains("stats_projection_applied_event"));
+        when(jdbc.queryForList(org.mockito.ArgumentMatchers.anyString(),
+                org.mockito.ArgumentMatchers.eq(String.class), org.mockito.ArgumentMatchers.<Object>any()))
+                .thenReturn(Collections.singletonList("memory-1"));
+        assertEquals(1, consumer.applyWithRetry(Collections.singletonList(event)).duplicates);
+        verify(jdbc, times(2)).batchUpdate(org.mockito.ArgumentMatchers.anyString(), anyList());
+    }
+
     private void configure(StatsProjectionDeltaConsumer consumer) {
         ReflectionTestUtils.setField(consumer, "enabled", true);
         ReflectionTestUtils.setField(consumer, "appliedTable", "stats_projection_applied_event");

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionPublisherWiringTest.java

@@ -0,0 +1,36 @@
+package com.common.service;
+
+import com.config.CacheConfig;
+import com.config.WebConfig;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+import org.springframework.mock.env.MockEnvironment;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/** 使用实际配置创建两个同类型Bean，覆盖现场启动时的构造器自动装配。 */
+class StatsProjectionPublisherWiringTest {
+    @Test
+    void startsWithCacheAndHttpMappersWithoutChangingSharedSerialization() {
+        try (AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext()) {
+            context.setEnvironment(new MockEnvironment()
+                    .withProperty("stats.projection-async.topic", "test.STATS.PROJECTION"));
+            context.registerBean("objectMapper", ObjectMapper.class, () -> new CacheConfig().objectMapper());
+            context.registerBean("httpObjectMapper", ObjectMapper.class, () -> new WebConfig().httpObjectMapper());
+            context.registerBean(KafkaProperties.class);
+            context.register(StatsProjectionAsyncPublisher.class);
+            assertDoesNotThrow(context::refresh);
+            assertEquals(2, context.getBeansOfType(ObjectMapper.class).size());
+            StatsProjectionAsyncPublisher publisher = context.getBean(StatsProjectionAsyncPublisher.class);
+            ObjectMapper shared = context.getBean("objectMapper", ObjectMapper.class);
+            ObjectMapper copied = (ObjectMapper) ReflectionTestUtils.getField(publisher, "mapper");
+            assertNotSame(shared, copied);
+            assertEquals(shared.getRegisteredModuleIds(), copied.getRegisteredModuleIds());
+            assertNotNull(shared.getSerializationConfig().getDefaultTyper(shared.constructType(Object.class)));
+            assertNull(copied.getSerializationConfig().getDefaultTyper(copied.constructType(Object.class)));
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/KafkaReliabilityConfigTest.java

@@ -8,12 +8,34 @@ import org.springframework.kafka.listener.ContainerProperties;
 import org.springframework.kafka.listener.DefaultErrorHandler;
 
 import java.util.Collections;
+import org.mockito.ArgumentCaptor;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.verify;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 class KafkaReliabilityConfigTest {
+    @Test
+    void rawDefaultIs500BaseAnd3000NativePollCeiling() throws Exception {
+        java.util.Properties properties = new java.util.Properties();
+        try (java.io.InputStream input = getClass().getResourceAsStream("/application.properties")) {
+            properties.load(input);
+        }
+        assertEquals("${SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS:500}",
+                properties.getProperty("syslog.raw-listener.max-poll-records"));
+        java.lang.reflect.Method method = java.util.Arrays.stream(KafkaReliabilityConfig.class.getDeclaredMethods())
+                .filter(candidate -> candidate.getName().equals("rawKafkaListenerContainerFactory")).findFirst().get();
+        org.springframework.beans.factory.annotation.Value fallback = method.getParameters()[4]
+                .getAnnotation(org.springframework.beans.factory.annotation.Value.class);
+        assertEquals("${syslog.raw-listener.max-poll-records:500}", fallback.value());
+        assertEquals("${SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS:3000}",
+                properties.getProperty("syslog.raw-listener.adaptive-max-records"));
+        assertEquals("${syslog.raw-listener.adaptive-max-records:3000}", method.getParameters()[5]
+                .getAnnotation(org.springframework.beans.factory.annotation.Value.class).value());
+    }
 
     @Test
     void retryListenerCommitsInsideAcknowledgment() {
@@ -39,13 +61,35 @@ class KafkaReliabilityConfigTest {
         @SuppressWarnings("unchecked")
         ConsumerFactory<Object, Object> consumerFactory = mock(ConsumerFactory.class);
         DefaultErrorHandler errorHandler = config.syslogDefaultErrorHandler();
+        when(consumerFactory.getConfigurationProperties()).thenReturn(
+                Collections.singletonMap(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 500));
 
         ConcurrentKafkaListenerContainerFactory<Object, Object> factory =
                 config.rawKafkaListenerContainerFactory(
-                        configurer, consumerFactory, errorHandler, 3);
+                        configurer, consumerFactory, errorHandler, 3, 500, 3000);
 
         assertEquals(ContainerProperties.AckMode.MANUAL_IMMEDIATE,
                 factory.getContainerProperties().getAckMode());
+        ArgumentCaptor<ConsumerFactory> actual = ArgumentCaptor.forClass(ConsumerFactory.class);
+        verify(configurer).configure(any(), actual.capture());
+        assertEquals(3000, actual.getValue().getConfigurationProperties().get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
+        assertEquals(false, actual.getValue().getConfigurationProperties().get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG));
+        assertEquals(500, consumerFactory.getConfigurationProperties().get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
+    }
+
+    @Test
+    void independentIngressOverridesAutoCommitAndOffsetResetFromBusinessConsumer() {
+        KafkaReliabilityConfig config = new KafkaReliabilityConfig();
+        ConcurrentKafkaListenerContainerFactoryConfigurer configurer = mock(ConcurrentKafkaListenerContainerFactoryConfigurer.class);
+        ConsumerFactory<Object, Object> source = mock(ConsumerFactory.class);
+        when(source.getConfigurationProperties()).thenReturn(Collections.singletonMap(
+                ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true));
+        config.ingressStatsKafkaListenerContainerFactory(configurer, source,
+                config.syslogDefaultErrorHandler(), 2000, "earliest");
+        ArgumentCaptor<ConsumerFactory> actual = ArgumentCaptor.forClass(ConsumerFactory.class);
+        verify(configurer).configure(any(), actual.capture());
+        assertEquals(false, actual.getValue().getConfigurationProperties().get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG));
+        assertEquals("earliest", actual.getValue().getConfigurationProperties().get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG));
     }
 
     @Test

haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/SyslogBuildStartupLoggerTest.java

@@ -0,0 +1,51 @@
+package com.config;
+
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
+import org.junit.jupiter.api.Test;
+import org.slf4j.LoggerFactory;
+import org.springframework.mock.env.MockEnvironment;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class SyslogBuildStartupLoggerTest {
+    @Test
+    void logsCodeRevisionAndEffectiveOverrides() {
+        MockEnvironment environment = new MockEnvironment()
+                .withProperty("syslog.raw-listener.max-poll-records", "700")
+                .withProperty("syslog.raw-listener.adaptive-max-records", "2500")
+                .withProperty("stats.projection-async.enabled", "false");
+        environment.setActiveProfiles("prod");
+        SyslogBuildStartupLogger reporter = new SyslogBuildStartupLogger(environment);
+        Logger logger = (Logger) LoggerFactory.getLogger(SyslogBuildStartupLogger.class);
+        ListAppender<ILoggingEvent> appender = new ListAppender<>();
+        appender.start();
+        logger.addAppender(appender);
+        try {
+            reporter.onApplicationReady();
+            assertEquals(1, appender.list.size());
+            String message = appender.list.get(0).getFormattedMessage();
+            assertTrue(message.contains("[SYSLOG_BUILD_READY] codeRevision=" + SyslogBuildStartupLogger.CODE_REVISION));
+            assertTrue(message.contains("application=syslog-consumer-rule"));
+            assertTrue(message.contains("profiles=prod"));
+            assertTrue(message.contains("baseProcessingRecords=700"));
+            assertTrue(message.contains("adaptiveMaxRecords=2500"));
+            assertTrue(message.contains("statsPublisherEnabled=false"));
+        } finally {
+            logger.detachAppender(appender);
+            appender.stop();
+        }
+    }
+
+    @Test
+    void worksWithoutBuildMetadataAndIgnoresConfiguredRevision() {
+        String summary = new SyslogBuildStartupLogger(new MockEnvironment()
+                .withProperty("codeRevision", "fake")).startupSummary();
+        assertTrue(summary.contains("codeRevision=" + SyslogBuildStartupLogger.CODE_REVISION));
+        assertFalse(summary.contains("buildTime="));
+        assertTrue(summary.contains("baseProcessingRecords=500"));
+        assertTrue(summary.contains("adaptiveMaxRecords=3000"));
+    }
+
+}

haobang-security-dm/upgrade-doc/v4/README.md

@@ -4,13 +4,16 @@
 
 v4 只升级 `syslog-consumer-rule` 和达梦数据库，不修改 `syslog-serve`，也不新增 Kafka Topic。
 
+后续吞吐优化（独立入口统计、事实计数/审计/分钟桶异步化、RAW底层poll上限3000/业务基础500自适应、精细慢批次日志）见
+[吞吐优化升级配置与验证](./config/02-throughput-upgrade.md)。已完成v4表升级的现场应用此补丁无需新增DDL。
+
 本次升级包含：
 
 - 根据消息 `receive_time` 与当前时间的差值，对积压消息分级采样；
 - `syslog_normal_alarm` 始终全量保留，并在 InfluxDB 原文、普通表和非标表之前单独提交；
 - 未被保留的 `syslog_normal_data`、`syslog_non_normal_message` 及对应原文不写入存储；
 - 按消息来源小时、Topic、Partition、策略版本和实际保留率记录采样审计；
-- 审计表不可用时自动回退为全量保留，不执行有损采样；
+- 最新吞吐补丁采用消费优先：审计后台尽力写入，失败不取消采样；分钟桶提交后后台重试，可能丢通知，详见上述文档；
 - InfluxDB 原文写入变为非关键旁路，启动检查或运行时写入失败只记录并继续主消费。
 
 ## 文件与执行顺序

haobang-security-dm/upgrade-doc/v4/config/01-new-environment-variables.md

@@ -18,7 +18,7 @@ syslog-consumer-rule/docker-compose/docker-compose-consumer-rule-prod.yaml
       - SYSLOG_BACKLOG_SAMPLING_DEGRADE_STEPS=10
       - SYSLOG_BACKLOG_SAMPLING_POLICY_VERSION=v2
 
-      # 小时审计最多等待 5 秒；失败后 60 秒内不重试，并回退全量保留。
+      # 最新吞吐补丁：小时审计在后台独立事务执行；失败后60秒跳过新审计，不取消采样。
       - SYSLOG_BACKLOG_SAMPLING_AUDIT_TIMEOUT_SECONDS=5
       - SYSLOG_BACKLOG_SAMPLING_AUDIT_RETRY_BACKOFF_MS=60000
 
@@ -36,13 +36,14 @@ syslog-consumer-rule/docker-compose/docker-compose-consumer-rule-prod.yaml
   每级4.5分钟；
 - `START_LAG_MS` 必须大于等于0并且小于 `LAG_THRESHOLD_MS`，配置非法时应用拒绝启动；
 - 修改起始延迟、结束阈值、最低保留率或级数时，必须同时修改 `POLICY_VERSION`，以便审计区分策略；
-- `AUDIT_TIMEOUT_SECONDS` 是单次达梦审计事务超时；审计失败时本批次全量保留；
+- `AUDIT_TIMEOUT_SECONDS` 是后台单次达梦审计事务超时；审计失败记录损失，业务仍按采样比例处理；
 - `INFLUXDB_RAW_WRITE_REQUEST_TIMEOUT` 是连接、写入、读取各网络阶段的上限，不是整个批次的总超时；
 - `INFLUXDB_RAW_FAILURE_BACKOFF_MS` 是确认 InfluxDB 依赖故障后的快速跳过时间。
 
 这些默认值已经写在 JAR 的 `application.properties` 中。现场只替换 JAR 时同样会默认开启采样；显式写入
 Compose 是为了便于后续核查和调整。启动新 JAR 前必须先创建
-`SYSLOG_BACKLOG_RETENTION_HOURLY`，否则应用会因审计失败回退全量保留，并周期性记录错误日志。
+`SYSLOG_BACKLOG_RETENTION_HOURLY`，否则审计会持续失败、丢失统计，但不会取消采样或阻止业务消费。
+最新普通RAW底层poll上限3000、业务基础500按保留率自适应，配置及精细日志说明见 [吞吐补丁](./02-throughput-upgrade.md)。
 
 关闭采样只需设置：
 

haobang-security-dm/upgrade-doc/v4/config/02-throughput-upgrade.md

@@ -0,0 +1,199 @@
+# 吞吐优化：统计退出事实写库路径、普通业务批次自适应500～3000条
+
+只升级 `syslog-consumer-rule`，保持 JDK 8。无需新增表或 Topic；不能删除现有
+`stats_projection_outbox`、`stats_projection_applied_event`：前者用于排空旧事件及兼容回滚，后者继续用于统计去重。
+
+## 本次行为
+
+- 普通 RAW 底层每次poll上限固定3000，业务块以500为基础按采样阶段动态扩大（默认最低档2500，最多3000）。
+  无积压时也可能一次poll返回3000条，随后分成500条业务块；Retry仍使用通用配置，大日志仍默认5条。
+- 普通业务处理按 Kafka value 序列化长度分成最多16MiB的块，单条不拆分；一个poll的全部块完成后ACK。
+- 每个业务块最多8个规则worker，按原顺序汇总结果。不会为2000条消息同时排队2000个任务。
+- 主处理方法不再调用入口统计写库，因此主批次日志 `ingressMs=0`。
+- 独立统计消费组读取原始普通/大日志Topic，解密后只读取设备头，不做规则或采样。
+  批内聚合后直接更新设备/采集任务MIN/MAX时间并提交达梦，再提交自己的offset；不写入口Outbox。
+- 独立统计数据库异常只重试统计组，不推进其offset。非法密文/头记录
+  `INGRESS_STATS_INVALID` 和错误尝试计数后跳过，不阻塞其它入口状态、不重复投递业务Retry。
+  密钥配置错误也会导致非法记录，发现该标记必须检查；原始记录只能在Kafka保留期内重放。
+- 标准化告警优先提交；告警分钟桶改为提交后入队，后台独立事务按设备、分钟固定顺序写入。
+  通知失败只后台重试，不回滚已入库告警；队列满/进程退出可能丢通知，导致部分迟到数据漏重算。
+- 采样审计也改为后台独立事务：失败不再取消采样，不回退全量入库。审计可能缺失或提交结果不确定，
+  对累加审计不盲目重试，避免重复计数；内存采样指标描述决策，不代表审计或业务数据已成功写库。
+- 生产默认 `stats.write-mode=DIRECT` 下，告警、普通、非标事实计数不再写Outbox，也不在事实事务内更新日统计表。
+  只做批内聚合并注册提交回调，事务成功提交后使用非等待式 `offer` 入有界内存队列。
+  Kafka发送、等待确认和重试在专用后台线程执行，使用独立Producer缓冲区；事实回滚不会入队。
+- 统计构造异常、队列满、关闭发布等情况记录异常/计数后放行，不回滚事实、不重试主消费。
+  Kafka发送失败暂存在有界队列内后台重试；饱和后丢弃新统计事件，不丢弃对应事实数据。
+  进程退出、崩溃及队列溢出可能造成统计缺失。这是本次明确选择的“消费连续性优先”，不保证统计精确。
+- 新统计事件标记 `outboxBacked=false`，独立投影消费者仅写统计及去重标记，不访问Outbox。
+  老事件默认 `true`，继续按原流程排空和标记Outbox。混合升级期间旧消费者仍可能执行无命中的Outbox UPDATE，
+  应完成所有消费节点升级。旧入口Outbox的MIN/MAX重放安全。
+
+## properties及变量
+
+均位于 `syslog-consumer-rule/src/main/resources/application.properties`，prod继承这些配置。
+
+| properties | 环境变量 | 默认值 |
+| --- | --- | --- |
+| syslog.raw-listener.max-poll-records | SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS | 500 |
+| syslog.raw-listener.adaptive-max-records | SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS | 3000 |
+| syslog.raw-listener.processing-batch-max-bytes | SYSLOG_RAW_PROCESSING_BATCH_MAX_BYTES | 16777216 |
+| syslog.normalization.workers-per-batch | SYSLOG_NORMALIZATION_WORKERS_PER_BATCH | 8 |
+| stats.ingress-metadata.group-id | STATS_INGRESS_GROUP_ID | 主消费组名加 `-ingress-v1` |
+| stats.ingress-metadata.large-group-id | STATS_INGRESS_LARGE_GROUP_ID | 入口统计组名加 `-large` |
+| stats.ingress-metadata.max-poll-records | STATS_INGRESS_MAX_POLL_RECORDS | 2000 |
+| stats.ingress-metadata.auto-offset-reset | STATS_INGRESS_AUTO_OFFSET_RESET | earliest |
+| stats.projection-async.memory-queue-capacity | STATS_PROJECTION_MEMORY_QUEUE_CAPACITY | 1000 |
+| stats.projection-async.memory-max-pending-groups | STATS_PROJECTION_MEMORY_MAX_PENDING_GROUPS | 50000 |
+| analysis.late-replay.notify-timeout-seconds | ANALYSIS_LATE_REPLAY_NOTIFY_TIMEOUT_SECONDS | 5 |
+
+只替换JAR也会使用上述默认值；已有 `SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS=2000` 时须删除或改成500。
+`spring.kafka.consumer.max-poll-records=500` 留给通用消费者，不再决定普通RAW批次上限。
+Kafka上限不保证每次拉满；业务块还受到16MiB限制。达梦每个小事务仍最多500行。
+保留旧配置名 `syslog.raw-listener.max-poll-records`，但其含义现在是**业务基础条数**。
+底层RAW的 `max.poll.records` 固定使用 `adaptive-max-records`，不重建consumer、不反射修改Kafka内部参数。
+基础值必须大于0且不超过上限，非法配置拒绝启动。独立入口统计默认2000条（不做规则计算），不采用该策略。
+
+业务块目标条数 = min(上限, ceil(基础条数 × 10000 / 较高保留万分比))。
+普通最低1000、非标最低2000，取较高2000，500×10000÷2000=2500；两者都为1000时命中3000上限。
+两者都为0时也只到上限，避免除0；告警仍全部保留。各阶段自动沿用现有采样决策，不另配倍率。
+
+实现采用**上一业务块的阶段反馈决定下一块大小**。每个RAW消费线程独立，首次或超过1分钟无有效反馈时从500探测。
+一个块内存在多个延迟阶段时取最高保留率，准备/规则异常保守回到500；恢复全量保存后的下一块回到500。
+例如首次poll3000条且均为默认最低档：先处理500，确认保留率后处理余下2500；后续poll可按2500+500处理。
+不会为了凑足目标条数等待新消息，也不会跳过已poll的尾部记录；每个poll所有块处理完才提交offset。
+启用连续性放行时失败块仍沿用既有DLT/记录策略，之后以基础批量继续，未增加新的业务丢弃规则。
+所有消息仍要解密、跑规则才能确认告警，扩大批量会增加整体poll处理耗时、内存和GC压力。
+整次poll的处理必须在max.poll.interval.ms内完成；16MiB仅限制单个业务块，不限制整个Kafka缓存。
+
+内存队列容量按批次聚合事件计，不是原始日志条数；另有最多1个发送中事件。
+聚合分组上限包含排队和发送中事件，不储存原文。默认每200ms调度发送，失败按既有
+`stats.projection-async.retry-backoff-ms`（默认1000ms）后台重试，保留同一eventId以便下游去重。
+`stats.projection-async.enabled=false` 不会回退到同步统计SQL，而是跳过统计、记录损失；正常保持true。
+历史 `DIRECT_SHADOW` / `LEGACY_CHANGE_LOG` 模式保留旧语义，本次消除统计同步写库针对默认DIRECT。
+
+保留 `stats.ingress-metadata.enabled=true`。此前建议用 `STATS_INGRESS_METADATA_ENABLED=false`
+应急关闭不适用：当正式统计回写启用时，现有启动校验会拒绝此组合。
+
+## 新消费组首次启动
+
+三个节点需使用相同的入口统计组名；它们不能与业务组相同，程序会拒绝这种错误配置。
+新组需要读取对应原始Topic和提交offset的Kafka权限。
+
+默认 `earliest` 从Kafka当前仍保留的最早记录开始，能够补齐保留期内入口时间；已有组offset时续跑。
+现场有大量历史数据时，新统计组也会回放这些记录，增加Kafka网络、解密CPU和数据库负载。
+只做MIN/MAX更新，不会根据原始消息再次增加成功入库条数。
+
+若现场明确只需接管上线后的接收时间，可在首次启动前设置 `STATS_INGRESS_AUTO_OFFSET_RESET=latest`；
+该选择不会回放启动前尚未更新的入口时间，且不影响已有offset。不能仅为了让LAG显示为零而选latest。
+Kafka保留期必须覆盖统计积压；过期记录无法由这个消费组恢复。
+
+不需要重置业务组offset。混合版本滚动期间，旧版仍可能写入口Outbox，新版直接更新MIN/MAX，
+这些时间字段允许重复；三个节点完成升级后主消费才全部移除入口统计等待。
+
+## 现场验证
+
+应用就绪时输出一条 `[SYSLOG_BUILD_READY]`，本次代码标识为
+`codeRevision=ASYNC_STATS_ADAPTIVE_RAW_V1`。同时打印应用名以及生效的基础条数、上限、并发和统计开关。
+标识固定在 Java 代码中，不依赖构建插件，无需修改 pom.xml，也不打印构建时间。
+代码标识证明包含本次功能修订，不代表唯一构建或未来永远是最新版本；后续功能修订需同步更新标识。
+只检查当前容器这次启动后的日志，避免把上次启动的记录当成本次证据：
+
+```bash
+docker logs --since "$(docker inspect -f '{{.State.StartedAt}}' syslog-consumer-rule-dm)" syslog-consumer-rule-dm 2>&1 | grep 'SYSLOG_BUILD_READY' | tail -1
+```
+
+该日志表示应用已就绪，不证明数据库写入正常或 Kafka 已分配分区，仍需检查以下运行指标。
+
+在各消费节点执行（宿主机，单行命令）：
+
+```bash
+docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'Raw Kafka batch configuration|Ingress stats consumer ready' | tail -10
+docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'SLOW_SYSLOG_BATCH|SLOW_DM_TRANSACTION|SLOW_DM_QUEUE' | tail -80
+docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'INGRESS_STATS_BATCH|INGRESS_STATS_INVALID|INGRESS_STATS_WRITE_FAILED' | tail -30
+docker exec syslog-consumer-rule-dm sh -c 'curl -fsS http://127.0.0.1:8289/xdrrule/api/syslog/stats/ingress/status'
+docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'STATS_ASYNC_LOSS|STATS_ASYNC_PUBLISH_FAILED|STATS_ASYNC_SHUTDOWN_PENDING' | tail -30
+docker exec syslog-consumer-rule-dm sh -c 'curl -fsS http://127.0.0.1:8289/xdrrule/api/syslog/stats/direct/status'
+```
+
+确认普通配置输出 `maxPollRecords=3000, baseProcessingRecords=500, adaptiveMaxRecords=3000`，
+主批次 `ingressMs=0`，入口状态模式为 `INDEPENDENT_KAFKA_DIRECT_MERGE`。
+`polledRecords`可大于500；`adaptiveTargetHistogram`和`actualChunkHistogram`分别表示业务目标与实际块大小。
+例如 `{500=1, 2500=1}` 表示对应大小各1块，不代表保留率。实际大小受剩余条数及字节上限限制。
+直接统计状态中 `daily_projection_write_mode=BEST_EFFORT_AFTER_COMMIT_KAFKA`，
+`async_publisher` 下有排队事件/分组数、已发布、失败重试、丢弃计数。
+`dropped_fact_rows_total` 指受统计缺失影响的事实行数/尝试行数，并不表示事实数据被删除。
+这些内存指标会在重启后清零，不能覆盖崩溃时丢失的统计。损失日志最多每10秒一次，计数逐次累加。
+`published_total` 仅表示Kafka确认，不代表统计已落达梦；统计最终入库还需检查投影消费组积压。
+`SLOW_DM_TRANSACTION` 自动沿用500ms阈值，无须打开所有事务INFO日志：
+`connectionBeginMs`包括取连接/事务初始化，`lateAlarmBucketMs`仅为分钟桶小键准备/注册提交回调，
+`jdbcExecuteMs`是JDBC批执行，`statsProjectionStageMs`在DIRECT下是统计事件构造/注册回调阶段，
+不含后台Kafka/统计写库；`commitMs`是事实事务提交阶段。状态中的统计成功指标表示聚合/注册完成，不保证投影落库。
+`SLOW_DM_QUEUE` 单独说明批写任务排队超过阈值。
+
+在有Kafka脚本的现场节点执行（已确认bootstrap地址）：
+
+```bash
+./kafka-consumer-groups.sh --bootstrap-server 10.141.121.35:9092 --group agent-01-syslog-group --describe
+./kafka-consumer-groups.sh --bootstrap-server 10.141.121.35:9092 --group agent-01-syslog-group-ingress-v1 --describe
+./kafka-consumer-groups.sh --bootstrap-server 10.141.121.35:9092 --group agent-01-syslog-group-ingress-v1-large --describe
+```
+
+如现场组名不同，使用启动日志中的真实组名。间隔60秒再查，比较所有分区CURRENT-OFFSET增量之和与
+LOG-END-OFFSET增量之和；前者持续大于后者才说明追赶积压。不要只比较筛选出的慢批次或单批毫秒数。
+
+入口统计已不再阻塞主线程，但仍共享Kafka、JVM和数据库资源，不能保证固定倍数提速。
+本地测试不替代现场达梦压测。主消费仍可能受事实表执行、COMMIT、规则计算、InfluxDB及异常路由等待限制。
+采样审计和告警分钟桶不再在主消费/事实事务执行SQL，但后台仍共享数据库和连接池资源。
+
+## 精细慢批次日志与后台状态
+
+```bash
+docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'SLOW_SYSLOG_BATCH|SLOW_DM_TRANSACTION|SLOW_DM_QUEUE|SLOW_NON_NORMAL_WRITE' | tail -80
+docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'SYSLOG_AUX_FAILURE|SYSLOG_AUX_LOSS|SYSLOG_AUX_SHUTDOWN_PENDING|SLOW_SYSLOG_AUX' | tail -40
+docker exec syslog-consumer-rule-dm sh -c 'curl -fsS http://127.0.0.1:8289/xdrrule/api/syslog/backlog-sampling/status'
+```
+
+拿到慢批次的batchId后，用 `docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep '实际batchId'`
+关联该批次的达梦事务/队列、非标异步写入日志。日志不打印原文。
+
+| 字段 | 如何判断 |
+| --- | --- |
+| batchId / offsetRanges | 本次监听调用的关联ID、各分区最小/最大offset（范围不代表中间全部连续） |
+| adaptiveRaw / baseProcessingRecords / adaptiveMaxRecords | 是否RAW自适应通道、基础条数、硬上限；Retry/大日志adaptiveRaw=false |
+| adaptiveTargetHistogram / actualChunkHistogram | 各业务目标条数/实际条数对应的块数，按整次poll累计 |
+| observedRetainBps / nextTargetRecords | 最近完成块观测的较高保留万分比及下一块目标；依赖上一块反馈，可能滞后一块 |
+| stageMs.ADAPTIVE_CHUNK_SELECTION | 计算RAW目标条数及按字节/条数选取业务块耗时 |
+| listenerGapMs | 同一监听线程上次调用结束到本次调用的间隔；首次为-1，不能据此直接判定Kafka空闲或提交慢 |
+| stageMs.PREPARE | 解密、元信息解析、原文点构造及准备失败路由 |
+| stageMs.RULE_SNAPSHOT | 规则快照及批次上下文准备 |
+| stageMs.RULE_EXECUTION_WAIT | 规则任务提交、排队和执行的整体墙钟时间 |
+| ruleQueueMaxMs / ruleWorkerBusySumMs | 规则worker最大排队时间/worker执行累计时间；累计值可大于墙钟时间，不可与stageMs相加 |
+| stageMs.SAMPLING_DECISION / AUDIT_ENQUEUE | 采样决策/审计聚合入队；没有审计SQL等待 |
+| stageMs.ALARM_DM / NORMAL_DM | 告警/普通表批写等待；用相同batchId的SLOW_DM_QUEUE和SLOW_DM_TRANSACTION继续分解 |
+| stageMs.DM_FAILURE_ROUTE / CONTINUITY_DLT | 业务坏行或整批异常的Retry/DLT处理耗时 |
+| stageMs.NON_NORMAL_ENQUEUE | 只表示非标异步提交耗时，不表示非标落库；看SLOW_NON_NORMAL_WRITE的queueMs/writeMs |
+| stageMs.INFLUX_AND_FAILURE_ROUTE | 仍然同步的原文写入及其失败处理耗时（此次未改成异步） |
+| alarmRequested/alarmSaved/normalRequested/normalSaved | 计划提交行数及成功返回的写入报告行数；异常中断时不是数据库最终事实数 |
+| rawSampledOut/normalSampledOut/nonNormalSampledOut | 已完成业务块的采样丢弃数，不包括异常丢弃 |
+| auditAcceptedChunks/auditLostChunks | 采样审计入队成功/失败块数；入队成功不代表后台已入库 |
+| failedChunks/failureStage/businessError | 中断的业务块数、最后失败阶段、异常类型；正常无失败时failureStage可为null |
+| kafkaCommitStatus | SUCCESS、FAILED、NOT_ATTEMPTED；业务中断不得误判为Kafka提交成功 |
+
+`stageMs`各阶段为互斥墙钟时间，跨业务块累加；旧有normalizeMs等字段包含部分子阶段，不能再与stageMs相加。
+兼容字段ruleComputeWorkerMs实际是整批规则计算墙钟耗时；新增ruleWorkerBusySumMs才是worker耗时之和，二者不要混淆。
+业务异常和提交异常即使小于500ms也输出诊断。`processedRecords/prepared`等旧计数仅覆盖完成的业务块；
+出现failedChunks时不能把它们理解为整次poll已成功入库。无限等待中的操作尚未结束，不会提前输出最终慢批次。
+
+采样审计接口新增 `audit_async`，直接统计接口的 `dm_transaction.late_alarm_bucket_async` 查看通知队列。
+两个队列各固定最多256个等待任务、最多50000个小分组（含发送中），各有独立单线程；满时记录后放行。
+审计失败后按既有audit-retry-backoff-ms暂停后台执行并跳过新审计，不取消采样；分钟桶每1秒后台重试失败任务。
+辅助异常日志最多每10秒一次，损失计数逐次累加，重启清零。进程退出可能丢失队列任务；未实现补偿扫描。
+新审计指标 `audit_lost_or_unconfirmed_messages_total` 表示未记录或提交结果未知的消息数，
+旧 `audit_fallback_messages_total` 本路径不再增加。
+
+## 回滚
+
+回滚旧JAR会恢复主消费同步入口统计、事实计数Outbox、同步审计及分钟桶保护，新独立listener随进程退出停止。
+新版本退出时尚未发送的内存统计可能丢失；回滚不自动恢复这些统计，后续需要另行校准。
+保留新增统计组offset、现有Outbox和审计表；不删除Topic、不重置主消费offset。