perf(syslog): 统计异步化并按采样保留率自适应调整业务批量

- 入口统计改用独立 Kafka 消费组，移除主消费同步统计写库
- DIRECT 模式入库计数改为事务提交后异步投递，取消事实事务内统计 Outbox 写入
- 采样审计、告警分钟桶通知改为后台处理，异常不阻断主消费或回滚告警
- RAW 基础业务批量默认 500，按保留率动态扩大，上限 3000
- 限制业务块字节数及规则任务数量，保留告警全量处理和整次 poll 完成后提交
- 细化 SLOW_SYSLOG_BATCH，补充批次关联、线程排队、数据库执行及提交耗时
- 更新 v4 升级说明，明确异步统计损失风险及现场验证方法

验证：JDK 8 构建成功，232 项测试通过。
升级：复用现有表和 Kafka Topic，无新增建表或建 Topic 脚本要求。

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/AdaptiveRawBatchSizerTest.java

@@ -0,0 +1,33 @@
+package com.Modules.NormalData;
+
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.*;
+
+class AdaptiveRawBatchSizerTest {
+    @Test void derivesMultiplierAndCapsWithoutPerStageConfiguration() {
+        AdaptiveRawBatchSizer sizer = new AdaptiveRawBatchSizer(500, 3000);
+        assertEquals(500, sizer.targetRecords(10000, 10000));
+        assertEquals(1000, sizer.targetRecords(5000, 5000));
+        assertEquals(2000, sizer.targetRecords(2500, 2500));
+        assertEquals(2500, sizer.targetRecords(1000, 2000));
+        assertEquals(3000, sizer.targetRecords(1000, 1000));
+        assertEquals(3000, sizer.targetRecords(0, 0));
+        assertEquals(500, sizer.targetRecords(0, 10000));
+        assertEquals(500, sizer.targetRecords(10000, 0));
+        assertEquals(834, sizer.targetRecords(5000, 6000));
+    }
+
+    @Test void supportsOtherBaseSizesWithoutOverflow() {
+        assertEquals(3000, new AdaptiveRawBatchSizer(1000, 3000).targetRecords(1000, 2000));
+        assertEquals(Integer.MAX_VALUE,
+                new AdaptiveRawBatchSizer(Integer.MAX_VALUE, Integer.MAX_VALUE).targetRecords(1, 1));
+    }
+
+    @Test void rejectsInvalidConfigurationAndRates() {
+        assertThrows(IllegalArgumentException.class, () -> new AdaptiveRawBatchSizer(0, 3000));
+        assertThrows(IllegalArgumentException.class, () -> new AdaptiveRawBatchSizer(500, 499));
+        AdaptiveRawBatchSizer sizer = new AdaptiveRawBatchSizer(500, 3000);
+        assertThrows(IllegalArgumentException.class, () -> sizer.targetRecords(-1, 2000));
+        assertThrows(IllegalArgumentException.class, () -> sizer.targetRecords(1000, 10001));
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogAdaptiveRawBatchTest.java

@@ -0,0 +1,155 @@
+package com.Modules.NormalData;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.kafka.support.Acknowledgment;
+import org.springframework.test.util.ReflectionTestUtils;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+class SysLogAdaptiveRawBatchTest {
+    private SysLogProcessor processor;
+    private Acknowledgment ack;
+
+    @BeforeEach void setup() {
+        processor = spy(new SysLogProcessor());
+        ReflectionTestUtils.setField(processor, "taskExecutor", (Executor) Runnable::run);
+        ReflectionTestUtils.setField(processor, "largeTaskExecutor", (Executor) Runnable::run);
+        ReflectionTestUtils.setField(processor, "largeProcessingBatchMaxBytes", 16777216L);
+        ReflectionTestUtils.setField(processor, "failureContinuityEnabled", false);
+        ack = mock(Acknowledgment.class);
+    }
+
+    @Test void livePollSplitsInto500WithoutEarlyAckOrRecordLoss() {
+        List<Integer> sizes = new ArrayList<>();
+        List<Long> offsets = new ArrayList<>();
+        doAnswer(call -> {
+            List<ConsumerRecord<String, String>> chunk = call.getArgument(0);
+            sizes.add(chunk.size());
+            chunk.forEach(record -> offsets.add(record.offset()));
+            verifyNoInteractions(ack);
+            processor.observeRawRetention(10000);
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(3000), ack);
+        assertEquals(Arrays.asList(500, 500, 500, 500, 500, 500), sizes);
+        for (int i = 0; i < 3000; i++) assertEquals((long) i, offsets.get(i).longValue());
+        verify(ack).acknowledge();
+    }
+
+    @Test void backlogExpandsNextChunkAndRecoveryShrinksFollowingChunk() {
+        List<Integer> sizes = new ArrayList<>();
+        doAnswer(call -> {
+            sizes.add(((List<?>) call.getArgument(0)).size());
+            processor.observeRawRetention(2000);
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(3000), ack);
+        assertEquals(Arrays.asList(500, 2500), sizes);
+        sizes.clear();
+        doAnswer(call -> {
+            sizes.add(((List<?>) call.getArgument(0)).size());
+            processor.observeRawRetention(10000);
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(3000), ack);
+        assertEquals(Arrays.asList(2500, 500), sizes);
+        sizes.clear();
+        processor.consumeAndParallelProcess(records(3000), ack);
+        assertEquals(Arrays.asList(500, 500, 500, 500, 500, 500), sizes);
+        verify(ack, times(3)).acknowledge();
+    }
+
+    @Test void tenPercentHits3000CapAndIncompleteTailIsNotHeldWaitingForMore() {
+        processor.observeRawRetention(1000);
+        List<Integer> sizes = new ArrayList<>();
+        doAnswer(call -> { sizes.add(((List<?>) call.getArgument(0)).size()); return null; })
+                .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(3000), ack);
+        processor.consumeAndParallelProcess(records(17), ack);
+        assertEquals(Arrays.asList(3000, 17), sizes);
+    }
+
+    @Test void bytesLimitStillWinsAndOversizedSingleRecordIsNotLost() {
+        List<ConsumerRecord<String, String>> records = Arrays.asList(
+                new ConsumerRecord<>("raw", 0, 0, "k", "中"),
+                new ConsumerRecord<>("raw", 0, 1, "k", "中"),
+                new ConsumerRecord<>("raw", 0, 2, "k", "oversized"),
+                new ConsumerRecord<>("raw", 0, 3, "k", "x"));
+        ReflectionTestUtils.setField(processor, "rawProcessingBatchMaxBytes", 3L);
+        processor.observeRawRetention(1000);
+        List<Integer> sizes = new ArrayList<>();
+        List<Long> offsets = new ArrayList<>();
+        doAnswer(call -> {
+            List<ConsumerRecord<String, String>> chunk = call.getArgument(0);
+            sizes.add(chunk.size());
+            chunk.forEach(record -> offsets.add(record.offset()));
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records, ack);
+        assertEquals(Arrays.asList(1, 1, 1, 1), sizes);
+        assertEquals(Arrays.asList(0L, 1L, 2L, 3L), offsets);
+        verify(ack).acknowledge();
+    }
+
+    @Test void failingLaterChunkDoesNotCommitUnfinishedPollAndResetsToBase() {
+        List<Integer> sizes = new ArrayList<>();
+        doAnswer(call -> {
+            sizes.add(((List<?>) call.getArgument(0)).size());
+            processor.observeRawRetention(2000);
+            if (sizes.size() == 2) throw new IllegalStateException("DM unavailable");
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        assertThrows(IllegalStateException.class, () -> processor.consumeAndParallelProcess(records(3000), ack));
+        verifyNoInteractions(ack);
+        assertEquals(Arrays.asList(500, 2500), sizes);
+        sizes.clear();
+        doAnswer(call -> { sizes.add(((List<?>) call.getArgument(0)).size()); return null; })
+                .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(1000), ack);
+        assertEquals(Arrays.asList(500, 500), sizes);
+    }
+
+    @Test void feedbackDoesNotLeakAcrossConsumerThreadsOrChangeRetryAndLargeLanes() throws Exception {
+        processor.observeRawRetention(1000);
+        List<Integer> sizes = Collections.synchronizedList(new ArrayList<>());
+        doAnswer(call -> { sizes.add(((List<?>) call.getArgument(0)).size()); return null; })
+                .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        ExecutorService otherConsumer = Executors.newSingleThreadExecutor();
+        try {
+            otherConsumer.submit(() -> processor.consumeAndParallelProcess(records(1000), ack)).get(3, TimeUnit.SECONDS);
+        } finally { otherConsumer.shutdownNow(); }
+        assertEquals(Arrays.asList(500, 500), sizes);
+        sizes.clear();
+        processor.consumeRetryAndReplay(records(1000), ack);
+        processor.consumeLargeMessages(records(1000), ack);
+        assertEquals(Arrays.asList(1000, 1000), sizes);
+    }
+
+    @Test void staleFeedbackResetsAfterOneMinuteWithoutObservation() {
+        processor.observeRawRetention(1000);
+        Object state = ((ThreadLocal<?>) ReflectionTestUtils.getField(processor, "rawAdaptiveState")).get();
+        ReflectionTestUtils.setField(state, "observedAt", System.nanoTime() - TimeUnit.MINUTES.toNanos(2));
+        List<Integer> sizes = new ArrayList<>();
+        doAnswer(call -> { sizes.add(((List<?>) call.getArgument(0)).size()); return null; })
+                .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        processor.consumeAndParallelProcess(records(1000), ack);
+        assertEquals(Arrays.asList(500, 500), sizes);
+    }
+
+    private List<ConsumerRecord<String, String>> records(int count) {
+        List<ConsumerRecord<String, String>> result = new ArrayList<>();
+        for (int i = 0; i < count; i++) result.add(new ConsumerRecord<>("raw", 0, i, "k", "value"));
+        return result;
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java

@@ -137,6 +137,32 @@ class SysLogProcessorParallelTest {
                 () -> processor.normalizePreparedMessages(messages(2), metrics));
     }
 
+    @Test
+    void concurrentLargePollsSubmitOnlyBoundedWorkersAndProcessEveryRecord() throws Exception {
+        AtomicInteger submissions = new AtomicInteger();
+        Executor countedExecutor = task -> {
+            submissions.incrementAndGet();
+            executor.execute(task);
+        };
+        ReflectionTestUtils.setField(processor, "taskExecutor", countedExecutor);
+        when(normalProcessor.prepare(any(PreparedNormalizationInput.class), any(), any(), any()))
+                .thenReturn(LogNormalProcessor.NormalizationResult.success(
+                        Collections.singletonList(normalizedRow())));
+        ExecutorService callers = Executors.newFixedThreadPool(3);
+        try {
+            List<java.util.concurrent.Future<?>> runs = new ArrayList<>();
+            for (int i = 0; i < 3; i++) {
+                runs.add(callers.submit(() -> processor.normalizePreparedMessages(messages(3000),
+                        new SysLogProcessor.BatchMetrics())));
+            }
+            for (java.util.concurrent.Future<?> run : runs) run.get(10, TimeUnit.SECONDS);
+            assertEquals(24, submissions.get(), "3个并发批次应提交24个worker，而非9000个任务");
+            verify(normalProcessor, times(9000)).prepare(any(PreparedNormalizationInput.class), any(), any(), any());
+        } finally {
+            callers.shutdownNow();
+        }
+    }
+
     @Test
     void recordsRuleComputeAndDmWriteTimingsSeparately() {
         when(normalProcessor.prepare(any(PreparedNormalizationInput.class), any(), any(), any()))
@@ -382,6 +408,31 @@ class SysLogProcessorParallelTest {
         assertEquals(1L, normalDropped);
         assertEquals(1L, nonNormalDropped);
         assertEquals(2L, rawDropped);
+        assertEquals(0, plan.maxRetainBps());
+        ReflectionTestUtils.setField(properties, "normalRetainBps", 1000);
+        ReflectionTestUtils.setField(properties, "nonNormalRetainBps", 2000);
+        assertEquals(2000, processor.buildSamplingPlan(computed).maxRetainBps(),
+                "放大倍率必须使用两类中较高保留率");
+        ReflectionTestUtils.setField(properties, "enabled", false);
+        assertEquals(10000, processor.buildSamplingPlan(computed).maxRetainBps());
+        ReflectionTestUtils.setField(properties, "enabled", true);
+        computed.add(new SysLogProcessor.PreparedNormalization(source.get(0), null,
+                SyslogFailureRouter.TerminalState.DLT_PUBLISHED));
+        assertEquals(10000, processor.buildSamplingPlan(computed).maxRetainBps(),
+                "规则结果未知时保守回到基础批量");
+        computed.remove(computed.size() - 1);
+        BacklogSamplingDecider mixed = mock(BacklogSamplingDecider.class);
+        LocalDateTime hour = LocalDateTime.of(2026, 9, 13, 0, 0);
+        BacklogSamplingDecision degraded = new BacklogSamplingDecision(true, true, true, true,
+                3600000L, hour, "test", 1000, 2000);
+        BacklogSamplingDecision recent = new BacklogSamplingDecision(false, true, true, true,
+                0L, hour, "test", 10000, 10000);
+        when(mixed.decide(any(), any(), org.mockito.ArgumentMatchers.anyBoolean(),
+                org.mockito.ArgumentMatchers.anyBoolean(), org.mockito.ArgumentMatchers.anyBoolean()))
+                .thenReturn(degraded, recent, degraded);
+        ReflectionTestUtils.setField(processor, "backlogSamplingDecider", mixed);
+        assertEquals(10000, processor.buildSamplingPlan(computed).maxRetainBps(),
+                "同块包含新旧不同阶段消息时取最高保留率");
     }
 
     private List<ConsumerRecord<String, String>> records(int count) {

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogSlowBatchTraceTest.java

@@ -0,0 +1,110 @@
+package com.Modules.NormalData;
+
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.kafka.support.Acknowledgment;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.stream.Collectors;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+import static org.mockito.ArgumentMatchers.*;
+
+class SysLogSlowBatchTraceTest {
+    private final Logger logger = (Logger) LoggerFactory.getLogger(SysLogProcessor.class);
+    private ListAppender<ILoggingEvent> appender;
+    private SysLogProcessor processor;
+
+    @BeforeEach void setup() {
+        appender = new ListAppender<>();
+        appender.start();
+        logger.addAppender(appender);
+        processor = spy(new SysLogProcessor());
+        ReflectionTestUtils.setField(processor, "taskExecutor", (Executor) Runnable::run);
+        ReflectionTestUtils.setField(processor, "slowBatchThresholdMs", 1L);
+        ReflectionTestUtils.setField(processor, "failureContinuityEnabled", false);
+        MDC.put("syslogBatchId", "previous-context");
+    }
+
+    @AfterEach void cleanup() {
+        logger.detachAppender(appender);
+        appender.stop();
+        MDC.remove("syslogBatchId");
+    }
+
+    @Test void successfulTraceContainsStageBreakdownOffsetsAndRestoresContext() {
+        doAnswer(call -> {
+            ReflectionTestUtils.invokeMethod(processor, "enterStage", "PREPARE");
+            Object trace = trace();
+            ReflectionTestUtils.setField(trace, "startedAtNanos", System.nanoTime() - 10_000_000L);
+            ReflectionTestUtils.setField(trace, "stageStarted", System.nanoTime() - 2_000_000L);
+            ReflectionTestUtils.invokeMethod(processor, "enterStage", "ALARM_DM");
+            return null;
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        Acknowledgment ack = mock(Acknowledgment.class);
+        processor.consumeAndParallelProcess(records(), ack);
+        verify(ack).acknowledge();
+        String log = onlyTrace();
+        assertTrue(log.contains("PREPARE="));
+        assertTrue(log.contains("ADAPTIVE_CHUNK_SELECTION="));
+        assertTrue(log.contains("adaptiveTargetHistogram={500=1}"));
+        assertTrue(log.contains("actualChunkHistogram={2=1}"));
+        assertTrue(log.contains("ALARM_DM="));
+        assertTrue(log.contains("raw:0=12..14"));
+        assertTrue(log.contains("kafkaCommitStatus=SUCCESS"));
+        assertTrue(log.contains("listenerGapMs=-1"));
+        assertFalse(log.contains("batchId=previous-context"));
+        assertEquals("previous-context", MDC.get("syslogBatchId"));
+        assertNull(trace());
+    }
+
+    @Test void businessAbortReportsFailingStageAndNoCommitInsteadOfFalseSuccess() {
+        doAnswer(call -> {
+            ReflectionTestUtils.invokeMethod(processor, "enterStage", "NORMAL_DM");
+            throw new IllegalStateException("DB stalled");
+        }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        Acknowledgment ack = mock(Acknowledgment.class);
+        assertThrows(IllegalStateException.class, () -> processor.consumeAndParallelProcess(records(), ack));
+        verifyNoInteractions(ack);
+        String log = onlyTrace();
+        assertTrue(log.contains("failureStage=NORMAL_DM"));
+        assertTrue(log.contains("kafkaCommitStatus=NOT_ATTEMPTED"));
+        assertTrue(log.contains("failedChunks=1"));
+        assertEquals("previous-context", MDC.get("syslogBatchId"));
+    }
+
+    @Test void commitFailureIsLoggedExactlyOnce() {
+        doNothing().when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
+        Acknowledgment ack = mock(Acknowledgment.class);
+        doThrow(new IllegalStateException("commit rejected")).when(ack).acknowledge();
+        assertThrows(IllegalStateException.class, () -> processor.consumeAndParallelProcess(records(), ack));
+        String log = onlyTrace();
+        assertTrue(log.contains("kafkaCommitStatus=FAILED"));
+        assertTrue(log.contains("kafkaCommitErrorType=java.lang.IllegalStateException"));
+        assertEquals("previous-context", MDC.get("syslogBatchId"));
+    }
+
+    private Object trace() {
+        return ((ThreadLocal<?>) ReflectionTestUtils.getField(processor, "slowBatchTrace")).get();
+    }
+    private String onlyTrace() {
+        List<String> logs = appender.list.stream().map(ILoggingEvent::getFormattedMessage)
+                .filter(line -> line.contains("[SLOW_SYSLOG_BATCH]")).collect(Collectors.toList());
+        assertEquals(1, logs.size());
+        assertFalse(logs.get(0).contains("batchId={}"));
+        return logs.get(0);
+    }
+    private List<ConsumerRecord<String, String>> records() {
+        return Arrays.asList(new ConsumerRecord<>("raw", 0, 12L, "k", "first"),
+                new ConsumerRecord<>("raw", 0, 14L, "k", "second"));
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BacklogSamplingAuditServiceTest.java

@@ -41,7 +41,7 @@ class BacklogSamplingAuditServiceTest {
     }
 
     @Test
-    void auditFailureReturnsFalseAndRecordsFullRetentionFallback() {
+    void auditFailureIsBackgroundOnlyAndNeverTriggersFullRetentionFallback() {
         BacklogRetentionAuditMapper mapper = mock(BacklogRetentionAuditMapper.class);
         when(mapper.upsert(any())).thenThrow(new IllegalStateException("table unavailable"));
         PlatformTransactionManager transactionManager = transactionManager();
@@ -49,13 +49,18 @@ class BacklogSamplingAuditServiceTest {
         BacklogSamplingAuditService service = new BacklogSamplingAuditService(
                 mapper, properties(), metrics, transactionManager);
 
-        boolean audited = service.recordBeforeDropping(Arrays.asList(
+        boolean audited = service.recordAsync(Arrays.asList(
                 delta(2, 1, 1, 1, 1)));
 
-        assertFalse(audited);
+        assertTrue(audited);
+        org.mockito.Mockito.verifyNoInteractions(mapper, transactionManager);
+        ((BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(service, "queue")).drain();
         Map<String, Object> snapshot = metrics.snapshot();
         assertEquals(1L, snapshot.get("audit_write_failures_total"));
-        assertEquals(2L, snapshot.get("audit_fallback_messages_total"));
+        assertEquals(0L, snapshot.get("audit_fallback_messages_total"));
+        assertEquals(2L, snapshot.get("audit_lost_or_unconfirmed_messages_total"));
+        assertFalse(service.recordAsync(Arrays.asList(delta(1, 0, 1, 0, 1))));
+        assertEquals(3L, metrics.snapshot().get("degraded_messages_total"));
     }
 
     @Test
@@ -67,10 +72,13 @@ class BacklogSamplingAuditServiceTest {
         BacklogSamplingAuditService service = new BacklogSamplingAuditService(
                 mapper, properties(), metrics, transactionManager);
 
-        assertTrue(service.recordBeforeDropping(Arrays.asList(
+        assertTrue(service.recordAsync(Arrays.asList(
                 delta(1, 0, 1, 0, 1))));
 
+        org.mockito.Mockito.verifyNoInteractions(mapper, transactionManager);
+        ((BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(service, "queue")).drain();
         verify(mapper).upsert(any());
+        verify(transactionManager).commit(any());
         assertEquals(1L, metrics.snapshot().get("degraded_messages_total"));
         assertEquals(1L, metrics.snapshot().get("normal_rows_dropped_total"));
     }

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BestEffortAuxiliaryQueueTest.java

@@ -0,0 +1,64 @@
+package com.common.service;
+
+import org.junit.jupiter.api.Test;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import static org.junit.jupiter.api.Assertions.*;
+
+class BestEffortAuxiliaryQueueTest {
+    @Test
+    void hungAuxiliaryDatabaseDoesNotBlockMainEnqueueAndCapacityIsBounded() throws Exception {
+        BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("TEST", 1, 2, true);
+        CountDownLatch entered = new CountDownLatch(1);
+        CountDownLatch release = new CountDownLatch(1);
+        assertTrue(queue.offer(1, () -> {
+            entered.countDown();
+            try { release.await(5, TimeUnit.SECONDS); }
+            catch (InterruptedException e) { Thread.currentThread().interrupt(); }
+        }));
+        CompletableFuture<Void> writer = CompletableFuture.runAsync(queue::drain);
+        try {
+            assertTrue(entered.await(2, TimeUnit.SECONDS));
+            CompletableFuture.runAsync(() -> {
+                assertTrue(queue.offer(1, () -> { }));
+                assertFalse(queue.offer(1, () -> fail("overflow must never run")));
+            }).get(2, TimeUnit.SECONDS);
+            assertEquals(2L, queue.status().get("pending_groups"));
+            assertEquals(1L, queue.status().get("lost_tasks_total"));
+        } finally {
+            release.countDown();
+            writer.get(3, TimeUnit.SECONDS);
+            queue.stop();
+        }
+        assertEquals(2L, queue.status().get("completed_total"));
+        assertEquals(0L, queue.status().get("pending_groups"));
+    }
+
+    @Test
+    void auditFailureIsNotAutomaticallyReplayedAndCapacityIsReleased() {
+        BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("AUDIT_TEST", 1, 1, false);
+        AtomicInteger attempts = new AtomicInteger();
+        assertTrue(queue.offer(1, () -> { attempts.incrementAndGet(); throw new IllegalStateException("unknown commit"); }));
+        assertDoesNotThrow(queue::drain);
+        queue.drain();
+        assertEquals(1, attempts.get());
+        assertEquals(0L, queue.status().get("pending_groups"));
+        assertTrue(queue.offer(1, () -> { }));
+        queue.drain();
+        assertEquals(1L, queue.status().get("completed_total"));
+        queue.stop();
+        assertFalse(queue.offer(1, () -> fail("closed")));
+    }
+
+    @Test
+    void queueCountLimitAlsoReleasesReservedGroups() {
+        BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("TEST", 1, 100, false);
+        assertTrue(queue.offer(1, () -> { }));
+        assertFalse(queue.offer(1, () -> fail("overflow")));
+        assertEquals(1L, queue.status().get("pending_groups"));
+        queue.drain();
+        assertEquals(0L, queue.status().get("pending_groups"));
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DirectStatsProjectionWriterTest.java

@@ -36,7 +36,7 @@ class DirectStatsProjectionWriterTest {
     }
 
     @Test
-    void directModeWritesOnlyDimensionBatchesAndMaterializesAffectedRows() {
+    void directModeAggregatesWithoutAnyStatisticsSql() {
         JdbcTemplate jdbc = mock(JdbcTemplate.class);
         DirectStatsProjectionWriter writer = writer(jdbc, "DIRECT", true);
         TransactionSynchronizationManager.setActualTransactionActive(true);
@@ -53,13 +53,7 @@ class DirectStatsProjectionWriterTest {
 
         writer.writeStandardizedRows(rows);
 
-        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
-        verify(jdbc, times(4)).batchUpdate(sql.capture(), anyList());
-        String combined = String.join("\n", sql.getAllValues());
-        assertTrue(combined.contains("MERGE INTO device_daily_stats"));
-        assertTrue(combined.contains("MERGE INTO normalize_rule_daily_stats"));
-        assertTrue(combined.contains("UPDATE device_device"));
-        assertTrue(combined.contains("UPDATE dm_normalize_rule"));
+        verifyNoInteractions(jdbc);
         assertTrue(((Number) writer.status().get("input_rows")).longValue() == 100L);
         assertTrue(((Number) writer.status().get("device_groups")).longValue() == 1L);
         assertTrue(((Number) writer.status().get("rule_groups")).longValue() == 2L);
@@ -76,7 +70,7 @@ class DirectStatsProjectionWriterTest {
     }
 
     @Test
-    void nonNormalUpdateRecomputesOnlyAffectedFactGroupsExactly() {
+    void shadowNonNormalUpdateRetainsExactRecomputation() {
         AtomicInteger exactQueries = new AtomicInteger();
         JdbcTemplate jdbc = mock(JdbcTemplate.class, invocation -> {
             if ("queryForMap".equals(invocation.getMethod().getName())) {
@@ -88,7 +82,7 @@ class DirectStatsProjectionWriterTest {
             }
             return Answers.RETURNS_DEFAULTS.answer(invocation);
         });
-        DirectStatsProjectionWriter writer = writer(jdbc, "DIRECT", true);
+        DirectStatsProjectionWriter writer = writer(jdbc, "DIRECT_SHADOW", true);
         TransactionSynchronizationManager.setActualTransactionActive(true);
         LocalDate today = LocalDate.now(ZoneId.of("Asia/Shanghai"));
         com.common.entity.SyslogNonNormalMessage oldValue =
@@ -104,9 +98,8 @@ class DirectStatsProjectionWriterTest {
 
         assertTrue(exactQueries.get() == 2);
         ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
-        verify(jdbc, times(2)).batchUpdate(sql.capture(), anyList());
+        verify(jdbc).batchUpdate(sql.capture(), anyList());
         assertTrue(sql.getAllValues().get(0).contains("parse_count=s.parse_count"));
-        assertTrue(sql.getAllValues().get(1).contains("UPDATE device_device"));
     }
 
     @Test
@@ -119,12 +112,11 @@ class DirectStatsProjectionWriterTest {
     }
 
     @Test
-    void durableOutboxRemovesDailyAndBusinessProjectionUpdatesFromFactTransaction() {
+    void afterCommitPublisherRemovesAllStatisticsSqlFromFactTransaction() {
         JdbcTemplate jdbc = mock(JdbcTemplate.class);
         StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
         when(publisher.isEnabled()).thenReturn(true);
-        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
-        when(outbox.isEnabled()).thenReturn(true);
+        StatsProjectionAsyncPublisher outbox = mock(StatsProjectionAsyncPublisher.class);
         DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(
                 jdbc, new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), publisher, outbox);
         configure(writer, true);
@@ -137,17 +129,17 @@ class DirectStatsProjectionWriterTest {
         row.put("normalize_rule_id", 11L);
         writer.writeStandardizedRow(row);
 
-        verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class));
+        verify(outbox).enqueueAfterCommit(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class),
+                org.mockito.ArgumentMatchers.eq(1));
         verifyNoInteractions(jdbc);
-        assertTrue("DURABLE_OUTBOX".equals(writer.status().get("daily_projection_write_mode")));
+        assertTrue("BEST_EFFORT_AFTER_COMMIT_KAFKA".equals(writer.status().get("daily_projection_write_mode")));
     }
 
     private DirectStatsProjectionWriter writer(JdbcTemplate jdbc, String mode, boolean readEnabled) {
         StatsWriteModeProvider modeProvider = new StatsWriteModeProvider(mode);
         StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
         when(publisher.isEnabled()).thenReturn(false);
-        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
-        when(outbox.isEnabled()).thenReturn(false);
+        StatsProjectionAsyncPublisher outbox = mock(StatsProjectionAsyncPublisher.class);
         DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(
                 jdbc, modeProvider, new DirectStatsWriteMetrics(), publisher, outbox);
         configure(writer, readEnabled);

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsConsumerTest.java

@@ -0,0 +1,90 @@
+package com.common.service;
+
+import com.Modules.NormalData.SyslogFailureRouter;
+import com.common.entity.IngressStatsEvent;
+import com.common.util.Sm4Util;
+import com.config.AppProperties;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.Acknowledgment;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+import static org.mockito.ArgumentMatchers.anyList;
+
+class IngressStatsConsumerTest {
+    private static final String KEY = "0123456789abcdeffedcba9876543210";
+    private final IngressStatsProjectionWriter writer = mock(IngressStatsProjectionWriter.class);
+    private final Acknowledgment ack = mock(Acknowledgment.class);
+
+    private IngressStatsConsumer consumer() {
+        AppProperties properties = mock(AppProperties.class);
+        when(properties.getSm4Key()).thenReturn(KEY);
+        return new IngressStatsConsumer(writer,
+                new SyslogFailureRouter(mock(KafkaTemplate.class), new ObjectMapper()), properties);
+    }
+
+    private ConsumerRecord<String, String> record(String topic, long offset, String time) {
+        String value = Sm4Util.encryptCbc("[device_id=126 device_collect_id=3 receive_time="
+                + time + " event_time=20000101000000000]正文不参与规则或时间统计", KEY);
+        return new ConsumerRecord<>(topic, 0, offset, "key", value);
+    }
+
+    @Test
+    void decodesActualEncryptedHeaderUsingReceiveTimeNotEventTime() {
+        IngressStatsEvent event = consumer().decode(record("raw", 0, "20260913044500000"));
+        assertEquals(Long.valueOf(126), event.getDeviceId());
+        assertEquals(Long.valueOf(3), event.getDeviceCollectId());
+        assertEquals(LocalDateTime.of(2026, 9, 13, 4, 45), event.getReceiveTime());
+    }
+
+    @Test
+    void commitsOnlyAfterStatsDatabaseReturnsAndSupportsLargeTopic() {
+        consumer().consumeLarge(Collections.singletonList(record("raw.large", 1, "20260913044500000")), ack);
+        InOrder order = inOrder(writer, ack);
+        order.verify(writer).writeBatch(anyList());
+        order.verify(ack).acknowledge();
+    }
+
+    @Test
+    void databaseFailureRetainsStatsOffsetForRetry() {
+        when(writer.writeBatch(anyList())).thenThrow(new IllegalStateException("database down"));
+        assertThrows(IllegalStateException.class, () -> consumer().consume(
+                Collections.singletonList(record("raw", 0, "20260913044500000")), ack));
+        verifyNoInteractions(ack);
+    }
+
+    @Test
+    void invalidHeaderIsCountedWithoutDiscardingOtherRecords() {
+        IngressStatsConsumer consumer = consumer();
+        consumer.consume(Arrays.asList(new ConsumerRecord<>("raw", 0, 0L, "k", "bad ciphertext"),
+                record("raw", 1L, "20260913044500000")), ack);
+        ArgumentCaptor<List<IngressStatsEvent>> events = ArgumentCaptor.forClass(List.class);
+        verify(writer).writeBatch(events.capture());
+        assertEquals(1, events.getValue().size());
+        assertEquals(1L, consumer.status().get("invalid_record_attempts"));
+        verify(ack).acknowledge();
+    }
+
+    @Test
+    void rejectsSharingBusinessConsumerGroup() {
+        IngressStatsConsumer consumer = consumer();
+        ReflectionTestUtils.setField(consumer, "rawGroup", "business");
+        ReflectionTestUtils.setField(consumer, "largeRawGroup", "business-large");
+        ReflectionTestUtils.setField(consumer, "group", "business");
+        ReflectionTestUtils.setField(consumer, "largeGroup", "stats-large");
+        assertThrows(IllegalStateException.class, consumer::validateGroups);
+        ReflectionTestUtils.setField(consumer, "group", "stats");
+        assertDoesNotThrow(consumer::validateGroups);
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsProjectionWriterTest.java

@@ -53,26 +53,27 @@ class IngressStatsProjectionWriterTest {
     }
 
     @Test
-    void directAsyncModeWritesOnlyOneOutboxEvent() {
+    void independentIngressWritesAggregatedMinMaxDirectlyAndCommits() {
         JdbcTemplate jdbc = mock(JdbcTemplate.class);
         PlatformTransactionManager manager = mock(PlatformTransactionManager.class);
         when(manager.getTransaction(any(TransactionDefinition.class)))
                 .thenReturn(new SimpleTransactionStatus());
-        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
-        when(outbox.isEnabled()).thenReturn(true);
         IngressStatsProjectionWriter writer = new IngressStatsProjectionWriter(
                 jdbc, manager, new StatsWriteModeProvider("DIRECT"),
-                mock(StatsMaterializationPublisher.class), outbox);
+                mock(StatsMaterializationPublisher.class));
         ReflectionTestUtils.setField(writer, "enabled", true);
         ReflectionTestUtils.setField(writer, "transactionTimeoutSeconds", 45);
+        ReflectionTestUtils.setField(writer, "businessZone", "Asia/Shanghai");
 
         int groups = writer.writeBatch(Arrays.asList(
                 new IngressStatsEvent("a", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 0)),
                 new IngressStatsEvent("b", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 1))));
 
         assertEquals(2, groups);
-        verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class));
-        verifyNoInteractions(jdbc);
+        verify(jdbc).update(org.mockito.ArgumentMatchers.contains("MERGE INTO device_daily_stats"),
+                any(), any(), any());
+        verify(jdbc).update(org.mockito.ArgumentMatchers.contains("MERGE INTO device_collect_stats"),
+                any(), any(), any());
         verify(manager).commit(any());
     }
 }

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/LateAlarmBucketWriterTest.java

@@ -6,6 +6,8 @@ import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.doAnswer;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import java.time.LocalDateTime;
 import java.util.Arrays;
@@ -16,14 +18,60 @@ import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.test.util.ReflectionTestUtils;
 
 class LateAlarmBucketWriterTest {
+  private final org.springframework.transaction.support.TransactionTemplate transaction =
+      new org.springframework.transaction.support.TransactionTemplate(new TestTransactionManager());
+
+  @org.junit.jupiter.api.AfterEach void clean() {
+    org.springframework.transaction.support.TransactionSynchronizationManager.clear();
+  }
+
+  private LateAlarmBucketWriter writer(JdbcTemplate jdbc) {
+    LateAlarmBucketWriter writer = new LateAlarmBucketWriter(jdbc, new TestTransactionManager());
+    ReflectionTestUtils.setField(writer, "enabled", true);
+    return writer;
+  }
+
+  private void committed(LateAlarmBucketWriter writer, java.util.List<Map<String, Object>> rows) {
+    transaction.execute(status -> { writer.record(rows); return null; });
+    ((BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(writer, "queue")).drain();
+  }
+
+  private static class TestTransactionManager extends org.springframework.transaction.support.AbstractPlatformTransactionManager {
+    protected Object doGetTransaction() { return new Object(); }
+    protected void doBegin(Object tx, org.springframework.transaction.TransactionDefinition def) { }
+    protected void doCommit(org.springframework.transaction.support.DefaultTransactionStatus status) { }
+    protected void doRollback(org.springframework.transaction.support.DefaultTransactionStatus status) { }
+  }
+
+  @Test
+  void takesBucketLocksInSameOrderForReversedBatches() {
+    JdbcTemplate jdbc = mock(JdbcTemplate.class);
+    LateAlarmBucketWriter writer = writer(jdbc);
+    ReflectionTestUtils.setField(writer, "enabled", true);
+    java.util.List<String> locks = new java.util.ArrayList<>();
+    doAnswer(call -> {
+      locks.add(call.getArgument(1) + ":" + call.getArgument(2));
+      return 1;
+    }).when(jdbc).update(anyString(), any(), any(), any(), any(), any(), any());
+    LocalDateTime time = LocalDateTime.of(2026, 9, 13, 4, 0);
+    java.util.List<Map<String, Object>> rows = new java.util.ArrayList<>(Arrays.asList(
+        row(126, time.plusMinutes(1), 1), row(125, time, 2), row(126, time, 3)));
+    committed(writer, rows);
+    java.util.List<String> first = new java.util.ArrayList<>(locks);
+    locks.clear();
+    java.util.Collections.reverse(rows);
+    committed(writer, rows);
+    assertEquals(first, locks);
+    assertEquals(3, locks.size());
+  }
 
   @Test
   void groupsRowsByDeviceAndEventMinute() {
     JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class);
-    LateAlarmBucketWriter writer = new LateAlarmBucketWriter(jdbcTemplate);
+    LateAlarmBucketWriter writer = writer(jdbcTemplate);
     ReflectionTestUtils.setField(writer, "enabled", true);
 
-    writer.record(
+    committed(writer,
         Arrays.asList(
             row(126, LocalDateTime.of(2026, 9, 4, 10, 1, 2), 3),
             row(126, LocalDateTime.of(2026, 9, 4, 10, 1, 59), 4),
@@ -41,13 +89,37 @@ class LateAlarmBucketWriterTest {
   }
 
   @Test
-  void rejectsAlarmRowsWithoutStableTimes() {
-    LateAlarmBucketWriter writer = new LateAlarmBucketWriter(mock(JdbcTemplate.class));
+  void invalidNotificationDoesNotAbortAlarmTransaction() {
+    LateAlarmBucketWriter writer = writer(mock(JdbcTemplate.class));
     ReflectionTestUtils.setField(writer, "enabled", true);
     Map<String, Object> invalid = new HashMap<>();
     invalid.put("device_id", 126);
 
-    assertThrows(IllegalArgumentException.class, () -> writer.record(Arrays.asList(invalid)));
+    org.junit.jupiter.api.Assertions.assertDoesNotThrow(() -> committed(writer, Arrays.asList(invalid)));
+    assertEquals(1L, writer.status().get("lost_tasks_total"));
+  }
+
+  @Test
+  void commitOnlyEnqueuesRollbackNeverEnqueuesAndDatabaseFailureOnlyRetriesBackground() {
+    JdbcTemplate jdbc = mock(JdbcTemplate.class);
+    LateAlarmBucketWriter writer = writer(jdbc);
+    java.util.List<Map<String, Object>> rows = Arrays.asList(row(126, LocalDateTime.now(), 1));
+    transaction.execute(status -> { writer.record(rows); status.setRollbackOnly(); return null; });
+    assertEquals(0L, writer.status().get("enqueued_total"));
+    transaction.execute(status -> { writer.record(rows); return null; });
+    org.mockito.Mockito.verifyNoInteractions(jdbc);
+    org.mockito.Mockito.doThrow(new IllegalStateException("database unavailable"))
+        .when(jdbc).update(anyString(), any(), any(), any(), any(), any(), any());
+    BestEffortAuxiliaryQueue queue = (BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(writer, "queue");
+    org.junit.jupiter.api.Assertions.assertDoesNotThrow(queue::drain);
+    assertEquals(1L, writer.status().get("failures_total"));
+    assertEquals(1, writer.status().get("pending_tasks"));
+    assertEquals(0L, writer.status().get("lost_tasks_total"));
+    Object current = ReflectionTestUtils.getField(queue, "current");
+    ReflectionTestUtils.setField(current, "retryAt", 0L);
+    org.mockito.Mockito.doReturn(1).when(jdbc).update(anyString(), any(), any(), any(), any(), any(), any());
+    queue.drain();
+    assertEquals(1L, writer.status().get("completed_total"));
   }
 
   private Map<String, Object> row(int deviceId, LocalDateTime eventTime, int receiveSecond) {

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncPublisherTest.java

@@ -0,0 +1,238 @@
+package com.common.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.SendResult;
+import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.support.AbstractPlatformTransactionManager;
+import org.springframework.transaction.support.DefaultTransactionStatus;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+import org.springframework.transaction.support.TransactionTemplate;
+import org.springframework.util.concurrent.SettableListenableFuture;
+
+import java.time.LocalDateTime;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+class StatsProjectionAsyncPublisherTest {
+    private final KafkaTemplate<String, String> kafka = mock(KafkaTemplate.class);
+    private final TransactionTemplate transaction = new TransactionTemplate(new TestTransactionManager());
+
+    @AfterEach
+    void cleanup() { TransactionSynchronizationManager.clear(); }
+
+    @Test
+    void productionConstructorUsesDedicatedProducerWithExistingKafkaSettings() {
+        org.springframework.boot.autoconfigure.kafka.KafkaProperties properties =
+                new org.springframework.boot.autoconfigure.kafka.KafkaProperties();
+        properties.setBootstrapServers(java.util.Collections.singletonList("broker.example:9092"));
+        properties.getProperties().put("security.protocol", "SASL_PLAINTEXT");
+        properties.getProducer().setAcks("all");
+        StatsProjectionAsyncPublisher publisher = new StatsProjectionAsyncPublisher(properties, new ObjectMapper(), 10, 10);
+        org.springframework.kafka.core.DefaultKafkaProducerFactory<?, ?> factory =
+                (org.springframework.kafka.core.DefaultKafkaProducerFactory<?, ?>)
+                        ReflectionTestUtils.getField(publisher, "ownedProducerFactory");
+        assertNotNull(factory);
+        assertEquals("SASL_PLAINTEXT", factory.getConfigurationProperties().get("security.protocol"));
+        assertEquals("all", factory.getConfigurationProperties().get("acks"));
+        assertEquals("syslog-stats-projection-memory", factory.getConfigurationProperties().get("client.id"));
+        assertEquals(java.util.Collections.singletonList("broker.example:9092"),
+                factory.getConfigurationProperties().get("bootstrap.servers"));
+        publisher.stop(); // 不创建连接即可构造/销毁，未依赖真实Kafka。
+    }
+
+    @Test
+    void commitOnlyOffersAndRollbackNeverEnqueues() throws Exception {
+        StatsProjectionAsyncPublisher publisher = publisher(10, 10);
+        StatsProjectionOutboxEvent event = event();
+        transaction.execute(status -> {
+            publisher.enqueueAfterCommit(event, 1);
+            assertEquals(0, metric(publisher, "pending_events"));
+            verifyNoInteractions(kafka);
+            return null;
+        });
+        assertEquals(1, metric(publisher, "pending_events"));
+        verifyNoInteractions(kafka); // 即使Kafka发送会阻塞，事实提交也不调用它。
+        transaction.execute(status -> {
+            publisher.enqueueAfterCommit(event(), 1);
+            status.setRollbackOnly();
+            return null;
+        });
+        assertEquals(1, metric(publisher, "enqueued_total"));
+        SettableListenableFuture<SendResult<String, String>> success = new SettableListenableFuture<>();
+        success.set(null);
+        when(kafka.send(anyString(), anyString(), anyString())).thenReturn(success);
+        publisher.flush();
+        assertEquals(1, metric(publisher, "published_total"));
+        assertEquals(0, metric(publisher, "pending_groups"));
+        ArgumentCaptor<String> payload = ArgumentCaptor.forClass(String.class);
+        verify(kafka).send(eq("stats-topic"), eq(StatsProjectionOutboxEvent.KAFKA_KEY), payload.capture());
+        StatsProjectionOutboxEvent sent = new ObjectMapper().readValue(payload.getValue(), StatsProjectionOutboxEvent.class);
+        assertFalse(sent.isOutboxBacked());
+        assertEquals(event.getEventId(), sent.getEventId());
+    }
+
+    @Test
+    void kafkaFailureRetriesSameEventWhileQueueOverflowLetsFactCommit() {
+        StatsProjectionAsyncPublisher publisher = publisher(1, 10);
+        SettableListenableFuture<SendResult<String, String>> failure = new SettableListenableFuture<>();
+        failure.setException(new IllegalStateException("Kafka unavailable"));
+        SettableListenableFuture<SendResult<String, String>> success = new SettableListenableFuture<>();
+        success.set(null);
+        when(kafka.send(anyString(), anyString(), anyString())).thenReturn(failure, success, success);
+        commit(publisher);
+        assertDoesNotThrow(publisher::flush);
+        assertEquals(1, metric(publisher, "publish_failures_total"));
+        commit(publisher);
+        commit(publisher); // queue=1，另有一个正在重试；第三条只损失统计。
+        assertEquals(2, metric(publisher, "pending_events"));
+        assertEquals(1, metric(publisher, "dropped_events_total"));
+        assertEquals(1, metric(publisher, "dropped_fact_rows_total"));
+        Object pending = ReflectionTestUtils.getField(publisher, "inFlight");
+        ReflectionTestUtils.setField(pending, "retryAt", 0L);
+        publisher.flush();
+        assertEquals(2, metric(publisher, "published_total"));
+        assertEquals(0, metric(publisher, "pending_groups"));
+        ArgumentCaptor<String> payload = ArgumentCaptor.forClass(String.class);
+        verify(kafka, times(3)).send(anyString(), anyString(), payload.capture());
+        assertEquals(payload.getAllValues().get(0), payload.getAllValues().get(1));
+    }
+
+    @Test
+    void blockedBackgroundSendDoesNotBlockNextCommit() throws Exception {
+        StatsProjectionAsyncPublisher publisher = publisher(1, 10);
+        SettableListenableFuture<SendResult<String, String>> blocked = new SettableListenableFuture<>();
+        java.util.concurrent.CountDownLatch entered = new java.util.concurrent.CountDownLatch(1);
+        when(kafka.send(anyString(), anyString(), anyString())).thenAnswer(call -> {
+            entered.countDown();
+            return blocked;
+        });
+        commit(publisher);
+        CompletableFuture<Void> worker = CompletableFuture.runAsync(publisher::flush);
+        try {
+            assertTrue(entered.await(3, TimeUnit.SECONDS));
+            // 使用另一线程+有界等待，防止错误地共享flush锁使测试永久挂住。
+            CompletableFuture.runAsync(() -> { commit(publisher); commit(publisher); }).get(2, TimeUnit.SECONDS);
+            assertEquals(1, metric(publisher, "dropped_events_total"));
+        } finally {
+            blocked.set(null);
+            worker.get(3, TimeUnit.SECONDS);
+        }
+    }
+
+    @Test
+    void groupBudgetIsBoundedAndReleasedAfterSend() {
+        StatsProjectionAsyncPublisher publisher = publisher(10, 1);
+        commit(publisher);
+        commit(publisher);
+        assertEquals(1, metric(publisher, "pending_groups"));
+        assertEquals(1, metric(publisher, "dropped_events_total"));
+        SettableListenableFuture<SendResult<String, String>> success = new SettableListenableFuture<>();
+        success.set(null);
+        when(kafka.send(anyString(), anyString(), anyString())).thenReturn(success);
+        publisher.flush();
+        commit(publisher);
+        assertEquals(2, metric(publisher, "enqueued_total"));
+    }
+
+    @Test
+    void missingTransactionDisabledPublisherAndShutdownFailOpen() {
+        StatsProjectionAsyncPublisher publisher = publisher(10, 10);
+        assertDoesNotThrow(() -> publisher.enqueueAfterCommit(event(), 1));
+        assertEquals(1, metric(publisher, "dropped_events_total"));
+        ReflectionTestUtils.setField(publisher, "enabled", false);
+        commit(publisher);
+        ReflectionTestUtils.setField(publisher, "enabled", true);
+        publisher.stop();
+        commit(publisher);
+        assertEquals(3, metric(publisher, "dropped_events_total"));
+        assertEquals(0, metric(publisher, "pending_events"));
+        verifyNoInteractions(kafka);
+    }
+
+    @Test
+    void directStatisticsAggregationFailureDoesNotRollbackFacts() {
+        StatsProjectionAsyncPublisher publisher = publisher(10, 10);
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(jdbc,
+                new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(),
+                mock(StatsMaterializationPublisher.class), publisher);
+        ReflectionTestUtils.setField(writer, "businessZone", "invalid-zone");
+        assertDoesNotThrow(() -> transaction.execute(status -> {
+            writer.writeStandardizedRow(row());
+            assertFalse(status.isRollbackOnly());
+            return null;
+        }));
+        assertEquals(1, metric(publisher, "dropped_events_total"));
+        verifyNoInteractions(jdbc, kafka);
+    }
+
+    @Test
+    void directNonNormalUpdateAndDisabledPublishingNeverUseStatisticsSql() {
+        StatsProjectionAsyncPublisher publisher = publisher(10, 10);
+        ReflectionTestUtils.setField(publisher, "enabled", false);
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(jdbc,
+                new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(),
+                mock(StatsMaterializationPublisher.class), publisher);
+        ReflectionTestUtils.setField(writer, "businessZone", "Asia/Shanghai");
+        com.common.entity.SyslogNonNormalMessage old = new com.common.entity.SyslogNonNormalMessage();
+        old.setDeviceId(7);
+        old.setCreatedAt(java.time.OffsetDateTime.now());
+        com.common.entity.SyslogNonNormalMessage patch = new com.common.entity.SyslogNonNormalMessage();
+        patch.setDeviceId(8);
+        transaction.execute(status -> {
+            writer.writeStandardizedRow(row());
+            writer.writeNonNormalInsert(old);
+            writer.writeNonNormalUpdate(old, patch);
+            return null;
+        });
+        assertEquals(3, metric(publisher, "dropped_events_total"));
+        verifyNoInteractions(jdbc, kafka);
+    }
+
+    private StatsProjectionAsyncPublisher publisher(int capacity, long groups) {
+        StatsProjectionAsyncPublisher publisher = new StatsProjectionAsyncPublisher(kafka, new ObjectMapper(), capacity, groups);
+        ReflectionTestUtils.setField(publisher, "enabled", true);
+        ReflectionTestUtils.setField(publisher, "topic", "stats-topic");
+        ReflectionTestUtils.setField(publisher, "timeoutMs", 10000L);
+        return publisher;
+    }
+
+    private void commit(StatsProjectionAsyncPublisher publisher) {
+        transaction.execute(status -> { publisher.enqueueAfterCommit(event(), 1); return null; });
+    }
+
+    private static Map<String, Object> row() {
+        Map<String, Object> row = new HashMap<>();
+        row.put("created_at", LocalDateTime.of(2026, 9, 13, 8, 0));
+        row.put("device_id", 7L);
+        return row;
+    }
+
+    private static StatsProjectionOutboxEvent event() {
+        return StatsProjectionOutboxEvent.fromAccumulator(new DirectStatsAccumulator().addStandardizedRow(row()));
+    }
+
+    private static long metric(StatsProjectionAsyncPublisher publisher, String key) {
+        return ((Number) publisher.status().get(key)).longValue();
+    }
+
+    private static class TestTransactionManager extends AbstractPlatformTransactionManager {
+        protected Object doGetTransaction() { return new Object(); }
+        protected void doBegin(Object transaction, TransactionDefinition definition) { }
+        protected void doCommit(DefaultTransactionStatus status) { }
+        protected void doRollback(DefaultTransactionStatus status) { }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncTest.java

@@ -138,6 +138,32 @@ class StatsProjectionAsyncTest {
         return event;
     }
 
+    @Test
+    void memoryEventUpdatesProjectionAndDedupMarkerWithoutTouchingOutbox() throws Exception {
+        StatsProjectionOutboxEvent event = oneDeviceEvent("memory-1");
+        event.setOutboxBacked(false);
+        ObjectMapper mapper = new ObjectMapper();
+        event = mapper.readValue(mapper.writeValueAsString(event), StatsProjectionOutboxEvent.class);
+        org.junit.jupiter.api.Assertions.assertFalse(event.isOutboxBacked());
+        assertTrue(mapper.readValue("{\"eventId\":\"legacy\"}", StatsProjectionOutboxEvent.class).isOutboxBacked());
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        PlatformTransactionManager manager = mock(PlatformTransactionManager.class);
+        when(manager.getTransaction(any(TransactionDefinition.class))).thenReturn(new SimpleTransactionStatus());
+        StatsProjectionDeltaConsumer consumer = new StatsProjectionDeltaConsumer(
+                jdbc, manager, mapper, mock(StatsMaterializationPublisher.class));
+        configure(consumer);
+        consumer.applyWithRetry(Collections.singletonList(event));
+        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
+        verify(jdbc, times(2)).batchUpdate(sql.capture(), anyList());
+        assertTrue(sql.getAllValues().get(0).contains("MERGE INTO device_daily_stats"));
+        assertTrue(sql.getAllValues().get(1).contains("stats_projection_applied_event"));
+        when(jdbc.queryForList(org.mockito.ArgumentMatchers.anyString(),
+                org.mockito.ArgumentMatchers.eq(String.class), org.mockito.ArgumentMatchers.<Object>any()))
+                .thenReturn(Collections.singletonList("memory-1"));
+        assertEquals(1, consumer.applyWithRetry(Collections.singletonList(event)).duplicates);
+        verify(jdbc, times(2)).batchUpdate(org.mockito.ArgumentMatchers.anyString(), anyList());
+    }
+
     private void configure(StatsProjectionDeltaConsumer consumer) {
         ReflectionTestUtils.setField(consumer, "enabled", true);
         ReflectionTestUtils.setField(consumer, "appliedTable", "stats_projection_applied_event");

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionPublisherWiringTest.java

@@ -0,0 +1,36 @@
+package com.common.service;
+
+import com.config.CacheConfig;
+import com.config.WebConfig;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+import org.springframework.mock.env.MockEnvironment;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/** 使用实际配置创建两个同类型Bean，覆盖现场启动时的构造器自动装配。 */
+class StatsProjectionPublisherWiringTest {
+    @Test
+    void startsWithCacheAndHttpMappersWithoutChangingSharedSerialization() {
+        try (AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext()) {
+            context.setEnvironment(new MockEnvironment()
+                    .withProperty("stats.projection-async.topic", "test.STATS.PROJECTION"));
+            context.registerBean("objectMapper", ObjectMapper.class, () -> new CacheConfig().objectMapper());
+            context.registerBean("httpObjectMapper", ObjectMapper.class, () -> new WebConfig().httpObjectMapper());
+            context.registerBean(KafkaProperties.class);
+            context.register(StatsProjectionAsyncPublisher.class);
+            assertDoesNotThrow(context::refresh);
+            assertEquals(2, context.getBeansOfType(ObjectMapper.class).size());
+            StatsProjectionAsyncPublisher publisher = context.getBean(StatsProjectionAsyncPublisher.class);
+            ObjectMapper shared = context.getBean("objectMapper", ObjectMapper.class);
+            ObjectMapper copied = (ObjectMapper) ReflectionTestUtils.getField(publisher, "mapper");
+            assertNotSame(shared, copied);
+            assertEquals(shared.getRegisteredModuleIds(), copied.getRegisteredModuleIds());
+            assertNotNull(shared.getSerializationConfig().getDefaultTyper(shared.constructType(Object.class)));
+            assertNull(copied.getSerializationConfig().getDefaultTyper(copied.constructType(Object.class)));
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/KafkaReliabilityConfigTest.java

@@ -8,12 +8,34 @@ import org.springframework.kafka.listener.ContainerProperties;
 import org.springframework.kafka.listener.DefaultErrorHandler;
 
 import java.util.Collections;
+import org.mockito.ArgumentCaptor;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.verify;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 class KafkaReliabilityConfigTest {
+    @Test
+    void rawDefaultIs500BaseAnd3000NativePollCeiling() throws Exception {
+        java.util.Properties properties = new java.util.Properties();
+        try (java.io.InputStream input = getClass().getResourceAsStream("/application.properties")) {
+            properties.load(input);
+        }
+        assertEquals("${SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS:500}",
+                properties.getProperty("syslog.raw-listener.max-poll-records"));
+        java.lang.reflect.Method method = java.util.Arrays.stream(KafkaReliabilityConfig.class.getDeclaredMethods())
+                .filter(candidate -> candidate.getName().equals("rawKafkaListenerContainerFactory")).findFirst().get();
+        org.springframework.beans.factory.annotation.Value fallback = method.getParameters()[4]
+                .getAnnotation(org.springframework.beans.factory.annotation.Value.class);
+        assertEquals("${syslog.raw-listener.max-poll-records:500}", fallback.value());
+        assertEquals("${SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS:3000}",
+                properties.getProperty("syslog.raw-listener.adaptive-max-records"));
+        assertEquals("${syslog.raw-listener.adaptive-max-records:3000}", method.getParameters()[5]
+                .getAnnotation(org.springframework.beans.factory.annotation.Value.class).value());
+    }
 
     @Test
     void retryListenerCommitsInsideAcknowledgment() {
@@ -39,13 +61,35 @@ class KafkaReliabilityConfigTest {
         @SuppressWarnings("unchecked")
         ConsumerFactory<Object, Object> consumerFactory = mock(ConsumerFactory.class);
         DefaultErrorHandler errorHandler = config.syslogDefaultErrorHandler();
+        when(consumerFactory.getConfigurationProperties()).thenReturn(
+                Collections.singletonMap(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 500));
 
         ConcurrentKafkaListenerContainerFactory<Object, Object> factory =
                 config.rawKafkaListenerContainerFactory(
-                        configurer, consumerFactory, errorHandler, 3);
+                        configurer, consumerFactory, errorHandler, 3, 500, 3000);
 
         assertEquals(ContainerProperties.AckMode.MANUAL_IMMEDIATE,
                 factory.getContainerProperties().getAckMode());
+        ArgumentCaptor<ConsumerFactory> actual = ArgumentCaptor.forClass(ConsumerFactory.class);
+        verify(configurer).configure(any(), actual.capture());
+        assertEquals(3000, actual.getValue().getConfigurationProperties().get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
+        assertEquals(false, actual.getValue().getConfigurationProperties().get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG));
+        assertEquals(500, consumerFactory.getConfigurationProperties().get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
+    }
+
+    @Test
+    void independentIngressOverridesAutoCommitAndOffsetResetFromBusinessConsumer() {
+        KafkaReliabilityConfig config = new KafkaReliabilityConfig();
+        ConcurrentKafkaListenerContainerFactoryConfigurer configurer = mock(ConcurrentKafkaListenerContainerFactoryConfigurer.class);
+        ConsumerFactory<Object, Object> source = mock(ConsumerFactory.class);
+        when(source.getConfigurationProperties()).thenReturn(Collections.singletonMap(
+                ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true));
+        config.ingressStatsKafkaListenerContainerFactory(configurer, source,
+                config.syslogDefaultErrorHandler(), 2000, "earliest");
+        ArgumentCaptor<ConsumerFactory> actual = ArgumentCaptor.forClass(ConsumerFactory.class);
+        verify(configurer).configure(any(), actual.capture());
+        assertEquals(false, actual.getValue().getConfigurationProperties().get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG));
+        assertEquals("earliest", actual.getValue().getConfigurationProperties().get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG));
     }
 
     @Test

haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/SyslogBuildStartupLoggerTest.java

@@ -0,0 +1,51 @@
+package com.config;
+
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
+import org.junit.jupiter.api.Test;
+import org.slf4j.LoggerFactory;
+import org.springframework.mock.env.MockEnvironment;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class SyslogBuildStartupLoggerTest {
+    @Test
+    void logsCodeRevisionAndEffectiveOverrides() {
+        MockEnvironment environment = new MockEnvironment()
+                .withProperty("syslog.raw-listener.max-poll-records", "700")
+                .withProperty("syslog.raw-listener.adaptive-max-records", "2500")
+                .withProperty("stats.projection-async.enabled", "false");
+        environment.setActiveProfiles("prod");
+        SyslogBuildStartupLogger reporter = new SyslogBuildStartupLogger(environment);
+        Logger logger = (Logger) LoggerFactory.getLogger(SyslogBuildStartupLogger.class);
+        ListAppender<ILoggingEvent> appender = new ListAppender<>();
+        appender.start();
+        logger.addAppender(appender);
+        try {
+            reporter.onApplicationReady();
+            assertEquals(1, appender.list.size());
+            String message = appender.list.get(0).getFormattedMessage();
+            assertTrue(message.contains("[SYSLOG_BUILD_READY] codeRevision=" + SyslogBuildStartupLogger.CODE_REVISION));
+            assertTrue(message.contains("application=syslog-consumer-rule"));
+            assertTrue(message.contains("profiles=prod"));
+            assertTrue(message.contains("baseProcessingRecords=700"));
+            assertTrue(message.contains("adaptiveMaxRecords=2500"));
+            assertTrue(message.contains("statsPublisherEnabled=false"));
+        } finally {
+            logger.detachAppender(appender);
+            appender.stop();
+        }
+    }
+
+    @Test
+    void worksWithoutBuildMetadataAndIgnoresConfiguredRevision() {
+        String summary = new SyslogBuildStartupLogger(new MockEnvironment()
+                .withProperty("codeRevision", "fake")).startupSummary();
+        assertTrue(summary.contains("codeRevision=" + SyslogBuildStartupLogger.CODE_REVISION));
+        assertFalse(summary.contains("buildTime="));
+        assertTrue(summary.contains("baseProcessingRecords=500"));
+        assertTrue(summary.contains("adaptiveMaxRecords=3000"));
+    }
+
+}