fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁

- 将设备、规则和采集任务业务统计改为专用 Kafka Topic 异步物化 - 使用维度 ID 作为 Kafka key，保证同一统计维度串行更新 - 物化过程读取小型统计表精确值，确保重复消费不会重复计数 - 增加提交后发布、失败重试及定时小表对账补偿 - 固定日统计聚合键写入顺序，完善达梦死锁识别和事务重试 - JSON 解析失败直接转为非标日志，避免进入 DLT 重试链路 - 非标日志使用独立异步队列，不再阻塞正常消息 Kafka ACK - Kafka 升级脚本新增 STATS.MATERIALIZE Topic - 增加统计物化状态指标、配置示例及单元测试
2026-09-11 20:42:22 +08:00
parent 2def445537
commit c1472b9a95
24 changed files with 1123 additions and 55 deletions
@@ -122,6 +122,23 @@ class LogNormalProcessorRuleSelectionTest {
        verify(fixture.nonNormalService, never()).saveMessage(any());
    }

+    @Test
+    void jsonWithoutObjectBoundaryAlsoGoesDirectlyToDeferredNonNormal() {
+        RuleSource invalidJson = new RuleSource(10L, "json-no-cropper", "json-no-cropper",
+                "json", "{\"mappers\":[" + equalMapping()
+                + "],\"complete_columns\":[]}", true, "0");
+        RuleSource regexWouldMatch = regexRule(20L, "regex-second", "value=(\\S+)");
+        Fixture fixture = fixture(Arrays.asList(invalidJson, regexWouldMatch), config(true));
+
+        LogNormalProcessor.NormalizationResult result = fixture.prepareActive("value=ok");
+
+        assertEquals(LogNormalProcessor.ProcessingOutcome.FILTERED, result.getOutcome());
+        assertTrue(result.getRows().isEmpty());
+        assertEquals(1, result.getNonNormalMessages().size());
+        assertEquals("JSON解析异常", result.getNonNormalMessages().get(0).getReason());
+        verify(fixture.nonNormalService, never()).saveMessage(any());
+    }
+
    @Test
    void legacyInvalidJsonIsNonNormalTerminalInsteadOfException() {
        Fixture fixture = fixture(Arrays.asList(
@@ -13,6 +13,7 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.dao.DataIntegrityViolationException;

 import java.time.Instant;
 import java.time.LocalDateTime;
@@ -63,6 +64,8 @@ class SysLogProcessorParallelTest {
        ReflectionTestUtils.setField(processor, "standardizedBatchWriter", batchWriter);
        ReflectionTestUtils.setField(processor, "nonNormalMessageService", nonNormalMessageService);
        ReflectionTestUtils.setField(processor, "nonNormalWriterExecutor", nonNormalExecutor);
+        ReflectionTestUtils.setField(processor, "nonNormalMaxConcurrencyRetries", 3);
+        ReflectionTestUtils.setField(processor, "nonNormalConcurrencyRetryBackoffMs", 0L);
        when(batchWriter.write(any())).thenReturn(new DmStandardizedBatchWriter.WriteReport<>());
        when(normalProcessor.captureRuleSnapshot(any())).thenReturn(
                new BatchRuleSnapshot(Collections.emptyMap(), System.currentTimeMillis()));
@@ -182,6 +185,31 @@ class SysLogProcessorParallelTest {
        assertEquals(2, metrics.filteredCount());
    }

+    @Test
+    void asynchronousNonNormalDeadlockRetriesInANewServiceTransaction() {
+        when(normalProcessor.prepare(any(PreparedNormalizationInput.class),
+                any(), any(), any())).thenAnswer(invocation -> {
+            SyslogNonNormalMessage message = new SyslogNonNormalMessage();
+            message.setId(java.util.UUID.randomUUID().toString());
+            return LogNormalProcessor.NormalizationResult.filtered(
+                    Collections.singletonList(message));
+        });
+        AtomicInteger attempts = new AtomicInteger();
+        when(nonNormalMessageService.saveBatchMessages(anyList())).thenAnswer(invocation -> {
+            if (attempts.incrementAndGet() == 1) {
+                throw new RuntimeException(new DataIntegrityViolationException("Deadlock"));
+            }
+            return ((List<?>) invocation.getArgument(0)).size();
+        });
+
+        SysLogProcessor.BatchMetrics metrics = new SysLogProcessor.BatchMetrics();
+        processor.normalizePreparedMessages(messages(2), metrics);
+
+        verify(nonNormalMessageService, timeout(1000).times(2)).saveBatchMessages(anyList());
+        assertEquals(2, attempts.get());
+        assertEquals(2, metrics.nonNormalQueued());
+    }
+
    @Test
    void rejectedNonNormalTaskDoesNotFailKafkaBatch() {
        when(normalProcessor.prepare(any(PreparedNormalizationInput.class),
@@ -9,6 +9,7 @@ import java.util.List;
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import com.common.service.StatsMaterializationConsumer;

 /** 防止多节点 consumer-rule 的监听器 ID 或显式消费组在后续合并中回退。 */
 class SyslogKafkaTopologyTest {
@@ -48,4 +49,20 @@ class SyslogKafkaTopologyTest {
                "${syslog.reliability.replay-topic:${spring.kafka.consumer.topic}.REPLAY}"
        }, listener.topics());
    }
+
+    @Test
+    void statsMaterializerUsesDedicatedTopicAndGroup() throws Exception {
+        Method method = StatsMaterializationConsumer.class.getMethod(
+                "consume", List.class, org.springframework.kafka.support.Acknowledgment.class);
+        KafkaListener listener = method.getAnnotation(KafkaListener.class);
+
+        assertNotNull(listener);
+        assertEquals("syslog-stats-materializer", listener.id());
+        assertEquals("${stats.materialization.group-id:${spring.kafka.consumer.group-id}-stats-materialize}",
+                listener.groupId());
+        assertEquals("statsMaterializationKafkaListenerContainerFactory", listener.containerFactory());
+        assertArrayEquals(new String[]{
+                "${stats.materialization.topic:${spring.kafka.consumer.topic}.STATS.MATERIALIZE}"
+        }, listener.topics());
+    }
 }
@@ -4,6 +4,7 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Answers;
+import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.test.util.ReflectionTestUtils;
 import org.springframework.transaction.support.TransactionSynchronizationManager;
@@ -11,6 +12,7 @@ import org.springframework.transaction.support.TransactionSynchronizationManager
 import java.time.LocalDateTime;
 import java.time.LocalDate;
 import java.time.ZoneId;
+import java.sql.BatchUpdateException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -19,7 +21,9 @@ import java.util.concurrent.atomic.AtomicInteger;

 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyCollection;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoInteractions;
@@ -105,10 +109,51 @@ class DirectStatsProjectionWriterTest {
        assertTrue(sql.getAllValues().get(1).contains("UPDATE device_device"));
    }

+    @Test
+    void damengBatchDeadlockWrappedAsIntegrityViolationIsRetryable() {
+        BatchUpdateException deadlock = new BatchUpdateException("Deadlock", null, 0, new int[0]);
+        DataIntegrityViolationException wrapped =
+                new DataIntegrityViolationException("PreparedStatementCallback", deadlock);
+
+        assertTrue(DirectStatsProjectionWriter.isConcurrentFailure(wrapped));
+    }
+
+    @Test
+    void kafkaMaterializationRemovesBusinessTableUpdatesFromFactTransaction() {
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
+        when(publisher.isEnabled()).thenReturn(true);
+        DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(
+                jdbc, new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), publisher);
+        configure(writer, true);
+        TransactionSynchronizationManager.setActualTransactionActive(true);
+
+        Map<String, Object> row = new HashMap<>();
+        row.put("created_at", LocalDateTime.now());
+        row.put("log_time", LocalDateTime.now());
+        row.put("device_id", 7L);
+        row.put("normalize_rule_id", 11L);
+        writer.writeStandardizedRow(row);
+
+        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
+        verify(jdbc, times(2)).batchUpdate(sql.capture(), anyList());
+        assertTrue(sql.getAllValues().get(0).contains("MERGE INTO device_daily_stats"));
+        assertTrue(sql.getAllValues().get(1).contains("MERGE INTO normalize_rule_daily_stats"));
+        verify(publisher).enqueueAfterCommit(anyCollection());
+        assertTrue("KAFKA_DIRTY_KEY".equals(writer.status().get("business_materialization_mode")));
+    }
+
    private DirectStatsProjectionWriter writer(JdbcTemplate jdbc, String mode, boolean readEnabled) {
        StatsWriteModeProvider modeProvider = new StatsWriteModeProvider(mode);
+        StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
+        when(publisher.isEnabled()).thenReturn(false);
        DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(
-                jdbc, modeProvider, new DirectStatsWriteMetrics());
+                jdbc, modeProvider, new DirectStatsWriteMetrics(), publisher);
+        configure(writer, readEnabled);
+        return writer;
+    }
+
+    private void configure(DirectStatsProjectionWriter writer, boolean readEnabled) {
        ReflectionTestUtils.setField(writer, "readEnabled", readEnabled);
        ReflectionTestUtils.setField(writer, "directDeviceTable", "device_daily_stats");
        ReflectionTestUtils.setField(writer, "directRuleTable", "normalize_rule_daily_stats");
@@ -117,6 +162,5 @@ class DirectStatsProjectionWriterTest {
        ReflectionTestUtils.setField(writer, "controlTable", "stats_direct_control");
        ReflectionTestUtils.setField(writer, "todayStatsDateEnabled", true);
        ReflectionTestUtils.setField(writer, "businessZone", "Asia/Shanghai");
-        return writer;
    }
 }
@@ -0,0 +1,28 @@
+package com.common.service;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.LocalDate;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class StatsMaterializationCommandTest {
+    @Test
+    void usesStableDimensionKeyAndValidatesDate() {
+        StatsMaterializationCommand command = StatsMaterializationCommand.device(
+                LocalDate.of(2026, 9, 11), 7L);
+
+        assertEquals("DEVICE:7", command.key());
+        assertEquals(LocalDate.of(2026, 9, 11), command.date());
+    }
+
+    @Test
+    void rejectsMissingDateForDailyDimension() {
+        StatsMaterializationCommand command = new StatsMaterializationCommand();
+        command.setType(StatsMaterializationCommand.Type.RULE);
+        command.setDimensionId(8L);
+
+        assertThrows(IllegalArgumentException.class, command::validate);
+    }
+}
@@ -0,0 +1,59 @@
+package com.common.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.support.SimpleTransactionStatus;
+
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class StatsMaterializationConsumerTest {
+    @Test
+    void materializesExactValuesInOneTransaction() {
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        PlatformTransactionManager manager = mock(PlatformTransactionManager.class);
+        when(manager.getTransaction(any(TransactionDefinition.class)))
+                .thenReturn(new SimpleTransactionStatus());
+        StatsMaterializationConsumer consumer = new StatsMaterializationConsumer(
+                jdbc, manager, new ObjectMapper(), new StatsWriteModeProvider("DIRECT"));
+        ReflectionTestUtils.setField(consumer, "readEnabled", true);
+        ReflectionTestUtils.setField(consumer, "deviceTable", "device_daily_stats");
+        ReflectionTestUtils.setField(consumer, "ruleTable", "normalize_rule_daily_stats");
+        ReflectionTestUtils.setField(consumer, "todayStatsDateEnabled", true);
+        ReflectionTestUtils.setField(consumer, "businessZone", "Asia/Shanghai");
+        ReflectionTestUtils.setField(consumer, "transactionTimeoutSeconds", 45);
+        ReflectionTestUtils.setField(consumer, "maxConcurrencyRetries", 3);
+        ReflectionTestUtils.setField(consumer, "retryBackoffMs", 0L);
+        LocalDate today = LocalDate.now(ZoneId.of("Asia/Shanghai"));
+
+        int count = consumer.materializeWithRetry(Arrays.asList(
+                StatsMaterializationCommand.device(today, 7L),
+                StatsMaterializationCommand.rule(today, 8L),
+                StatsMaterializationCommand.collect(2L)));
+
+        assertEquals(3, count);
+        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
+        verify(jdbc, times(3)).batchUpdate(sql.capture(), anyList());
+        List<String> statements = sql.getAllValues();
+        assertTrue(statements.get(0).contains("MERGE INTO device_device"));
+        assertTrue(statements.get(1).contains("MAX(max_log_time)"));
+        assertTrue(statements.get(2).contains("MERGE INTO device_collect_task"));
+        verify(manager).commit(any());
+    }
+}
@@ -0,0 +1,41 @@
+package com.common.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.transaction.support.TransactionSynchronization;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+
+import java.time.LocalDate;
+import java.util.Collections;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+
+class StatsMaterializationPublisherTest {
+    @AfterEach
+    void clearTransactionState() {
+        TransactionSynchronizationManager.clear();
+    }
+
+    @Test
+    void dirtyKeyBecomesPublishableOnlyAfterDatabaseCommit() {
+        @SuppressWarnings("unchecked")
+        KafkaTemplate<String, String> kafka = mock(KafkaTemplate.class);
+        StatsMaterializationPublisher publisher = new StatsMaterializationPublisher(kafka, new ObjectMapper());
+        ReflectionTestUtils.setField(publisher, "enabled", true);
+        TransactionSynchronizationManager.initSynchronization();
+
+        publisher.enqueueAfterCommit(Collections.singletonList(
+                StatsMaterializationCommand.device(LocalDate.of(2026, 9, 11), 7L)));
+        assertEquals(0, publisher.status().get("pending_keys"));
+
+        for (TransactionSynchronization synchronization
+                : TransactionSynchronizationManager.getSynchronizations()) {
+            synchronization.afterCommit();
+        }
+        assertEquals(1, publisher.status().get("pending_keys"));
+    }
+}
@@ -0,0 +1,43 @@
+package com.common.service;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+
+class StatsMaterializationReconcilerTest {
+    @Test
+    void scansOnlyAggregateTablesAndBuildsExactDirtyKeys() {
+        JdbcTemplate jdbc = mock(JdbcTemplate.class, invocation -> {
+            if (!"queryForList".equals(invocation.getMethod().getName())) {
+                return org.mockito.Answers.RETURNS_DEFAULTS.answer(invocation);
+            }
+            String sql = (String) invocation.getArguments()[0];
+            Map<String, Object> row = new HashMap<>();
+            if (sql.contains("device_daily_stats")) row.put("DIMENSION_ID", 7L);
+            else if (sql.contains("normalize_rule_daily_stats")) row.put("dimension_id", 8L);
+            else if (sql.contains("device_collect_stats")) row.put("DIMENSION_ID", 2L);
+            else return Collections.emptyList();
+            return Collections.singletonList(row);
+        });
+        StatsMaterializationReconciler reconciler = new StatsMaterializationReconciler(
+                jdbc, new StatsWriteModeProvider("DIRECT"), mock(StatsMaterializationPublisher.class));
+        ReflectionTestUtils.setField(reconciler, "deviceTable", "device_daily_stats");
+        ReflectionTestUtils.setField(reconciler, "ruleTable", "normalize_rule_daily_stats");
+        ReflectionTestUtils.setField(reconciler, "businessZone", "Asia/Shanghai");
+
+        List<StatsMaterializationCommand> commands = reconciler.findDifferences();
+
+        assertEquals(3, commands.size());
+        assertEquals("DEVICE:7", commands.get(0).key());
+        assertEquals("RULE:8", commands.get(1).key());
+        assertEquals("COLLECT:2", commands.get(2).key());
+    }
+}