perf(syslog-consumer-rule): 非标日志改为单线程异步批量写入

- 非标写库结果不再参与 Kafka ACK
- 使用独立单线程和有界队列处理非标日志
- 非标数据按固定大小分批写入达梦
- 增加异步成功、失败和队列拒绝指标
- 补充异步解耦及批量写入测试

perf(syslog-consumer-rule): 按批次字段并集优化达梦批量写入

- 普通日志和告警日志按非空字段并集生成批写模板
- 为同批记录补齐缺失字段并绑定 NULL
- 使用 MyBatis ExecutorType.BATCH 复用 PreparedStatement
- 避免固定写入全部 373 个候选字段
- 非标日志保留固定 24 列多行批量写入
- 批量大小和非标队列容量默认调整为 500
- 补充字段并集与动态 SQL 一致性测试

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java

@@ -4,6 +4,7 @@ import com.Modules.NormalData.compiled.BatchEnrichmentContext;
 import com.Modules.NormalData.compiled.BatchRuleSnapshot;
 import com.Modules.NormalData.compiled.CompiledRuleCache;
 import com.Modules.NormalData.compiled.CompiledRuleCompiler;
+import com.Modules.NormalData.compiled.PreparedNormalizationInput;
 import com.Modules.NormalData.compiled.RuleExecutionMetrics;
 import com.Modules.NormalData.compiled.RuleSource;
 import com.common.entity.SyslogNonNormalMessage;
@@ -90,6 +91,20 @@ class LogNormalProcessorRuleSelectionTest {
         verify(fixture.nonNormalService).saveMessage(any(SyslogNonNormalMessage.class));
     }
 
+    @Test
+    void activeKafkaPreparationDefersNonNormalPersistence() {
+        Fixture fixture = fixture(Arrays.asList(
+                regexRule(10L, "first", "first=(\\S+)"),
+                regexRule(20L, "second", "second=(\\S+)")), config(true));
+
+        LogNormalProcessor.NormalizationResult result = fixture.prepareActive("nothing matches");
+
+        assertEquals(LogNormalProcessor.ProcessingOutcome.FILTERED, result.getOutcome());
+        assertEquals(1, result.getNonNormalMessages().size());
+        assertEquals("未命中规则", result.getNonNormalMessages().get(0).getReason());
+        verify(fixture.nonNormalService, never()).saveMessage(any());
+    }
+
     @Test
     void nonNormalSaveFailureRemainsRetryable() {
         Fixture fixture = fixture(Arrays.asList(
@@ -232,5 +247,23 @@ class LogNormalProcessorRuleSelectionTest {
                     new BatchEnrichmentContext(false, taskService, deviceService),
                     new RuleExecutionMetrics());
         }
+
+        private LogNormalProcessor.NormalizationResult prepareActive(String body) {
+            LocalDateTime receivedAt = LocalDateTime.of(2026, 8, 21, 15, 0);
+            Map<String, String> deviceInfo = new LinkedHashMap<>();
+            deviceInfo.put("receive_time", "20260821150000000");
+            deviceInfo.put("event_id", "event-selection");
+            deviceInfo.put("device_id", "7");
+            deviceInfo.put("device_name", "fw");
+            deviceInfo.put("vendor", "vendor");
+            deviceInfo.put("data_type", "json");
+            deviceInfo.put("device_collect_id", "3");
+            PreparedNormalizationInput input = new PreparedNormalizationInput(
+                    "event-selection", "raw", receivedAt, receivedAt,
+                    7L, 3L, "json", body, body, "", deviceInfo);
+            return processor.prepare(input, snapshot,
+                    new BatchEnrichmentContext(false, taskService, deviceService),
+                    new RuleExecutionMetrics());
+        }
     }
 }

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/QuanxiNormalizationTest.java

@@ -59,7 +59,7 @@ class QuanxiNormalizationTest {
                 assertTrue(f.store.isAlarmData(row));
                 assertEquals(1, f.store.insertDynamic(row));
             }
-            verify(f.alarm, times(20)).insertDynamic(anyMap());
+            verify(f.alarm, times(20)).insertBatchPlanned(anyMap());
             verifyNoInteractions(f.normal);
         }
     }
@@ -107,7 +107,9 @@ class QuanxiNormalizationTest {
     @Test void ruleDestinationsAreActualAlarmSqlColumnsAndHttpPathsAreExact() throws Exception {
         JsonNode rule = JSON.readTree(file("normalization-rule.json").toFile());
         String sql = new String(Files.readAllBytes(file("../../../syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml")), "UTF-8");
-        for (JsonNode m : rule.path("mappers")) assertTrue(sql.contains("dataMap." + m.path("dest_field").asText() + " "));
+        for (JsonNode m : rule.path("mappers")) {
+            assertTrue(sql.contains("dataMap." + m.path("dest_field").asText()));
+        }
         for (String[] field : HTTP_FIELDS) {
             boolean found = false;
             for (JsonNode m : rule.path("mappers")) if (field[1].equals(m.path("dest_field").asText())) {
@@ -138,7 +140,7 @@ class QuanxiNormalizationTest {
             ReflectionTestUtils.setField(store, "syslogNormalDataMapper", normal);
             ReflectionTestUtils.setField(store, "statsChangeCaptureService", mock(StatsChangeCaptureService.class));
             ReflectionTestUtils.setField(store, "directStatsProjectionWriter", mock(DirectStatsProjectionWriter.class));
-            when(alarm.insertDynamic(anyMap())).thenReturn(1);
+            when(alarm.insertBatchPlanned(anyMap())).thenReturn(1);
             NormalizationOptimizationProperties p = new NormalizationOptimizationProperties();
             p.getCompiledRule().setEnabled(compiled); p.getCompiledRule().setShadowCompareEnabled(false);
             p.getRuleSelection().setEnabled(true); p.getRuleSelection().setShadowEnabled(false);

haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java

@@ -3,9 +3,11 @@ package com.Modules.NormalData;
 import com.Modules.NormalData.compiled.BatchEnrichmentContext;
 import com.Modules.NormalData.compiled.BatchRuleSnapshot;
 import com.Modules.NormalData.compiled.PreparedNormalizationInput;
+import com.common.entity.SyslogNonNormalMessage;
 import com.common.service.DeviceCollectTaskService;
 import com.common.service.DeviceDeviceService;
 import com.common.service.DmStandardizedBatchWriter;
+import com.common.service.SyslogNonNormalMessageService;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -19,12 +21,18 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -32,23 +40,29 @@ import static org.mockito.Mockito.when;
 class SysLogProcessorParallelTest {
 
     private ExecutorService executor;
+    private ExecutorService nonNormalExecutor;
     private SysLogProcessor processor;
     private LogNormalProcessor normalProcessor;
     private SyslogFailureRouter failureRouter;
     private DmStandardizedBatchWriter batchWriter;
+    private SyslogNonNormalMessageService nonNormalMessageService;
 
     @BeforeEach
     void setUp() {
         executor = Executors.newFixedThreadPool(4);
+        nonNormalExecutor = Executors.newSingleThreadExecutor();
         processor = new SysLogProcessor();
         normalProcessor = mock(LogNormalProcessor.class);
         failureRouter = mock(SyslogFailureRouter.class);
         batchWriter = mock(DmStandardizedBatchWriter.class);
+        nonNormalMessageService = mock(SyslogNonNormalMessageService.class);
         ReflectionTestUtils.setField(processor, "taskExecutor", executor);
         ReflectionTestUtils.setField(processor, "processTimeoutMs", 5000L);
         ReflectionTestUtils.setField(processor, "logNormalProcessor", normalProcessor);
         ReflectionTestUtils.setField(processor, "failureRouter", failureRouter);
         ReflectionTestUtils.setField(processor, "standardizedBatchWriter", batchWriter);
+        ReflectionTestUtils.setField(processor, "nonNormalMessageService", nonNormalMessageService);
+        ReflectionTestUtils.setField(processor, "nonNormalWriterExecutor", nonNormalExecutor);
         when(batchWriter.write(any())).thenReturn(new DmStandardizedBatchWriter.WriteReport<>());
         when(normalProcessor.captureRuleSnapshot(any())).thenReturn(
                 new BatchRuleSnapshot(Collections.emptyMap(), System.currentTimeMillis()));
@@ -59,7 +73,9 @@ class SysLogProcessorParallelTest {
     @AfterEach
     void tearDown() throws InterruptedException {
         executor.shutdownNow();
+        nonNormalExecutor.shutdownNow();
         executor.awaitTermination(5, TimeUnit.SECONDS);
+        nonNormalExecutor.awaitTermination(5, TimeUnit.SECONDS);
     }
 
     @Test
@@ -121,6 +137,73 @@ class SysLogProcessorParallelTest {
                 "达梦写入耗时应独立覆盖批写调用");
     }
 
+    @Test
+    void deferredNonNormalMessagesAreSavedOncePerKafkaBatch() {
+        when(normalProcessor.prepare(any(PreparedNormalizationInput.class),
+                any(), any(), any())).thenAnswer(invocation -> {
+            SyslogNonNormalMessage message = new SyslogNonNormalMessage();
+            message.setId(java.util.UUID.randomUUID().toString());
+            return LogNormalProcessor.NormalizationResult.filtered(
+                    Collections.singletonList(message));
+        });
+        when(nonNormalMessageService.saveBatchMessages(anyList()))
+                .thenAnswer(invocation -> ((List<?>) invocation.getArgument(0)).size());
+
+        SysLogProcessor.BatchMetrics metrics = new SysLogProcessor.BatchMetrics();
+        processor.normalizePreparedMessages(messages(8), metrics);
+
+        @SuppressWarnings("unchecked")
+        org.mockito.ArgumentCaptor<List<SyslogNonNormalMessage>> captor =
+                org.mockito.ArgumentCaptor.forClass(List.class);
+        verify(nonNormalMessageService, timeout(1000)).saveBatchMessages(captor.capture());
+        assertEquals(8, captor.getValue().size());
+        assertEquals(8, metrics.nonNormalQueued());
+        assertEquals(0, metrics.nonNormalRejected());
+        assertEquals(0, metrics.successCount());
+    }
+
+    @Test
+    void asynchronousNonNormalFailureDoesNotFailKafkaBatch() {
+        when(normalProcessor.prepare(any(PreparedNormalizationInput.class),
+                any(), any(), any())).thenAnswer(invocation -> {
+            SyslogNonNormalMessage message = new SyslogNonNormalMessage();
+            message.setId(java.util.UUID.randomUUID().toString());
+            return LogNormalProcessor.NormalizationResult.filtered(
+                    Collections.singletonList(message));
+        });
+        when(nonNormalMessageService.saveBatchMessages(anyList()))
+                .thenThrow(new IllegalStateException("dm unavailable"));
+
+        SysLogProcessor.BatchMetrics metrics = new SysLogProcessor.BatchMetrics();
+        processor.normalizePreparedMessages(messages(2), metrics);
+
+        verify(nonNormalMessageService, timeout(1000)).saveBatchMessages(anyList());
+        assertEquals(2, metrics.nonNormalQueued());
+        assertEquals(2, metrics.filteredCount());
+    }
+
+    @Test
+    void rejectedNonNormalTaskDoesNotFailKafkaBatch() {
+        when(normalProcessor.prepare(any(PreparedNormalizationInput.class),
+                any(), any(), any())).thenAnswer(invocation -> {
+            SyslogNonNormalMessage message = new SyslogNonNormalMessage();
+            return LogNormalProcessor.NormalizationResult.filtered(
+                    Collections.singletonList(message));
+        });
+        ReflectionTestUtils.setField(processor, "nonNormalWriterExecutor",
+                (java.util.concurrent.Executor) command -> {
+                    throw new RejectedExecutionException("queue full");
+                });
+
+        SysLogProcessor.BatchMetrics metrics = new SysLogProcessor.BatchMetrics();
+        processor.normalizePreparedMessages(messages(3), metrics);
+
+        verify(nonNormalMessageService, never()).saveBatchMessages(anyList());
+        assertEquals(0, metrics.nonNormalQueued());
+        assertEquals(3, metrics.nonNormalRejected());
+        assertEquals(3, metrics.filteredCount());
+    }
+
     private List<SysLogProcessor.PreparedMessage> messages(int count) {
         List<SysLogProcessor.PreparedMessage> messages = new ArrayList<>();
         for (int i = 0; i < count; i++) {

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/mapper/FixedColumnBatchMapperTest.java

@@ -0,0 +1,127 @@
+package com.common.mapper;
+
+import org.apache.ibatis.builder.xml.XMLMapperBuilder;
+import org.apache.ibatis.mapping.MappedStatement;
+import org.apache.ibatis.session.Configuration;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class FixedColumnBatchMapperTest {
+
+    @Test
+    void normalDataUsesWhitelistedBatchUnionTemplate() {
+        assertBatchUnionMapper(
+                "/mapper/SyslogNormalDataMapper.xml", "syslog_normal_data");
+    }
+
+    @Test
+    void normalAlarmUsesWhitelistedBatchUnionTemplate() {
+        assertBatchUnionMapper(
+                "/mapper/SyslogNormalAlarmMapper.xml", "syslog_normal_alarm");
+    }
+
+    @Test
+    void alignedRowsProduceIdenticalMyBatisSqlEvenWhenValuesAreNull() {
+        assertSameBoundSql(
+                "/mapper/SyslogNormalDataMapper.xml",
+                "com.common.mapper.SyslogNormalDataMapper.insertBatchPlanned");
+        assertSameBoundSql(
+                "/mapper/SyslogNormalAlarmMapper.xml",
+                "com.common.mapper.SyslogNormalAlarmMapper.insertBatchPlanned");
+    }
+
+    private void assertBatchUnionMapper(String resourcePath, String table) {
+        String xml = resource(resourcePath);
+        Matcher statement = Pattern.compile(
+                "<insert id=\"insertBatchPlanned\".*?INSERT INTO " + table
+                        + "\\s*<trim.*?</trim>\\s*VALUES\\s*<trim.*?</trim>\\s*</insert>",
+                Pattern.DOTALL).matcher(xml);
+
+        assertTrue(statement.find(), "缺少 " + table + " 批次列并集 INSERT");
+        String insert = statement.group(0);
+        assertTrue(insert.contains("dataMap.containsKey('src_ip')"));
+        assertTrue(insert.contains("dataMap.containsKey('dest_ip')"));
+        assertTrue(insert.contains("dataMap.containsKey('rule_version')"));
+        assertFalse(insert.contains("dataMap.src_ip != null"),
+                "列选择必须依据调用方补齐后的统一键集合，不能再按各行值判断");
+
+        Matcher conditions = Pattern.compile("dataMap\\.containsKey\\('([a-zA-Z0-9_]+)'\\)")
+                .matcher(insert);
+        java.util.Map<String, Integer> occurrences = new java.util.HashMap<>();
+        while (conditions.find()) {
+            occurrences.merge(conditions.group(1), 1, Integer::sum);
+        }
+        assertTrue(occurrences.size() > 300, "白名单必须保留历史标准化候选字段");
+        assertTrue(occurrences.values().stream().allMatch(count -> count == 2),
+                "每个允许字段必须同时出现在列清单和值清单中");
+    }
+
+    private void assertSameBoundSql(String resourcePath, String statementId) {
+        String xml = resource(resourcePath);
+        Configuration configuration = new Configuration();
+        try (InputStream stream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) {
+            new XMLMapperBuilder(stream, configuration, resourcePath,
+                    configuration.getSqlFragments()).parse();
+        } catch (Exception parseFailure) {
+            throw new AssertionError("Mapper XML 解析失败: " + resourcePath, parseFailure);
+        }
+        MappedStatement statement = configuration.getMappedStatement(statementId);
+
+        Map<String, Object> first = new HashMap<>();
+        first.put("id", "row-1");
+        first.put("src_ip", "10.0.0.1");
+        first.put("dest_ip", null);
+        first.put("rule_version", "v1");
+        Map<String, Object> second = new HashMap<>();
+        second.put("id", "row-2");
+        second.put("src_ip", null);
+        second.put("dest_ip", "10.0.0.2");
+        second.put("rule_version", "v1");
+
+        String firstSql = normalizedSql(statement, first);
+        String secondSql = normalizedSql(statement, second);
+        assertEquals(firstSql, secondSql, "同一字段并集的各行必须生成完全相同的 JDBC Batch SQL");
+        assertTrue(firstSql.contains("src_ip"));
+        assertTrue(firstSql.contains("dest_ip"));
+        assertFalse(firstSql.contains("webshell_type"));
+    }
+
+    private String normalizedSql(MappedStatement statement, Map<String, Object> row) {
+        Map<String, Object> parameter = new HashMap<>();
+        parameter.put("dataMap", row);
+        return statement.getBoundSql(parameter).getSql().replaceAll("\\s+", " ").trim();
+    }
+
+    @Test
+    void nonNormalDataUsesFixedColumnsAndMultiRowValues() {
+        String xml = resource("/mapper/SyslogNonNormalMessageMapper.xml");
+        Matcher statement = Pattern.compile(
+                "<insert id=\"batchInsert\".*?</insert>", Pattern.DOTALL).matcher(xml);
+
+        assertTrue(statement.find(), "缺少 syslog_non_normal_message 批量 INSERT");
+        assertFalse(statement.group(0).contains("<if"), "非标批量 INSERT 不能动态裁剪列");
+        assertTrue(statement.group(0).contains("<foreach collection=\"list\""));
+        assertTrue(statement.group(0).contains("reason_detail"));
+    }
+
+    private String resource(String path) {
+        InputStream stream = FixedColumnBatchMapperTest.class.getResourceAsStream(path);
+        assertNotNull(stream, "找不到测试资源: " + path);
+        try (Scanner scanner = new Scanner(stream, "UTF-8").useDelimiter("\\A")) {
+            return scanner.hasNext() ? scanner.next() : "";
+        }
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmStandardizedBatchExecutorTest.java

@@ -0,0 +1,41 @@
+package com.common.service;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+class DmStandardizedBatchExecutorTest {
+
+    @Test
+    void alignsEveryRowToTheNonNullColumnUnionWithoutMutatingSources() {
+        Map<String, Object> first = new HashMap<>();
+        first.put("id", "row-1");
+        first.put("src_ip", "10.0.0.1");
+        first.put("unused_null", null);
+
+        Map<String, Object> second = new HashMap<>();
+        second.put("id", "row-2");
+        second.put("dest_ip", "10.0.0.2");
+
+        List<Map<String, Object>> aligned =
+                DmStandardizedBatchExecutor.alignBatchColumns(Arrays.asList(first, second));
+
+        assertEquals(aligned.get(0).keySet(), aligned.get(1).keySet());
+        assertEquals(Arrays.asList("dest_ip", "id", "src_ip"),
+                aligned.get(0).keySet().stream().sorted().collect(java.util.stream.Collectors.toList()));
+        assertEquals("10.0.0.1", aligned.get(0).get("src_ip"));
+        assertNull(aligned.get(0).get("dest_ip"));
+        assertNull(aligned.get(1).get("src_ip"));
+        assertEquals("10.0.0.2", aligned.get(1).get("dest_ip"));
+        assertFalse(aligned.get(0).containsKey("unused_null"));
+        assertFalse(first.containsKey("dest_ip"), "不得修改上游标准化结果 Map");
+        assertFalse(second.containsKey("src_ip"), "不得修改上游标准化结果 Map");
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmStandardizedBatchWriterTest.java

@@ -100,6 +100,31 @@ class DmStandardizedBatchWriterTest {
         verify(normalDataService, never()).insertDynamic(any());
     }
 
+    @Test
+    void normalDataWithDifferentPopulatedColumnsSharesOneUnionPlannedBatch() {
+        List<DmStandardizedBatchWriter.WriteRequest<String>> requests = requests(2, -1);
+        requests.get(0).getData().put("src_ip", "10.0.0.1");
+        requests.get(1).getData().put("dest_ip", "10.0.0.2");
+
+        DmStandardizedBatchWriter.WriteReport<String> report = writer.write(requests);
+
+        assertEquals(2, report.getSuccessfulRows());
+        verify(batchExecutor, times(1)).execute(anyBoolean(), anyList(), anyInt());
+    }
+
+    @Test
+    void alarmDataWithDifferentPopulatedColumnsSharesOneUnionPlannedBatch() {
+        when(normalDataService.isAlarmData(any())).thenReturn(true);
+        List<DmStandardizedBatchWriter.WriteRequest<String>> requests = requests(2, -1);
+        requests.get(0).getData().put("src_ip", "10.0.0.1");
+        requests.get(1).getData().put("dest_ip", "10.0.0.2");
+
+        DmStandardizedBatchWriter.WriteReport<String> report = writer.write(requests);
+
+        assertEquals(2, report.getSuccessfulRows());
+        verify(batchExecutor, times(1)).execute(anyBoolean(), anyList(), anyInt());
+    }
+
     private List<DmStandardizedBatchWriter.WriteRequest<String>> requests(int count, int badIndex) {
         List<DmStandardizedBatchWriter.WriteRequest<String>> requests = new ArrayList<>();
         for (int i = 0; i < count; i++) {
@@ -108,7 +133,7 @@ class DmStandardizedBatchWriterTest {
             row.put("rule_version", "legacy");
             row.put("created_at", LocalDateTime.of(2026, 8, 11, 3, 55));
             row.put("log_time", LocalDateTime.of(2026, 8, 11, 3, 55));
-            // 所有行都保留相同列签名；坏行只改变值，确保它真的混在同一个 JDBC batch 中。
+            // 普通日志按当前小批次字段并集生成模板；额外键不会再造成批次拆分。
             row.put("bad", i == badIndex);
             requests.add(new DmStandardizedBatchWriter.WriteRequest<>("source-" + i, row));
         }

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNonNormalMessageServiceBatchTest.java

@@ -0,0 +1,61 @@
+package com.common.service;
+
+import com.common.entity.SyslogNonNormalMessage;
+import com.common.mapper.SyslogNonNormalMessageMapper;
+import com.common.service.impl.SyslogNonNormalMessageServiceImpl;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class SyslogNonNormalMessageServiceBatchTest {
+
+    private SyslogNonNormalMessageServiceImpl service;
+    private SyslogNonNormalMessageMapper mapper;
+    private DirectStatsProjectionWriter directStatsProjectionWriter;
+
+    @BeforeEach
+    void setUp() {
+        service = new SyslogNonNormalMessageServiceImpl();
+        mapper = mock(SyslogNonNormalMessageMapper.class);
+        directStatsProjectionWriter = mock(DirectStatsProjectionWriter.class);
+        ReflectionTestUtils.setField(service, "messageMapper", mapper);
+        ReflectionTestUtils.setField(service, "statsChangeCaptureService",
+                mock(StatsChangeCaptureService.class));
+        ReflectionTestUtils.setField(service, "directStatsProjectionWriter",
+                directStatsProjectionWriter);
+        ReflectionTestUtils.setField(service, "nonNormalBatchSize", 100);
+        when(mapper.batchInsert(anyList()))
+                .thenAnswer(invocation -> ((List<?>) invocation.getArgument(0)).size());
+    }
+
+    @Test
+    void chunksLargePollInsideOneServiceTransactionAndAggregatesStatsOnce() {
+        List<SyslogNonNormalMessage> messages = new ArrayList<>();
+        for (int i = 0; i < 250; i++) {
+            SyslogNonNormalMessage message = new SyslogNonNormalMessage();
+            message.setId("non-normal-" + i);
+            messages.add(message);
+        }
+
+        assertEquals(250, service.saveBatchMessages(messages));
+
+        @SuppressWarnings("unchecked")
+        ArgumentCaptor<List<SyslogNonNormalMessage>> captor = ArgumentCaptor.forClass(List.class);
+        verify(mapper, times(3)).batchInsert(captor.capture());
+        assertEquals(100, captor.getAllValues().get(0).size());
+        assertEquals(100, captor.getAllValues().get(1).size());
+        assertEquals(50, captor.getAllValues().get(2).size());
+        verify(directStatsProjectionWriter).writeNonNormalRows(messages);
+    }
+}

haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNormalDataServiceIdempotencyTest.java

@@ -38,7 +38,7 @@ class SyslogNormalDataServiceIdempotencyTest {
     @Test
     void dmUniqueViolationWithExistingNormalResultIsIdempotentSuccess() {
         Map<String, Object> data = idempotencyData(false);
-        when(normalDataMapper.insertDynamic(anyMap())).thenThrow(dmUniqueViolation());
+        when(normalDataMapper.insertBatchPlanned(anyMap())).thenThrow(dmUniqueViolation());
         when(normalDataMapper.countByLogTimeAndId(
                 (LocalDateTime) data.get("log_time"), (String) data.get("id")))
                 .thenReturn(1);
@@ -53,7 +53,7 @@ class SyslogNormalDataServiceIdempotencyTest {
     @Test
     void dmUniqueViolationWithExistingAlarmResultIsIdempotentSuccess() {
         Map<String, Object> data = idempotencyData(true);
-        when(normalAlarmMapper.insertDynamic(anyMap())).thenThrow(dmUniqueViolation());
+        when(normalAlarmMapper.insertBatchPlanned(anyMap())).thenThrow(dmUniqueViolation());
         when(normalAlarmMapper.countByLogTimeAndId(
                 (LocalDateTime) data.get("log_time"), (String) data.get("id")))
                 .thenReturn(1);
@@ -70,7 +70,7 @@ class SyslogNormalDataServiceIdempotencyTest {
         Map<String, Object> data = idempotencyData(false);
         DataIntegrityViolationException invalidData = new DataIntegrityViolationException(
                 "value too long", new SQLException("data too long"));
-        when(normalDataMapper.insertDynamic(anyMap())).thenThrow(invalidData);
+        when(normalDataMapper.insertBatchPlanned(anyMap())).thenThrow(invalidData);
 
         assertThrows(DataIntegrityViolationException.class, () -> service.insertDynamic(data));