perf(syslog-consumer-rule): 消除达梦批写动态SQL性能瓶颈

- 使用原生JDBC PreparedStatement执行标准化事实批写 - 按批次字段并集只生成一次INSERT语句 - 移除批写主路径逐行执行MyBatis OGNL动态SQL的开销 - 从达梦表元数据缓存合法列名和NULL字段类型 - 保留告警优先、事务一致性及失败隔离逻辑 - 默认关闭统计变更捕获时跳过无效MyBatis BatchSession - 增加SQL规划、参数绑定和批量执行耗时指标 - 补充JDBC批写、字段白名单及参数绑定测试
2026-09-13 03:05:42 +08:00
parent 87fee6fe69
commit d3bc083b88
10 changed files with 440 additions and 112 deletions
@@ -0,0 +1,125 @@
+package com.common.service;
+
+import org.junit.jupiter.api.Test;
+
+import java.sql.PreparedStatement;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.Types;
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class DmPreparedBatchInserterTest {
+
+    @Test
+    void buildsOneCanonicalInsertFromBatchUnionAndDatabaseWhitelist() {
+        Map<String, Integer> columns = new LinkedHashMap<>();
+        columns.put("id", Types.VARCHAR);
+        columns.put("created_at", Types.TIMESTAMP);
+        columns.put("src_ip", Types.VARCHAR);
+        columns.put("dest_ip", Types.VARCHAR);
+
+        Map<String, Object> first = new LinkedHashMap<>();
+        first.put("src_ip", "10.0.0.1");
+        first.put("id", "row-1");
+        first.put("unknown_rule_field", "ignored");
+        Map<String, Object> second = new LinkedHashMap<>();
+        second.put("id", "row-2");
+        second.put("dest_ip", "10.0.0.2");
+        second.put("src_ip", null);
+
+        DmPreparedBatchInserter.BatchPlan plan = DmPreparedBatchInserter.buildPlan(
+                DmPreparedBatchInserter.NORMAL_TABLE, Arrays.asList(first, second),
+                DmPreparedBatchInserter.TableMetadata.of(columns));
+
+        assertEquals("INSERT INTO syslog_normal_data (id,src_ip,dest_ip) VALUES (?,?,?)", plan.sql);
+        assertEquals(3, plan.columns.size());
+    }
+
+    @Test
+    void bindsDatesAndUsesDatabaseTypeForMissingValues() throws Exception {
+        PreparedStatement statement = mock(PreparedStatement.class);
+        LocalDateTime createdAt = LocalDateTime.of(2026, 9, 13, 2, 30);
+        Date logTime = new Date(1_700_000_000_000L);
+        Map<String, Object> row = new LinkedHashMap<>();
+        row.put("id", "row-1");
+        row.put("created_at", createdAt);
+        row.put("log_time", logTime);
+
+        List<DmPreparedBatchInserter.ColumnMetadata> columns = Arrays.asList(
+                new DmPreparedBatchInserter.ColumnMetadata("id", Types.VARCHAR),
+                new DmPreparedBatchInserter.ColumnMetadata("created_at", Types.TIMESTAMP),
+                new DmPreparedBatchInserter.ColumnMetadata("log_time", Types.TIMESTAMP),
+                new DmPreparedBatchInserter.ColumnMetadata("dest_ip", Types.VARCHAR));
+
+        DmPreparedBatchInserter.bindRow(statement, row, columns);
+
+        verify(statement).setObject(1, "row-1");
+        verify(statement).setTimestamp(2, java.sql.Timestamp.valueOf(createdAt));
+        verify(statement).setTimestamp(3, new java.sql.Timestamp(logTime.getTime()));
+        verify(statement).setNull(4, Types.VARCHAR);
+    }
+
+    @Test
+    void rejectsDynamicTableNames() {
+        Map<String, Integer> columns = new LinkedHashMap<>();
+        columns.put("id", Types.VARCHAR);
+        Map<String, Object> row = new LinkedHashMap<>();
+        row.put("id", "row-1");
+
+        assertThrows(IllegalArgumentException.class, () -> DmPreparedBatchInserter.buildPlan(
+                "syslog_normal_data;drop table x", Arrays.asList(row),
+                DmPreparedBatchInserter.TableMetadata.of(columns)));
+    }
+
+    @Test
+    void preparesOneStatementAndExecutesOneJdbcBatchForAllRows() throws Exception {
+        Connection connection = mock(Connection.class);
+        PreparedStatement metadataStatement = mock(PreparedStatement.class);
+        ResultSet metadataResult = mock(ResultSet.class);
+        ResultSetMetaData resultSetMetaData = mock(ResultSetMetaData.class);
+        PreparedStatement insertStatement = mock(PreparedStatement.class);
+        when(connection.prepareStatement("SELECT * FROM syslog_normal_data WHERE 1=0"))
+                .thenReturn(metadataStatement);
+        when(metadataStatement.executeQuery()).thenReturn(metadataResult);
+        when(metadataResult.getMetaData()).thenReturn(resultSetMetaData);
+        when(resultSetMetaData.getColumnCount()).thenReturn(2);
+        when(resultSetMetaData.getColumnName(1)).thenReturn("ID");
+        when(resultSetMetaData.getColumnName(2)).thenReturn("SRC_IP");
+        when(resultSetMetaData.getColumnType(1)).thenReturn(Types.VARCHAR);
+        when(resultSetMetaData.getColumnType(2)).thenReturn(Types.VARCHAR);
+        when(connection.prepareStatement(
+                "INSERT INTO syslog_normal_data (id,src_ip) VALUES (?,?)"))
+                .thenReturn(insertStatement);
+        when(insertStatement.executeBatch()).thenReturn(new int[]{1, 1});
+
+        Map<String, Object> first = new LinkedHashMap<>();
+        first.put("id", "row-1");
+        first.put("src_ip", "10.0.0.1");
+        Map<String, Object> second = new LinkedHashMap<>();
+        second.put("id", "row-2");
+
+        DmPreparedBatchInserter inserter = new DmPreparedBatchInserter(mock(
+                org.springframework.jdbc.core.JdbcTemplate.class));
+        DmPreparedBatchInserter.BatchExecution result = inserter.execute(connection,
+                DmPreparedBatchInserter.NORMAL_TABLE, Arrays.asList(first, second), 45);
+
+        assertEquals(2, result.getRows());
+        assertEquals(2, result.getColumns());
+        verify(insertStatement, times(2)).addBatch();
+        verify(insertStatement).executeBatch();
+        verify(insertStatement).setNull(2, Types.VARCHAR);
+    }
+}
@@ -1,41 +0,0 @@
-package com.common.service;
-
-import org.junit.jupiter.api.Test;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
-
-class DmStandardizedBatchExecutorTest {
-
-    @Test
-    void alignsEveryRowToTheNonNullColumnUnionWithoutMutatingSources() {
-        Map<String, Object> first = new HashMap<>();
-        first.put("id", "row-1");
-        first.put("src_ip", "10.0.0.1");
-        first.put("unused_null", null);
-
-        Map<String, Object> second = new HashMap<>();
-        second.put("id", "row-2");
-        second.put("dest_ip", "10.0.0.2");
-
-        List<Map<String, Object>> aligned =
-                DmStandardizedBatchExecutor.alignBatchColumns(Arrays.asList(first, second));
-
-        assertEquals(aligned.get(0).keySet(), aligned.get(1).keySet());
-        assertEquals(Arrays.asList("dest_ip", "id", "src_ip"),
-                aligned.get(0).keySet().stream().sorted().collect(java.util.stream.Collectors.toList()));
-        assertEquals("10.0.0.1", aligned.get(0).get("src_ip"));
-        assertNull(aligned.get(0).get("dest_ip"));
-        assertNull(aligned.get(1).get("src_ip"));
-        assertEquals("10.0.0.2", aligned.get(1).get("dest_ip"));
-        assertFalse(aligned.get(0).containsKey("unused_null"));
-        assertFalse(first.containsKey("dest_ip"), "不得修改上游标准化结果 Map");
-        assertFalse(second.containsKey("src_ip"), "不得修改上游标准化结果 Map");
-    }
-}