王绍宏
d3bc083b88
- 使用原生JDBC PreparedStatement执行标准化事实批写 - 按批次字段并集只生成一次INSERT语句 - 移除批写主路径逐行执行MyBatis OGNL动态SQL的开销 - 从达梦表元数据缓存合法列名和NULL字段类型 - 保留告警优先、事务一致性及失败隔离逻辑 - 默认关闭统计变更捕获时跳过无效MyBatis BatchSession - 增加SQL规划、参数绑定和批量执行耗时指标 - 补充JDBC批写、字段白名单及参数绑定测试
260 lines
10 KiB
Java
260 lines
10 KiB
Java
package com.common.service;
|
|
|
|
import org.springframework.jdbc.core.ConnectionCallback;
|
|
import org.springframework.jdbc.core.JdbcTemplate;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import java.sql.Connection;
|
|
import java.sql.PreparedStatement;
|
|
import java.sql.ResultSet;
|
|
import java.sql.ResultSetMetaData;
|
|
import java.sql.SQLException;
|
|
import java.sql.Statement;
|
|
import java.sql.Timestamp;
|
|
import java.time.LocalDate;
|
|
import java.time.LocalDateTime;
|
|
import java.util.ArrayList;
|
|
import java.util.Collections;
|
|
import java.util.HashSet;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.List;
|
|
import java.util.Locale;
|
|
import java.util.Map;
|
|
import java.util.Set;
|
|
import java.util.concurrent.ConcurrentHashMap;
|
|
import java.util.concurrent.ConcurrentMap;
|
|
|
|
/**
|
|
* 使用一个固定 {@link PreparedStatement} 写入一批标准化事实。
|
|
*
|
|
* <p>字段白名单和 NULL 的 JDBC 类型从目标表元数据读取并缓存。每个批次只根据字段并集生成
|
|
* 一次 INSERT，后续每行只绑定参数和 {@code addBatch()}，避免 MyBatis 动态 XML 为每行执行
|
|
* 数百次 OGNL 表达式和反射方法查找。</p>
|
|
*/
|
|
@Component
|
|
public class DmPreparedBatchInserter {
|
|
|
|
static final String NORMAL_TABLE = "syslog_normal_data";
|
|
static final String ALARM_TABLE = "syslog_normal_alarm";
|
|
|
|
private final JdbcTemplate jdbcTemplate;
|
|
private final ConcurrentMap<String, TableMetadata> metadataCache = new ConcurrentHashMap<>();
|
|
|
|
public DmPreparedBatchInserter(JdbcTemplate jdbcTemplate) {
|
|
this.jdbcTemplate = jdbcTemplate;
|
|
}
|
|
|
|
public BatchExecution insert(boolean alarmTable, List<Map<String, Object>> rows,
|
|
int statementTimeoutSeconds) {
|
|
if (rows == null || rows.isEmpty()) {
|
|
return BatchExecution.EMPTY;
|
|
}
|
|
final String table = alarmTable ? ALARM_TABLE : NORMAL_TABLE;
|
|
return jdbcTemplate.execute((ConnectionCallback<BatchExecution>) connection ->
|
|
execute(connection, table, rows, statementTimeoutSeconds));
|
|
}
|
|
|
|
BatchExecution execute(Connection connection, String table, List<Map<String, Object>> rows,
|
|
int statementTimeoutSeconds) throws SQLException {
|
|
long planStarted = System.nanoTime();
|
|
TableMetadata metadata = metadata(connection, table);
|
|
BatchPlan plan = buildPlan(table, rows, metadata);
|
|
long planMs = elapsedMillis(planStarted);
|
|
|
|
long bindMs;
|
|
long executeMs;
|
|
try (PreparedStatement statement = connection.prepareStatement(plan.sql)) {
|
|
statement.setQueryTimeout(Math.max(1, statementTimeoutSeconds));
|
|
long bindStarted = System.nanoTime();
|
|
for (Map<String, Object> row : rows) {
|
|
bindRow(statement, row, plan.columns);
|
|
statement.addBatch();
|
|
}
|
|
bindMs = elapsedMillis(bindStarted);
|
|
|
|
long executeStarted = System.nanoTime();
|
|
int[] results = statement.executeBatch();
|
|
executeMs = elapsedMillis(executeStarted);
|
|
rejectFailedResult(results);
|
|
}
|
|
return new BatchExecution(rows.size(), plan.columns.size(), planMs, bindMs, executeMs);
|
|
}
|
|
|
|
private TableMetadata metadata(Connection connection, String table) throws SQLException {
|
|
TableMetadata cached = metadataCache.get(table);
|
|
if (cached != null) {
|
|
return cached;
|
|
}
|
|
TableMetadata loaded = loadMetadata(connection, table);
|
|
TableMetadata raced = metadataCache.putIfAbsent(table, loaded);
|
|
return raced == null ? loaded : raced;
|
|
}
|
|
|
|
private static TableMetadata loadMetadata(Connection connection, String table) throws SQLException {
|
|
validateIdentifier(table);
|
|
List<ColumnMetadata> columns = new ArrayList<>();
|
|
try (PreparedStatement statement = connection.prepareStatement(
|
|
"SELECT * FROM " + table + " WHERE 1=0");
|
|
ResultSet resultSet = statement.executeQuery()) {
|
|
ResultSetMetaData metadata = resultSet.getMetaData();
|
|
for (int index = 1; index <= metadata.getColumnCount(); index++) {
|
|
String name = metadata.getColumnName(index);
|
|
validateIdentifier(name);
|
|
columns.add(new ColumnMetadata(name.toLowerCase(Locale.ROOT), metadata.getColumnType(index)));
|
|
}
|
|
}
|
|
if (columns.isEmpty()) {
|
|
throw new SQLException("目标表没有可写字段: " + table);
|
|
}
|
|
return new TableMetadata(columns);
|
|
}
|
|
|
|
static BatchPlan buildPlan(String table, List<Map<String, Object>> rows, TableMetadata metadata) {
|
|
validateIdentifier(table);
|
|
Set<String> requestedColumns = new HashSet<>();
|
|
for (Map<String, Object> row : rows) {
|
|
if (row == null) {
|
|
throw new IllegalArgumentException("达梦批写行不能为空");
|
|
}
|
|
for (Map.Entry<String, Object> entry : row.entrySet()) {
|
|
if (entry.getKey() != null && entry.getValue() != null) {
|
|
// 标准化字段契约使用小写 snake_case；与旧 Mapper 一致，未知或大小写错误的键忽略。
|
|
requestedColumns.add(entry.getKey());
|
|
}
|
|
}
|
|
}
|
|
|
|
List<ColumnMetadata> selected = new ArrayList<>();
|
|
for (ColumnMetadata column : metadata.columns) {
|
|
if (requestedColumns.contains(column.name)) {
|
|
selected.add(column);
|
|
}
|
|
}
|
|
if (selected.isEmpty()) {
|
|
throw new IllegalArgumentException("达梦批写没有目标表允许的非空字段: " + table);
|
|
}
|
|
|
|
StringBuilder sql = new StringBuilder(64 + selected.size() * 12);
|
|
sql.append("INSERT INTO ").append(table).append(" (");
|
|
for (int index = 0; index < selected.size(); index++) {
|
|
if (index > 0) sql.append(',');
|
|
sql.append(selected.get(index).name);
|
|
}
|
|
sql.append(") VALUES (");
|
|
for (int index = 0; index < selected.size(); index++) {
|
|
if (index > 0) sql.append(',');
|
|
sql.append('?');
|
|
}
|
|
sql.append(')');
|
|
return new BatchPlan(sql.toString(), selected);
|
|
}
|
|
|
|
static void bindRow(PreparedStatement statement, Map<String, Object> row,
|
|
List<ColumnMetadata> columns) throws SQLException {
|
|
for (int index = 0; index < columns.size(); index++) {
|
|
ColumnMetadata column = columns.get(index);
|
|
Object value = row.get(column.name);
|
|
int parameterIndex = index + 1;
|
|
if (value == null) {
|
|
statement.setNull(parameterIndex, column.jdbcType);
|
|
} else if (value instanceof Timestamp) {
|
|
statement.setTimestamp(parameterIndex, (Timestamp) value);
|
|
} else if (value instanceof java.sql.Date) {
|
|
statement.setDate(parameterIndex, (java.sql.Date) value);
|
|
} else if (value instanceof java.sql.Time) {
|
|
statement.setTime(parameterIndex, (java.sql.Time) value);
|
|
} else if (value instanceof java.util.Date) {
|
|
statement.setTimestamp(parameterIndex,
|
|
new Timestamp(((java.util.Date) value).getTime()));
|
|
} else if (value instanceof LocalDateTime) {
|
|
statement.setTimestamp(parameterIndex, Timestamp.valueOf((LocalDateTime) value));
|
|
} else if (value instanceof LocalDate) {
|
|
statement.setDate(parameterIndex, java.sql.Date.valueOf((LocalDate) value));
|
|
} else if (value instanceof byte[]) {
|
|
statement.setBytes(parameterIndex, (byte[]) value);
|
|
} else {
|
|
statement.setObject(parameterIndex, value);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void rejectFailedResult(int[] results) throws SQLException {
|
|
if (results == null) return;
|
|
for (int result : results) {
|
|
if (result == Statement.EXECUTE_FAILED) {
|
|
throw new SQLException("达梦 JDBC batch 返回 EXECUTE_FAILED");
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void validateIdentifier(String identifier) {
|
|
if (identifier == null || !identifier.matches("[A-Za-z_][A-Za-z0-9_$#]*")) {
|
|
throw new IllegalArgumentException("非法达梦标识符: " + identifier);
|
|
}
|
|
}
|
|
|
|
private static long elapsedMillis(long startedAt) {
|
|
return (System.nanoTime() - startedAt) / 1_000_000L;
|
|
}
|
|
|
|
static final class TableMetadata {
|
|
final List<ColumnMetadata> columns;
|
|
|
|
TableMetadata(List<ColumnMetadata> columns) {
|
|
this.columns = Collections.unmodifiableList(new ArrayList<>(columns));
|
|
}
|
|
|
|
static TableMetadata of(Map<String, Integer> columns) {
|
|
List<ColumnMetadata> result = new ArrayList<>();
|
|
for (Map.Entry<String, Integer> entry : new LinkedHashMap<>(columns).entrySet()) {
|
|
result.add(new ColumnMetadata(entry.getKey().toLowerCase(Locale.ROOT), entry.getValue()));
|
|
}
|
|
return new TableMetadata(result);
|
|
}
|
|
}
|
|
|
|
static final class ColumnMetadata {
|
|
final String name;
|
|
final int jdbcType;
|
|
|
|
ColumnMetadata(String name, int jdbcType) {
|
|
validateIdentifier(name);
|
|
this.name = name.toLowerCase(Locale.ROOT);
|
|
this.jdbcType = jdbcType;
|
|
}
|
|
}
|
|
|
|
static final class BatchPlan {
|
|
final String sql;
|
|
final List<ColumnMetadata> columns;
|
|
|
|
BatchPlan(String sql, List<ColumnMetadata> columns) {
|
|
this.sql = sql;
|
|
this.columns = Collections.unmodifiableList(new ArrayList<>(columns));
|
|
}
|
|
}
|
|
|
|
public static final class BatchExecution {
|
|
static final BatchExecution EMPTY = new BatchExecution(0, 0, 0L, 0L, 0L);
|
|
private final int rows;
|
|
private final int columns;
|
|
private final long planMs;
|
|
private final long bindMs;
|
|
private final long executeMs;
|
|
|
|
BatchExecution(int rows, int columns, long planMs, long bindMs, long executeMs) {
|
|
this.rows = rows;
|
|
this.columns = columns;
|
|
this.planMs = planMs;
|
|
this.bindMs = bindMs;
|
|
this.executeMs = executeMs;
|
|
}
|
|
|
|
public int getRows() { return rows; }
|
|
public int getColumns() { return columns; }
|
|
public long getPlanMs() { return planMs; }
|
|
public long getBindMs() { return bindMs; }
|
|
public long getExecuteMs() { return executeMs; }
|
|
}
|
|
}
|