perf(syslog-consumer-rule): 非标日志改为单线程异步批量写入
- 非标写库结果不再参与 Kafka ACK - 使用独立单线程和有界队列处理非标日志 - 非标数据按固定大小分批写入达梦 - 增加异步成功、失败和队列拒绝指标 - 补充异步解耦及批量写入测试 perf(syslog-consumer-rule): 按批次字段并集优化达梦批量写入 - 普通日志和告警日志按非空字段并集生成批写模板 - 为同批记录补齐缺失字段并绑定 NULL - 使用 MyBatis ExecutorType.BATCH 复用 PreparedStatement - 避免固定写入全部 373 个候选字段 - 非标日志保留固定 24 列多行批量写入 - 批量大小和非标队列容量默认调整为 500 - 补充字段并集与动态 SQL 一致性测试
This commit is contained in:
+21
@@ -31,6 +31,9 @@ public class ThreadPoolConfig {
|
||||
@Value("${dm.standardized-write.writer-queue-capacity:100}")
|
||||
private int dmWriterQueueCapacity;
|
||||
|
||||
@Value("${dm.non-normal-write.queue-capacity:500}")
|
||||
private int nonNormalWriterQueueCapacity;
|
||||
|
||||
@Bean("logNormalProcessorExecutor")
|
||||
@Primary
|
||||
public Executor logNormalProcessorExecutor() {
|
||||
@@ -71,4 +74,22 @@ public class ThreadPoolConfig {
|
||||
executor.initialize();
|
||||
return executor;
|
||||
}
|
||||
|
||||
/**
|
||||
* 非标日志使用单独的串行写线程,与正常事实批写解耦。
|
||||
* listener 不等待该线程,队列溢出或异步写入失败只记录错误,不阻塞 Kafka ACK。
|
||||
*/
|
||||
@Bean("nonNormalWriterExecutor")
|
||||
public Executor nonNormalWriterExecutor() {
|
||||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||||
executor.setCorePoolSize(1);
|
||||
executor.setMaxPoolSize(1);
|
||||
executor.setQueueCapacity(Math.max(1, nonNormalWriterQueueCapacity));
|
||||
executor.setThreadNamePrefix("dm-non-normal-writer-");
|
||||
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
|
||||
executor.setWaitForTasksToCompleteOnShutdown(true);
|
||||
executor.setAwaitTerminationSeconds(60);
|
||||
executor.initialize();
|
||||
return executor;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user