王绍宏
ce473b2c7d
- 非标写库结果不再参与 Kafka ACK - 使用独立单线程和有界队列处理非标日志 - 非标数据按固定大小分批写入达梦 - 增加异步成功、失败和队列拒绝指标 - 补充异步解耦及批量写入测试 perf(syslog-consumer-rule): 按批次字段并集优化达梦批量写入 - 普通日志和告警日志按非空字段并集生成批写模板 - 为同批记录补齐缺失字段并绑定 NULL - 使用 MyBatis ExecutorType.BATCH 复用 PreparedStatement - 避免固定写入全部 373 个候选字段 - 非标日志保留固定 24 列多行批量写入 - 批量大小和非标队列容量默认调整为 500 - 补充字段并集与动态 SQL 一致性测试
96 lines
3.9 KiB
Java
96 lines
3.9 KiB
Java
package com.config;
|
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.context.annotation.Bean;
|
|
import org.springframework.context.annotation.Configuration;
|
|
import org.springframework.scheduling.annotation.EnableAsync;
|
|
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
|
|
|
import java.util.concurrent.Executor;
|
|
import java.util.concurrent.ThreadPoolExecutor;
|
|
import org.springframework.context.annotation.Primary;
|
|
@Configuration
|
|
@EnableAsync
|
|
public class ThreadPoolConfig {
|
|
|
|
@Value("${app.processor.thread-pool.core-pool-size:10}")
|
|
private int corePoolSize;
|
|
|
|
@Value("${app.processor.thread-pool.max-pool-size:20}")
|
|
private int maxPoolSize;
|
|
|
|
@Value("${app.processor.thread-pool.queue-capacity:2000}")
|
|
private int queueCapacity;
|
|
|
|
@Value("${app.processor.thread-pool.keep-alive-seconds:60}")
|
|
private int keepAliveSeconds;
|
|
|
|
@Value("${dm.standardized-write.writer-concurrency:2}")
|
|
private int dmWriterConcurrency;
|
|
|
|
@Value("${dm.standardized-write.writer-queue-capacity:100}")
|
|
private int dmWriterQueueCapacity;
|
|
|
|
@Value("${dm.non-normal-write.queue-capacity:500}")
|
|
private int nonNormalWriterQueueCapacity;
|
|
|
|
@Bean("logNormalProcessorExecutor")
|
|
@Primary
|
|
public Executor logNormalProcessorExecutor() {
|
|
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
|
executor.setCorePoolSize(corePoolSize);
|
|
executor.setMaxPoolSize(maxPoolSize);
|
|
executor.setQueueCapacity(queueCapacity);
|
|
executor.setKeepAliveSeconds(keepAliveSeconds);
|
|
executor.setThreadNamePrefix("log-processor-");
|
|
|
|
// 队列耗尽时明确失败并保留 Kafka offset，不能在 listener 线程中无限兜底执行而掩盖过载。
|
|
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
|
|
|
|
// 等待所有任务结束后再关闭线程池
|
|
executor.setWaitForTasksToCompleteOnShutdown(true);
|
|
executor.setAwaitTerminationSeconds(60);
|
|
executor.initialize();
|
|
return executor;
|
|
}
|
|
|
|
/**
|
|
* 达梦批写使用独立的有界线程池，不能复用规则计算线程池。
|
|
*
|
|
* <p>分离后，少量慢 SQL 不会占满所有标准化计算线程；AbortPolicy 会把过载明确返回到
|
|
* Kafka ACK 边界，由消费者保留 offset 形成背压，而不是在内存中无限排队。</p>
|
|
*/
|
|
@Bean("dmStandardizedWriterExecutor")
|
|
public Executor dmStandardizedWriterExecutor() {
|
|
int concurrency = Math.max(1, dmWriterConcurrency);
|
|
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
|
executor.setCorePoolSize(concurrency);
|
|
executor.setMaxPoolSize(concurrency);
|
|
executor.setQueueCapacity(Math.max(1, dmWriterQueueCapacity));
|
|
executor.setThreadNamePrefix("dm-batch-writer-");
|
|
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
|
|
executor.setWaitForTasksToCompleteOnShutdown(true);
|
|
executor.setAwaitTerminationSeconds(60);
|
|
executor.initialize();
|
|
return executor;
|
|
}
|
|
|
|
/**
|
|
* 非标日志使用单独的串行写线程，与正常事实批写解耦。
|
|
* listener 不等待该线程，队列溢出或异步写入失败只记录错误，不阻塞 Kafka ACK。
|
|
*/
|
|
@Bean("nonNormalWriterExecutor")
|
|
public Executor nonNormalWriterExecutor() {
|
|
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
|
executor.setCorePoolSize(1);
|
|
executor.setMaxPoolSize(1);
|
|
executor.setQueueCapacity(Math.max(1, nonNormalWriterQueueCapacity));
|
|
executor.setThreadNamePrefix("dm-non-normal-writer-");
|
|
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
|
|
executor.setWaitForTasksToCompleteOnShutdown(true);
|
|
executor.setAwaitTerminationSeconds(60);
|
|
executor.initialize();
|
|
return executor;
|
|
}
|
|
}
|