王绍宏
29445c6825
- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
75 lines
3.0 KiB
Java
75 lines
3.0 KiB
Java
package com.config;
|
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.context.annotation.Bean;
|
|
import org.springframework.context.annotation.Configuration;
|
|
import org.springframework.scheduling.annotation.EnableAsync;
|
|
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
|
|
|
import java.util.concurrent.Executor;
|
|
import java.util.concurrent.ThreadPoolExecutor;
|
|
import org.springframework.context.annotation.Primary;
|
|
@Configuration
|
|
@EnableAsync
|
|
public class ThreadPoolConfig {
|
|
|
|
@Value("${app.processor.thread-pool.core-pool-size:10}")
|
|
private int corePoolSize;
|
|
|
|
@Value("${app.processor.thread-pool.max-pool-size:20}")
|
|
private int maxPoolSize;
|
|
|
|
@Value("${app.processor.thread-pool.queue-capacity:2000}")
|
|
private int queueCapacity;
|
|
|
|
@Value("${app.processor.thread-pool.keep-alive-seconds:60}")
|
|
private int keepAliveSeconds;
|
|
|
|
@Value("${dm.standardized-write.writer-concurrency:2}")
|
|
private int dmWriterConcurrency;
|
|
|
|
@Value("${dm.standardized-write.writer-queue-capacity:100}")
|
|
private int dmWriterQueueCapacity;
|
|
|
|
@Bean("logNormalProcessorExecutor")
|
|
@Primary
|
|
public Executor logNormalProcessorExecutor() {
|
|
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
|
executor.setCorePoolSize(corePoolSize);
|
|
executor.setMaxPoolSize(maxPoolSize);
|
|
executor.setQueueCapacity(queueCapacity);
|
|
executor.setKeepAliveSeconds(keepAliveSeconds);
|
|
executor.setThreadNamePrefix("log-processor-");
|
|
|
|
// 队列耗尽时明确失败并保留 Kafka offset，不能在 listener 线程中无限兜底执行而掩盖过载。
|
|
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
|
|
|
|
// 等待所有任务结束后再关闭线程池
|
|
executor.setWaitForTasksToCompleteOnShutdown(true);
|
|
executor.setAwaitTerminationSeconds(60);
|
|
executor.initialize();
|
|
return executor;
|
|
}
|
|
|
|
/**
|
|
* 达梦批写使用独立的有界线程池，不能复用规则计算线程池。
|
|
*
|
|
* <p>分离后，少量慢 SQL 不会占满所有标准化计算线程；AbortPolicy 会把过载明确返回到
|
|
* Kafka ACK 边界，由消费者保留 offset 形成背压，而不是在内存中无限排队。</p>
|
|
*/
|
|
@Bean("dmStandardizedWriterExecutor")
|
|
public Executor dmStandardizedWriterExecutor() {
|
|
int concurrency = Math.max(1, dmWriterConcurrency);
|
|
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
|
executor.setCorePoolSize(concurrency);
|
|
executor.setMaxPoolSize(concurrency);
|
|
executor.setQueueCapacity(Math.max(1, dmWriterQueueCapacity));
|
|
executor.setThreadNamePrefix("dm-batch-writer-");
|
|
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
|
|
executor.setWaitForTasksToCompleteOnShutdown(true);
|
|
executor.setAwaitTerminationSeconds(60);
|
|
executor.initialize();
|
|
return executor;
|
|
}
|
|
}
|