feat(syslog): 完成高吞吐可靠链路与统计存储优化
- 新增接入 WAL、检查点、背压恢复及可靠分发机制,完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存,支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型,统一接收时间与 Point 时间,支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路,支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
+31
-4
@@ -25,8 +25,14 @@ public class ThreadPoolConfig {
|
||||
@Value("${app.processor.thread-pool.keep-alive-seconds:60}")
|
||||
private int keepAliveSeconds;
|
||||
|
||||
@Value("${dm.standardized-write.writer-concurrency:2}")
|
||||
private int dmWriterConcurrency;
|
||||
|
||||
@Value("${dm.standardized-write.writer-queue-capacity:100}")
|
||||
private int dmWriterQueueCapacity;
|
||||
|
||||
@Bean("logNormalProcessorExecutor")
|
||||
@Primary // 添加这个注解
|
||||
@Primary
|
||||
public Executor logNormalProcessorExecutor() {
|
||||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||||
executor.setCorePoolSize(corePoolSize);
|
||||
@@ -35,8 +41,8 @@ public class ThreadPoolConfig {
|
||||
executor.setKeepAliveSeconds(keepAliveSeconds);
|
||||
executor.setThreadNamePrefix("log-processor-");
|
||||
|
||||
// 拒绝策略:由调用线程处理该任务(保证不丢失消息)
|
||||
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
// 队列耗尽时明确失败并保留 Kafka offset,不能在 listener 线程中无限兜底执行而掩盖过载。
|
||||
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
|
||||
|
||||
// 等待所有任务结束后再关闭线程池
|
||||
executor.setWaitForTasksToCompleteOnShutdown(true);
|
||||
@@ -44,4 +50,25 @@ public class ThreadPoolConfig {
|
||||
executor.initialize();
|
||||
return executor;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 达梦批写使用独立的有界线程池,不能复用规则计算线程池。
|
||||
*
|
||||
* <p>分离后,少量慢 SQL 不会占满所有标准化计算线程;AbortPolicy 会把过载明确返回到
|
||||
* Kafka ACK 边界,由消费者保留 offset 形成背压,而不是在内存中无限排队。</p>
|
||||
*/
|
||||
@Bean("dmStandardizedWriterExecutor")
|
||||
public Executor dmStandardizedWriterExecutor() {
|
||||
int concurrency = Math.max(1, dmWriterConcurrency);
|
||||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||||
executor.setCorePoolSize(concurrency);
|
||||
executor.setMaxPoolSize(concurrency);
|
||||
executor.setQueueCapacity(Math.max(1, dmWriterQueueCapacity));
|
||||
executor.setThreadNamePrefix("dm-batch-writer-");
|
||||
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
|
||||
executor.setWaitForTasksToCompleteOnShutdown(true);
|
||||
executor.setAwaitTerminationSeconds(60);
|
||||
executor.initialize();
|
||||
return executor;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user