feat(syslog): 完成高吞吐可靠链路与统计存储优化

- 新增接入 WAL、检查点、背压恢复及可靠分发机制,完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程
- 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存,支持缓存失效广播、影子对账和执行指标
- 升级 InfluxDB V2 原始日志模型,统一接收时间与 Point 时间,支持按日 measurement、UUID 分桶及 V1/V2 双读
- 重构达梦标准化批量写入与统计链路,支持事务内直写、增量投影、幂等控制、窗口水位和日期切换
- 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
王绍宏
2026-08-20 00:18:33 +08:00
parent a3852fc0f9
commit 29445c6825
303 changed files with 20419 additions and 4481 deletions
@@ -25,8 +25,14 @@ public class ThreadPoolConfig {
@Value("${app.processor.thread-pool.keep-alive-seconds:60}")
private int keepAliveSeconds;
@Value("${dm.standardized-write.writer-concurrency:2}")
private int dmWriterConcurrency;
@Value("${dm.standardized-write.writer-queue-capacity:100}")
private int dmWriterQueueCapacity;
@Bean("logNormalProcessorExecutor")
@Primary // 添加这个注解
@Primary
public Executor logNormalProcessorExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(corePoolSize);
@@ -35,8 +41,8 @@ public class ThreadPoolConfig {
executor.setKeepAliveSeconds(keepAliveSeconds);
executor.setThreadNamePrefix("log-processor-");
// 拒绝策略:由调用线程处理该任务(保证不丢失消息)
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
// 队列耗尽时明确失败并保留 Kafka offset,不能在 listener 线程中无限兜底执行而掩盖过载。
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
// 等待所有任务结束后再关闭线程池
executor.setWaitForTasksToCompleteOnShutdown(true);
@@ -44,4 +50,25 @@ public class ThreadPoolConfig {
executor.initialize();
return executor;
}
}
/**
* 达梦批写使用独立的有界线程池,不能复用规则计算线程池。
*
* <p>分离后,少量慢 SQL 不会占满所有标准化计算线程;AbortPolicy 会把过载明确返回到
* Kafka ACK 边界,由消费者保留 offset 形成背压,而不是在内存中无限排队。</p>
*/
@Bean("dmStandardizedWriterExecutor")
public Executor dmStandardizedWriterExecutor() {
int concurrency = Math.max(1, dmWriterConcurrency);
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(concurrency);
executor.setMaxPoolSize(concurrency);
executor.setQueueCapacity(Math.max(1, dmWriterQueueCapacity));
executor.setThreadNamePrefix("dm-batch-writer-");
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
executor.setWaitForTasksToCompleteOnShutdown(true);
executor.setAwaitTerminationSeconds(60);
executor.initialize();
return executor;
}
}