perf(syslog-consumer-rule): 非标日志改为单线程异步批量写入
- 非标写库结果不再参与 Kafka ACK - 使用独立单线程和有界队列处理非标日志 - 非标数据按固定大小分批写入达梦 - 增加异步成功、失败和队列拒绝指标 - 补充异步解耦及批量写入测试 perf(syslog-consumer-rule): 按批次字段并集优化达梦批量写入 - 普通日志和告警日志按非空字段并集生成批写模板 - 为同批记录补齐缺失字段并绑定 NULL - 使用 MyBatis ExecutorType.BATCH 复用 PreparedStatement - 避免固定写入全部 373 个候选字段 - 非标日志保留固定 24 列多行批量写入 - 批量大小和非标队列容量默认调整为 500 - 补充字段并集与动态 SQL 一致性测试
This commit is contained in:
王绍宏
+43
-12
@@ -50,18 +50,31 @@ public class LogNormalProcessor {
|
||||
public static final class NormalizationResult {
|
||||
private final ProcessingOutcome outcome;
|
||||
private final List<Map<String, Object>> rows;
|
||||
private final List<SyslogNonNormalMessage> nonNormalMessages;
|
||||
|
||||
private NormalizationResult(ProcessingOutcome outcome, List<Map<String, Object>> rows) {
|
||||
private NormalizationResult(ProcessingOutcome outcome, List<Map<String, Object>> rows,
|
||||
List<SyslogNonNormalMessage> nonNormalMessages) {
|
||||
this.outcome = outcome;
|
||||
this.rows = Collections.unmodifiableList(new ArrayList<>(rows));
|
||||
this.nonNormalMessages = Collections.unmodifiableList(new ArrayList<>(nonNormalMessages));
|
||||
}
|
||||
|
||||
static NormalizationResult success(List<Map<String, Object>> rows) {
|
||||
return new NormalizationResult(ProcessingOutcome.SUCCESS, rows);
|
||||
return success(rows, Collections.emptyList());
|
||||
}
|
||||
|
||||
static NormalizationResult success(List<Map<String, Object>> rows,
|
||||
List<SyslogNonNormalMessage> nonNormalMessages) {
|
||||
return new NormalizationResult(ProcessingOutcome.SUCCESS, rows, nonNormalMessages);
|
||||
}
|
||||
|
||||
static NormalizationResult filtered() {
|
||||
return new NormalizationResult(ProcessingOutcome.FILTERED, Collections.emptyList());
|
||||
return filtered(Collections.emptyList());
|
||||
}
|
||||
|
||||
static NormalizationResult filtered(List<SyslogNonNormalMessage> nonNormalMessages) {
|
||||
return new NormalizationResult(ProcessingOutcome.FILTERED,
|
||||
Collections.emptyList(), nonNormalMessages);
|
||||
}
|
||||
|
||||
public ProcessingOutcome getOutcome() {
|
||||
@@ -71,6 +84,14 @@ public class LogNormalProcessor {
|
||||
public List<Map<String, Object>> getRows() {
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* 活动 Kafka 批处理链路中需要集中落库的非标日志。
|
||||
* 历史单条入口仍会立即保存，因此其结果通常为空。
|
||||
*/
|
||||
public List<SyslogNonNormalMessage> getNonNormalMessages() {
|
||||
return nonNormalMessages;
|
||||
}
|
||||
}
|
||||
|
||||
private static final ZoneId RECEIVE_TIME_ZONE = ZoneId.of("Asia/Shanghai");
|
||||
@@ -126,7 +147,9 @@ public class LogNormalProcessor {
|
||||
RuleExecutionMetrics executionMetrics;
|
||||
MessageParseContext messageParseContext;
|
||||
boolean shadowMode;
|
||||
boolean deferNonNormalPersistence;
|
||||
final List<Map<String, Object>> normalizedRows = new ArrayList<>();
|
||||
final List<SyslogNonNormalMessage> deferredNonNormalMessages = new ArrayList<>();
|
||||
}
|
||||
|
||||
// ==================== 构造器注入（Spring 自动完成）====================
|
||||
@@ -254,7 +277,7 @@ public class LogNormalProcessor {
|
||||
* 只执行标准化计算并返回待写入行，不在本方法中写两张标准化表。
|
||||
*
|
||||
* <p>这是批量写入的关键边界：并行线程只负责 CPU/规则计算，达梦事务由 Kafka 批次汇总后的
|
||||
* 批写器统一管理。非标日志仍沿用原有独立保存路径，不属于本次两张标准化表的改造范围。</p>
|
||||
* 批写器统一管理。活动 Kafka 链路产生的非标日志也随结果返回，在 poll 边界集中批写。</p>
|
||||
*/
|
||||
public NormalizationResult prepare(String logMsg, String syslogUUID, String syslogTopic,
|
||||
LocalDateTime fallbackFirstReceivedAt) {
|
||||
@@ -326,6 +349,8 @@ public class LogNormalProcessor {
|
||||
ctx.executionMetrics = executionMetrics == null ? new RuleExecutionMetrics() : executionMetrics;
|
||||
ctx.messageParseContext = new MessageParseContext(
|
||||
optimizationProperties.getMessageParseCache().isEnabled(), globalMetrics);
|
||||
// 活动 Kafka 链路必须把非标日志带回 poll 边界集中批写，禁止计算线程逐条开启达梦事务。
|
||||
ctx.deferNonNormalPersistence = true;
|
||||
return init(ctx);
|
||||
}
|
||||
|
||||
@@ -373,7 +398,7 @@ public class LogNormalProcessor {
|
||||
// 原文已在进入标准化前写入 InfluxDB。设备没有绑定规则是可预期的业务终态，
|
||||
// 继续 Retry/DLT 不会凭空产生规则，只会重复消耗 Kafka、线程和日志 I/O。
|
||||
// 因此只跳过达梦标准化并允许 ACK；系统依赖和规则执行异常仍由下方异常链处理。
|
||||
return NormalizationResult.filtered();
|
||||
return NormalizationResult.filtered(ctx.deferredNonNormalMessages);
|
||||
}
|
||||
NormalizationOptimizationProperties.RuleSelection selectionConfig =
|
||||
optimizationProperties.getRuleSelection();
|
||||
@@ -420,9 +445,9 @@ public class LogNormalProcessor {
|
||||
if (!decision.isSelected()) {
|
||||
saveSelectionFailure(ctx, deviceId, "未命中规则",
|
||||
"同设备多规则均未形成可用归属候选");
|
||||
// 规则均未命中是确定性的业务终态；非标保存成功后直接 ACK，避免 Retry
|
||||
// 为同一个 syslog_uuid 重复写入随机 ID 的非标记录。保存失败仍会由上层异常链重试。
|
||||
return NormalizationResult.filtered();
|
||||
// 规则均未命中是确定性的业务终态；非标记录随结果返回并交给独立线程批写。
|
||||
// 按现场临时策略，非标写入结果不参与原 Kafka 消息的 ACK。
|
||||
return NormalizationResult.filtered(ctx.deferredNonNormalMessages);
|
||||
}
|
||||
|
||||
RuleSelectionEngine.Candidate<ExecutionPlan> selected = decision.getWinner();
|
||||
@@ -442,7 +467,7 @@ public class LogNormalProcessor {
|
||||
saveSelectionFailure(ctx, deviceId, "多规则归属歧义",
|
||||
"候选评分相同但业务结果不同, candidateRuleIds="
|
||||
+ candidateRuleIds(decision.getTiedCandidates()));
|
||||
return NormalizationResult.filtered();
|
||||
return NormalizationResult.filtered(ctx.deferredNonNormalMessages);
|
||||
}
|
||||
globalMetrics.ruleSelection("equivalent_tie");
|
||||
prepared = byRule.get(selected.getSource().getId());
|
||||
@@ -716,8 +741,12 @@ public class LogNormalProcessor {
|
||||
}
|
||||
|
||||
private NormalizationResult finish(ProcessContext ctx) {
|
||||
if (ctx.savedCount > 0) return NormalizationResult.success(ctx.normalizedRows);
|
||||
if (ctx.filteredCount > 0 && ctx.rejectedCount == 0) return NormalizationResult.filtered();
|
||||
if (ctx.savedCount > 0) {
|
||||
return NormalizationResult.success(ctx.normalizedRows, ctx.deferredNonNormalMessages);
|
||||
}
|
||||
if (ctx.filteredCount > 0 && ctx.rejectedCount == 0) {
|
||||
return NormalizationResult.filtered(ctx.deferredNonNormalMessages);
|
||||
}
|
||||
throw new IllegalArgumentException("Syslog 未产生可用标准化结果: eventId=" + ctx.strSyslogUUID);
|
||||
}
|
||||
|
||||
@@ -1133,7 +1162,9 @@ public class LogNormalProcessor {
|
||||
normalMessage.setCollectTaskId(collect_id);
|
||||
if (collectTask != null)
|
||||
normalMessage.setCollectTaskName(collectTask.getTaskName());
|
||||
if (!this.messageService.saveMessage(normalMessage)) {
|
||||
if (ctx.deferNonNormalPersistence) {
|
||||
ctx.deferredNonNormalMessages.add(normalMessage);
|
||||
} else if (!this.messageService.saveMessage(normalMessage)) {
|
||||
throw new IllegalStateException("非标日志服务返回保存失败");
|
||||
}
|
||||
ctx.isSaveNonNormal = true;
|
||||
|
||||
+89
-4
@@ -9,7 +9,9 @@ import com.config.AppProperties;
|
||||
import com.config.KafkaConsumerProperties;
|
||||
import com.common.service.DmStandardizedBatchWriter;
|
||||
import com.common.service.IngressStatsProjectionWriter;
|
||||
import com.common.service.SyslogNonNormalMessageService;
|
||||
import com.common.entity.IngressStatsEvent;
|
||||
import com.common.entity.SyslogNonNormalMessage;
|
||||
import com.influxdb.client.write.Point;
|
||||
import com.influx.raw.InfluxRawPointFactory;
|
||||
import com.influx.raw.PointTimeResolver;
|
||||
@@ -71,6 +73,13 @@ public class SysLogProcessor {
|
||||
@Autowired
|
||||
private DmStandardizedBatchWriter standardizedBatchWriter;
|
||||
|
||||
@Autowired
|
||||
private SyslogNonNormalMessageService nonNormalMessageService;
|
||||
|
||||
@Autowired
|
||||
@Qualifier("nonNormalWriterExecutor")
|
||||
private Executor nonNormalWriterExecutor;
|
||||
|
||||
@Autowired
|
||||
private IngressStatsProjectionWriter ingressStatsProjectionWriter;
|
||||
|
||||
@@ -83,6 +92,9 @@ public class SysLogProcessor {
|
||||
@Autowired private RawInfluxMetrics rawInfluxMetrics;
|
||||
|
||||
private final AtomicInteger totalProcessed = new AtomicInteger(0);
|
||||
private final AtomicLong nonNormalAsyncCompletedTotal = new AtomicLong();
|
||||
private final AtomicLong nonNormalAsyncFailedTotal = new AtomicLong();
|
||||
private final AtomicLong nonNormalAsyncRejectedTotal = new AtomicLong();
|
||||
// [SYSLOG-ALARM][KACK][已修复-主链路停用] 历史辅助方法仍保留此计数用于兼容，但唯一活动 listener 不再用全局计数决定 ACK。
|
||||
private final AtomicInteger currentBatchCount = new AtomicInteger(0);
|
||||
|
||||
@@ -133,8 +145,9 @@ public class SysLogProcessor {
|
||||
}
|
||||
|
||||
/**
|
||||
* 统一批处理边界：原文批量写入 InfluxDB 后，有界并行执行标准化；
|
||||
* 只有整批全部到达可靠终态才提交 offset。
|
||||
* 统一批处理边界：原文批量写入 InfluxDB 后，有界并行执行标准化。
|
||||
* 正常消息仍需到达可靠终态才提交 offset；非标消息按现场临时策略异步落库，
|
||||
* 其写库结果不参与 Kafka ACK。
|
||||
*/
|
||||
private void consumeBatch(List<ConsumerRecord<String, String>> records, Acknowledgment ack,
|
||||
boolean enforceRetryDue, String lane) {
|
||||
@@ -180,7 +193,9 @@ public class SysLogProcessor {
|
||||
+ "ruleComputeMs={}, ruleSnapshotMs={}, messageParseMs={}, fieldMappingMs={}, "
|
||||
+ "filterCompleteMs={}, enrichmentMs={}, ruleAssembleMs={}, compiledRuleHit={}, "
|
||||
+ "compiledRuleMiss={}, messageParseHit={}, messageParseMiss={}, deviceUnique={}, "
|
||||
+ "collectTaskUnique={}, shadowCompared={}, shadowMismatch={}, dmWriteMs={}, totalMs={}",
|
||||
+ "collectTaskUnique={}, shadowCompared={}, shadowMismatch={}, dmWriteMs={}, "
|
||||
+ "nonNormalQueued={}, nonNormalRejected={}, nonNormalAsyncCompletedTotal={}, "
|
||||
+ "nonNormalAsyncFailedTotal={}, nonNormalAsyncRejectedTotal={}, totalMs={}",
|
||||
lane, topicPartitions(records), records.size(), preparedMessages.size(), persistedMessages.size(),
|
||||
metrics.success.get(), metrics.filtered.get(), metrics.retry.get(), metrics.dlt.get(),
|
||||
metrics.normalizedRows.get(), metrics.dmSuccessfulRows.get(), metrics.dmFailedMessages.get(), ingressGroups,
|
||||
@@ -190,6 +205,9 @@ public class SysLogProcessor {
|
||||
metrics.compiledRuleMisses(), metrics.messageParseHits(), metrics.messageParseMisses(),
|
||||
metrics.deviceUnique(), metrics.collectTaskUnique(), metrics.shadowCompared(),
|
||||
metrics.shadowDifferences(), metrics.dmWriteMs(),
|
||||
metrics.nonNormalQueued(), metrics.nonNormalRejected(),
|
||||
nonNormalAsyncCompletedTotal.get(), nonNormalAsyncFailedTotal.get(),
|
||||
nonNormalAsyncRejectedTotal.get(),
|
||||
System.currentTimeMillis() - batchStartedAt);
|
||||
}
|
||||
|
||||
@@ -492,15 +510,18 @@ public class SysLogProcessor {
|
||||
metrics.recordRuleExecution(executionMetrics, enrichmentContext);
|
||||
|
||||
List<DmStandardizedBatchWriter.WriteRequest<PreparedMessage>> writeRequests = new ArrayList<>();
|
||||
List<SyslogNonNormalMessage> nonNormalMessages = new ArrayList<>();
|
||||
Set<PreparedMessage> pendingDatabaseMessages = new HashSet<>();
|
||||
int filteredMessages = 0;
|
||||
for (CompletableFuture<PreparedNormalization> future : futures) {
|
||||
PreparedNormalization prepared = future.join();
|
||||
if (prepared.terminalState != null) {
|
||||
metrics.record(prepared.terminalState);
|
||||
continue;
|
||||
}
|
||||
nonNormalMessages.addAll(prepared.result.getNonNormalMessages());
|
||||
if (prepared.result.getOutcome() == LogNormalProcessor.ProcessingOutcome.FILTERED) {
|
||||
metrics.record(SyslogFailureRouter.TerminalState.FILTERED);
|
||||
filteredMessages++;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -528,6 +549,14 @@ public class SysLogProcessor {
|
||||
metrics.record(SyslogFailureRouter.TerminalState.SUCCESS);
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < filteredMessages; i++) {
|
||||
metrics.record(SyslogFailureRouter.TerminalState.FILTERED);
|
||||
}
|
||||
|
||||
// 正常链路先达到可靠终态，再把非标日志交给独立单线程，避免正常写入失败重试时
|
||||
// 提前制造非标重复数据。这里只提交任务，不等待结果，非标写入不参与 Kafka ACK。
|
||||
// 队列满、写库失败或进程异常退出都可能丢非标数据，这是现场选择的性能优先策略。
|
||||
submitNonNormalMessages(nonNormalMessages, metrics);
|
||||
} catch (TimeoutException timeout) {
|
||||
cancelOutstanding(futures);
|
||||
throw new IllegalStateException("标准化批次处理超时，禁止提交 offset", timeout);
|
||||
@@ -544,6 +573,48 @@ public class SysLogProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 把非标批次交给独立线程后立即返回。任何提交或执行失败只记录，不传播到 Kafka ACK 边界。
|
||||
*/
|
||||
private void submitNonNormalMessages(List<SyslogNonNormalMessage> messages, BatchMetrics metrics) {
|
||||
if (messages.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<SyslogNonNormalMessage> batch = new ArrayList<>(messages);
|
||||
try {
|
||||
nonNormalWriterExecutor.execute(() -> {
|
||||
long startedAt = System.nanoTime();
|
||||
try {
|
||||
int inserted = nonNormalMessageService.saveBatchMessages(batch);
|
||||
if (inserted != batch.size()) {
|
||||
nonNormalAsyncFailedTotal.addAndGet(batch.size());
|
||||
log.error("非标日志异步批写结果异常，Kafka offset 不回退: rows={}, inserted={}, "
|
||||
+ "durationMs={}, failedTotal={}",
|
||||
batch.size(), inserted, elapsedMillis(startedAt),
|
||||
nonNormalAsyncFailedTotal.get());
|
||||
return;
|
||||
}
|
||||
nonNormalAsyncCompletedTotal.addAndGet(inserted);
|
||||
log.debug("非标日志异步批写完成: rows={}, durationMs={}, completedTotal={}",
|
||||
inserted, elapsedMillis(startedAt), nonNormalAsyncCompletedTotal.get());
|
||||
} catch (Exception failure) {
|
||||
nonNormalAsyncFailedTotal.addAndGet(batch.size());
|
||||
log.error("非标日志异步批写失败，Kafka offset 不回退，当前非标数据可能丢失: "
|
||||
+ "rows={}, durationMs={}, failedTotal={}",
|
||||
batch.size(), elapsedMillis(startedAt),
|
||||
nonNormalAsyncFailedTotal.get(), failure);
|
||||
}
|
||||
});
|
||||
metrics.recordNonNormalSubmission(batch.size(), 0);
|
||||
} catch (RuntimeException rejected) {
|
||||
nonNormalAsyncRejectedTotal.addAndGet(batch.size());
|
||||
metrics.recordNonNormalSubmission(0, batch.size());
|
||||
log.error("非标日志异步队列提交失败，当前非标数据已丢弃且不阻塞 Kafka ACK: "
|
||||
+ "rows={}, rejectedTotal={}",
|
||||
batch.size(), nonNormalAsyncRejectedTotal.get(), rejected);
|
||||
}
|
||||
}
|
||||
|
||||
private static long elapsedMillis(long startedAtNanos) {
|
||||
return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAtNanos);
|
||||
}
|
||||
@@ -681,6 +752,8 @@ public class SysLogProcessor {
|
||||
private final AtomicLong ruleComputeMs = new AtomicLong();
|
||||
private final AtomicLong ruleSnapshotMs = new AtomicLong();
|
||||
private final AtomicLong dmWriteMs = new AtomicLong();
|
||||
private final AtomicInteger nonNormalQueued = new AtomicInteger();
|
||||
private final AtomicInteger nonNormalRejected = new AtomicInteger();
|
||||
private volatile RuleExecutionMetrics ruleExecutionMetrics = new RuleExecutionMetrics();
|
||||
private volatile int deviceUnique;
|
||||
private volatile int collectTaskUnique;
|
||||
@@ -703,6 +776,10 @@ public class SysLogProcessor {
|
||||
return success.get();
|
||||
}
|
||||
|
||||
int filteredCount() {
|
||||
return filtered.get();
|
||||
}
|
||||
|
||||
long ruleComputeMs() {
|
||||
return ruleComputeMs.get();
|
||||
}
|
||||
@@ -711,6 +788,9 @@ public class SysLogProcessor {
|
||||
return dmWriteMs.get();
|
||||
}
|
||||
|
||||
int nonNormalQueued() { return nonNormalQueued.get(); }
|
||||
int nonNormalRejected() { return nonNormalRejected.get(); }
|
||||
|
||||
long ruleSnapshotMs() { return ruleSnapshotMs.get(); }
|
||||
long messageParseMs() { return ruleExecutionMetrics.getMessageParseMs(); }
|
||||
long fieldMappingMs() { return ruleExecutionMetrics.getFieldMappingMs(); }
|
||||
@@ -747,6 +827,11 @@ public class SysLogProcessor {
|
||||
dmWriteMs.set(Math.max(0L, elapsedMs));
|
||||
}
|
||||
|
||||
void recordNonNormalSubmission(int queued, int rejected) {
|
||||
nonNormalQueued.addAndGet(Math.max(0, queued));
|
||||
nonNormalRejected.addAndGet(Math.max(0, rejected));
|
||||
}
|
||||
|
||||
/** 仅用于批次汇总观测，不参与 ACK 判断；ACK 仍由每条消息的可靠终态决定。 */
|
||||
void recordDmWrite(int rows, int successfulRows, int failedMessages) {
|
||||
this.normalizedRows.addAndGet(rows);
|
||||
|
||||
+3
-8
@@ -15,21 +15,16 @@ import java.util.Map;
|
||||
public interface SyslogNormalAlarmMapper {
|
||||
|
||||
/**
|
||||
* 动态插入数据 - 使用 Map
|
||||
* 使用当前小批次的字段并集模板插入告警日志。调用方保证同批 Map 具有相同键集合，
|
||||
* 缺失值绑定为 NULL，使 MyBatis ExecutorType.BATCH 能复用 PreparedStatement。
|
||||
*/
|
||||
int insertDynamic(@Param("dataMap") Map<String, Object> dataMap);
|
||||
int insertBatchPlanned(@Param("dataMap") Map<String, Object> dataMap);
|
||||
|
||||
/**
|
||||
* 动态插入数据 - 使用实体类
|
||||
*/
|
||||
int insertByEntity(SyslogNormalAlarm entity);
|
||||
|
||||
/**
|
||||
* 批量插入数据
|
||||
*/
|
||||
int batchInsert(@Param("dataList") List<Map<String, Object>> dataList);
|
||||
|
||||
|
||||
/**
|
||||
* 统计24小时内分组数据量
|
||||
*/
|
||||
|
||||
+3
-8
@@ -12,21 +12,16 @@ import java.util.List;
|
||||
public interface SyslogNormalDataMapper {
|
||||
|
||||
/**
|
||||
* 动态插入数据 - 使用 Map
|
||||
* 使用当前小批次的字段并集模板插入普通日志。调用方保证同批 Map 具有相同键集合，
|
||||
* 缺失值绑定为 NULL，使 MyBatis ExecutorType.BATCH 能复用 PreparedStatement。
|
||||
*/
|
||||
int insertDynamic(@Param("dataMap") Map<String, Object> dataMap);
|
||||
int insertBatchPlanned(@Param("dataMap") Map<String, Object> dataMap);
|
||||
|
||||
/**
|
||||
* 动态插入数据 - 使用实体类
|
||||
*/
|
||||
int insertByEntity(SyslogNormalData entity);
|
||||
|
||||
/**
|
||||
* 批量插入数据
|
||||
*/
|
||||
int batchInsert(@Param("dataList") java.util.List<Map<String, Object>> dataList);
|
||||
|
||||
|
||||
/**
|
||||
* 统计24小时内分组数据量
|
||||
*/
|
||||
|
||||
+44
-5
@@ -13,13 +13,18 @@ import org.springframework.transaction.support.TransactionSynchronizationAdapter
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 在一个达梦小事务中执行真正的 MyBatis JDBC batch。
|
||||
*
|
||||
* <p>这个类只负责数据库执行，不负责错误分类、拆分或 Kafka 状态。上层写入器保证传入的所有行
|
||||
* 属于同一张表和同一个动态列签名，因此 MyBatis BatchExecutor 可以复用同一个 PreparedStatement。
|
||||
* 普通日志和告警日志分别使用当前小批次的非空字段并集模板，因此同一目标表、同一分区日期内
|
||||
* MyBatis BatchExecutor 可以复用同一个 PreparedStatement。
|
||||
* 每个 {@link #execute(boolean, List, int)} 调用都会创建独立事务；失败时整个小批次回滚，上层随后
|
||||
* 决定二分定位还是按系统故障停止。</p>
|
||||
*/
|
||||
@@ -63,7 +68,7 @@ public class DmStandardizedBatchExecutor {
|
||||
* 执行一个同构小批次。
|
||||
*
|
||||
* @param alarmTable true 写 syslog_normal_alarm，false 写 syslog_normal_data
|
||||
* @param rows 已按相同动态列集合分组的行
|
||||
* @param rows 已按相同目标表和分区日期分组的行
|
||||
* @param transactionTimeoutSeconds 当前小事务硬超时
|
||||
*/
|
||||
public void execute(boolean alarmTable, List<Map<String, Object>> rows,
|
||||
@@ -72,10 +77,11 @@ public class DmStandardizedBatchExecutor {
|
||||
return;
|
||||
}
|
||||
|
||||
List<Map<String, Object>> alignedRows = alignBatchColumns(rows);
|
||||
int attempt = 0;
|
||||
while (true) {
|
||||
try {
|
||||
executeTransaction(alarmTable, rows, transactionTimeoutSeconds, attempt);
|
||||
executeTransaction(alarmTable, alignedRows, transactionTimeoutSeconds, attempt);
|
||||
return;
|
||||
} catch (DirectStatsProjectionWriter.ConcurrentStatsWriteException concurrentFailure) {
|
||||
if (attempt >= Math.max(0, maxConcurrencyRetries)) {
|
||||
@@ -89,6 +95,39 @@ public class DmStandardizedBatchExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算当前小批次所有非空字段的并集，并为每行建立相同键集合的浅拷贝。
|
||||
*
|
||||
* <p>Mapper XML 只允许声明过的列，额外业务键即使出现在 Map 中也不会进入 SQL。
|
||||
* 同一键在某行不存在时，Map#get 返回 null，最终由 JDBC 按 SQL NULL 绑定。</p>
|
||||
*/
|
||||
static List<Map<String, Object>> alignBatchColumns(List<Map<String, Object>> rows) {
|
||||
Set<String> columnUnion = new LinkedHashSet<>();
|
||||
for (Map<String, Object> row : rows) {
|
||||
if (row == null) {
|
||||
throw new IllegalArgumentException("达梦批写行不能为空");
|
||||
}
|
||||
for (Map.Entry<String, Object> entry : row.entrySet()) {
|
||||
if (entry.getKey() != null && entry.getValue() != null) {
|
||||
columnUnion.add(entry.getKey());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (columnUnion.isEmpty()) {
|
||||
throw new IllegalArgumentException("达梦批写没有可插入的非空字段");
|
||||
}
|
||||
|
||||
List<Map<String, Object>> alignedRows = new ArrayList<>(rows.size());
|
||||
for (Map<String, Object> row : rows) {
|
||||
Map<String, Object> aligned = new LinkedHashMap<>(columnUnion.size());
|
||||
for (String column : columnUnion) {
|
||||
aligned.put(column, row.get(column));
|
||||
}
|
||||
alignedRows.add(aligned);
|
||||
}
|
||||
return alignedRows;
|
||||
}
|
||||
|
||||
private void executeTransaction(boolean alarmTable, List<Map<String, Object>> rows,
|
||||
int transactionTimeoutSeconds, int retries) {
|
||||
|
||||
@@ -112,12 +151,12 @@ public class DmStandardizedBatchExecutor {
|
||||
SyslogNormalAlarmMapper mapper = batchSqlSession.getMapper(SyslogNormalAlarmMapper.class);
|
||||
for (Map<String, Object> row : rows) {
|
||||
// BatchExecutor 下这里不会逐条提交；返回值也不是最终影响行数，不能据此判断成功。
|
||||
mapper.insertDynamic(row);
|
||||
mapper.insertBatchPlanned(row);
|
||||
}
|
||||
} else {
|
||||
SyslogNormalDataMapper mapper = batchSqlSession.getMapper(SyslogNormalDataMapper.class);
|
||||
for (Map<String, Object> row : rows) {
|
||||
mapper.insertDynamic(row);
|
||||
mapper.insertBatchPlanned(row);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+8
-24
@@ -32,8 +32,6 @@ import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Executor;
|
||||
@@ -48,7 +46,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
* <p>写入器遵守三条不能破坏的可靠性规则：</p>
|
||||
*
|
||||
* <ol>
|
||||
* <li>正常行按目标表、分区日期和动态列签名分组后批量提交；</li>
|
||||
* <li>普通日志和告警日志分别按目标表、分区日期计算批次字段并集后批量提交；</li>
|
||||
* <li>数据类错误回滚当前小批次并有限二分，最终只把坏行交给 Retry/DLT；</li>
|
||||
* <li>连接、超时、提交未知等系统故障绝不降级为大量单条请求，而是立即失败并保留 Kafka offset。</li>
|
||||
* </ol>
|
||||
@@ -67,7 +65,7 @@ public class DmStandardizedBatchWriter {
|
||||
@Value("${dm.standardized-write.mode:BATCH}")
|
||||
private String mode;
|
||||
|
||||
@Value("${dm.standardized-write.batch-size:100}")
|
||||
@Value("${dm.standardized-write.batch-size:500}")
|
||||
private int batchSize;
|
||||
|
||||
@Value("${dm.standardized-write.transaction-timeout-seconds:45}")
|
||||
@@ -161,7 +159,7 @@ public class DmStandardizedBatchWriter {
|
||||
}
|
||||
|
||||
/**
|
||||
* 相同目标表、log_time 日期和非空列集合才会进入同一批次。
|
||||
* 两张标准化表在每个小批次计算字段并集，只按目标表和 log_time 日期分组。
|
||||
* LinkedHashMap 保持稳定遍历顺序，便于测试和现场日志复现。
|
||||
*/
|
||||
private <T> void writeGroupedBatches(List<WriteRequest<T>> requests, WriteReport<T> report,
|
||||
@@ -173,7 +171,7 @@ public class DmStandardizedBatchWriter {
|
||||
groups.computeIfAbsent(key, ignored -> new ArrayList<>()).add(request);
|
||||
} catch (RuntimeException invalidGroupingKey) {
|
||||
report.recordFailure(request.getSource(), new PermanentRowWriteException(
|
||||
"标准化结果缺少稳定分区时间或列签名无效", invalidGroupingKey));
|
||||
"标准化结果缺少稳定分区时间", invalidGroupingKey));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -294,18 +292,7 @@ public class DmStandardizedBatchWriter {
|
||||
boolean alarmTable = normalDataService.isAlarmData(data);
|
||||
LocalDate partitionDate = partitionDate(data.get("log_time"));
|
||||
|
||||
// MyBatis XML 的动态列判断只包含非 null 值，所以签名也只记录非 null 列。
|
||||
// TreeSet 提供固定顺序，避免 HashMap 插入顺序不同却被错误拆成多个批次。
|
||||
Set<String> columns = new TreeSet<>();
|
||||
for (Map.Entry<String, Object> entry : data.entrySet()) {
|
||||
if (entry.getValue() != null) {
|
||||
columns.add(entry.getKey());
|
||||
}
|
||||
}
|
||||
if (columns.isEmpty()) {
|
||||
throw new IllegalArgumentException("标准化结果没有可写入列");
|
||||
}
|
||||
return new BatchKey(alarmTable, partitionDate, String.join(",", columns));
|
||||
return new BatchKey(alarmTable, partitionDate);
|
||||
}
|
||||
|
||||
private LocalDate partitionDate(Object value) {
|
||||
@@ -499,12 +486,10 @@ public class DmStandardizedBatchWriter {
|
||||
private static final class BatchKey {
|
||||
private final boolean alarmTable;
|
||||
private final LocalDate partitionDate;
|
||||
private final String columnSignature;
|
||||
|
||||
private BatchKey(boolean alarmTable, LocalDate partitionDate, String columnSignature) {
|
||||
private BatchKey(boolean alarmTable, LocalDate partitionDate) {
|
||||
this.alarmTable = alarmTable;
|
||||
this.partitionDate = partitionDate;
|
||||
this.columnSignature = columnSignature;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -513,13 +498,12 @@ public class DmStandardizedBatchWriter {
|
||||
if (!(other instanceof BatchKey)) return false;
|
||||
BatchKey that = (BatchKey) other;
|
||||
return alarmTable == that.alarmTable
|
||||
&& Objects.equals(partitionDate, that.partitionDate)
|
||||
&& Objects.equals(columnSignature, that.columnSignature);
|
||||
&& Objects.equals(partitionDate, that.partitionDate);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(alarmTable, partitionDate, columnSignature);
|
||||
return Objects.hash(alarmTable, partitionDate);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+19
-7
@@ -15,6 +15,7 @@ import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.time.LocalDateTime;
|
||||
@@ -35,21 +36,22 @@ public class SyslogNormalDataService {
|
||||
|
||||
|
||||
/**
|
||||
* 动态插入单条数据 - Map方式
|
||||
* 标准化单条兼容入口：按 event_level 路由后，两张目标表均使用受白名单约束的动态列模板。
|
||||
*/
|
||||
@Transactional
|
||||
public int insertDynamic(Map<String, Object> dataMap) {
|
||||
// [流程节点7] 根据标准化结果分类落库：安全事件写入 syslog_normal_alarm，访问事件写入 syslog_normal_data。
|
||||
// [SYSLOG-ALARM][DBBATCH] 每条标准化结果单独开启事务并执行动态 INSERT，会产生大量数据库往返、SQL 解析和提交开销；
|
||||
// 高峰期 DB 变慢又会放大 Kafka 批次超时。建议按目标表和固定列集合聚合批次写入，并用 event_id + 规则版本唯一键保证重试幂等。
|
||||
// 高峰期 DB 变慢又会放大 Kafka 批次超时。建议按目标表和批次字段并集聚合写入，并用 event_id + 规则版本唯一键保证重试幂等。
|
||||
prepareForInsert(dataMap);
|
||||
|
||||
// [流程节点7.2] event_level 非空走安全事件表；为空走访问事件表，这是两条后续分析链路的分叉点。
|
||||
boolean alarmData = isAlarmData(dataMap);
|
||||
Map<String, Object> plannedData = nonNullColumnMap(dataMap);
|
||||
try {
|
||||
int inserted = alarmData
|
||||
? syslogNormalAlarmMapper.insertDynamic(dataMap)
|
||||
: syslogNormalDataMapper.insertDynamic(dataMap);
|
||||
? syslogNormalAlarmMapper.insertBatchPlanned(plannedData)
|
||||
: syslogNormalDataMapper.insertBatchPlanned(plannedData);
|
||||
if (inserted > 0) {
|
||||
if (alarmData && lateAlarmBucketWriter != null) {
|
||||
lateAlarmBucketWriter.record(java.util.Collections.singletonList(dataMap));
|
||||
@@ -72,6 +74,17 @@ public class SyslogNormalDataService {
|
||||
}
|
||||
}
|
||||
|
||||
/** 单条兼容路径也只把实际非空字段交给列模板，避免原 Map 中的空键误触发列选择。 */
|
||||
private Map<String, Object> nonNullColumnMap(Map<String, Object> dataMap) {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, Object> entry : dataMap.entrySet()) {
|
||||
if (entry.getKey() != null && entry.getValue() != null) {
|
||||
result.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在单条和批量写入之前执行完全相同的数据准备与关键字段校验。
|
||||
*
|
||||
@@ -220,9 +233,8 @@ public class SyslogNormalDataService {
|
||||
data.put("log_time", new java.util.Date());
|
||||
}
|
||||
}
|
||||
// 旧 mapper 的 batchInsert 只写固定的少数字段，DIRECT 下会造成“统计有 device/rule，
|
||||
// 事实行却丢了 device/rule”的永久差异。主 Kafka 链路使用 DmStandardizedBatchExecutor；
|
||||
// 这个低频兼容 API 逐条复用完整动态 INSERT，但仍处于本方法的同一个 Spring 事务中。
|
||||
// 主 Kafka 链路使用 DmStandardizedBatchExecutor 执行真正的 JDBC batch；这个低频兼容
|
||||
// API 在同一个 Spring 事务内复用受白名单约束的列模板，不能再调用旧的五字段 batch SQL。
|
||||
int inserted = 0;
|
||||
for (Map<String, Object> data : dataList) {
|
||||
inserted += insertDynamic(data);
|
||||
|
||||
+19
-2
@@ -11,12 +11,14 @@ import com.common.service.StatsChangeCaptureService;
|
||||
import com.common.service.DirectStatsProjectionWriter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
@@ -38,6 +40,9 @@ public class SyslogNonNormalMessageServiceImpl
|
||||
@Autowired
|
||||
private DirectStatsProjectionWriter directStatsProjectionWriter;
|
||||
|
||||
@Value("${dm.non-normal-write.batch-size:500}")
|
||||
private int nonNormalBatchSize;
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public boolean saveMessage(SyslogNonNormalMessage message) {
|
||||
@@ -125,8 +130,20 @@ public class SyslogNonNormalMessageServiceImpl
|
||||
}
|
||||
}
|
||||
|
||||
// 批量插入
|
||||
int inserted = messageMapper.batchInsert(messages);
|
||||
// 同一个服务事务内分块执行固定列多行 INSERT，避免单条提交，也避免一个 Kafka poll
|
||||
// 直接生成过大的 SQL。所有分块成功后才统一写 DIRECT 统计并提交事务。
|
||||
int inserted = 0;
|
||||
int effectiveBatchSize = Math.max(1, nonNormalBatchSize);
|
||||
for (int start = 0; start < messages.size(); start += effectiveBatchSize) {
|
||||
int end = Math.min(start + effectiveBatchSize, messages.size());
|
||||
int affected = messageMapper.batchInsert(
|
||||
new ArrayList<>(messages.subList(start, end)));
|
||||
if (affected <= 0) {
|
||||
throw new IllegalStateException("非标日志批量 INSERT 未写入记录: start="
|
||||
+ start + ", end=" + end);
|
||||
}
|
||||
inserted += affected;
|
||||
}
|
||||
if (inserted > 0) {
|
||||
if (statsChangeCaptureService.isEnabled()) {
|
||||
for (SyslogNonNormalMessage message : messages) {
|
||||
|
||||
+21
@@ -31,6 +31,9 @@ public class ThreadPoolConfig {
|
||||
@Value("${dm.standardized-write.writer-queue-capacity:100}")
|
||||
private int dmWriterQueueCapacity;
|
||||
|
||||
@Value("${dm.non-normal-write.queue-capacity:500}")
|
||||
private int nonNormalWriterQueueCapacity;
|
||||
|
||||
@Bean("logNormalProcessorExecutor")
|
||||
@Primary
|
||||
public Executor logNormalProcessorExecutor() {
|
||||
@@ -71,4 +74,22 @@ public class ThreadPoolConfig {
|
||||
executor.initialize();
|
||||
return executor;
|
||||
}
|
||||
|
||||
/**
|
||||
* 非标日志使用单独的串行写线程，与正常事实批写解耦。
|
||||
* listener 不等待该线程，队列溢出或异步写入失败只记录错误，不阻塞 Kafka ACK。
|
||||
*/
|
||||
@Bean("nonNormalWriterExecutor")
|
||||
public Executor nonNormalWriterExecutor() {
|
||||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||||
executor.setCorePoolSize(1);
|
||||
executor.setMaxPoolSize(1);
|
||||
executor.setQueueCapacity(Math.max(1, nonNormalWriterQueueCapacity));
|
||||
executor.setThreadNamePrefix("dm-non-normal-writer-");
|
||||
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
|
||||
executor.setWaitForTasksToCompleteOnShutdown(true);
|
||||
executor.setAwaitTerminationSeconds(60);
|
||||
executor.initialize();
|
||||
return executor;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,8 +56,11 @@ mybatis.type-aliases-package=com.common.entity
|
||||
mybatis.configuration.default-statement-timeout=30
|
||||
mybatis.configuration.default-fetch-size=1000
|
||||
mybatis.configuration.map-underscore-to-camel-case=true
|
||||
# 批次列并集写入会为部分行绑定空字段；达梦对未指定类型的 NULL 参数兼容性较差，统一使用 JDBC NULL。
|
||||
mybatis.configuration.jdbc-type-for-null=NULL
|
||||
mybatis.type-handlers-package=com.Modules.etl.handler
|
||||
mybatis-plus.configuration.map-underscore-to-camel-case=true
|
||||
mybatis-plus.configuration.jdbc-type-for-null=NULL
|
||||
mybatis-plus.type-handlers-package=com.Modules.etl.handler
|
||||
|
||||
# kafka Configuration
|
||||
@@ -142,10 +145,11 @@ app.processor.batch-size=100
|
||||
app.processor.process-timeout-ms=120000
|
||||
|
||||
# 达梦标准化表默认启用批量写入；现场需要紧急回退时可显式切换为 SINGLE。
|
||||
# BATCH 模式按目标表、log_time 日期和动态列集合分组，失败时只对数据类错误有限二分；
|
||||
# BATCH 模式按目标表和 log_time 日期分组，按小批次实际非空字段并集执行 JDBC Batch；
|
||||
# 失败时只对数据类错误有限二分；
|
||||
# 连接、超时或提交结果未知会立即停止并保留 Kafka offset，禁止放大为逐条请求风暴。
|
||||
dm.standardized-write.mode=BATCH
|
||||
dm.standardized-write.batch-size=100
|
||||
dm.standardized-write.batch-size=${DM_STANDARDIZED_WRITE_BATCH_SIZE:500}
|
||||
dm.standardized-write.writer-concurrency=2
|
||||
dm.standardized-write.writer-queue-capacity=100
|
||||
dm.standardized-write.transaction-timeout-seconds=45
|
||||
@@ -154,6 +158,8 @@ dm.standardized-write.max-split-depth=10
|
||||
dm.standardized-write.max-fallback-operations=100
|
||||
dm.standardized-write.circuit-breaker-failure-threshold=3
|
||||
dm.standardized-write.circuit-breaker-backoff-ms=30000
|
||||
dm.non-normal-write.batch-size=${DM_NON_NORMAL_WRITE_BATCH_SIZE:500}
|
||||
dm.non-normal-write.queue-capacity=${DM_NON_NORMAL_WRITE_QUEUE_CAPACITY:500}
|
||||
|
||||
# ETL配置
|
||||
etl.batch.page-size=1000
|
||||
|
||||
+748
-755
File diff suppressed because it is too large
Load Diff
+748
-755
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user