From 8173c2ff3f4517a411e7298438e6f42e24e7ccce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E7=BB=8D=E5=AE=8F?= <994233303@qq.com> Date: Sun, 13 Sep 2026 06:11:12 +0800 Subject: [PATCH] =?UTF-8?q?perf(syslog):=20=E7=BB=9F=E8=AE=A1=E5=BC=82?= =?UTF-8?q?=E6=AD=A5=E5=8C=96=E5=B9=B6=E6=8C=89=E9=87=87=E6=A0=B7=E4=BF=9D?= =?UTF-8?q?=E7=95=99=E7=8E=87=E8=87=AA=E9=80=82=E5=BA=94=E8=B0=83=E6=95=B4?= =?UTF-8?q?=E4=B8=9A=E5=8A=A1=E6=89=B9=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 入口统计改用独立 Kafka 消费组，移除主消费同步统计写库 - DIRECT 模式入库计数改为事务提交后异步投递，取消事实事务内统计 Outbox 写入 - 采样审计、告警分钟桶通知改为后台处理，异常不阻断主消费或回滚告警 - RAW 基础业务批量默认 500，按保留率动态扩大，上限 3000 - 限制业务块字节数及规则任务数量，保留告警全量处理和整次 poll 完成后提交 - 细化 SLOW_SYSLOG_BATCH，补充批次关联、线程排队、数据库执行及提交耗时 - 更新 v4 升级说明，明确异步统计损失风险及现场验证方法 验证：JDK 8 构建成功，232 项测试通过。 升级：复用现有表和 Kafka Topic，无新增建表或建 Topic 脚本要求。 --- .../NormalData/AdaptiveRawBatchSizer.java | 35 ++ .../NormalData/LogNormalProcessor.java | 2 - .../Modules/NormalData/SysLogProcessor.java | 354 +++++++++++++++--- .../service/BacklogSamplingAuditService.java | 47 ++- .../service/BacklogSamplingMetrics.java | 6 + .../service/BestEffortAuxiliaryQueue.java | 147 ++++++++ .../service/DirectStatsProjectionWriter.java | 102 +++-- .../service/DmStandardizedBatchExecutor.java | 48 ++- .../service/DmStandardizedBatchWriter.java | 18 + .../common/service/IngressStatsConsumer.java | 140 +++++++ .../service/IngressStatsProjectionWriter.java | 17 +- .../common/service/LateAlarmBucketWriter.java | 58 ++- .../StatsProjectionAsyncPublisher.java | 217 +++++++++++ .../service/StatsProjectionDeltaConsumer.java | 10 +- .../service/StatsProjectionOutboxEvent.java | 7 +- .../com/config/KafkaReliabilityConfig.java | 59 ++- .../com/config/SyslogBuildStartupLogger.java | 46 +++ .../BacklogSamplingController.java | 6 +- .../controllers/IngressStatsController.java | 9 +- .../resources/application-prod.properties | 2 +- .../src/main/resources/application.properties | 32 +- .../NormalData/AdaptiveRawBatchSizerTest.java | 33 ++ .../SysLogAdaptiveRawBatchTest.java | 155 ++++++++ .../SysLogProcessorParallelTest.java | 51 +++ .../NormalData/SysLogSlowBatchTraceTest.java | 110 ++++++ .../BacklogSamplingAuditServiceTest.java | 18 +- .../service/BestEffortAuxiliaryQueueTest.java | 64 ++++ .../DirectStatsProjectionWriterTest.java | 30 +- .../service/IngressStatsConsumerTest.java | 90 +++++ .../IngressStatsProjectionWriterTest.java | 13 +- .../service/LateAlarmBucketWriterTest.java | 82 +++- .../StatsProjectionAsyncPublisherTest.java | 238 ++++++++++++ .../service/StatsProjectionAsyncTest.java | 26 ++ .../StatsProjectionPublisherWiringTest.java | 36 ++ .../config/KafkaReliabilityConfigTest.java | 46 ++- .../config/SyslogBuildStartupLoggerTest.java | 51 +++ haobang-security-dm/upgrade-doc/v4/README.md | 5 +- .../v4/config/01-new-environment-variables.md | 7 +- .../v4/config/02-throughput-upgrade.md | 199 ++++++++++ 39 files changed, 2400 insertions(+), 216 deletions(-) create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/AdaptiveRawBatchSizer.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BestEffortAuxiliaryQueue.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsConsumer.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionAsyncPublisher.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/SyslogBuildStartupLogger.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/AdaptiveRawBatchSizerTest.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogAdaptiveRawBatchTest.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogSlowBatchTraceTest.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BestEffortAuxiliaryQueueTest.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsConsumerTest.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncPublisherTest.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionPublisherWiringTest.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/SyslogBuildStartupLoggerTest.java create mode 100644 haobang-security-dm/upgrade-doc/v4/config/02-throughput-upgrade.md diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/AdaptiveRawBatchSizer.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/AdaptiveRawBatchSizer.java new file mode 100644 index 0000000..fb149d4 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/AdaptiveRawBatchSizer.java @@ -0,0 +1,35 @@ +package com.Modules.NormalData; + +/** 普通/非标采用较高保留率，倍率由当前策略自动计算；不改变实际采样或告警保留规则。 */ +@org.springframework.stereotype.Component +public final class AdaptiveRawBatchSizer { + private final int baseRecords; + private final int maxRecords; + + public AdaptiveRawBatchSizer( + @org.springframework.beans.factory.annotation.Value("${syslog.raw-listener.max-poll-records:500}") int baseRecords, + @org.springframework.beans.factory.annotation.Value("${syslog.raw-listener.adaptive-max-records:3000}") int maxRecords) { + if (baseRecords < 1 || maxRecords < baseRecords) { + throw new IllegalArgumentException("普通自适应批量要求 1 <= 基础条数 <= 上限条数"); + } + this.baseRecords = baseRecords; + this.maxRecords = maxRecords; + } + + public int targetRecords(int normalRetainBps, int nonNormalRetainBps) { + validateRate(normalRetainBps); + validateRate(nonNormalRetainBps); + int retainBps = Math.max(normalRetainBps, nonNormalRetainBps); + if (retainBps == 0) return maxRecords; + long numerator = (long) baseRecords * 10000L; + long calculated = (numerator + retainBps - 1L) / retainBps; + return (int) Math.min(maxRecords, calculated); + } + + public int getBaseRecords() { return baseRecords; } + public int getMaxRecords() { return maxRecords; } + + private static void validateRate(int rate) { + if (rate < 0 || rate > 10000) throw new IllegalArgumentException("保留万分比必须在0~10000之间"); + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java index f73e8b0..c640e71 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java @@ -525,10 +525,8 @@ public class LogNormalProcessor { if (compiledEnabled) { try { CompiledRule compiled = compiledRuleCache.get(source, ctx.executionMetrics); - long parseStarted = System.nanoTime(); ParsedMessage parsed = ctx.messageParseContext.parse( compiled, ctx.strLogMsg, ctx.executionMetrics); - ctx.executionMetrics.addMessageParseNanos(System.nanoTime() - parseStarted); if (parsed == null) { if (isJsonRule(source)) { markInvalidJson(ctx, source); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java index b76d896..c718d5e 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java @@ -8,13 +8,11 @@ import com.common.util.SyslogParser; import com.config.AppProperties; import com.config.KafkaConsumerProperties; import com.common.service.DmStandardizedBatchWriter; -import com.common.service.IngressStatsProjectionWriter; import com.common.service.DirectStatsProjectionWriter; import com.common.service.SyslogNonNormalMessageService; import com.common.service.SyslogNormalDataService; import com.common.service.BacklogRetentionDelta; import com.common.service.BacklogSamplingAuditService; -import com.common.entity.IngressStatsEvent; import com.common.entity.SyslogNonNormalMessage; import com.influxdb.client.write.Point; import com.influx.raw.InfluxRawPointFactory; @@ -46,6 +44,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.Arrays; import java.time.LocalDateTime; import java.nio.charset.StandardCharsets; @@ -68,6 +68,17 @@ public class SysLogProcessor { @Value("${app.processor.process-timeout-ms:120000}") private long processTimeoutMs; + @Value("${syslog.normalization.workers-per-batch:8}") + private int normalizationWorkersPerBatch = 8; + + @Value("${syslog.raw-listener.processing-batch-max-bytes:16777216}") + private long rawProcessingBatchMaxBytes = 16777216L; + + @Autowired + private AdaptiveRawBatchSizer adaptiveRawBatchSizer = new AdaptiveRawBatchSizer(500, 3000); + // 每个普通listener线程独立学习上一个业务块的阶段，避免不同节点/通道互相放大批量。 + private final ThreadLocal rawAdaptiveState = ThreadLocal.withInitial(RawAdaptiveState::new); + @Value("${syslog.large-listener.processing-batch-max-bytes:8388608}") private long largeProcessingBatchMaxBytes; @@ -79,6 +90,7 @@ public class SysLogProcessor { /** listener、业务处理和 Kafka commit 均在同一个 consumer 线程，ThreadLocal 用于拼接端到端慢批次。 */ private final ThreadLocal slowBatchTrace = new ThreadLocal<>(); + private final ThreadLocal lastListenerFinished = new ThreadLocal<>(); @Autowired private AppProperties appProperties; @@ -108,9 +120,6 @@ public class SysLogProcessor { @Value("${dm.non-normal-write.concurrency-retry-backoff-ms:50}") private long nonNormalConcurrencyRetryBackoffMs; - @Autowired - private IngressStatsProjectionWriter ingressStatsProjectionWriter; - @Autowired private com.influx.InfluxDBClient influxClient; @@ -201,8 +210,11 @@ public class SysLogProcessor { } long processingMs = elapsedMillis(processingStartedAt); acknowledgeAndCompleteSlowBatchTrace(ack, processingMs); + } catch (RuntimeException aborted) { + completeSlowBatchTrace(elapsedMillis(processingStartedAt), 0L, aborted, "NOT_ATTEMPTED"); + throw aborted; } finally { - slowBatchTrace.remove(); + clearSlowBatchTrace(); } } @@ -236,14 +248,69 @@ public class SysLogProcessor { beginSlowBatchTrace(lane, records); long processingStartedAt = System.nanoTime(); try { - processBatchWithContinuity(records, enforceRetryDue, lane, normalizationExecutor); + if ("RAW".equals(lane)) { + processAdaptiveRawChunks(records, normalizationExecutor); + } else { + for (List> chunk : partitionRecordsByBytes( + records, Math.max(1L, rawProcessingBatchMaxBytes))) { + processBatchWithContinuity(chunk, enforceRetryDue, lane, normalizationExecutor); + } + } long processingMs = elapsedMillis(processingStartedAt); acknowledgeAndCompleteSlowBatchTrace(ack, processingMs); + } catch (RuntimeException aborted) { + if ("RAW".equals(lane)) observeRawRetention(10000); + completeSlowBatchTrace(elapsedMillis(processingStartedAt), 0L, aborted, "NOT_ATTEMPTED"); + throw aborted; } finally { - slowBatchTrace.remove(); + clearSlowBatchTrace(); } } + private void processAdaptiveRawChunks(List> records, Executor executor) { + RawAdaptiveState state = rawAdaptiveState.get(); + // 首次启动、长时间无反馈后从基础批量探测，避免沿用过时的积压阶段。 + if (state.observedAt == 0L || System.nanoTime() - state.observedAt > TimeUnit.MINUTES.toNanos(1)) { + state.retainBps = 10000; + } + for (int start = 0; start < records.size();) { + enterStage("ADAPTIVE_CHUNK_SELECTION"); + int target = adaptiveRawBatchSizer.targetRecords(state.retainBps, state.retainBps); + int end = adaptiveChunkEnd(records, start, target, Math.max(1L, rawProcessingBatchMaxBytes)); + SlowBatchTrace trace = slowBatchTrace.get(); + if (trace != null) { + trace.adaptiveTargets.merge(target, 1, Integer::sum); + trace.actualChunkSizes.merge(end - start, 1, Integer::sum); + } + processBatchWithContinuity(new ArrayList<>(records.subList(start, end)), false, "RAW", executor); + start = end; + } + } + + static int adaptiveChunkEnd(List> records, int start, int target, long maxBytes) { + if (target < 1 || maxBytes < 1 || start < 0 || start >= records.size()) { + throw new IllegalArgumentException("自适应分块参数非法"); + } + long bytes = 0L; + int end = start; + while (end < records.size() && end - start < target) { + ConsumerRecord record = records.get(end); + long size = record.serializedValueSize(); + if (size < 0L) size = record.value() == null ? 0L : record.value().getBytes(StandardCharsets.UTF_8).length; + if (end > start && size > maxBytes - bytes) break; + bytes += size; + end++; + if (bytes >= maxBytes) break; // 单条超限仍独立处理，不切断、不丢弃。 + } + return end; + } + + void observeRawRetention(int retainBps) { + RawAdaptiveState state = rawAdaptiveState.get(); + state.retainBps = retainBps; + state.observedAt = System.nanoTime(); + } + private void processBatchWithContinuity(List> records, boolean enforceRetryDue, String lane, Executor normalizationExecutor) { @@ -253,16 +320,27 @@ public class SysLogProcessor { // Retry Topic 的计划时间尚未到达，不属于处理异常，仍按原延时语义保留 offset。 throw notDue; } catch (RuntimeException failure) { + if ("RAW".equals(lane)) observeRawRetention(10000); + SlowBatchTrace trace = slowBatchTrace.get(); + if (trace != null) { + trace.failedChunks++; + trace.failureStage = trace.activeStage; + trace.businessError = failure.getClass().getName(); + } if (!failureContinuityEnabled) throw failure; + enterStage("CONTINUITY_DLT"); routeContinuityBatch(records, failure, "BATCH_" + lane); log.error("[SYSLOG_CONTINUITY] 批次特殊异常已记录，继续消费后续消息: lane={}, " + "records={}, errorType={}", lane, records.size(), failure.getClass().getName(), failure); + } finally { + endStage(); } } void processBatch(List> records, boolean enforceRetryDue, String lane, Executor normalizationExecutor) { + enterStage("PREPARE"); long batchStartedAt = System.currentTimeMillis(); long prepareStartedAt = batchStartedAt; BatchMetrics metrics = new BatchMetrics(); @@ -282,39 +360,48 @@ public class SysLogProcessor { } long prepareMs = System.currentTimeMillis() - prepareStartedAt; - // 入口状态是 Kafka 已可靠接收的事实。批内按设备/探针聚合，失败时禁止 ACK；重投只重复 MIN/MAX。 - long ingressStartedAt = System.nanoTime(); - int ingressGroups = ingressStatsProjectionWriter.writeBatch(ingressEvents(preparedMessages)); - long ingressMs = elapsedMillis(ingressStartedAt); + // 入口统计由独立 Kafka 消费组处理；主链路不再调用统计写库或 Outbox。 + int ingressGroups = 0; + long ingressMs = 0L; // 必须先完成规则计算，才能确保告警全量保留，并让未保留消息同时跳过 InfluxDB 和达梦。 long normalizeStartedAt = System.currentTimeMillis(); List computed = computePreparedNormalizations( preparedMessages, metrics, normalizationExecutor); + enterStage("SAMPLING_DECISION"); SamplingPlan samplingPlan = buildSamplingPlan(computed); if (!samplingPlan.auditDeltas.isEmpty()) { - boolean audited = backlogSamplingAuditService.recordBeforeDropping(samplingPlan.auditDeltas); - if (!audited) { - samplingPlan = allRetainedPlan(computed); - log.error("[BACKLOG_SAMPLING_FALLBACK] 采样审计不可用，当前批次已回退全量保留: " - + "lane={}, records={}", lane, records.size()); - } else { - metrics.recordSampling(samplingPlan); + enterStage("AUDIT_ENQUEUE"); + boolean accepted = backlogSamplingAuditService.recordAsync(samplingPlan.auditDeltas); + SlowBatchTrace trace = slowBatchTrace.get(); + if (trace != null) { + if (accepted) trace.auditAcceptedChunks++; + else trace.auditLostChunks++; } + // 审计失败不回退全量保存，避免进一步放大业务写入压力。 + metrics.recordSampling(samplingPlan); } long normalizeMs = System.currentTimeMillis() - normalizeStartedAt; // 告警是最高优先级：先独立提交告警表，不能让 InfluxDB 或普通表的慢写阻塞告警入库。 long persistencePlanStartedAt = System.nanoTime(); + enterStage("PERSISTENCE_PLAN"); StandardizedPersistencePlan standardizedPlan = prepareStandardizedPersistence( samplingPlan.normalizations, metrics); long persistencePlanMs = elapsedMillis(persistencePlanStartedAt); long alarmWriteStartedAt = System.nanoTime(); + enterStage("ALARM_DM"); + SlowBatchTrace rowTrace = slowBatchTrace.get(); + if (rowTrace != null) rowTrace.alarmRequested += standardizedPlan.alarmWriteRequests.size(); DmStandardizedBatchWriter.WriteReport alarmReport = standardizedBatchWriter.write(standardizedPlan.alarmWriteRequests); + if (rowTrace != null) { + rowTrace.alarmSaved += alarmReport.getSuccessfulRows(); + } long alarmWriteMs = elapsedMillis(alarmWriteStartedAt); long influxStartedAt = System.currentTimeMillis(); + enterStage("INFLUX_AND_FAILURE_ROUTE"); List persistedMessages = persistRawWithoutBlockingMainFlow( samplingPlan.rawMessages, metrics, lane); long influxMs = System.currentTimeMillis() - influxStartedAt; @@ -322,6 +409,12 @@ public class SysLogProcessor { long remainingDmStartedAt = System.nanoTime(); persistRemainingStandardizedData(standardizedPlan, alarmReport, alarmWriteMs, metrics); long remainingDmMs = elapsedMillis(remainingDmStartedAt); + enterStage("BATCH_FINISH"); + + if ("RAW".equals(lane)) { + // 任意准备/规则失败采用基础批量；多个阶段混合时取更高保留率保守放大。 + observeRawRetention(preparedMessages.size() == records.size() ? samplingPlan.maxRetainBps : 10000); + } totalProcessed.addAndGet(records.size()); long businessTotalMs = System.currentTimeMillis() - batchStartedAt; @@ -369,8 +462,46 @@ public class SysLogProcessor { polledMessageBytes += messageBytes; maxMessageBytes = Math.max(maxMessageBytes, messageBytes); } - slowBatchTrace.set(new SlowBatchTrace(lane, topicPartitions(records), records.size(), - polledMessageBytes, maxMessageBytes, System.nanoTime())); + long now = System.nanoTime(); + SlowBatchTrace trace = new SlowBatchTrace(lane, topicPartitions(records), records.size(), + polledMessageBytes, maxMessageBytes, now); + Long last = lastListenerFinished.get(); + trace.listenerGapMs = last == null ? -1 : TimeUnit.NANOSECONDS.toMillis(now - last); + trace.previousMdc = org.slf4j.MDC.get("syslogBatchId"); + org.slf4j.MDC.put("syslogBatchId", trace.batchId); + Map ranges = new LinkedHashMap<>(); + for (ConsumerRecord record : records) { + String key = record.topic() + ":" + record.partition(); + long[] range = ranges.computeIfAbsent(key, ignored -> new long[]{record.offset(), record.offset()}); + range[0] = Math.min(range[0], record.offset()); + range[1] = Math.max(range[1], record.offset()); + } + for (Map.Entry range : ranges.entrySet()) { + trace.offsetRanges.put(range.getKey(), range.getValue()[0] + ".." + range.getValue()[1]); + } + slowBatchTrace.set(trace); + } + + private void clearSlowBatchTrace() { + SlowBatchTrace trace = slowBatchTrace.get(); + if (trace == null || trace.previousMdc == null) org.slf4j.MDC.remove("syslogBatchId"); + else org.slf4j.MDC.put("syslogBatchId", trace.previousMdc); + slowBatchTrace.remove(); + lastListenerFinished.set(System.nanoTime()); + } + + private void enterStage(String name) { + endStage(); + SlowBatchTrace trace = slowBatchTrace.get(); + if (trace != null) { trace.activeStage = name; trace.stageStarted = System.nanoTime(); } + } + + private void endStage() { + SlowBatchTrace trace = slowBatchTrace.get(); + if (trace != null && trace.activeStage != null) { + trace.stageNanos.merge(trace.activeStage, System.nanoTime() - trace.stageStarted, Long::sum); + trace.activeStage = null; + } } private void recordSlowBatchStages(int records, int prepared, int persisted, int normalizedRows, @@ -394,6 +525,11 @@ public class SysLogProcessor { trace.ruleComputeMs += metrics.ruleComputeMs(); trace.messageParseMs += metrics.messageParseMs(); trace.fieldMappingMs += metrics.fieldMappingMs(); + trace.ruleQueueMaxMs = Math.max(trace.ruleQueueMaxMs, metrics.ruleQueueMaxMs.get()); + trace.ruleWorkerBusySumMs += metrics.ruleWorkerBusySumMs.get(); + trace.rawSampledOut += metrics.rawSampledOut.get(); + trace.normalSampledOut += metrics.normalRowsSampledOut.get(); + trace.nonNormalSampledOut += metrics.nonNormalRowsSampledOut.get(); } private void acknowledgeAndCompleteSlowBatchTrace(Acknowledgment ack, long businessProcessMs) { @@ -409,25 +545,53 @@ public class SysLogProcessor { private void completeSlowBatchTrace(long businessProcessMs, long kafkaCommitMs, RuntimeException commitFailure) { + completeSlowBatchTrace(businessProcessMs, kafkaCommitMs, commitFailure, + commitFailure == null ? "SUCCESS" : "FAILED"); + } + + private void completeSlowBatchTrace(long businessProcessMs, long kafkaCommitMs, + RuntimeException commitFailure, String commitStatus) { SlowBatchTrace trace = slowBatchTrace.get(); - if (trace == null) return; + if (trace == null || trace.completed) return; + if ("NOT_ATTEMPTED".equals(commitStatus) && trace.failureStage == null) trace.failureStage = trace.activeStage; + endStage(); + trace.completed = true; long endToEndMs = elapsedMillis(trace.startedAtNanos); - if (slowBatchThresholdMs <= 0L || endToEndMs < slowBatchThresholdMs) return; - log.warn("[SLOW_SYSLOG_BATCH] lane={}, topicPartitions={}, polledRecords={}, " + if (commitFailure == null && trace.failedChunks == 0 + && (slowBatchThresholdMs <= 0L || endToEndMs < slowBatchThresholdMs)) return; + Map stages = new LinkedHashMap<>(); + trace.stageNanos.forEach((key, value) -> stages.put(key, TimeUnit.NANOSECONDS.toMillis(value))); + log.warn("[SLOW_SYSLOG_BATCH] batchId={}, lane={}, topicPartitions={}, polledRecords={}, " + "polledMessageBytes={}, maxMessageBytes={}, processedRecords={}, chunks={}, " + "prepared={}, persisted={}, normalizedRows={}, prepareMs={}, ingressMs={}, " + "normalizeMs={}, ruleComputeWorkerMs={}, messageParseWorkerMs={}, " + "fieldMappingWorkerMs={}, persistencePlanMs={}, alarmDmWriteMs={}, influxMs={}, " + "remainingDmWriteMs={}, businessProcessMs={}, kafkaCommitMs={}, " - + "kafkaCommitStatus={}, kafkaCommitErrorType={}, endToEndMs={}", - trace.lane, trace.topicPartitions, trace.polledRecords, + + "kafkaCommitStatus={}, kafkaCommitErrorType={}, endToEndMs={}, " + + "offsetRanges={}, listenerGapMs={}, stageMs={}, failedChunks={}, failureStage={}, businessError={}, " + + "alarmRequested={}, alarmSaved={}, normalRequested={}, normalSaved={}, nonNormalQueued={}, nonNormalRejected={}, " + + "auditAcceptedChunks={}, auditLostChunks={}, rawSampledOut={}, normalSampledOut={}, nonNormalSampledOut={}, " + + "ruleQueueMaxMs={}, ruleWorkerBusySumMs={}, adaptiveRaw={}, " + + "baseProcessingRecords={}, adaptiveMaxRecords={}, adaptiveTargetHistogram={}, actualChunkHistogram={}, " + + "nextTargetRecords={}, observedRetainBps={}", + trace.batchId, trace.lane, trace.topicPartitions, trace.polledRecords, trace.polledMessageBytes, trace.maxMessageBytes, trace.recordsProcessed, trace.chunks, trace.prepared, trace.persisted, trace.normalizedRows, trace.prepareMs, trace.ingressMs, trace.normalizeMs, trace.ruleComputeMs, trace.messageParseMs, trace.fieldMappingMs, trace.persistencePlanMs, trace.alarmDmMs, trace.influxMs, trace.remainingDmMs, - businessProcessMs, kafkaCommitMs, commitFailure == null ? "SUCCESS" : "FAILED", - commitFailure == null ? "none" : commitFailure.getClass().getName(), endToEndMs); + businessProcessMs, kafkaCommitMs, commitStatus, + "FAILED".equals(commitStatus) ? commitFailure.getClass().getName() : "none", endToEndMs, + trace.offsetRanges, trace.listenerGapMs, stages, trace.failedChunks, trace.failureStage, trace.businessError, + trace.alarmRequested, trace.alarmSaved, trace.normalRequested, trace.normalSaved, + trace.nonNormalQueued, trace.nonNormalRejected, trace.auditAcceptedChunks, trace.auditLostChunks, + trace.rawSampledOut, trace.normalSampledOut, trace.nonNormalSampledOut, + trace.ruleQueueMaxMs, trace.ruleWorkerBusySumMs, "RAW".equals(trace.lane), + adaptiveRawBatchSizer.getBaseRecords(), adaptiveRawBatchSizer.getMaxRecords(), + trace.adaptiveTargets, trace.actualChunkSizes, + "RAW".equals(trace.lane) ? adaptiveRawBatchSizer.targetRecords(rawAdaptiveState.get().retainBps, + rawAdaptiveState.get().retainBps) : 0, + "RAW".equals(trace.lane) ? rawAdaptiveState.get().retainBps : 10000); } private void routeContinuityBatch(List> records, @@ -504,8 +668,11 @@ public class SysLogProcessor { List> current = new ArrayList<>(); long currentBytes = 0L; for (ConsumerRecord record : records) { - String value = record.value(); - long bytes = value == null ? 0L : value.getBytes(StandardCharsets.UTF_8).length; + long bytes = record.serializedValueSize(); + if (bytes < 0L) { + String value = record.value(); + bytes = value == null ? 0L : value.getBytes(StandardCharsets.UTF_8).length; + } if (!current.isEmpty() && bytes > maxBytes - currentBytes) { result.add(current); current = new ArrayList<>(); @@ -859,6 +1026,7 @@ public class SysLogProcessor { private List computePreparedNormalizations( List messages, BatchMetrics metrics, Executor normalizationExecutor) { if (messages.isEmpty()) return Collections.emptyList(); + enterStage("RULE_SNAPSHOT"); long snapshotStartedAt = System.nanoTime(); Set deviceIds = new HashSet<>(); for (PreparedMessage message : messages) { @@ -869,17 +1037,34 @@ public class SysLogProcessor { RuleExecutionMetrics executionMetrics = new RuleExecutionMetrics(); metrics.recordRuleSnapshotMs(elapsedMillis(snapshotStartedAt)); long ruleComputeStartedAt = System.nanoTime(); - List> futures = new ArrayList<>(messages.size()); + enterStage("RULE_EXECUTION_WAIT"); + // 每批固定数量的 worker，避免多个 consumer 按日志条数同时向线程池提交任务。 + int workers = Math.min(messages.size(), Math.max(1, normalizationWorkersPerBatch)); + List> futures = new ArrayList<>(workers); + PreparedNormalization[] results = new PreparedNormalization[messages.size()]; + AtomicInteger next = new AtomicInteger(); + AtomicBoolean stopped = new AtomicBoolean(); try { - for (PreparedMessage message : messages) { - futures.add(CompletableFuture.supplyAsync(() -> { + for (int worker = 0; worker < workers; worker++) { + long queuedAt = System.nanoTime(); + futures.add(CompletableFuture.runAsync(() -> { + long workerBegin = System.nanoTime(); + metrics.ruleQueueMaxMs.accumulateAndGet(elapsedMillis(queuedAt), Math::max); try { - return prepareNormalization(message, ruleSnapshot, enrichmentContext, executionMetrics); - } catch (Exception failure) { - // 解析/规则计算失败仍按原消息可靠路由。只有取得 Retry/DLT Broker ACK， - // 这个消息才会携带 terminalState 返回并允许原批次继续。 - return new PreparedNormalization(message, null, - routeFailure(message.record, message.context, failure)); + while (!stopped.get() && !Thread.currentThread().isInterrupted()) { + int index = next.getAndIncrement(); + if (index >= messages.size()) return; + PreparedMessage message = messages.get(index); + try { + results[index] = prepareNormalization(message, ruleSnapshot, + enrichmentContext, executionMetrics); + } catch (Exception failure) { + results[index] = new PreparedNormalization(message, null, + routeFailure(message.record, message.context, failure)); + } + } + } finally { + metrics.ruleWorkerBusySumMs.addAndGet(elapsedMillis(workerBegin)); } }, normalizationExecutor)); } @@ -887,11 +1072,10 @@ public class SysLogProcessor { .get(processTimeoutMs, TimeUnit.MILLISECONDS); metrics.recordRuleComputeMs(elapsedMillis(ruleComputeStartedAt)); metrics.recordRuleExecution(executionMetrics, enrichmentContext); - List computed = new ArrayList<>(futures.size()); - for (CompletableFuture future : futures) { - computed.add(future.join()); + for (PreparedNormalization result : results) { + if (result == null) throw new IllegalStateException("标准化工作任务提前结束"); } - return computed; + return new ArrayList<>(Arrays.asList(results)); } catch (TimeoutException timeout) { cancelOutstanding(futures); throw new IllegalStateException("标准化批次处理超时，禁止提交 offset", timeout); @@ -905,10 +1089,12 @@ public class SysLogProcessor { } catch (RuntimeException submitFailure) { cancelOutstanding(futures); throw submitFailure; + } finally { + stopped.set(true); } } - /** 根据已审计的保留决策写两张标准化表和非标表。 */ + /** 根据保留决策写两张标准化表和非标表，不等待后台采样审计。 */ private void persistComputedNormalizations(List normalizations, BatchMetrics metrics) { StandardizedPersistencePlan plan = prepareStandardizedPersistence(normalizations, metrics); @@ -963,8 +1149,14 @@ public class SysLogProcessor { DmStandardizedBatchWriter.WriteReport alarmReport, long alarmWriteMs, BatchMetrics metrics) { long normalWriteStartedAt = System.nanoTime(); + enterStage("NORMAL_DM"); + SlowBatchTrace rowTrace = slowBatchTrace.get(); + if (rowTrace != null) rowTrace.normalRequested += plan.normalWriteRequests.size(); DmStandardizedBatchWriter.WriteReport normalReport = standardizedBatchWriter.write(plan.normalWriteRequests); + if (rowTrace != null) { + rowTrace.normalSaved += normalReport.getSuccessfulRows(); + } metrics.recordDmWriteMs(alarmWriteMs + elapsedMillis(normalWriteStartedAt)); Map failures = new LinkedHashMap<>(); failures.putAll(alarmReport.getFailures()); @@ -976,6 +1168,7 @@ public class SysLogProcessor { int successfulRows = alarmReport.getSuccessfulRows() + normalReport.getSuccessfulRows(); metrics.recordDmWrite(requestedRows, successfulRows, failedMessages.size()); + enterStage("DM_FAILURE_ROUTE"); for (Map.Entry failure : failures.entrySet()) { PreparedMessage message = failure.getKey(); metrics.record(routeFailure(message.record, message.context, failure.getValue())); @@ -990,15 +1183,22 @@ public class SysLogProcessor { } // 非标日志仍使用现有独立线程；采样决策已经在进入队列前完成。 + enterStage("NON_NORMAL_ENQUEUE"); submitNonNormalMessages(plan.nonNormalMessages, metrics); + if (rowTrace != null) { + rowTrace.nonNormalQueued += metrics.nonNormalQueued(); + rowTrace.nonNormalRejected += metrics.nonNormalRejected(); + } } SamplingPlan buildSamplingPlan(List computed) { List retained = new ArrayList<>(computed.size()); List rawMessages = new ArrayList<>(computed.size()); List auditDeltas = new ArrayList<>(); + int maxRetainBps = computed.isEmpty() ? 10000 : 0; for (PreparedNormalization prepared : computed) { if (prepared.terminalState != null) { + maxRetainBps = 10000; retained.add(new RetentionPreparedNormalization(prepared, false, false)); // 规则异常时无法证明该消息不是告警，保留已构造的 InfluxDB 原文点。 rawMessages.add(prepared.message); @@ -1014,6 +1214,8 @@ public class SysLogProcessor { BacklogSamplingDecision decision = backlogSamplingDecider.decide( prepared.message.eventId, prepared.message.receivedAt, alarmRows > 0L, normalRows > 0L, nonNormalRows > 0L); + maxRetainBps = Math.max(maxRetainBps, + Math.max(decision.getNormalRetainBps(), decision.getNonNormalRetainBps())); retained.add(new RetentionPreparedNormalization( prepared, decision.isKeepNormal(), decision.isKeepNonNormal())); if (decision.isKeepRaw()) rawMessages.add(prepared.message); @@ -1032,7 +1234,7 @@ public class SysLogProcessor { decision.getLagMs(), decision.getLagMs())); } } - return new SamplingPlan(retained, rawMessages, auditDeltas); + return new SamplingPlan(retained, rawMessages, auditDeltas, maxRetainBps); } private SamplingPlan allRetainedPlan(List computed) { @@ -1053,9 +1255,12 @@ public class SysLogProcessor { return; } List batch = new ArrayList<>(messages); + final String batchId = org.slf4j.MDC.get("syslogBatchId"); + final long queuedAt = System.nanoTime(); try { nonNormalWriterExecutor.execute(() -> { long startedAt = System.nanoTime(); + long queueMs = elapsedMillis(queuedAt); try { int inserted = saveNonNormalBatchWithConcurrencyRetry(batch); if (inserted != batch.size()) { @@ -1067,13 +1272,17 @@ public class SysLogProcessor { return; } nonNormalAsyncCompletedTotal.addAndGet(inserted); + if (slowBatchThresholdMs > 0 && queueMs + elapsedMillis(startedAt) >= slowBatchThresholdMs) { + log.warn("[SLOW_NON_NORMAL_WRITE] batchId={}, rows={}, inserted={}, queueMs={}, writeMs={}, status=SUCCESS", + batchId, batch.size(), inserted, queueMs, elapsedMillis(startedAt)); + } log.debug("非标日志异步批写完成: rows={}, durationMs={}, completedTotal={}", inserted, elapsedMillis(startedAt), nonNormalAsyncCompletedTotal.get()); } catch (Exception failure) { nonNormalAsyncFailedTotal.addAndGet(batch.size()); - log.error("非标日志异步批写失败，Kafka offset 不回退，当前非标数据可能丢失: " + log.error("[SLOW_NON_NORMAL_WRITE] batchId={}, queueMs={}, status=FAILED, 非标日志异步批写失败，Kafka offset 不回退，当前非标数据可能丢失: " + "rows={}, durationMs={}, failedTotal={}", - batch.size(), elapsedMillis(startedAt), + batchId, queueMs, batch.size(), elapsedMillis(startedAt), nonNormalAsyncFailedTotal.get(), failure); } }); @@ -1209,15 +1418,6 @@ public class SysLogProcessor { } } - private List ingressEvents(List messages) { - List events = new ArrayList<>(messages.size()); - for (PreparedMessage message : messages) { - events.add(new IngressStatsEvent(message.eventId, message.deviceId, - message.deviceCollectId, message.receivedAt)); - } - return events; - } - private static Long optionalLong(String value) { if (value == null || value.trim().isEmpty()) return null; try { return Long.valueOf(value.trim()); } @@ -1256,6 +1456,7 @@ public class SysLogProcessor { } static final class SamplingPlan { + private final int maxRetainBps; private final List normalizations; private final List rawMessages; private final List auditDeltas; @@ -1263,14 +1464,22 @@ public class SysLogProcessor { private SamplingPlan(List normalizations, List rawMessages, List auditDeltas) { + this(normalizations, rawMessages, auditDeltas, 10000); + } + + private SamplingPlan(List normalizations, + List rawMessages, List auditDeltas, + int maxRetainBps) { this.normalizations = normalizations; this.rawMessages = rawMessages; this.auditDeltas = auditDeltas; + this.maxRetainBps = maxRetainBps; } int rawMessageCount() { return rawMessages.size(); } int auditMessageCount() { return auditDeltas.size(); } List auditDeltas() { return auditDeltas; } + int maxRetainBps() { return maxRetainBps; } } private static final class StandardizedPersistencePlan { @@ -1293,7 +1502,38 @@ public class SysLogProcessor { } } + private static final class RawAdaptiveState { + private int retainBps = 10000; + private long observedAt; + } + private static final class SlowBatchTrace { + private final Map adaptiveTargets = new LinkedHashMap<>(); + private final Map actualChunkSizes = new LinkedHashMap<>(); + private final String batchId = java.util.UUID.randomUUID().toString(); + private String previousMdc; + private String activeStage; + private long stageStarted; + private final Map stageNanos = new LinkedHashMap<>(); + private final Map offsetRanges = new LinkedHashMap<>(); + private long listenerGapMs; + private boolean completed; + private int failedChunks; + private String failureStage; + private String businessError; + private int auditAcceptedChunks; + private int auditLostChunks; + private int alarmRequested; + private int alarmSaved; + private int normalRequested; + private int normalSaved; + private int nonNormalQueued; + private int nonNormalRejected; + private long ruleQueueMaxMs; + private long ruleWorkerBusySumMs; + private int rawSampledOut; + private int normalSampledOut; + private int nonNormalSampledOut; private final String lane; private final String topicPartitions; private final int polledRecords; @@ -1343,6 +1583,8 @@ public class SysLogProcessor { private final AtomicInteger normalRowsSampledOut = new AtomicInteger(); private final AtomicInteger nonNormalRowsSampledOut = new AtomicInteger(); private final AtomicLong ruleComputeMs = new AtomicLong(); + private final AtomicLong ruleQueueMaxMs = new AtomicLong(); + private final AtomicLong ruleWorkerBusySumMs = new AtomicLong(); private final AtomicLong ruleSnapshotMs = new AtomicLong(); private final AtomicLong dmWriteMs = new AtomicLong(); private final AtomicInteger nonNormalQueued = new AtomicInteger(); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BacklogSamplingAuditService.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BacklogSamplingAuditService.java index 41ad01c..c92438e 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BacklogSamplingAuditService.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BacklogSamplingAuditService.java @@ -12,7 +12,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -/** 先提交小时审计，再允许调用方执行有损采样；审计失败时调用方必须回退全量保留。 */ +/** 消费优先：小时采样审计后台尽力保存，失败不改变采样决策，不等待主消费。 */ @Slf4j @Service public class BacklogSamplingAuditService { @@ -21,6 +21,11 @@ public class BacklogSamplingAuditService { private final BacklogSamplingMetrics metrics; private final PlatformTransactionManager transactionManager; private volatile long retryAfterMs; + private final BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("SAMPLING_AUDIT", 256, 50000, false); + + @javax.annotation.PostConstruct public void start() { queue.start(); } + @javax.annotation.PreDestroy public void stop() { queue.stop(); } + public Map status() { return queue.status(); } public BacklogSamplingAuditService(BacklogRetentionAuditMapper mapper, BacklogSamplingProperties properties, @@ -32,34 +37,46 @@ public class BacklogSamplingAuditService { this.transactionManager = transactionManager; } - /** 返回 false 表示不能安全采样，调用方应保留当前批次全部数据。 */ - public boolean recordBeforeDropping(List deltas) { + /** 返回值仅代表入队是否成功，不影响调用方原采样决策。 */ + public boolean recordAsync(List deltas) { if (deltas == null || deltas.isEmpty()) return true; - long affectedMessages = messageCount(deltas); - long now = System.currentTimeMillis(); - if (now < retryAfterMs) { - metrics.auditFailure(affectedMessages); + long affectedMessages = 0L; + try { + affectedMessages = messageCount(deltas); + final long taskMessages = affectedMessages; + List aggregated = aggregate(deltas); + metrics.record(aggregated); // 决策指标，不再假称这些数字已写入审计数据库。 + if (System.currentTimeMillis() < retryAfterMs) { + queue.loss("DATABASE_BACKOFF", null); + metrics.auditLoss(affectedMessages); + return false; + } + boolean offered = queue.offer(aggregated.size(), () -> writeAudit(aggregated, taskMessages)); + if (!offered) metrics.auditLoss(affectedMessages); + return offered; + } catch (RuntimeException failure) { + queue.loss("PREPARE_FAILED", failure); + metrics.auditLoss(affectedMessages); return false; } + } - List aggregated = aggregate(deltas); + private void writeAudit(List aggregated, long affectedMessages) { try { TransactionTemplate transaction = new TransactionTemplate(transactionManager); + transaction.setPropagationBehavior(org.springframework.transaction.TransactionDefinition.PROPAGATION_REQUIRES_NEW); transaction.setTimeout(Math.max(1, properties.getAuditTimeoutSeconds())); transaction.execute(status -> { for (BacklogRetentionDelta delta : aggregated) mapper.upsert(delta); return null; }); retryAfterMs = 0L; - metrics.record(aggregated); - return true; } catch (RuntimeException failure) { retryAfterMs = System.currentTimeMillis() + Math.max(1L, properties.getAuditRetryBackoffMs()); - metrics.auditFailure(affectedMessages); - log.error("[BACKLOG_SAMPLING_AUDIT_FAILURE] 小时采样审计写入失败，当前批次自动回退全量保留: " - + "messages={}, groups={}, retryBackoffMs={}", - affectedMessages, aggregated.size(), properties.getAuditRetryBackoffMs(), failure); - return false; + queue.pause(properties.getAuditRetryBackoffMs()); + metrics.auditWriteFailed(affectedMessages); + // 审计为累加UPSERT；提交结果不确定时自动重放可能双计数，当前选择记录损失而不重放。 + throw failure; } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BacklogSamplingMetrics.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BacklogSamplingMetrics.java index 2ddf078..ab02de7 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BacklogSamplingMetrics.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BacklogSamplingMetrics.java @@ -20,6 +20,10 @@ public class BacklogSamplingMetrics { private final LongAdder rawDropped = new LongAdder(); private final LongAdder auditWriteFailures = new LongAdder(); private final LongAdder auditFallbackMessages = new LongAdder(); + private final LongAdder auditLostMessages = new LongAdder(); + + public void auditLoss(long messages) { auditLostMessages.add(Math.max(0L, messages)); } + public void auditWriteFailed(long messages) { auditWriteFailures.increment(); auditLoss(messages); } public void record(List deltas) { for (BacklogRetentionDelta delta : deltas) { @@ -51,6 +55,8 @@ public class BacklogSamplingMetrics { values.put("raw_dropped_total", rawDropped.sum()); values.put("audit_write_failures_total", auditWriteFailures.sum()); values.put("audit_fallback_messages_total", auditFallbackMessages.sum()); + values.put("audit_lost_or_unconfirmed_messages_total", auditLostMessages.sum()); + values.put("counter_semantics", "SAMPLING_DECISIONS_NOT_DB_CONFIRMATIONS"); return values; } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BestEffortAuxiliaryQueue.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BestEffortAuxiliaryQueue.java new file mode 100644 index 0000000..72dcf83 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/BestEffortAuxiliaryQueue.java @@ -0,0 +1,147 @@ +package com.common.service; + +import lombok.extern.slf4j.Slf4j; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** 辅助写库专用有界队列。入队不执行任务、不等待数据库、不使用CallerRuns。 */ +@Slf4j +final class BestEffortAuxiliaryQueue { + private final String kind; + private final ArrayBlockingQueue queue; + private final long maxGroups; + private final boolean retry; + private final AtomicLong groups = new AtomicLong(); + private final AtomicLong accepted = new AtomicLong(); + private final AtomicLong completed = new AtomicLong(); + private final AtomicLong failures = new AtomicLong(); + private final AtomicLong lost = new AtomicLong(); + private final AtomicLong logAfter = new AtomicLong(); + private volatile Task current; + private volatile boolean closed; + private volatile long resumeAt; + private ScheduledExecutorService executor; + + BestEffortAuxiliaryQueue(String kind, int capacity, long maxGroups, boolean retry) { + this.kind = kind; + this.queue = new ArrayBlockingQueue<>(capacity); + this.maxGroups = maxGroups; + this.retry = retry; + } + + void start() { + executor = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "syslog-aux-" + kind.toLowerCase(java.util.Locale.ROOT)); + t.setDaemon(true); + return t; + }); + executor.scheduleWithFixedDelay(this::drain, 0, 200, TimeUnit.MILLISECONDS); + } + + boolean offer(int weight, Runnable work) { + boolean reserved = false; + try { + if (closed || weight < 1) { loss("CLOSED_OR_INVALID", null); return false; } + long value; + do { + value = groups.get(); + if (weight > maxGroups - value) { loss("GROUP_LIMIT", null); return false; } + } while (!groups.compareAndSet(value, value + weight)); + reserved = true; + if (!queue.offer(new Task(weight, work))) { loss("QUEUE_FULL", null); return false; } + reserved = false; + accepted.incrementAndGet(); + return true; + } catch (RuntimeException failure) { + loss("ENQUEUE_FAILED", failure); + return false; + } finally { + if (reserved) groups.addAndGet(-weight); + } + } + + // 只有后台线程/测试调用；入队和状态查询不获取该锁。 + synchronized void drain() { + if (resumeAt != 0L && System.nanoTime() < resumeAt) return; + for (int n = 0; n < 32 && !closed; n++) { + if (current == null) current = queue.poll(); + Task task = current; + if (task == null || (task.retryAt != 0L && System.nanoTime() < task.retryAt)) return; + long begin = System.nanoTime(); + long queueMs = TimeUnit.NANOSECONDS.toMillis(begin - task.createdAt); + try { + task.work.run(); + completed.incrementAndGet(); + groups.addAndGet(-task.weight); + current = null; + long workMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - begin); + if (workMs >= 500 || queueMs >= 500) log.warn( + "[SLOW_SYSLOG_AUX] kind={}, groups={}, queueAgeMs={}, executeMs={}, status=SUCCESS", + kind, task.weight, queueMs, workMs); + } catch (RuntimeException failure) { + failures.incrementAndGet(); + if (retry) { + task.retryAt = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + } else { + groups.addAndGet(-task.weight); + current = null; + lost.incrementAndGet(); + } + if (mayLog()) log.error("[SYSLOG_AUX_FAILURE] kind={}, groups={}, queueAgeMs={}, executeMs={}, " + + "action={}, failuresTotal={}", kind, task.weight, queueMs, + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - begin), + retry ? "BACKGROUND_RETRY" : "DROP_AUX_ONLY", failures.get(), failure); + return; + } + } + } + + void loss(String reason, RuntimeException failure) { + lost.incrementAndGet(); + if (mayLog()) log.error("[SYSLOG_AUX_LOSS] kind={}, reason={}, lostTasksTotal={}, action=CONTINUE_FACT_CONSUMPTION", + kind, reason, lost.get(), failure); + } + + void pause(long millis) { + resumeAt = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(Math.max(1L, millis)); + } + + private boolean mayLog() { + long now = System.nanoTime(); + long after = logAfter.get(); + return (after == 0 || now >= after) && logAfter.compareAndSet(after, now + TimeUnit.SECONDS.toNanos(10)); + } + + Map status() { + Map result = new LinkedHashMap<>(); + result.put("mode", "BEST_EFFORT_BACKGROUND"); + result.put("pending_tasks", queue.size() + (current == null ? 0 : 1)); + result.put("pending_groups", groups.get()); + result.put("max_groups", maxGroups); + result.put("enqueued_total", accepted.get()); + result.put("completed_total", completed.get()); + result.put("failures_total", failures.get()); + result.put("lost_tasks_total", lost.get()); + return result; + } + + void stop() { + closed = true; + if (executor != null) executor.shutdownNow(); + if (groups.get() > 0) log.warn("[SYSLOG_AUX_SHUTDOWN_PENDING] kind={}, pendingGroups={}, action=AUX_DATA_MAY_BE_LOST", + kind, groups.get()); + } + + private static final class Task { + final int weight; + final Runnable work; + final long createdAt = System.nanoTime(); + long retryAt; + Task(int weight, Runnable work) { this.weight = weight; this.work = work; } + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java index 9b06959..ddbe597 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java @@ -21,12 +21,13 @@ import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; +import java.util.function.Supplier; /** - * 把当前事实事务的内存聚合结果写入同步投影或可靠异步投影入口。 + * 把当前事实事务的内存聚合结果交给提交后的有界异步统计队列。 * - *

DIRECT 默认只追加一条批次 Outbox；SHADOW/显式回退仍同步更新统计表。调用方必须 - * 已经开启事实事务，本类不会吞掉任何数据库异常。

+ *

DIRECT 连续消费优先，不写 Outbox 或统计表，统计异常记录后放行。 + * 历史 SHADOW 模式仍同步写影子投影，保留原有校验语义。

*/ @Slf4j @Service @@ -64,7 +65,7 @@ public class DirectStatsProjectionWriter { private final StatsWriteModeProvider modeProvider; private final DirectStatsWriteMetrics metrics; private final StatsMaterializationPublisher materializationPublisher; - private final StatsProjectionOutboxWriter projectionOutboxWriter; + private final StatsProjectionAsyncPublisher asyncPublisher; @Value("${stats.incremental.read-enabled:false}") private boolean readEnabled; @@ -94,62 +95,51 @@ public class DirectStatsProjectionWriter { StatsWriteModeProvider modeProvider, DirectStatsWriteMetrics metrics, StatsMaterializationPublisher materializationPublisher, - StatsProjectionOutboxWriter projectionOutboxWriter) { + StatsProjectionAsyncPublisher asyncPublisher) { this.jdbc = jdbc; this.modeProvider = modeProvider; this.metrics = metrics; this.materializationPublisher = materializationPublisher; - this.projectionOutboxWriter = projectionOutboxWriter; + this.asyncPublisher = asyncPublisher; } public WriteResult writeStandardizedRows(List> rows) { - if (!shouldBuildAggregation()) { - handleFactOnlyMode(); - return WriteResult.inactive(rows == null ? 0 : rows.size()); - } - long aggregateStarted = System.nanoTime(); - DirectStatsAccumulator accumulator = accumulator().addStandardizedRows(rows); - return write(accumulator, elapsedMs(aggregateStarted)); + return project(rows == null ? 0 : rows.size(), () -> accumulator().addStandardizedRows(rows), false); } public WriteResult writeStandardizedRow(Map row) { - if (!shouldBuildAggregation()) { - handleFactOnlyMode(); - return WriteResult.inactive(row == null ? 0 : 1); - } - long aggregateStarted = System.nanoTime(); - DirectStatsAccumulator accumulator = accumulator().addStandardizedRow(row); - return write(accumulator, elapsedMs(aggregateStarted)); + return project(row == null ? 0 : 1, () -> accumulator().addStandardizedRow(row), false); } public WriteResult writeNonNormalRows(List messages) { - if (!shouldBuildAggregation()) { - handleFactOnlyMode(); - return WriteResult.inactive(messages == null ? 0 : messages.size()); - } - long aggregateStarted = System.nanoTime(); - DirectStatsAccumulator accumulator = accumulator().addNonNormalRows(messages); - return write(accumulator, elapsedMs(aggregateStarted)); + return project(messages == null ? 0 : messages.size(), () -> accumulator().addNonNormalRows(messages), false); } public WriteResult writeNonNormalInsert(SyslogNonNormalMessage message) { - if (!shouldBuildAggregation()) { - handleFactOnlyMode(); - return WriteResult.inactive(message == null ? 0 : 1); - } - long aggregateStarted = System.nanoTime(); - DirectStatsAccumulator accumulator = accumulator().addNonNormalInsert(message); - return write(accumulator, elapsedMs(aggregateStarted)); + return project(message == null ? 0 : 1, () -> accumulator().addNonNormalInsert(message), false); } public WriteResult writeNonNormalUpdate(SyslogNonNormalMessage oldValue, SyslogNonNormalMessage patch) { - if (!shouldBuildAggregation()) { - handleFactOnlyMode(); - return WriteResult.inactive(oldValue == null || patch == null ? 0 : 1); + return project(oldValue == null || patch == null ? 0 : 1, + () -> accumulator().addNonNormalUpdate(oldValue, patch), true); + } + + private WriteResult project(int rows, Supplier build, boolean exact) { + try { + if (!shouldBuildAggregation()) { + handleFactOnlyMode(); + return WriteResult.inactive(rows); + } + long started = System.nanoTime(); + DirectStatsAccumulator aggregate = build.get(); + return exact ? writeExactDeviceGroups(aggregate, elapsedMs(started)) + : write(aggregate, elapsedMs(started)); + } catch (RuntimeException failure) { + if (!asynchronousProjection()) throw failure; + metrics.recordFailure(); + asyncPublisher.recordLoss(rows, "AGGREGATION_OR_REGISTRATION_FAILED", failure); + return WriteResult.inactive(rows); } - long aggregateStarted = System.nanoTime(); - DirectStatsAccumulator accumulator = accumulator().addNonNormalUpdate(oldValue, patch); - return writeExactDeviceGroups(accumulator, elapsedMs(aggregateStarted)); } public void recordTransactionRetry() { @@ -165,7 +155,8 @@ public class DirectStatsProjectionWriter { status.put("business_materialization_mode", asynchronousMaterialization() ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS"); status.put("daily_projection_write_mode", asynchronousProjection() - ? "DURABLE_OUTBOX" : "SYNCHRONOUS_MERGE"); + ? "BEST_EFFORT_AFTER_COMMIT_KAFKA" : "SYNCHRONOUS_MERGE"); + status.put("async_publisher", asyncPublisher.status()); return status; } @@ -201,21 +192,16 @@ public class DirectStatsProjectionWriter { requireTransaction(); if (asynchronousProjection()) { - try { - long outboxStarted = System.nanoTime(); - projectionOutboxWriter.append(StatsProjectionOutboxEvent.fromAccumulator(accumulator)); - long outboxMs = elapsedMs(outboxStarted); - int deviceGroups = effectiveDeviceEntries(accumulator).size(); - int ruleGroups = effectiveRuleEntries(accumulator).size(); - metrics.recordSuccess(accumulator.getInputRows(), deviceGroups, ruleGroups, - aggregateMs, outboxMs, 0L); - return WriteResult.direct(accumulator.getInputRows(), deviceGroups, ruleGroups, - aggregateMs, outboxMs, 0L); - } catch (RuntimeException failure) { - metrics.recordFailure(); - throw new DirectStatsWriteException( - "统计投影 Outbox 写入失败，事实事务必须整体回滚", failure); - } + long enqueueStarted = System.nanoTime(); + asyncPublisher.enqueueAfterCommit(StatsProjectionOutboxEvent.fromAccumulator(accumulator), + accumulator.getInputRows()); + long enqueueMs = elapsedMs(enqueueStarted); + int deviceGroups = effectiveDeviceEntries(accumulator).size(); + int ruleGroups = effectiveRuleEntries(accumulator).size(); + metrics.recordSuccess(accumulator.getInputRows(), deviceGroups, ruleGroups, + aggregateMs, enqueueMs, 0L); + return WriteResult.direct(accumulator.getInputRows(), deviceGroups, ruleGroups, + aggregateMs, enqueueMs, 0L); } List> devices = @@ -438,8 +424,8 @@ public class DirectStatsProjectionWriter { } private boolean asynchronousProjection() { - return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT - && projectionOutboxWriter != null && projectionOutboxWriter.isEnabled(); + // 关闭统计发布也绝不回退同步统计SQL，以免再次阻塞事实消费。 + return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT; } private List materializationCommands( diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java index 08d0a98..044bd1e 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java @@ -36,6 +36,9 @@ public class DmStandardizedBatchExecutor { @org.springframework.beans.factory.annotation.Value("${stats.direct.transaction-log-enabled:false}") private boolean transactionLogEnabled; + @org.springframework.beans.factory.annotation.Value("${syslog.slow-batch.threshold-ms:500}") + private long slowTransactionThresholdMs; + @org.springframework.beans.factory.annotation.Value("${stats.direct.max-concurrency-retries:1}") private int maxConcurrencyRetries; @@ -96,11 +99,14 @@ public class DmStandardizedBatchExecutor { transaction.setTimeout(Math.max(1, transactionTimeoutSeconds)); long transactionStarted = System.nanoTime(); final TransactionTiming timing = new TransactionTiming(); + try { transaction.execute(status -> { + timing.beginMs = elapsedMs(transactionStarted); TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() { @Override public void beforeCommit(boolean readOnly) { timing.commitStarted = System.nanoTime(); + timing.phase("FACT_COMMIT"); } @Override @@ -110,31 +116,56 @@ public class DmStandardizedBatchExecutor { }); long flushStarted = System.nanoTime(); // 事实表使用一个固定 PreparedStatement 直接 addBatch，彻底绕开逐行 OGNL 动态 SQL。 + timing.phase("JDBC_FACT_WRITE"); timing.batchExecution = preparedBatchInserter.insert( alarmTable, rows, transactionTimeoutSeconds); // 兼容期变更日志仍与事实行共用当前事务；默认关闭时完全不创建 MyBatis BatchSession。 if (statsChangeCaptureService.isEnabled()) { + timing.phase("LEGACY_STATS_CAPTURE"); statsChangeCaptureService.captureStandardizedBatch(batchSqlSession, rows, alarmTable); batchSqlSession.flushStatements(); } timing.factFlushMs = elapsedMs(flushStarted); - // 与标准化告警事实共用同一事务写入事件时间桶。事实成功而通知失败时必须整体回滚， - // 否则迟到数据可能永久失去重放机会。 + // 只注册提交后入队；分钟桶在独立后台事务写入，失败不回滚当前告警。 if (alarmTable) { + timing.phase("LATE_BUCKET_REGISTER"); + long bucketStarted = System.nanoTime(); lateAlarmBucketWriter.record(rows); + timing.lateBucketMs = elapsedMs(bucketStarted); } - // DIRECT 异步投影模式只在当前事实事务追加一条聚合 Outbox；独立 Kafka consumer - // 再更新日统计。SHADOW/回退模式仍保留原同步 MERGE。 + // DIRECT 只聚合并注册afterCommit入队；此处不写Outbox/统计表、不发送Kafka。 + // 独立后台线程发送统计事件，失败不影响事实提交；历史SHADOW模式保留同步MERGE。 + timing.phase("STATS_REGISTER"); timing.statsResult = directStatsProjectionWriter.writeStandardizedRows(rows); return null; }); + } catch (RuntimeException failure) { + log.error("[SLOW_DM_TRANSACTION] batchId={}, table={}, factRows={}, status=FAILED, " + + "failureStage={}, phaseElapsedMs={}, transactionMs={}, connectionBeginMs={}, " + + "jdbcExecuteMs={}, lateAlarmBucketMs={}, retry={}, errorType={}", + org.slf4j.MDC.get("syslogBatchId"), alarmTable ? "syslog_normal_alarm" : "syslog_normal_data", + rows.size(), timing.phase, elapsedMs(timing.phaseStarted), elapsedMs(transactionStarted), + timing.beginMs, timing.batchExecution.getExecuteMs(), timing.lateBucketMs, retries, + failure.getClass().getName()); + throw failure; + } long transactionMs = elapsedMs(transactionStarted); transactionMetrics.recordSuccess(rows.size(), timing.batchExecution.getPlanMs(), timing.batchExecution.getBindMs(), timing.batchExecution.getExecuteMs(), timing.factFlushMs, timing.commitMs, transactionMs); + if (slowTransactionThresholdMs > 0 && transactionMs >= slowTransactionThresholdMs) { + log.warn("[SLOW_DM_TRANSACTION] batchId={}, table={}, factRows={}, status=SUCCESS, connectionBeginMs={}, jdbcPlanMs={}, " + + "jdbcBindMs={}, jdbcExecuteMs={}, lateAlarmBucketMs={}, statsProjectionStageMs={}, " + + "commitMs={}, transactionMs={}, retry={}", + org.slf4j.MDC.get("syslogBatchId"), alarmTable ? "syslog_normal_alarm" : "syslog_normal_data", rows.size(), timing.beginMs, + timing.batchExecution.getPlanMs(), timing.batchExecution.getBindMs(), + timing.batchExecution.getExecuteMs(), timing.lateBucketMs, + timing.statsResult == null ? 0L : timing.statsResult.getMergeMs(), + timing.commitMs, transactionMs, retries); + } if (transactionLogEnabled && timing.statsResult != null && timing.statsResult.isDirect()) { log.info("DIRECT 达梦事务完成: factRows={}, deviceGroups={}, ruleGroups={}, aggregateMs={}, " + "jdbcPlanMs={}, jdbcBindMs={}, jdbcExecuteMs={}, factFlushMs={}, " @@ -148,7 +179,9 @@ public class DmStandardizedBatchExecutor { } public Map status() { - return transactionMetrics.snapshot(); + Map result = transactionMetrics.snapshot(); + result.put("late_alarm_bucket_async", lateAlarmBucketWriter.status()); + return result; } private static long elapsedMs(long startedNanos) { @@ -156,6 +189,11 @@ public class DmStandardizedBatchExecutor { } private static final class TransactionTiming { + private String phase = "CONNECTION_BEGIN"; + private long phaseStarted = System.nanoTime(); + private void phase(String value) { phase = value; phaseStarted = System.nanoTime(); } + private long beginMs; + private long lateBucketMs; private long factFlushMs; private long commitStarted; private long commitMs; diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java index cae6c0b..bce7847 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java @@ -69,6 +69,9 @@ public class DmStandardizedBatchWriter { @Value("${dm.standardized-write.batch-size:500}") private int batchSize; + @Value("${syslog.slow-batch.threshold-ms:500}") + private long slowQueueThresholdMs; + @Value("${dm.standardized-write.transaction-timeout-seconds:45}") private int transactionTimeoutSeconds; @@ -190,10 +193,25 @@ public class DmStandardizedBatchWriter { List>> futures = new ArrayList<>(groups.size()); try { for (Map.Entry>> entry : groups.entrySet()) { + long queuedAt = System.nanoTime(); + String batchId = org.slf4j.MDC.get("syslogBatchId"); futures.add(CompletableFuture.supplyAsync(() -> { + String previous = org.slf4j.MDC.get("syslogBatchId"); + if (batchId != null) org.slf4j.MDC.put("syslogBatchId", batchId); + else org.slf4j.MDC.remove("syslogBatchId"); + try { + long queueMs = (System.nanoTime() - queuedAt) / 1_000_000L; + if (slowQueueThresholdMs > 0 && queueMs >= slowQueueThresholdMs) { + log.warn("[SLOW_DM_QUEUE] batchId={}, alarmTable={}, rows={}, queueMs={}", + batchId, entry.getKey().alarmTable, entry.getValue().size(), queueMs); + } WriteReport groupReport = new WriteReport<>(); writeGroup(entry.getKey(), entry.getValue(), groupReport, budget); return groupReport; + } finally { + if (previous == null) org.slf4j.MDC.remove("syslogBatchId"); + else org.slf4j.MDC.put("syslogBatchId", previous); + } }, writerExecutor)); } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsConsumer.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsConsumer.java new file mode 100644 index 0000000..0e276cd --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsConsumer.java @@ -0,0 +1,140 @@ +package com.common.service; + +import com.Modules.NormalData.SyslogFailureRouter; +import com.common.entity.IngressStatsEvent; +import com.common.util.Sm4Util; +import com.common.util.SyslogParser; +import com.config.AppProperties; +import com.influx.raw.ReceiveTimeParser; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.Acknowledgment; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** 独立消费原始Topic，只维护接收时间；不执行规则、采样或事实表写入。 */ +@Component +@Slf4j +public class IngressStatsConsumer { + private final IngressStatsProjectionWriter writer; + private final SyslogFailureRouter failureRouter; + private final AppProperties appProperties; + private final AtomicLong batches = new AtomicLong(); + private final AtomicLong invalidAttempts = new AtomicLong(); + private final AtomicLong writeFailures = new AtomicLong(); + + @Value("${spring.kafka.consumer.group-id}") private String rawGroup; + @Value("${syslog.large-listener.group-id:${spring.kafka.consumer.group-id}-large}") + private String largeRawGroup; + @Value("${stats.ingress-metadata.group-id:${spring.kafka.consumer.group-id}-ingress-v1}") + private String group; + @Value("${stats.ingress-metadata.large-group-id:${stats.ingress-metadata.group-id}-large}") + private String largeGroup; + + public IngressStatsConsumer(IngressStatsProjectionWriter writer, + SyslogFailureRouter failureRouter, AppProperties appProperties) { + this.writer = writer; + this.failureRouter = failureRouter; + this.appProperties = appProperties; + } + + @PostConstruct + public void validateGroups() { + if (group == null || group.trim().isEmpty() || largeGroup == null || largeGroup.trim().isEmpty() + || group.equals(largeGroup) || group.equals(rawGroup) || group.equals(largeRawGroup) + || largeGroup.equals(rawGroup) || largeGroup.equals(largeRawGroup)) { + throw new IllegalStateException("入口统计必须使用与主消费隔离的非空 Kafka group-id"); + } + log.info("Ingress stats consumer ready: group={}, largeGroup={}, mode=INDEPENDENT_KAFKA_DIRECT_MERGE", + group, largeGroup); + } + + @KafkaListener(id = "syslog-ingress-stats", groupId = "${stats.ingress-metadata.group-id}", + topics = "${spring.kafka.consumer.topic}", + autoStartup = "${stats.ingress-metadata.enabled:true}", + containerFactory = "ingressStatsKafkaListenerContainerFactory") + public void consume(List> records, Acknowledgment ack) { + consumeBatch(records, ack, "RAW"); + } + + @KafkaListener(id = "syslog-ingress-stats-large", groupId = "${stats.ingress-metadata.large-group-id}", + topics = "${spring.kafka.consumer.large-topic}", + autoStartup = "#{${stats.ingress-metadata.enabled:true} && ${syslog.large-listener.enabled:true}}", + containerFactory = "largeIngressStatsKafkaListenerContainerFactory") + public void consumeLarge(List> records, Acknowledgment ack) { + consumeBatch(records, ack, "RAW_LARGE"); + } + + private void consumeBatch(List> records, Acknowledgment ack, String lane) { + long started = System.nanoTime(); + List events = new ArrayList<>(records.size()); + int invalid = 0; + String firstInvalid = null; + for (ConsumerRecord record : records) { + try { + events.add(decode(record)); + } catch (RuntimeException malformed) { + invalid++; + if (firstInvalid == null) firstInvalid = record.topic() + "-" + record.partition() + + "@" + record.offset() + ":" + malformed.getClass().getSimpleName(); + } + } + if (invalid > 0) { + invalidAttempts.addAndGet(invalid); + // 原始消息仍留在Kafka；坏头不会永久挡住其它设备接收状态，也不向业务Retry重复投递。 + log.error("[INGRESS_STATS_INVALID] lane={}, invalidRecords={}, firstInvalid={}, " + + "action=SKIP_INVALID_METADATA", lane, invalid, firstInvalid); + } + int groups; + try { + groups = writer.writeBatch(events); + } catch (RuntimeException failure) { + writeFailures.incrementAndGet(); + log.error("[INGRESS_STATS_WRITE_FAILED] lane={}, records={}, action=RETRY_STATS_GROUP", + lane, records.size(), failure); + throw failure; + } + // MIN/MAX更新和达梦提交均已完成，才推进独立统计组；重复消费不重复增加事实数量。 + ack.acknowledge(); + batches.incrementAndGet(); + log.info("[INGRESS_STATS_BATCH] lane={}, records={}, valid={}, invalid={}, groups={}, totalMs={}", + lane, records.size(), events.size(), invalid, groups, (System.nanoTime() - started) / 1_000_000L); + } + + IngressStatsEvent decode(ConsumerRecord record) { + SyslogFailureRouter.MessageContext context = failureRouter.resolve(record); + String decrypted = Sm4Util.decryptCbc(context.getPayload(), appProperties.getSm4Key()); + if (decrypted.indexOf(']') < 0) throw new IllegalArgumentException("缺少设备元数据头"); + Map header = SyslogParser.parseKeyValuePairs( + SyslogParser.substringBeforeFirstChar(decrypted, ']')); + Long device = positiveId(header.get("device_id")); + Long collector = positiveId(header.get("device_collect_id")); + if (device == null && collector == null) throw new IllegalArgumentException("缺少有效设备/采集任务ID"); + return new IngressStatsEvent(context.getEventId(), device, collector, + ReceiveTimeParser.parse(header.get("receive_time")).getLocalDateTime()); + } + + private static Long positiveId(String value) { + if (value == null || value.trim().isEmpty()) return null; + long parsed = Long.parseLong(value); + return parsed > 0L ? parsed : null; + } + + public Map status() { + Map result = new LinkedHashMap<>(); + result.put("group_id", group); + result.put("large_group_id", largeGroup); + result.put("successful_batches", batches.get()); + result.put("invalid_record_attempts", invalidAttempts.get()); + result.put("write_failures", writeFailures.get()); + return result; + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java index 86c60cf..11bcdc2 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java @@ -29,7 +29,6 @@ public class IngressStatsProjectionWriter { private final PlatformTransactionManager transactionManager; private final StatsWriteModeProvider modeProvider; private final StatsMaterializationPublisher materializationPublisher; - private final StatsProjectionOutboxWriter projectionOutboxWriter; @Value("${stats.ingress-metadata.enabled:false}") private boolean enabled; @@ -51,13 +50,11 @@ public class IngressStatsProjectionWriter { public IngressStatsProjectionWriter(JdbcTemplate jdbc, PlatformTransactionManager transactionManager, StatsWriteModeProvider modeProvider, - StatsMaterializationPublisher materializationPublisher, - StatsProjectionOutboxWriter projectionOutboxWriter) { + StatsMaterializationPublisher materializationPublisher) { this.jdbc = jdbc; this.transactionManager = transactionManager; this.modeProvider = modeProvider; this.materializationPublisher = materializationPublisher; - this.projectionOutboxWriter = projectionOutboxWriter; } public int writeBatch(List events) { @@ -95,8 +92,7 @@ public class IngressStatsProjectionWriter { ? "device_daily_stats+" + shadowDeviceTable : "device_daily_stats"); status.put("business_materialization_mode", asynchronousMaterialization() ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS"); - status.put("daily_projection_write_mode", asynchronousProjection() - ? "DURABLE_OUTBOX" : "SYNCHRONOUS_MERGE"); + status.put("daily_projection_write_mode", "INDEPENDENT_KAFKA_DIRECT_MERGE"); status.put("successful_batches", successfulBatches.get()); status.put("received_events", receivedEvents.get()); status.put("updated_groups", updatedGroups.get()); @@ -114,10 +110,6 @@ public class IngressStatsProjectionWriter { TransactionTemplate transaction = new TransactionTemplate(transactionManager); transaction.setTimeout(Math.max(1, transactionTimeoutSeconds)); Integer updated = transaction.execute(status -> { - if (asynchronousProjection()) { - projectionOutboxWriter.append(StatsProjectionOutboxEvent.fromIngress(aggregation)); - return aggregation.devices.size() + aggregation.collects.size(); - } int groups = 0; boolean asyncMaterialization = asynchronousMaterialization(); List commands = asyncMaterialization @@ -246,11 +238,6 @@ public class IngressStatsProjectionWriter { && materializationPublisher.isEnabled(); } - private boolean asynchronousProjection() { - return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT - && projectionOutboxWriter != null && projectionOutboxWriter.isEnabled(); - } - private static String safeTable(String table) { if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) { throw new IllegalStateException("非法入口设备统计表名: " + table); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/LateAlarmBucketWriter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/LateAlarmBucketWriter.java index 54cb485..b16829d 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/LateAlarmBucketWriter.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/LateAlarmBucketWriter.java @@ -3,7 +3,8 @@ package com.common.service; import java.sql.Timestamp; import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; -import java.util.LinkedHashSet; +import java.util.TreeSet; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -12,11 +13,15 @@ import java.util.UUID; import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.springframework.transaction.support.TransactionTemplate; /** - * 在标准化告警事实事务内登记事件时间桶。 + * 告警提交后登记事件时间桶，后台独立事务失败只重试通知，不回滚告警。 * - *

该通知不是“告警一定迟到”的结论。通知与事实在同一事务提交；下游能领取通知时，事实必然 + *

该通知不是“告警一定迟到”的结论。事实提交后才入队；下游能领取通知时，事实必然 * 已经可见，再根据领取时已经完成的规则窗口决定是否重放。这里刻意不使用事实的 * {@code created_at} 判断提交先后，因为 Kafka 积压时接收时间可能早于实际入库时间。 */ @@ -38,12 +43,22 @@ public class LateAlarmBucketWriter { + "VALUES (?,?,?,?,SYSDATE,1,0,0,'PENDING',0,0,SYSDATE,SYSDATE)"; private final JdbcTemplate jdbcTemplate; + private final PlatformTransactionManager transactionManager; + private final BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("LATE_ALARM_BUCKET", 256, 50000, true); + + @Value("${analysis.late-replay.notify-timeout-seconds:5}") + private int transactionTimeoutSeconds = 5; + + @javax.annotation.PostConstruct public void start() { queue.start(); } + @javax.annotation.PreDestroy public void stop() { queue.stop(); } + public Map status() { return queue.status(); } @Value("${analysis.late-replay.outbox-enabled:true}") private boolean enabled; - public LateAlarmBucketWriter(JdbcTemplate jdbcTemplate) { + public LateAlarmBucketWriter(JdbcTemplate jdbcTemplate, PlatformTransactionManager transactionManager) { this.jdbcTemplate = jdbcTemplate; + this.transactionManager = transactionManager; } /** @@ -56,7 +71,27 @@ public class LateAlarmBucketWriter { return; } - Set buckets = new LinkedHashSet<>(); + try { + // 只保留设备/分钟的小键，不把原始日志和事实行滞留在后台队列。 + Set buckets = buckets(rows); + if (!TransactionSynchronizationManager.isActualTransactionActive() + || !TransactionSynchronizationManager.isSynchronizationActive()) { + throw new IllegalStateException("迟到告警通知必须在事实提交后入队"); + } + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override public void afterCommit() { + queue.offer(buckets.size(), () -> writeBuckets(buckets)); + } + }); + } catch (RuntimeException failure) { + queue.loss("PREPARE_OR_REGISTER_FAILED", failure); + } + } + + private Set buckets(List> rows) { + // 不同Kafka批次可能含有相反顺序的设备/分钟，统一加锁顺序避免交叉等待。 + Set buckets = new TreeSet<>(Comparator + .comparing((BucketKey key) -> key.deviceId).thenComparing(key -> key.bucketStart)); for (Map row : rows) { Integer deviceId = integerValue(row.get("device_id")); LocalDateTime eventTime = timeValue(row.get("log_time")); @@ -65,7 +100,14 @@ public class LateAlarmBucketWriter { } buckets.add(new BucketKey(deviceId, eventTime.truncatedTo(ChronoUnit.MINUTES))); } + return buckets; + } + private void writeBuckets(Set buckets) { + TransactionTemplate transaction = new TransactionTemplate(transactionManager); + transaction.setPropagationBehavior(org.springframework.transaction.TransactionDefinition.PROPAGATION_REQUIRES_NEW); + transaction.setTimeout(Math.max(1, transactionTimeoutSeconds)); + transaction.execute(status -> { for (BucketKey key : buckets) { LocalDateTime bucketEnd = key.bucketStart.plusMinutes(1); try { @@ -79,14 +121,16 @@ public class LateAlarmBucketWriter { Timestamp.valueOf(bucketEnd)); } catch (RuntimeException failure) { // 多个 Kafka 分区可能同时首次写同一设备分钟，MERGE 的并发插入会竞争唯一键。 - // 交给外层事实事务整体回滚并有限重试，不能只吞掉 outbox 失败。 + // 仅后台通知事务回滚重试，绝不返回事实事务。 if (isConcurrentWrite(failure)) { throw new DirectStatsProjectionWriter.ConcurrentStatsWriteException( - "迟到告警分钟桶并发写入失败，可安全回滚后有限重试", failure); + "迟到告警分钟桶并发写入失败，仅后台通知事务回滚重试", failure); } throw failure; } } + return null; + }); } private Integer integerValue(Object value) { diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionAsyncPublisher.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionAsyncPublisher.java new file mode 100644 index 0000000..c710718 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionAsyncPublisher.java @@ -0,0 +1,217 @@ +package com.common.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** 连续消费优先：事实提交后只做有界内存offer，Kafka发送与重试在独立线程执行。 */ +@Slf4j +@Component +public class StatsProjectionAsyncPublisher { + private final KafkaTemplate kafka; + private final ObjectMapper mapper; + private final ArrayBlockingQueue queue; + private final long maxPendingGroups; + private final AtomicLong pendingGroups = new AtomicLong(); + private final AtomicLong enqueued = new AtomicLong(); + private final AtomicLong published = new AtomicLong(); + private final AtomicLong failures = new AtomicLong(); + private final AtomicLong droppedEvents = new AtomicLong(); + private final AtomicLong droppedFactRows = new AtomicLong(); + private final AtomicLong nextLossLogAt = new AtomicLong(); + private volatile boolean closed; + private volatile Pending inFlight; + private ScheduledExecutorService executor; + private DefaultKafkaProducerFactory ownedProducerFactory; + + // 与旧统计路径使用相同的 mapper；显式区分 WebConfig 的 httpObjectMapper。 + // 下方构造器复制后关闭类型信息，不修改共享缓存 mapper。 + @Autowired + public StatsProjectionAsyncPublisher(KafkaProperties properties, + @Qualifier("objectMapper") ObjectMapper mapper, + @Value("${stats.projection-async.memory-queue-capacity:1000}") int capacity, + @Value("${stats.projection-async.memory-max-pending-groups:50000}") long maxPendingGroups) { + this(dedicatedFactory(properties), mapper, capacity, maxPendingGroups); + } + + private StatsProjectionAsyncPublisher(DefaultKafkaProducerFactory factory, + ObjectMapper mapper, int capacity, long maxPendingGroups) { + this(new KafkaTemplate<>(factory), mapper, capacity, maxPendingGroups); + this.ownedProducerFactory = factory; + } + + private static DefaultKafkaProducerFactory dedicatedFactory(KafkaProperties properties) { + Map config = new LinkedHashMap<>(properties.buildProducerProperties()); + config.put(ProducerConfig.CLIENT_ID_CONFIG, "syslog-stats-projection-memory"); + // 不与主业务Retry/DLT/告警发送共用KafkaProducer缓冲区，统计断链只影响后台发送。 + return new DefaultKafkaProducerFactory<>(config); + } + + @Value("${stats.projection-async.enabled:true}") private boolean enabled; + @Value("${stats.projection-async.topic:${spring.kafka.consumer.topic}.STATS.PROJECTION}") private String topic; + @Value("${stats.projection-async.publish-timeout-ms:10000}") private long timeoutMs; + @Value("${stats.projection-async.retry-backoff-ms:1000}") private long retryBackoffMs; + + StatsProjectionAsyncPublisher(KafkaTemplate kafka, ObjectMapper mapper, + int capacity, long maxPendingGroups) { + if (capacity < 1 || maxPendingGroups < 1) throw new IllegalArgumentException("统计内存队列容量必须大于0"); + this.kafka = kafka; + this.mapper = mapper.copy(); + this.mapper.deactivateDefaultTyping(); + this.queue = new ArrayBlockingQueue<>(capacity); + this.maxPendingGroups = maxPendingGroups; + } + + @PostConstruct + public void start() { + executor = Executors.newSingleThreadScheduledExecutor(task -> { + Thread thread = new Thread(task, "stats-projection-publisher"); + thread.setDaemon(true); + return thread; + }); + executor.scheduleWithFixedDelay(this::flush, 0, 200, TimeUnit.MILLISECONDS); + log.info("Stats projection publisher ready: mode=BEST_EFFORT_AFTER_COMMIT_KAFKA, enabled={}, " + + "queueCapacity={}, maxPendingGroups={}, topic={}", + enabled, queue.remainingCapacity(), maxPendingGroups, topic); + } + + public void enqueueAfterCommit(StatsProjectionOutboxEvent event, int factRows) { + try { + event.validate(); + event.setOutboxBacked(false); + if (!TransactionSynchronizationManager.isActualTransactionActive() + || !TransactionSynchronizationManager.isSynchronizationActive()) { + throw new IllegalStateException("统计事件必须在事实事务提交后入队"); + } + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override public void afterCommit() { + // 不做序列化、网络发送或数据库写入，任何运行时失败均不得影响已提交事实。 + offer(event, factRows); + } + }); + } catch (RuntimeException failure) { + recordLoss(factRows, "PREPARE_OR_REGISTER_FAILED", failure); + } + } + + private void offer(StatsProjectionOutboxEvent event, int factRows) { + long groups = (long) event.getDevices().size() + event.getRules().size() + event.getCollects().size(); + boolean reserved = false; + try { + if (!enabled || closed) { + recordLoss(factRows, closed ? "SHUTTING_DOWN" : "PROJECTION_DISABLED", null); + return; + } + long current; + do { + current = pendingGroups.get(); + if (groups > maxPendingGroups - current) { + recordLoss(factRows, "GROUP_CAPACITY", null); + return; + } + } while (!pendingGroups.compareAndSet(current, current + groups)); + reserved = true; + if (queue.offer(new Pending(event, groups))) { + reserved = false; // 预算归队列/发送线程所有，发送成功后释放。 + enqueued.incrementAndGet(); + } else { + recordLoss(factRows, "QUEUE_FULL", null); + } + } catch (RuntimeException failure) { + recordLoss(factRows, "ENQUEUE_FAILED", failure); + } finally { + if (reserved) pendingGroups.addAndGet(-groups); + } + } + + /** 独立线程每轮最多发送100个聚合事件；失败保留原eventId，下游可去重。 */ + synchronized void flush() { + if (!enabled || closed) return; + for (int i = 0; i < 100 && !closed; i++) { + if (inFlight == null) inFlight = queue.poll(); + Pending pending = inFlight; + if (pending == null || pending.retryAt > System.currentTimeMillis()) return; + try { + if (pending.payload == null) pending.payload = mapper.writeValueAsString(pending.event); + kafka.send(topic, StatsProjectionOutboxEvent.KAFKA_KEY, pending.payload) + .get(Math.max(1L, timeoutMs), TimeUnit.MILLISECONDS); + published.incrementAndGet(); + pendingGroups.addAndGet(-pending.groups); + inFlight = null; + } catch (Exception failure) { + failures.incrementAndGet(); + pending.retryAt = System.currentTimeMillis() + Math.max(1000L, retryBackoffMs); + log.warn("[STATS_ASYNC_PUBLISH_FAILED] eventId={}, pendingEvents={}, action=BACKGROUND_RETRY, errorType={}", + pending.event.getEventId(), queue.size() + 1, failure.getClass().getName()); + if (failure instanceof InterruptedException) Thread.currentThread().interrupt(); + return; + } + } + } + + public void recordLoss(int factRows, String reason, RuntimeException failure) { + droppedEvents.incrementAndGet(); + droppedFactRows.addAndGet(Math.max(0, factRows)); + long now = System.currentTimeMillis(); + long next = nextLossLogAt.get(); + if (now >= next && nextLossLogAt.compareAndSet(next, now + 10000L)) { + log.error("[STATS_ASYNC_LOSS] reason={}, droppedEventsTotal={}, droppedFactRowsTotal={}, " + + "action=CONTINUE_FACT_CONSUMPTION, errorType={}", reason, droppedEvents.get(), + droppedFactRows.get(), failure == null ? "none" : failure.getClass().getName()); + } + } + + public Map status() { + Map result = new LinkedHashMap<>(); + result.put("mode", "BEST_EFFORT_AFTER_COMMIT_KAFKA"); + result.put("enabled", enabled); + result.put("topic", topic); + result.put("pending_events", queue.size() + (inFlight == null ? 0 : 1)); + result.put("queue_capacity", queue.size() + queue.remainingCapacity()); + result.put("pending_groups", pendingGroups.get()); + result.put("max_pending_groups", maxPendingGroups); + result.put("enqueued_total", enqueued.get()); + result.put("published_total", published.get()); + result.put("publish_failures_total", failures.get()); + result.put("dropped_events_total", droppedEvents.get()); + result.put("dropped_fact_rows_total", droppedFactRows.get()); + return result; + } + + @PreDestroy + public void stop() { + closed = true; + if (executor != null) executor.shutdownNow(); + int pending = queue.size() + (inFlight == null ? 0 : 1); + if (pending > 0) log.warn("[STATS_ASYNC_SHUTDOWN_PENDING] pendingEvents={}, " + + "pendingGroups={}, action=STATISTICS_MAY_BE_LOST", pending, pendingGroups.get()); + if (ownedProducerFactory != null) ownedProducerFactory.destroy(); + } + + private static final class Pending { + final StatsProjectionOutboxEvent event; + final long groups; + String payload; + long retryAt; + Pending(StatsProjectionOutboxEvent event, long groups) { this.event = event; this.groups = groups; } + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionDeltaConsumer.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionDeltaConsumer.java index 3ae37c0..5ae87df 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionDeltaConsumer.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionDeltaConsumer.java @@ -211,11 +211,11 @@ public class StatsProjectionDeltaConsumer { jdbc.batchUpdate("INSERT INTO " + StatsProjectionOutboxWriter.safeTable(appliedTable) + " (event_id,event_schema,applied_at) VALUES (?,?,SYSDATE)", markers); } - if (!events.isEmpty()) { - List acknowledged = new ArrayList<>(events.size()); - for (StatsProjectionOutboxEvent event : events) { - acknowledged.add(new Object[]{event.getEventId()}); - } + List acknowledged = new ArrayList<>(); + for (StatsProjectionOutboxEvent event : events) { + if (event.isOutboxBacked()) acknowledged.add(new Object[]{event.getEventId()}); + } + if (!acknowledged.isEmpty()) { jdbc.batchUpdate("UPDATE " + StatsProjectionOutboxWriter.safeTable(outboxTable) + " SET status='APPLIED',applied_at=SYSDATE,next_attempt_at=NULL," + "updated_at=SYSDATE WHERE event_id=?", acknowledged); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxEvent.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxEvent.java index c6c22af..ef7be4e 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxEvent.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxEvent.java @@ -10,7 +10,7 @@ import java.util.Map; import java.util.UUID; /** - * A durable, batch-aggregated statistics delta written beside the fact rows. + * Batch-aggregated delta shared by the after-commit memory publisher and legacy durable Outbox. * *

The event is additive/max-min only, so events from different application nodes may be * published in a different order without changing the final projection. Kafka duplicates are @@ -23,6 +23,11 @@ public class StatsProjectionOutboxEvent { private String schema = SCHEMA; private String eventId = UUID.randomUUID().toString(); + // 老版消息没有此字段，默认true，兼容清理升级前的Outbox。 + private boolean outboxBacked = true; + + public boolean isOutboxBacked() { return outboxBacked; } + public void setOutboxBacked(boolean outboxBacked) { this.outboxBacked = outboxBacked; } private List devices = new ArrayList<>(); private List rules = new ArrayList<>(); private List collects = new ArrayList<>(); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/KafkaReliabilityConfig.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/KafkaReliabilityConfig.java index 0c84123..3d1c621 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/KafkaReliabilityConfig.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/KafkaReliabilityConfig.java @@ -15,9 +15,11 @@ import org.springframework.util.backoff.FixedBackOff; import java.util.HashMap; import java.util.Map; +import lombok.extern.slf4j.Slf4j; /** Kafka 消费失败边界：业务未确认的消息绝不能被框架默认恢复器静默跳过。 */ @Configuration +@Slf4j public class KafkaReliabilityConfig { @Bean @@ -50,16 +52,26 @@ public class KafkaReliabilityConfig { ConcurrentKafkaListenerContainerFactoryConfigurer configurer, ConsumerFactory consumerFactory, DefaultErrorHandler syslogDefaultErrorHandler, - @Value("${syslog.raw-listener.concurrency:3}") int concurrency) { + @Value("${syslog.raw-listener.concurrency:3}") int concurrency, + @Value("${syslog.raw-listener.max-poll-records:500}") int baseRecords, + @Value("${syslog.raw-listener.adaptive-max-records:3000}") int maxPollRecords) { if (concurrency < 1) { throw new IllegalArgumentException("syslog.raw-listener.concurrency 必须大于 0"); } - ConcurrentKafkaListenerContainerFactory factory = - configuredFactory(configurer, consumerFactory, syslogDefaultErrorHandler); + new com.Modules.NormalData.AdaptiveRawBatchSizer(baseRecords, maxPollRecords); + Map properties = new HashMap<>(consumerFactory.getConfigurationProperties()); + properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); + // 只在所有业务子块完成后手动提交本次poll，禁止自动提交尚未处理的尾部记录。 + properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + ConcurrentKafkaListenerContainerFactory factory = configuredFactory( + configurer, new DefaultKafkaConsumerFactory<>(properties), syslogDefaultErrorHandler); factory.setConcurrency(concurrency); // 在 listener 线程调用 acknowledge() 时立即同步提交，使慢批次日志能够准确覆盖 // “业务处理完成 -> Kafka offset 提交完成”的全部时间；可靠性语义仍是处理成功后提交。 factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE); + log.info("Raw Kafka batch configuration: maxPollRecords={}, baseProcessingRecords={}, " + + "adaptiveMaxRecords={}, concurrency={}", + maxPollRecords, baseRecords, maxPollRecords, concurrency); return factory; } @@ -90,6 +102,47 @@ public class KafkaReliabilityConfig { return factory; } + /** 统计脏键与原始日志隔离；同一维度 key 由 Kafka 分区保证只被一个组成员处理。 */ + @Bean("ingressStatsKafkaListenerContainerFactory") + public ConcurrentKafkaListenerContainerFactory ingressStatsKafkaListenerContainerFactory( + ConcurrentKafkaListenerContainerFactoryConfigurer configurer, + ConsumerFactory consumerFactory, + DefaultErrorHandler syslogDefaultErrorHandler, + @Value("${stats.ingress-metadata.max-poll-records:2000}") int maxPollRecords, + @Value("${stats.ingress-metadata.auto-offset-reset:earliest}") String reset) { + return ingressFactory(configurer, consumerFactory, syslogDefaultErrorHandler, maxPollRecords, reset); + } + + @Bean("largeIngressStatsKafkaListenerContainerFactory") + public ConcurrentKafkaListenerContainerFactory largeIngressStatsKafkaListenerContainerFactory( + ConcurrentKafkaListenerContainerFactoryConfigurer configurer, + ConsumerFactory consumerFactory, + DefaultErrorHandler syslogDefaultErrorHandler, + @Value("${stats.ingress-metadata.auto-offset-reset:earliest}") String reset) { + return ingressFactory(configurer, consumerFactory, syslogDefaultErrorHandler, 5, reset); + } + + private ConcurrentKafkaListenerContainerFactory ingressFactory( + ConcurrentKafkaListenerContainerFactoryConfigurer configurer, + ConsumerFactory source, DefaultErrorHandler errorHandler, + int maxPollRecords, String reset) { + if (maxPollRecords < 1 || !("earliest".equals(reset) || "latest".equals(reset))) { + throw new IllegalArgumentException("入口统计 poll/reset 配置非法"); + } + Map properties = new HashMap<>(source.getConfigurationProperties()); + properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); + properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, reset); + properties.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, 33554432); + properties.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, 67108864); + ConcurrentKafkaListenerContainerFactory factory = configuredFactory( + configurer, new DefaultKafkaConsumerFactory<>(properties), errorHandler); + factory.setBatchListener(true); + factory.setConcurrency(1); + factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE); + return factory; + } + /** 统计脏键与原始日志隔离；同一维度 key 由 Kafka 分区保证只被一个组成员处理。 */ @Bean("statsMaterializationKafkaListenerContainerFactory") public ConcurrentKafkaListenerContainerFactory statsMaterializationKafkaListenerContainerFactory( diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/SyslogBuildStartupLogger.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/SyslogBuildStartupLogger.java new file mode 100644 index 0000000..bc64e7a --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/SyslogBuildStartupLogger.java @@ -0,0 +1,46 @@ +package com.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +/** 标识运行代码的功能修订；不依赖构建插件、镜像标签或可覆盖的配置版本号。 */ +@Slf4j +@Component +public class SyslogBuildStartupLogger { + + // 确认包含此功能修订，不代表唯一构建；后续功能修订时同步更新此标识。 + static final String CODE_REVISION = "ASYNC_STATS_ADAPTIVE_RAW_V20260913_56"; + private final Environment environment; + + public SyslogBuildStartupLogger(Environment environment) { + this.environment = environment; + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationReady() { + log.info("[SYSLOG_BUILD_READY] {}", startupSummary()); + } + + String startupSummary() { + return "codeRevision=" + CODE_REVISION + + ", application=syslog-consumer-rule" + + ", profiles=" + String.join(",", environment.getActiveProfiles()) + + ", rawEnabled=" + value("syslog.raw-consumer.enabled", "true") + + ", rawConcurrency=" + value("syslog.raw-listener.concurrency", "3") + + ", baseProcessingRecords=" + value("syslog.raw-listener.max-poll-records", "500") + + ", adaptiveMaxRecords=" + value("syslog.raw-listener.adaptive-max-records", "3000") + + ", samplingEnabled=" + value("syslog.backlog-sampling.enabled", "true") + + ", statsWriteMode=" + value("stats.write-mode", "DIRECT") + + ", ingressStatsEnabled=" + value("stats.ingress-metadata.enabled", "true") + + ", statsPublisherEnabled=" + value("stats.projection-async.enabled", "true") + + ", lateBucketEnabled=" + value("analysis.late-replay.outbox-enabled", "true") + + ", slowBatchThresholdMs=" + value("syslog.slow-batch.threshold-ms", "500"); + } + + private String value(String key, String fallback) { + return environment.getProperty(key, fallback); + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/BacklogSamplingController.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/BacklogSamplingController.java index 022d08f..972ef3b 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/BacklogSamplingController.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/BacklogSamplingController.java @@ -14,11 +14,14 @@ import java.util.Map; public class BacklogSamplingController { private final BacklogSamplingProperties properties; private final BacklogSamplingMetrics metrics; + private final com.common.service.BacklogSamplingAuditService auditService; public BacklogSamplingController(BacklogSamplingProperties properties, - BacklogSamplingMetrics metrics) { + BacklogSamplingMetrics metrics, + com.common.service.BacklogSamplingAuditService auditService) { this.properties = properties; this.metrics = metrics; + this.auditService = auditService; } @GetMapping("/status") @@ -32,6 +35,7 @@ public class BacklogSamplingController { result.put("degrade_steps", properties.getDegradeSteps()); result.put("policy_version", properties.getPolicyVersion()); result.putAll(metrics.snapshot()); + result.put("audit_async", auditService.status()); return result; } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/IngressStatsController.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/IngressStatsController.java index 29bef2e..0b552bb 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/IngressStatsController.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/IngressStatsController.java @@ -1,6 +1,7 @@ package com.controllers; import com.common.service.IngressStatsProjectionWriter; +import com.common.service.IngressStatsConsumer; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -13,13 +14,17 @@ import java.util.Map; @RequestMapping("/api/syslog/stats/ingress") public class IngressStatsController { private final IngressStatsProjectionWriter writer; + private final IngressStatsConsumer consumer; - public IngressStatsController(IngressStatsProjectionWriter writer) { + public IngressStatsController(IngressStatsProjectionWriter writer, IngressStatsConsumer consumer) { this.writer = writer; + this.consumer = consumer; } @GetMapping("/status") public ResponseEntity> status() { - return ResponseEntity.ok(writer.status()); + Map result = writer.status(); + result.put("consumer", consumer.status()); + return ResponseEntity.ok(result); } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties index e78089a..43c7881 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties @@ -63,7 +63,7 @@ spring.kafka.consumer.topic=agent-01-syslog-topic syslog.raw-consumer.enabled=true -# 调试修改：prod 环境普通 Kafka Consumer 单次最多拉取条数；如需改为2000，将 500 改为 2000。 +# 通用/Retry消费者上限。RAW底层上限由adaptive-max-records决定（3000），业务基础批量由max-poll-records决定（500）。 spring.kafka.consumer.max-poll-records=500 spring.kafka.consumer.properties.max.poll.interval.ms=900000 spring.kafka.consumer.properties.session.timeout.ms=45000 diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties index 0055ec8..8045d9e 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties @@ -84,6 +84,16 @@ syslog.raw-consumer.enabled=true syslog.raw-listener.concurrency=${SYSLOG_RAW_LISTENER_CONCURRENCY:3} # 端到端批次耗时达到该阈值时，打印业务各阶段及 Kafka offset 同步提交耗时；0 表示关闭。 syslog.slow-batch.threshold-ms=${SYSLOG_SLOW_BATCH_THRESHOLD_MS:500} +# 普通业务基础批量500（保留旧配置名）；不再表示底层poll上限。已有同名环境变量仍覆盖基础值。 +# 目标业务批量=ceil(基础条数*10000/max(普通保留万分比,非标保留万分比))，并受下面3000上限限制。 +syslog.raw-listener.max-poll-records=${SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS:500} +# 底层RAW max.poll.records固定为此值；无积压时也可能poll到3000，再按500分块，全部处理后才ACK。 +# 基础值必须大于0且不超过上限；Retry/大日志/独立统计不采用此自适应策略。 +syslog.raw-listener.adaptive-max-records=${SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS:3000} +# 大批次按 Kafka value 序列化字节拆成16MiB业务块，全部完成后提交本次poll；单条不会切断。 +syslog.raw-listener.processing-batch-max-bytes=${SYSLOG_RAW_PROCESSING_BATCH_MAX_BYTES:16777216} +# 每个业务块只向共享规则线程池提交8个worker，避免2000条对应2000个排队任务。 +syslog.normalization.workers-per-batch=${SYSLOG_NORMALIZATION_WORKERS_PER_BATCH:8} # 连续性优先：特殊/系统性异常优先写 DLT；即使 DLT 失败也记录告警并推进原 offset，避免阻塞分区。 syslog.failure-continuity.enabled=${SYSLOG_FAILURE_CONTINUITY_ENABLED:true} # DLT 故障时最多快速等待 1 秒；首次失败后整批直接记录并放行。 @@ -146,7 +156,7 @@ syslog.large-reliability.consumer-group-id=${spring.kafka.consumer.group-id}-lar syslog.large-reliability.consumer.enabled=true -# 调试修改：普通 Kafka Consumer 单次最多拉取条数。如需改为2000，将 500 改为 2000。 +# 通用消费者（如Retry）的上限；RAW底层上限用adaptive-max-records，业务基础值用syslog.raw-listener.max-poll-records。 # 使用 prod profile 时 application-prod.properties 中存在同名配置，必须同步修改，否则 prod 值会覆盖本项。 spring.kafka.consumer.max-poll-records=500 spring.kafka.consumer.properties.max.poll.interval.ms=900000 @@ -163,6 +173,8 @@ spring.task.scheduling.pool.size=10 # 标准化告警事实与迟到重放通知使用同一个达梦事务。 analysis.late-replay.outbox-enabled=${ANALYSIS_LATE_REPLAY_OUTBOX_ENABLED:true} +# 告警提交后后台写分钟桶的事务超时；不在主消费线程等待。JDBC/连接池自身超时仍独立生效。 +analysis.late-replay.notify-timeout-seconds=${ANALYSIS_LATE_REPLAY_NOTIFY_TIMEOUT_SECONDS:5} # 日志配置 logging.level.com.common.schedule=INFO @@ -240,11 +252,18 @@ spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule spring.datasource.hikari.auto-commit=false -# DIRECT 不再写逐事实 change_log；默认只在事实事务追加一条批次 Outbox，日统计异步投影。 +# DIRECT 连续消费优先：事实事务不再写统计Outbox/日统计表，提交后通过有界内存队列异步发布统计。 # 首次部署 DIRECT 前必须先执行 08/09/10/11 脚本并按设计文档排空旧 pending。 stats.write-mode=DIRECT stats.incremental.capture-enabled=false stats.ingress-metadata.enabled=true +# 入口统计独立消费原始Topic，直接聚合写统计表；主消费不再写入口Outbox。 +# 各节点必须使用同一组名，且不得与业务主消费组相同。 +stats.ingress-metadata.group-id=${STATS_INGRESS_GROUP_ID:${spring.kafka.consumer.group-id}-ingress-v1} +stats.ingress-metadata.large-group-id=${STATS_INGRESS_LARGE_GROUP_ID:${stats.ingress-metadata.group-id}-large} +stats.ingress-metadata.max-poll-records=${STATS_INGRESS_MAX_POLL_RECORDS:2000} +# 新组默认从Kafka仍保留的最早消息开始，确保不跳过历史接收时间；已有offset继续原位置。 +stats.ingress-metadata.auto-offset-reset=${STATS_INGRESS_AUTO_OFFSET_RESET:earliest} stats.incremental.read-enabled=true stats.incremental.transaction-timeout-seconds=45 stats.device-receive.change-capture-enabled=false @@ -275,10 +294,13 @@ stats.materialization.concurrency-retry-backoff-ms=${STATS_MATERIALIZATION_CONCU stats.materialization.reconcile-initial-delay-ms=${STATS_MATERIALIZATION_RECONCILE_INITIAL_DELAY_MS:30000} stats.materialization.reconcile-interval-ms=${STATS_MATERIALIZATION_RECONCILE_INTERVAL_MS:60000} -# Fact transactions append one aggregated durable event instead of locking the daily projection rows. -# A dedicated single-lane Kafka consumer applies device/rule/collector projection deltas and then -# emits the existing idempotent business-materialization dirty keys. +# 事实提交后只做内存offer，独立线程发送到统计Kafka；满队列/进程退出可能少统计，不阻碍事实消费。 +# enabled=false 会跳过统计并记录损失，绝不回退为同步统计写库。正常保持true。 +# 旧Outbox继续后台排空，applied表仍用于统计事件去重，不要删表。 stats.projection-async.enabled=${STATS_PROJECTION_ASYNC_ENABLED:true} +# 队列最多1000个批次聚合事件；含正在重试事件的聚合分组总数最多50000，限制内存占用。 +stats.projection-async.memory-queue-capacity=${STATS_PROJECTION_MEMORY_QUEUE_CAPACITY:1000} +stats.projection-async.memory-max-pending-groups=${STATS_PROJECTION_MEMORY_MAX_PENDING_GROUPS:50000} stats.projection-async.outbox-table=${STATS_PROJECTION_OUTBOX_TABLE:stats_projection_outbox} stats.projection-async.applied-table=${STATS_PROJECTION_APPLIED_TABLE:stats_projection_applied_event} stats.projection-async.topic=${STATS_PROJECTION_TOPIC:${spring.kafka.consumer.topic}.STATS.PROJECTION} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/AdaptiveRawBatchSizerTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/AdaptiveRawBatchSizerTest.java new file mode 100644 index 0000000..f4b34c4 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/AdaptiveRawBatchSizerTest.java @@ -0,0 +1,33 @@ +package com.Modules.NormalData; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +class AdaptiveRawBatchSizerTest { + @Test void derivesMultiplierAndCapsWithoutPerStageConfiguration() { + AdaptiveRawBatchSizer sizer = new AdaptiveRawBatchSizer(500, 3000); + assertEquals(500, sizer.targetRecords(10000, 10000)); + assertEquals(1000, sizer.targetRecords(5000, 5000)); + assertEquals(2000, sizer.targetRecords(2500, 2500)); + assertEquals(2500, sizer.targetRecords(1000, 2000)); + assertEquals(3000, sizer.targetRecords(1000, 1000)); + assertEquals(3000, sizer.targetRecords(0, 0)); + assertEquals(500, sizer.targetRecords(0, 10000)); + assertEquals(500, sizer.targetRecords(10000, 0)); + assertEquals(834, sizer.targetRecords(5000, 6000)); + } + + @Test void supportsOtherBaseSizesWithoutOverflow() { + assertEquals(3000, new AdaptiveRawBatchSizer(1000, 3000).targetRecords(1000, 2000)); + assertEquals(Integer.MAX_VALUE, + new AdaptiveRawBatchSizer(Integer.MAX_VALUE, Integer.MAX_VALUE).targetRecords(1, 1)); + } + + @Test void rejectsInvalidConfigurationAndRates() { + assertThrows(IllegalArgumentException.class, () -> new AdaptiveRawBatchSizer(0, 3000)); + assertThrows(IllegalArgumentException.class, () -> new AdaptiveRawBatchSizer(500, 499)); + AdaptiveRawBatchSizer sizer = new AdaptiveRawBatchSizer(500, 3000); + assertThrows(IllegalArgumentException.class, () -> sizer.targetRecords(-1, 2000)); + assertThrows(IllegalArgumentException.class, () -> sizer.targetRecords(1000, 10001)); + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogAdaptiveRawBatchTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogAdaptiveRawBatchTest.java new file mode 100644 index 0000000..4dd250c --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogAdaptiveRawBatchTest.java @@ -0,0 +1,155 @@ +package com.Modules.NormalData; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.kafka.support.Acknowledgment; +import org.springframework.test.util.ReflectionTestUtils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class SysLogAdaptiveRawBatchTest { + private SysLogProcessor processor; + private Acknowledgment ack; + + @BeforeEach void setup() { + processor = spy(new SysLogProcessor()); + ReflectionTestUtils.setField(processor, "taskExecutor", (Executor) Runnable::run); + ReflectionTestUtils.setField(processor, "largeTaskExecutor", (Executor) Runnable::run); + ReflectionTestUtils.setField(processor, "largeProcessingBatchMaxBytes", 16777216L); + ReflectionTestUtils.setField(processor, "failureContinuityEnabled", false); + ack = mock(Acknowledgment.class); + } + + @Test void livePollSplitsInto500WithoutEarlyAckOrRecordLoss() { + List sizes = new ArrayList<>(); + List offsets = new ArrayList<>(); + doAnswer(call -> { + List> chunk = call.getArgument(0); + sizes.add(chunk.size()); + chunk.forEach(record -> offsets.add(record.offset())); + verifyNoInteractions(ack); + processor.observeRawRetention(10000); + return null; + }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + processor.consumeAndParallelProcess(records(3000), ack); + assertEquals(Arrays.asList(500, 500, 500, 500, 500, 500), sizes); + for (int i = 0; i < 3000; i++) assertEquals((long) i, offsets.get(i).longValue()); + verify(ack).acknowledge(); + } + + @Test void backlogExpandsNextChunkAndRecoveryShrinksFollowingChunk() { + List sizes = new ArrayList<>(); + doAnswer(call -> { + sizes.add(((List) call.getArgument(0)).size()); + processor.observeRawRetention(2000); + return null; + }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + processor.consumeAndParallelProcess(records(3000), ack); + assertEquals(Arrays.asList(500, 2500), sizes); + sizes.clear(); + doAnswer(call -> { + sizes.add(((List) call.getArgument(0)).size()); + processor.observeRawRetention(10000); + return null; + }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + processor.consumeAndParallelProcess(records(3000), ack); + assertEquals(Arrays.asList(2500, 500), sizes); + sizes.clear(); + processor.consumeAndParallelProcess(records(3000), ack); + assertEquals(Arrays.asList(500, 500, 500, 500, 500, 500), sizes); + verify(ack, times(3)).acknowledge(); + } + + @Test void tenPercentHits3000CapAndIncompleteTailIsNotHeldWaitingForMore() { + processor.observeRawRetention(1000); + List sizes = new ArrayList<>(); + doAnswer(call -> { sizes.add(((List) call.getArgument(0)).size()); return null; }) + .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + processor.consumeAndParallelProcess(records(3000), ack); + processor.consumeAndParallelProcess(records(17), ack); + assertEquals(Arrays.asList(3000, 17), sizes); + } + + @Test void bytesLimitStillWinsAndOversizedSingleRecordIsNotLost() { + List> records = Arrays.asList( + new ConsumerRecord<>("raw", 0, 0, "k", "中"), + new ConsumerRecord<>("raw", 0, 1, "k", "中"), + new ConsumerRecord<>("raw", 0, 2, "k", "oversized"), + new ConsumerRecord<>("raw", 0, 3, "k", "x")); + ReflectionTestUtils.setField(processor, "rawProcessingBatchMaxBytes", 3L); + processor.observeRawRetention(1000); + List sizes = new ArrayList<>(); + List offsets = new ArrayList<>(); + doAnswer(call -> { + List> chunk = call.getArgument(0); + sizes.add(chunk.size()); + chunk.forEach(record -> offsets.add(record.offset())); + return null; + }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + processor.consumeAndParallelProcess(records, ack); + assertEquals(Arrays.asList(1, 1, 1, 1), sizes); + assertEquals(Arrays.asList(0L, 1L, 2L, 3L), offsets); + verify(ack).acknowledge(); + } + + @Test void failingLaterChunkDoesNotCommitUnfinishedPollAndResetsToBase() { + List sizes = new ArrayList<>(); + doAnswer(call -> { + sizes.add(((List) call.getArgument(0)).size()); + processor.observeRawRetention(2000); + if (sizes.size() == 2) throw new IllegalStateException("DM unavailable"); + return null; + }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + assertThrows(IllegalStateException.class, () -> processor.consumeAndParallelProcess(records(3000), ack)); + verifyNoInteractions(ack); + assertEquals(Arrays.asList(500, 2500), sizes); + sizes.clear(); + doAnswer(call -> { sizes.add(((List) call.getArgument(0)).size()); return null; }) + .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + processor.consumeAndParallelProcess(records(1000), ack); + assertEquals(Arrays.asList(500, 500), sizes); + } + + @Test void feedbackDoesNotLeakAcrossConsumerThreadsOrChangeRetryAndLargeLanes() throws Exception { + processor.observeRawRetention(1000); + List sizes = Collections.synchronizedList(new ArrayList<>()); + doAnswer(call -> { sizes.add(((List) call.getArgument(0)).size()); return null; }) + .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + ExecutorService otherConsumer = Executors.newSingleThreadExecutor(); + try { + otherConsumer.submit(() -> processor.consumeAndParallelProcess(records(1000), ack)).get(3, TimeUnit.SECONDS); + } finally { otherConsumer.shutdownNow(); } + assertEquals(Arrays.asList(500, 500), sizes); + sizes.clear(); + processor.consumeRetryAndReplay(records(1000), ack); + processor.consumeLargeMessages(records(1000), ack); + assertEquals(Arrays.asList(1000, 1000), sizes); + } + + @Test void staleFeedbackResetsAfterOneMinuteWithoutObservation() { + processor.observeRawRetention(1000); + Object state = ((ThreadLocal) ReflectionTestUtils.getField(processor, "rawAdaptiveState")).get(); + ReflectionTestUtils.setField(state, "observedAt", System.nanoTime() - TimeUnit.MINUTES.toNanos(2)); + List sizes = new ArrayList<>(); + doAnswer(call -> { sizes.add(((List) call.getArgument(0)).size()); return null; }) + .when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + processor.consumeAndParallelProcess(records(1000), ack); + assertEquals(Arrays.asList(500, 500), sizes); + } + + private List> records(int count) { + List> result = new ArrayList<>(); + for (int i = 0; i < count; i++) result.add(new ConsumerRecord<>("raw", 0, i, "k", "value")); + return result; + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java index 9afb0f0..d24e8f1 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java @@ -137,6 +137,32 @@ class SysLogProcessorParallelTest { () -> processor.normalizePreparedMessages(messages(2), metrics)); } + @Test + void concurrentLargePollsSubmitOnlyBoundedWorkersAndProcessEveryRecord() throws Exception { + AtomicInteger submissions = new AtomicInteger(); + Executor countedExecutor = task -> { + submissions.incrementAndGet(); + executor.execute(task); + }; + ReflectionTestUtils.setField(processor, "taskExecutor", countedExecutor); + when(normalProcessor.prepare(any(PreparedNormalizationInput.class), any(), any(), any())) + .thenReturn(LogNormalProcessor.NormalizationResult.success( + Collections.singletonList(normalizedRow()))); + ExecutorService callers = Executors.newFixedThreadPool(3); + try { + List> runs = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + runs.add(callers.submit(() -> processor.normalizePreparedMessages(messages(3000), + new SysLogProcessor.BatchMetrics()))); + } + for (java.util.concurrent.Future run : runs) run.get(10, TimeUnit.SECONDS); + assertEquals(24, submissions.get(), "3个并发批次应提交24个worker，而非9000个任务"); + verify(normalProcessor, times(9000)).prepare(any(PreparedNormalizationInput.class), any(), any(), any()); + } finally { + callers.shutdownNow(); + } + } + @Test void recordsRuleComputeAndDmWriteTimingsSeparately() { when(normalProcessor.prepare(any(PreparedNormalizationInput.class), any(), any(), any())) @@ -382,6 +408,31 @@ class SysLogProcessorParallelTest { assertEquals(1L, normalDropped); assertEquals(1L, nonNormalDropped); assertEquals(2L, rawDropped); + assertEquals(0, plan.maxRetainBps()); + ReflectionTestUtils.setField(properties, "normalRetainBps", 1000); + ReflectionTestUtils.setField(properties, "nonNormalRetainBps", 2000); + assertEquals(2000, processor.buildSamplingPlan(computed).maxRetainBps(), + "放大倍率必须使用两类中较高保留率"); + ReflectionTestUtils.setField(properties, "enabled", false); + assertEquals(10000, processor.buildSamplingPlan(computed).maxRetainBps()); + ReflectionTestUtils.setField(properties, "enabled", true); + computed.add(new SysLogProcessor.PreparedNormalization(source.get(0), null, + SyslogFailureRouter.TerminalState.DLT_PUBLISHED)); + assertEquals(10000, processor.buildSamplingPlan(computed).maxRetainBps(), + "规则结果未知时保守回到基础批量"); + computed.remove(computed.size() - 1); + BacklogSamplingDecider mixed = mock(BacklogSamplingDecider.class); + LocalDateTime hour = LocalDateTime.of(2026, 9, 13, 0, 0); + BacklogSamplingDecision degraded = new BacklogSamplingDecision(true, true, true, true, + 3600000L, hour, "test", 1000, 2000); + BacklogSamplingDecision recent = new BacklogSamplingDecision(false, true, true, true, + 0L, hour, "test", 10000, 10000); + when(mixed.decide(any(), any(), org.mockito.ArgumentMatchers.anyBoolean(), + org.mockito.ArgumentMatchers.anyBoolean(), org.mockito.ArgumentMatchers.anyBoolean())) + .thenReturn(degraded, recent, degraded); + ReflectionTestUtils.setField(processor, "backlogSamplingDecider", mixed); + assertEquals(10000, processor.buildSamplingPlan(computed).maxRetainBps(), + "同块包含新旧不同阶段消息时取最高保留率"); } private List> records(int count) { diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogSlowBatchTraceTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogSlowBatchTraceTest.java new file mode 100644 index 0000000..824ec19 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogSlowBatchTraceTest.java @@ -0,0 +1,110 @@ +package com.Modules.NormalData; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.kafka.support.Acknowledgment; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.stream.Collectors; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.*; + +class SysLogSlowBatchTraceTest { + private final Logger logger = (Logger) LoggerFactory.getLogger(SysLogProcessor.class); + private ListAppender appender; + private SysLogProcessor processor; + + @BeforeEach void setup() { + appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + processor = spy(new SysLogProcessor()); + ReflectionTestUtils.setField(processor, "taskExecutor", (Executor) Runnable::run); + ReflectionTestUtils.setField(processor, "slowBatchThresholdMs", 1L); + ReflectionTestUtils.setField(processor, "failureContinuityEnabled", false); + MDC.put("syslogBatchId", "previous-context"); + } + + @AfterEach void cleanup() { + logger.detachAppender(appender); + appender.stop(); + MDC.remove("syslogBatchId"); + } + + @Test void successfulTraceContainsStageBreakdownOffsetsAndRestoresContext() { + doAnswer(call -> { + ReflectionTestUtils.invokeMethod(processor, "enterStage", "PREPARE"); + Object trace = trace(); + ReflectionTestUtils.setField(trace, "startedAtNanos", System.nanoTime() - 10_000_000L); + ReflectionTestUtils.setField(trace, "stageStarted", System.nanoTime() - 2_000_000L); + ReflectionTestUtils.invokeMethod(processor, "enterStage", "ALARM_DM"); + return null; + }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + Acknowledgment ack = mock(Acknowledgment.class); + processor.consumeAndParallelProcess(records(), ack); + verify(ack).acknowledge(); + String log = onlyTrace(); + assertTrue(log.contains("PREPARE=")); + assertTrue(log.contains("ADAPTIVE_CHUNK_SELECTION=")); + assertTrue(log.contains("adaptiveTargetHistogram={500=1}")); + assertTrue(log.contains("actualChunkHistogram={2=1}")); + assertTrue(log.contains("ALARM_DM=")); + assertTrue(log.contains("raw:0=12..14")); + assertTrue(log.contains("kafkaCommitStatus=SUCCESS")); + assertTrue(log.contains("listenerGapMs=-1")); + assertFalse(log.contains("batchId=previous-context")); + assertEquals("previous-context", MDC.get("syslogBatchId")); + assertNull(trace()); + } + + @Test void businessAbortReportsFailingStageAndNoCommitInsteadOfFalseSuccess() { + doAnswer(call -> { + ReflectionTestUtils.invokeMethod(processor, "enterStage", "NORMAL_DM"); + throw new IllegalStateException("DB stalled"); + }).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + Acknowledgment ack = mock(Acknowledgment.class); + assertThrows(IllegalStateException.class, () -> processor.consumeAndParallelProcess(records(), ack)); + verifyNoInteractions(ack); + String log = onlyTrace(); + assertTrue(log.contains("failureStage=NORMAL_DM")); + assertTrue(log.contains("kafkaCommitStatus=NOT_ATTEMPTED")); + assertTrue(log.contains("failedChunks=1")); + assertEquals("previous-context", MDC.get("syslogBatchId")); + } + + @Test void commitFailureIsLoggedExactlyOnce() { + doNothing().when(processor).processBatch(anyList(), anyBoolean(), anyString(), any()); + Acknowledgment ack = mock(Acknowledgment.class); + doThrow(new IllegalStateException("commit rejected")).when(ack).acknowledge(); + assertThrows(IllegalStateException.class, () -> processor.consumeAndParallelProcess(records(), ack)); + String log = onlyTrace(); + assertTrue(log.contains("kafkaCommitStatus=FAILED")); + assertTrue(log.contains("kafkaCommitErrorType=java.lang.IllegalStateException")); + assertEquals("previous-context", MDC.get("syslogBatchId")); + } + + private Object trace() { + return ((ThreadLocal) ReflectionTestUtils.getField(processor, "slowBatchTrace")).get(); + } + private String onlyTrace() { + List logs = appender.list.stream().map(ILoggingEvent::getFormattedMessage) + .filter(line -> line.contains("[SLOW_SYSLOG_BATCH]")).collect(Collectors.toList()); + assertEquals(1, logs.size()); + assertFalse(logs.get(0).contains("batchId={}")); + return logs.get(0); + } + private List> records() { + return Arrays.asList(new ConsumerRecord<>("raw", 0, 12L, "k", "first"), + new ConsumerRecord<>("raw", 0, 14L, "k", "second")); + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BacklogSamplingAuditServiceTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BacklogSamplingAuditServiceTest.java index c5670ee..508c377 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BacklogSamplingAuditServiceTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BacklogSamplingAuditServiceTest.java @@ -41,7 +41,7 @@ class BacklogSamplingAuditServiceTest { } @Test - void auditFailureReturnsFalseAndRecordsFullRetentionFallback() { + void auditFailureIsBackgroundOnlyAndNeverTriggersFullRetentionFallback() { BacklogRetentionAuditMapper mapper = mock(BacklogRetentionAuditMapper.class); when(mapper.upsert(any())).thenThrow(new IllegalStateException("table unavailable")); PlatformTransactionManager transactionManager = transactionManager(); @@ -49,13 +49,18 @@ class BacklogSamplingAuditServiceTest { BacklogSamplingAuditService service = new BacklogSamplingAuditService( mapper, properties(), metrics, transactionManager); - boolean audited = service.recordBeforeDropping(Arrays.asList( + boolean audited = service.recordAsync(Arrays.asList( delta(2, 1, 1, 1, 1))); - assertFalse(audited); + assertTrue(audited); + org.mockito.Mockito.verifyNoInteractions(mapper, transactionManager); + ((BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(service, "queue")).drain(); Map snapshot = metrics.snapshot(); assertEquals(1L, snapshot.get("audit_write_failures_total")); - assertEquals(2L, snapshot.get("audit_fallback_messages_total")); + assertEquals(0L, snapshot.get("audit_fallback_messages_total")); + assertEquals(2L, snapshot.get("audit_lost_or_unconfirmed_messages_total")); + assertFalse(service.recordAsync(Arrays.asList(delta(1, 0, 1, 0, 1)))); + assertEquals(3L, metrics.snapshot().get("degraded_messages_total")); } @Test @@ -67,10 +72,13 @@ class BacklogSamplingAuditServiceTest { BacklogSamplingAuditService service = new BacklogSamplingAuditService( mapper, properties(), metrics, transactionManager); - assertTrue(service.recordBeforeDropping(Arrays.asList( + assertTrue(service.recordAsync(Arrays.asList( delta(1, 0, 1, 0, 1)))); + org.mockito.Mockito.verifyNoInteractions(mapper, transactionManager); + ((BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(service, "queue")).drain(); verify(mapper).upsert(any()); + verify(transactionManager).commit(any()); assertEquals(1L, metrics.snapshot().get("degraded_messages_total")); assertEquals(1L, metrics.snapshot().get("normal_rows_dropped_total")); } diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BestEffortAuxiliaryQueueTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BestEffortAuxiliaryQueueTest.java new file mode 100644 index 0000000..365839c --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/BestEffortAuxiliaryQueueTest.java @@ -0,0 +1,64 @@ +package com.common.service; + +import org.junit.jupiter.api.Test; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import static org.junit.jupiter.api.Assertions.*; + +class BestEffortAuxiliaryQueueTest { + @Test + void hungAuxiliaryDatabaseDoesNotBlockMainEnqueueAndCapacityIsBounded() throws Exception { + BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("TEST", 1, 2, true); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + assertTrue(queue.offer(1, () -> { + entered.countDown(); + try { release.await(5, TimeUnit.SECONDS); } + catch (InterruptedException e) { Thread.currentThread().interrupt(); } + })); + CompletableFuture writer = CompletableFuture.runAsync(queue::drain); + try { + assertTrue(entered.await(2, TimeUnit.SECONDS)); + CompletableFuture.runAsync(() -> { + assertTrue(queue.offer(1, () -> { })); + assertFalse(queue.offer(1, () -> fail("overflow must never run"))); + }).get(2, TimeUnit.SECONDS); + assertEquals(2L, queue.status().get("pending_groups")); + assertEquals(1L, queue.status().get("lost_tasks_total")); + } finally { + release.countDown(); + writer.get(3, TimeUnit.SECONDS); + queue.stop(); + } + assertEquals(2L, queue.status().get("completed_total")); + assertEquals(0L, queue.status().get("pending_groups")); + } + + @Test + void auditFailureIsNotAutomaticallyReplayedAndCapacityIsReleased() { + BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("AUDIT_TEST", 1, 1, false); + AtomicInteger attempts = new AtomicInteger(); + assertTrue(queue.offer(1, () -> { attempts.incrementAndGet(); throw new IllegalStateException("unknown commit"); })); + assertDoesNotThrow(queue::drain); + queue.drain(); + assertEquals(1, attempts.get()); + assertEquals(0L, queue.status().get("pending_groups")); + assertTrue(queue.offer(1, () -> { })); + queue.drain(); + assertEquals(1L, queue.status().get("completed_total")); + queue.stop(); + assertFalse(queue.offer(1, () -> fail("closed"))); + } + + @Test + void queueCountLimitAlsoReleasesReservedGroups() { + BestEffortAuxiliaryQueue queue = new BestEffortAuxiliaryQueue("TEST", 1, 100, false); + assertTrue(queue.offer(1, () -> { })); + assertFalse(queue.offer(1, () -> fail("overflow"))); + assertEquals(1L, queue.status().get("pending_groups")); + queue.drain(); + assertEquals(0L, queue.status().get("pending_groups")); + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DirectStatsProjectionWriterTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DirectStatsProjectionWriterTest.java index 837a704..c7befde 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DirectStatsProjectionWriterTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DirectStatsProjectionWriterTest.java @@ -36,7 +36,7 @@ class DirectStatsProjectionWriterTest { } @Test - void directModeWritesOnlyDimensionBatchesAndMaterializesAffectedRows() { + void directModeAggregatesWithoutAnyStatisticsSql() { JdbcTemplate jdbc = mock(JdbcTemplate.class); DirectStatsProjectionWriter writer = writer(jdbc, "DIRECT", true); TransactionSynchronizationManager.setActualTransactionActive(true); @@ -53,13 +53,7 @@ class DirectStatsProjectionWriterTest { writer.writeStandardizedRows(rows); - ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); - verify(jdbc, times(4)).batchUpdate(sql.capture(), anyList()); - String combined = String.join("\n", sql.getAllValues()); - assertTrue(combined.contains("MERGE INTO device_daily_stats")); - assertTrue(combined.contains("MERGE INTO normalize_rule_daily_stats")); - assertTrue(combined.contains("UPDATE device_device")); - assertTrue(combined.contains("UPDATE dm_normalize_rule")); + verifyNoInteractions(jdbc); assertTrue(((Number) writer.status().get("input_rows")).longValue() == 100L); assertTrue(((Number) writer.status().get("device_groups")).longValue() == 1L); assertTrue(((Number) writer.status().get("rule_groups")).longValue() == 2L); @@ -76,7 +70,7 @@ class DirectStatsProjectionWriterTest { } @Test - void nonNormalUpdateRecomputesOnlyAffectedFactGroupsExactly() { + void shadowNonNormalUpdateRetainsExactRecomputation() { AtomicInteger exactQueries = new AtomicInteger(); JdbcTemplate jdbc = mock(JdbcTemplate.class, invocation -> { if ("queryForMap".equals(invocation.getMethod().getName())) { @@ -88,7 +82,7 @@ class DirectStatsProjectionWriterTest { } return Answers.RETURNS_DEFAULTS.answer(invocation); }); - DirectStatsProjectionWriter writer = writer(jdbc, "DIRECT", true); + DirectStatsProjectionWriter writer = writer(jdbc, "DIRECT_SHADOW", true); TransactionSynchronizationManager.setActualTransactionActive(true); LocalDate today = LocalDate.now(ZoneId.of("Asia/Shanghai")); com.common.entity.SyslogNonNormalMessage oldValue = @@ -104,9 +98,8 @@ class DirectStatsProjectionWriterTest { assertTrue(exactQueries.get() == 2); ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); - verify(jdbc, times(2)).batchUpdate(sql.capture(), anyList()); + verify(jdbc).batchUpdate(sql.capture(), anyList()); assertTrue(sql.getAllValues().get(0).contains("parse_count=s.parse_count")); - assertTrue(sql.getAllValues().get(1).contains("UPDATE device_device")); } @Test @@ -119,12 +112,11 @@ class DirectStatsProjectionWriterTest { } @Test - void durableOutboxRemovesDailyAndBusinessProjectionUpdatesFromFactTransaction() { + void afterCommitPublisherRemovesAllStatisticsSqlFromFactTransaction() { JdbcTemplate jdbc = mock(JdbcTemplate.class); StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class); when(publisher.isEnabled()).thenReturn(true); - StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class); - when(outbox.isEnabled()).thenReturn(true); + StatsProjectionAsyncPublisher outbox = mock(StatsProjectionAsyncPublisher.class); DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter( jdbc, new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), publisher, outbox); configure(writer, true); @@ -137,17 +129,17 @@ class DirectStatsProjectionWriterTest { row.put("normalize_rule_id", 11L); writer.writeStandardizedRow(row); - verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class)); + verify(outbox).enqueueAfterCommit(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class), + org.mockito.ArgumentMatchers.eq(1)); verifyNoInteractions(jdbc); - assertTrue("DURABLE_OUTBOX".equals(writer.status().get("daily_projection_write_mode"))); + assertTrue("BEST_EFFORT_AFTER_COMMIT_KAFKA".equals(writer.status().get("daily_projection_write_mode"))); } private DirectStatsProjectionWriter writer(JdbcTemplate jdbc, String mode, boolean readEnabled) { StatsWriteModeProvider modeProvider = new StatsWriteModeProvider(mode); StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class); when(publisher.isEnabled()).thenReturn(false); - StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class); - when(outbox.isEnabled()).thenReturn(false); + StatsProjectionAsyncPublisher outbox = mock(StatsProjectionAsyncPublisher.class); DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter( jdbc, modeProvider, new DirectStatsWriteMetrics(), publisher, outbox); configure(writer, readEnabled); diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsConsumerTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsConsumerTest.java new file mode 100644 index 0000000..e196aff --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsConsumerTest.java @@ -0,0 +1,90 @@ +package com.common.service; + +import com.Modules.NormalData.SyslogFailureRouter; +import com.common.entity.IngressStatsEvent; +import com.common.util.Sm4Util; +import com.config.AppProperties; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.Acknowledgment; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.anyList; + +class IngressStatsConsumerTest { + private static final String KEY = "0123456789abcdeffedcba9876543210"; + private final IngressStatsProjectionWriter writer = mock(IngressStatsProjectionWriter.class); + private final Acknowledgment ack = mock(Acknowledgment.class); + + private IngressStatsConsumer consumer() { + AppProperties properties = mock(AppProperties.class); + when(properties.getSm4Key()).thenReturn(KEY); + return new IngressStatsConsumer(writer, + new SyslogFailureRouter(mock(KafkaTemplate.class), new ObjectMapper()), properties); + } + + private ConsumerRecord record(String topic, long offset, String time) { + String value = Sm4Util.encryptCbc("[device_id=126 device_collect_id=3 receive_time=" + + time + " event_time=20000101000000000]正文不参与规则或时间统计", KEY); + return new ConsumerRecord<>(topic, 0, offset, "key", value); + } + + @Test + void decodesActualEncryptedHeaderUsingReceiveTimeNotEventTime() { + IngressStatsEvent event = consumer().decode(record("raw", 0, "20260913044500000")); + assertEquals(Long.valueOf(126), event.getDeviceId()); + assertEquals(Long.valueOf(3), event.getDeviceCollectId()); + assertEquals(LocalDateTime.of(2026, 9, 13, 4, 45), event.getReceiveTime()); + } + + @Test + void commitsOnlyAfterStatsDatabaseReturnsAndSupportsLargeTopic() { + consumer().consumeLarge(Collections.singletonList(record("raw.large", 1, "20260913044500000")), ack); + InOrder order = inOrder(writer, ack); + order.verify(writer).writeBatch(anyList()); + order.verify(ack).acknowledge(); + } + + @Test + void databaseFailureRetainsStatsOffsetForRetry() { + when(writer.writeBatch(anyList())).thenThrow(new IllegalStateException("database down")); + assertThrows(IllegalStateException.class, () -> consumer().consume( + Collections.singletonList(record("raw", 0, "20260913044500000")), ack)); + verifyNoInteractions(ack); + } + + @Test + void invalidHeaderIsCountedWithoutDiscardingOtherRecords() { + IngressStatsConsumer consumer = consumer(); + consumer.consume(Arrays.asList(new ConsumerRecord<>("raw", 0, 0L, "k", "bad ciphertext"), + record("raw", 1L, "20260913044500000")), ack); + ArgumentCaptor> events = ArgumentCaptor.forClass(List.class); + verify(writer).writeBatch(events.capture()); + assertEquals(1, events.getValue().size()); + assertEquals(1L, consumer.status().get("invalid_record_attempts")); + verify(ack).acknowledge(); + } + + @Test + void rejectsSharingBusinessConsumerGroup() { + IngressStatsConsumer consumer = consumer(); + ReflectionTestUtils.setField(consumer, "rawGroup", "business"); + ReflectionTestUtils.setField(consumer, "largeRawGroup", "business-large"); + ReflectionTestUtils.setField(consumer, "group", "business"); + ReflectionTestUtils.setField(consumer, "largeGroup", "stats-large"); + assertThrows(IllegalStateException.class, consumer::validateGroups); + ReflectionTestUtils.setField(consumer, "group", "stats"); + assertDoesNotThrow(consumer::validateGroups); + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsProjectionWriterTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsProjectionWriterTest.java index e846156..298bdbf 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsProjectionWriterTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsProjectionWriterTest.java @@ -53,26 +53,27 @@ class IngressStatsProjectionWriterTest { } @Test - void directAsyncModeWritesOnlyOneOutboxEvent() { + void independentIngressWritesAggregatedMinMaxDirectlyAndCommits() { JdbcTemplate jdbc = mock(JdbcTemplate.class); PlatformTransactionManager manager = mock(PlatformTransactionManager.class); when(manager.getTransaction(any(TransactionDefinition.class))) .thenReturn(new SimpleTransactionStatus()); - StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class); - when(outbox.isEnabled()).thenReturn(true); IngressStatsProjectionWriter writer = new IngressStatsProjectionWriter( jdbc, manager, new StatsWriteModeProvider("DIRECT"), - mock(StatsMaterializationPublisher.class), outbox); + mock(StatsMaterializationPublisher.class)); ReflectionTestUtils.setField(writer, "enabled", true); ReflectionTestUtils.setField(writer, "transactionTimeoutSeconds", 45); + ReflectionTestUtils.setField(writer, "businessZone", "Asia/Shanghai"); int groups = writer.writeBatch(Arrays.asList( new IngressStatsEvent("a", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 0)), new IngressStatsEvent("b", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 1)))); assertEquals(2, groups); - verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class)); - verifyNoInteractions(jdbc); + verify(jdbc).update(org.mockito.ArgumentMatchers.contains("MERGE INTO device_daily_stats"), + any(), any(), any()); + verify(jdbc).update(org.mockito.ArgumentMatchers.contains("MERGE INTO device_collect_stats"), + any(), any(), any()); verify(manager).commit(any()); } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/LateAlarmBucketWriterTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/LateAlarmBucketWriterTest.java index e7923d1..9a1df96 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/LateAlarmBucketWriterTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/LateAlarmBucketWriterTest.java @@ -6,6 +6,8 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.doAnswer; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.time.LocalDateTime; import java.util.Arrays; @@ -16,14 +18,60 @@ import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.test.util.ReflectionTestUtils; class LateAlarmBucketWriterTest { + private final org.springframework.transaction.support.TransactionTemplate transaction = + new org.springframework.transaction.support.TransactionTemplate(new TestTransactionManager()); + + @org.junit.jupiter.api.AfterEach void clean() { + org.springframework.transaction.support.TransactionSynchronizationManager.clear(); + } + + private LateAlarmBucketWriter writer(JdbcTemplate jdbc) { + LateAlarmBucketWriter writer = new LateAlarmBucketWriter(jdbc, new TestTransactionManager()); + ReflectionTestUtils.setField(writer, "enabled", true); + return writer; + } + + private void committed(LateAlarmBucketWriter writer, java.util.List> rows) { + transaction.execute(status -> { writer.record(rows); return null; }); + ((BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(writer, "queue")).drain(); + } + + private static class TestTransactionManager extends org.springframework.transaction.support.AbstractPlatformTransactionManager { + protected Object doGetTransaction() { return new Object(); } + protected void doBegin(Object tx, org.springframework.transaction.TransactionDefinition def) { } + protected void doCommit(org.springframework.transaction.support.DefaultTransactionStatus status) { } + protected void doRollback(org.springframework.transaction.support.DefaultTransactionStatus status) { } + } + + @Test + void takesBucketLocksInSameOrderForReversedBatches() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + LateAlarmBucketWriter writer = writer(jdbc); + ReflectionTestUtils.setField(writer, "enabled", true); + java.util.List locks = new java.util.ArrayList<>(); + doAnswer(call -> { + locks.add(call.getArgument(1) + ":" + call.getArgument(2)); + return 1; + }).when(jdbc).update(anyString(), any(), any(), any(), any(), any(), any()); + LocalDateTime time = LocalDateTime.of(2026, 9, 13, 4, 0); + java.util.List> rows = new java.util.ArrayList<>(Arrays.asList( + row(126, time.plusMinutes(1), 1), row(125, time, 2), row(126, time, 3))); + committed(writer, rows); + java.util.List first = new java.util.ArrayList<>(locks); + locks.clear(); + java.util.Collections.reverse(rows); + committed(writer, rows); + assertEquals(first, locks); + assertEquals(3, locks.size()); + } @Test void groupsRowsByDeviceAndEventMinute() { JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); - LateAlarmBucketWriter writer = new LateAlarmBucketWriter(jdbcTemplate); + LateAlarmBucketWriter writer = writer(jdbcTemplate); ReflectionTestUtils.setField(writer, "enabled", true); - writer.record( + committed(writer, Arrays.asList( row(126, LocalDateTime.of(2026, 9, 4, 10, 1, 2), 3), row(126, LocalDateTime.of(2026, 9, 4, 10, 1, 59), 4), @@ -41,13 +89,37 @@ class LateAlarmBucketWriterTest { } @Test - void rejectsAlarmRowsWithoutStableTimes() { - LateAlarmBucketWriter writer = new LateAlarmBucketWriter(mock(JdbcTemplate.class)); + void invalidNotificationDoesNotAbortAlarmTransaction() { + LateAlarmBucketWriter writer = writer(mock(JdbcTemplate.class)); ReflectionTestUtils.setField(writer, "enabled", true); Map invalid = new HashMap<>(); invalid.put("device_id", 126); - assertThrows(IllegalArgumentException.class, () -> writer.record(Arrays.asList(invalid))); + org.junit.jupiter.api.Assertions.assertDoesNotThrow(() -> committed(writer, Arrays.asList(invalid))); + assertEquals(1L, writer.status().get("lost_tasks_total")); + } + + @Test + void commitOnlyEnqueuesRollbackNeverEnqueuesAndDatabaseFailureOnlyRetriesBackground() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + LateAlarmBucketWriter writer = writer(jdbc); + java.util.List> rows = Arrays.asList(row(126, LocalDateTime.now(), 1)); + transaction.execute(status -> { writer.record(rows); status.setRollbackOnly(); return null; }); + assertEquals(0L, writer.status().get("enqueued_total")); + transaction.execute(status -> { writer.record(rows); return null; }); + org.mockito.Mockito.verifyNoInteractions(jdbc); + org.mockito.Mockito.doThrow(new IllegalStateException("database unavailable")) + .when(jdbc).update(anyString(), any(), any(), any(), any(), any(), any()); + BestEffortAuxiliaryQueue queue = (BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(writer, "queue"); + org.junit.jupiter.api.Assertions.assertDoesNotThrow(queue::drain); + assertEquals(1L, writer.status().get("failures_total")); + assertEquals(1, writer.status().get("pending_tasks")); + assertEquals(0L, writer.status().get("lost_tasks_total")); + Object current = ReflectionTestUtils.getField(queue, "current"); + ReflectionTestUtils.setField(current, "retryAt", 0L); + org.mockito.Mockito.doReturn(1).when(jdbc).update(anyString(), any(), any(), any(), any(), any(), any()); + queue.drain(); + assertEquals(1L, writer.status().get("completed_total")); } private Map row(int deviceId, LocalDateTime eventTime, int receiveSecond) { diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncPublisherTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncPublisherTest.java new file mode 100644 index 0000000..b752c03 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncPublisherTest.java @@ -0,0 +1,238 @@ +package com.common.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.SendResult; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.AbstractPlatformTransactionManager; +import org.springframework.transaction.support.DefaultTransactionStatus; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.util.concurrent.SettableListenableFuture; + +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class StatsProjectionAsyncPublisherTest { + private final KafkaTemplate kafka = mock(KafkaTemplate.class); + private final TransactionTemplate transaction = new TransactionTemplate(new TestTransactionManager()); + + @AfterEach + void cleanup() { TransactionSynchronizationManager.clear(); } + + @Test + void productionConstructorUsesDedicatedProducerWithExistingKafkaSettings() { + org.springframework.boot.autoconfigure.kafka.KafkaProperties properties = + new org.springframework.boot.autoconfigure.kafka.KafkaProperties(); + properties.setBootstrapServers(java.util.Collections.singletonList("broker.example:9092")); + properties.getProperties().put("security.protocol", "SASL_PLAINTEXT"); + properties.getProducer().setAcks("all"); + StatsProjectionAsyncPublisher publisher = new StatsProjectionAsyncPublisher(properties, new ObjectMapper(), 10, 10); + org.springframework.kafka.core.DefaultKafkaProducerFactory factory = + (org.springframework.kafka.core.DefaultKafkaProducerFactory) + ReflectionTestUtils.getField(publisher, "ownedProducerFactory"); + assertNotNull(factory); + assertEquals("SASL_PLAINTEXT", factory.getConfigurationProperties().get("security.protocol")); + assertEquals("all", factory.getConfigurationProperties().get("acks")); + assertEquals("syslog-stats-projection-memory", factory.getConfigurationProperties().get("client.id")); + assertEquals(java.util.Collections.singletonList("broker.example:9092"), + factory.getConfigurationProperties().get("bootstrap.servers")); + publisher.stop(); // 不创建连接即可构造/销毁，未依赖真实Kafka。 + } + + @Test + void commitOnlyOffersAndRollbackNeverEnqueues() throws Exception { + StatsProjectionAsyncPublisher publisher = publisher(10, 10); + StatsProjectionOutboxEvent event = event(); + transaction.execute(status -> { + publisher.enqueueAfterCommit(event, 1); + assertEquals(0, metric(publisher, "pending_events")); + verifyNoInteractions(kafka); + return null; + }); + assertEquals(1, metric(publisher, "pending_events")); + verifyNoInteractions(kafka); // 即使Kafka发送会阻塞，事实提交也不调用它。 + transaction.execute(status -> { + publisher.enqueueAfterCommit(event(), 1); + status.setRollbackOnly(); + return null; + }); + assertEquals(1, metric(publisher, "enqueued_total")); + SettableListenableFuture> success = new SettableListenableFuture<>(); + success.set(null); + when(kafka.send(anyString(), anyString(), anyString())).thenReturn(success); + publisher.flush(); + assertEquals(1, metric(publisher, "published_total")); + assertEquals(0, metric(publisher, "pending_groups")); + ArgumentCaptor payload = ArgumentCaptor.forClass(String.class); + verify(kafka).send(eq("stats-topic"), eq(StatsProjectionOutboxEvent.KAFKA_KEY), payload.capture()); + StatsProjectionOutboxEvent sent = new ObjectMapper().readValue(payload.getValue(), StatsProjectionOutboxEvent.class); + assertFalse(sent.isOutboxBacked()); + assertEquals(event.getEventId(), sent.getEventId()); + } + + @Test + void kafkaFailureRetriesSameEventWhileQueueOverflowLetsFactCommit() { + StatsProjectionAsyncPublisher publisher = publisher(1, 10); + SettableListenableFuture> failure = new SettableListenableFuture<>(); + failure.setException(new IllegalStateException("Kafka unavailable")); + SettableListenableFuture> success = new SettableListenableFuture<>(); + success.set(null); + when(kafka.send(anyString(), anyString(), anyString())).thenReturn(failure, success, success); + commit(publisher); + assertDoesNotThrow(publisher::flush); + assertEquals(1, metric(publisher, "publish_failures_total")); + commit(publisher); + commit(publisher); // queue=1，另有一个正在重试；第三条只损失统计。 + assertEquals(2, metric(publisher, "pending_events")); + assertEquals(1, metric(publisher, "dropped_events_total")); + assertEquals(1, metric(publisher, "dropped_fact_rows_total")); + Object pending = ReflectionTestUtils.getField(publisher, "inFlight"); + ReflectionTestUtils.setField(pending, "retryAt", 0L); + publisher.flush(); + assertEquals(2, metric(publisher, "published_total")); + assertEquals(0, metric(publisher, "pending_groups")); + ArgumentCaptor payload = ArgumentCaptor.forClass(String.class); + verify(kafka, times(3)).send(anyString(), anyString(), payload.capture()); + assertEquals(payload.getAllValues().get(0), payload.getAllValues().get(1)); + } + + @Test + void blockedBackgroundSendDoesNotBlockNextCommit() throws Exception { + StatsProjectionAsyncPublisher publisher = publisher(1, 10); + SettableListenableFuture> blocked = new SettableListenableFuture<>(); + java.util.concurrent.CountDownLatch entered = new java.util.concurrent.CountDownLatch(1); + when(kafka.send(anyString(), anyString(), anyString())).thenAnswer(call -> { + entered.countDown(); + return blocked; + }); + commit(publisher); + CompletableFuture worker = CompletableFuture.runAsync(publisher::flush); + try { + assertTrue(entered.await(3, TimeUnit.SECONDS)); + // 使用另一线程+有界等待，防止错误地共享flush锁使测试永久挂住。 + CompletableFuture.runAsync(() -> { commit(publisher); commit(publisher); }).get(2, TimeUnit.SECONDS); + assertEquals(1, metric(publisher, "dropped_events_total")); + } finally { + blocked.set(null); + worker.get(3, TimeUnit.SECONDS); + } + } + + @Test + void groupBudgetIsBoundedAndReleasedAfterSend() { + StatsProjectionAsyncPublisher publisher = publisher(10, 1); + commit(publisher); + commit(publisher); + assertEquals(1, metric(publisher, "pending_groups")); + assertEquals(1, metric(publisher, "dropped_events_total")); + SettableListenableFuture> success = new SettableListenableFuture<>(); + success.set(null); + when(kafka.send(anyString(), anyString(), anyString())).thenReturn(success); + publisher.flush(); + commit(publisher); + assertEquals(2, metric(publisher, "enqueued_total")); + } + + @Test + void missingTransactionDisabledPublisherAndShutdownFailOpen() { + StatsProjectionAsyncPublisher publisher = publisher(10, 10); + assertDoesNotThrow(() -> publisher.enqueueAfterCommit(event(), 1)); + assertEquals(1, metric(publisher, "dropped_events_total")); + ReflectionTestUtils.setField(publisher, "enabled", false); + commit(publisher); + ReflectionTestUtils.setField(publisher, "enabled", true); + publisher.stop(); + commit(publisher); + assertEquals(3, metric(publisher, "dropped_events_total")); + assertEquals(0, metric(publisher, "pending_events")); + verifyNoInteractions(kafka); + } + + @Test + void directStatisticsAggregationFailureDoesNotRollbackFacts() { + StatsProjectionAsyncPublisher publisher = publisher(10, 10); + JdbcTemplate jdbc = mock(JdbcTemplate.class); + DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(jdbc, + new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), + mock(StatsMaterializationPublisher.class), publisher); + ReflectionTestUtils.setField(writer, "businessZone", "invalid-zone"); + assertDoesNotThrow(() -> transaction.execute(status -> { + writer.writeStandardizedRow(row()); + assertFalse(status.isRollbackOnly()); + return null; + })); + assertEquals(1, metric(publisher, "dropped_events_total")); + verifyNoInteractions(jdbc, kafka); + } + + @Test + void directNonNormalUpdateAndDisabledPublishingNeverUseStatisticsSql() { + StatsProjectionAsyncPublisher publisher = publisher(10, 10); + ReflectionTestUtils.setField(publisher, "enabled", false); + JdbcTemplate jdbc = mock(JdbcTemplate.class); + DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(jdbc, + new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), + mock(StatsMaterializationPublisher.class), publisher); + ReflectionTestUtils.setField(writer, "businessZone", "Asia/Shanghai"); + com.common.entity.SyslogNonNormalMessage old = new com.common.entity.SyslogNonNormalMessage(); + old.setDeviceId(7); + old.setCreatedAt(java.time.OffsetDateTime.now()); + com.common.entity.SyslogNonNormalMessage patch = new com.common.entity.SyslogNonNormalMessage(); + patch.setDeviceId(8); + transaction.execute(status -> { + writer.writeStandardizedRow(row()); + writer.writeNonNormalInsert(old); + writer.writeNonNormalUpdate(old, patch); + return null; + }); + assertEquals(3, metric(publisher, "dropped_events_total")); + verifyNoInteractions(jdbc, kafka); + } + + private StatsProjectionAsyncPublisher publisher(int capacity, long groups) { + StatsProjectionAsyncPublisher publisher = new StatsProjectionAsyncPublisher(kafka, new ObjectMapper(), capacity, groups); + ReflectionTestUtils.setField(publisher, "enabled", true); + ReflectionTestUtils.setField(publisher, "topic", "stats-topic"); + ReflectionTestUtils.setField(publisher, "timeoutMs", 10000L); + return publisher; + } + + private void commit(StatsProjectionAsyncPublisher publisher) { + transaction.execute(status -> { publisher.enqueueAfterCommit(event(), 1); return null; }); + } + + private static Map row() { + Map row = new HashMap<>(); + row.put("created_at", LocalDateTime.of(2026, 9, 13, 8, 0)); + row.put("device_id", 7L); + return row; + } + + private static StatsProjectionOutboxEvent event() { + return StatsProjectionOutboxEvent.fromAccumulator(new DirectStatsAccumulator().addStandardizedRow(row())); + } + + private static long metric(StatsProjectionAsyncPublisher publisher, String key) { + return ((Number) publisher.status().get(key)).longValue(); + } + + private static class TestTransactionManager extends AbstractPlatformTransactionManager { + protected Object doGetTransaction() { return new Object(); } + protected void doBegin(Object transaction, TransactionDefinition definition) { } + protected void doCommit(DefaultTransactionStatus status) { } + protected void doRollback(DefaultTransactionStatus status) { } + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncTest.java index 1b30807..c29762e 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncTest.java @@ -138,6 +138,32 @@ class StatsProjectionAsyncTest { return event; } + @Test + void memoryEventUpdatesProjectionAndDedupMarkerWithoutTouchingOutbox() throws Exception { + StatsProjectionOutboxEvent event = oneDeviceEvent("memory-1"); + event.setOutboxBacked(false); + ObjectMapper mapper = new ObjectMapper(); + event = mapper.readValue(mapper.writeValueAsString(event), StatsProjectionOutboxEvent.class); + org.junit.jupiter.api.Assertions.assertFalse(event.isOutboxBacked()); + assertTrue(mapper.readValue("{\"eventId\":\"legacy\"}", StatsProjectionOutboxEvent.class).isOutboxBacked()); + JdbcTemplate jdbc = mock(JdbcTemplate.class); + PlatformTransactionManager manager = mock(PlatformTransactionManager.class); + when(manager.getTransaction(any(TransactionDefinition.class))).thenReturn(new SimpleTransactionStatus()); + StatsProjectionDeltaConsumer consumer = new StatsProjectionDeltaConsumer( + jdbc, manager, mapper, mock(StatsMaterializationPublisher.class)); + configure(consumer); + consumer.applyWithRetry(Collections.singletonList(event)); + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(jdbc, times(2)).batchUpdate(sql.capture(), anyList()); + assertTrue(sql.getAllValues().get(0).contains("MERGE INTO device_daily_stats")); + assertTrue(sql.getAllValues().get(1).contains("stats_projection_applied_event")); + when(jdbc.queryForList(org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.eq(String.class), org.mockito.ArgumentMatchers.any())) + .thenReturn(Collections.singletonList("memory-1")); + assertEquals(1, consumer.applyWithRetry(Collections.singletonList(event)).duplicates); + verify(jdbc, times(2)).batchUpdate(org.mockito.ArgumentMatchers.anyString(), anyList()); + } + private void configure(StatsProjectionDeltaConsumer consumer) { ReflectionTestUtils.setField(consumer, "enabled", true); ReflectionTestUtils.setField(consumer, "appliedTable", "stats_projection_applied_event"); diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionPublisherWiringTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionPublisherWiringTest.java new file mode 100644 index 0000000..346b87f --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionPublisherWiringTest.java @@ -0,0 +1,36 @@ +package com.common.service; + +import com.config.CacheConfig; +import com.config.WebConfig; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.mock.env.MockEnvironment; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.junit.jupiter.api.Assertions.*; + +/** 使用实际配置创建两个同类型Bean，覆盖现场启动时的构造器自动装配。 */ +class StatsProjectionPublisherWiringTest { + @Test + void startsWithCacheAndHttpMappersWithoutChangingSharedSerialization() { + try (AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext()) { + context.setEnvironment(new MockEnvironment() + .withProperty("stats.projection-async.topic", "test.STATS.PROJECTION")); + context.registerBean("objectMapper", ObjectMapper.class, () -> new CacheConfig().objectMapper()); + context.registerBean("httpObjectMapper", ObjectMapper.class, () -> new WebConfig().httpObjectMapper()); + context.registerBean(KafkaProperties.class); + context.register(StatsProjectionAsyncPublisher.class); + assertDoesNotThrow(context::refresh); + assertEquals(2, context.getBeansOfType(ObjectMapper.class).size()); + StatsProjectionAsyncPublisher publisher = context.getBean(StatsProjectionAsyncPublisher.class); + ObjectMapper shared = context.getBean("objectMapper", ObjectMapper.class); + ObjectMapper copied = (ObjectMapper) ReflectionTestUtils.getField(publisher, "mapper"); + assertNotSame(shared, copied); + assertEquals(shared.getRegisteredModuleIds(), copied.getRegisteredModuleIds()); + assertNotNull(shared.getSerializationConfig().getDefaultTyper(shared.constructType(Object.class))); + assertNull(copied.getSerializationConfig().getDefaultTyper(copied.constructType(Object.class))); + } + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/KafkaReliabilityConfigTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/KafkaReliabilityConfigTest.java index 530d20b..1534b36 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/KafkaReliabilityConfigTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/KafkaReliabilityConfigTest.java @@ -8,12 +8,34 @@ import org.springframework.kafka.listener.ContainerProperties; import org.springframework.kafka.listener.DefaultErrorHandler; import java.util.Collections; +import org.mockito.ArgumentCaptor; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; class KafkaReliabilityConfigTest { + @Test + void rawDefaultIs500BaseAnd3000NativePollCeiling() throws Exception { + java.util.Properties properties = new java.util.Properties(); + try (java.io.InputStream input = getClass().getResourceAsStream("/application.properties")) { + properties.load(input); + } + assertEquals("${SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS:500}", + properties.getProperty("syslog.raw-listener.max-poll-records")); + java.lang.reflect.Method method = java.util.Arrays.stream(KafkaReliabilityConfig.class.getDeclaredMethods()) + .filter(candidate -> candidate.getName().equals("rawKafkaListenerContainerFactory")).findFirst().get(); + org.springframework.beans.factory.annotation.Value fallback = method.getParameters()[4] + .getAnnotation(org.springframework.beans.factory.annotation.Value.class); + assertEquals("${syslog.raw-listener.max-poll-records:500}", fallback.value()); + assertEquals("${SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS:3000}", + properties.getProperty("syslog.raw-listener.adaptive-max-records")); + assertEquals("${syslog.raw-listener.adaptive-max-records:3000}", method.getParameters()[5] + .getAnnotation(org.springframework.beans.factory.annotation.Value.class).value()); + } @Test void retryListenerCommitsInsideAcknowledgment() { @@ -39,13 +61,35 @@ class KafkaReliabilityConfigTest { @SuppressWarnings("unchecked") ConsumerFactory consumerFactory = mock(ConsumerFactory.class); DefaultErrorHandler errorHandler = config.syslogDefaultErrorHandler(); + when(consumerFactory.getConfigurationProperties()).thenReturn( + Collections.singletonMap(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 500)); ConcurrentKafkaListenerContainerFactory factory = config.rawKafkaListenerContainerFactory( - configurer, consumerFactory, errorHandler, 3); + configurer, consumerFactory, errorHandler, 3, 500, 3000); assertEquals(ContainerProperties.AckMode.MANUAL_IMMEDIATE, factory.getContainerProperties().getAckMode()); + ArgumentCaptor actual = ArgumentCaptor.forClass(ConsumerFactory.class); + verify(configurer).configure(any(), actual.capture()); + assertEquals(3000, actual.getValue().getConfigurationProperties().get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG)); + assertEquals(false, actual.getValue().getConfigurationProperties().get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)); + assertEquals(500, consumerFactory.getConfigurationProperties().get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG)); + } + + @Test + void independentIngressOverridesAutoCommitAndOffsetResetFromBusinessConsumer() { + KafkaReliabilityConfig config = new KafkaReliabilityConfig(); + ConcurrentKafkaListenerContainerFactoryConfigurer configurer = mock(ConcurrentKafkaListenerContainerFactoryConfigurer.class); + ConsumerFactory source = mock(ConsumerFactory.class); + when(source.getConfigurationProperties()).thenReturn(Collections.singletonMap( + ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true)); + config.ingressStatsKafkaListenerContainerFactory(configurer, source, + config.syslogDefaultErrorHandler(), 2000, "earliest"); + ArgumentCaptor actual = ArgumentCaptor.forClass(ConsumerFactory.class); + verify(configurer).configure(any(), actual.capture()); + assertEquals(false, actual.getValue().getConfigurationProperties().get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)); + assertEquals("earliest", actual.getValue().getConfigurationProperties().get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)); } @Test diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/SyslogBuildStartupLoggerTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/SyslogBuildStartupLoggerTest.java new file mode 100644 index 0000000..3837632 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/config/SyslogBuildStartupLoggerTest.java @@ -0,0 +1,51 @@ +package com.config; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.springframework.mock.env.MockEnvironment; + +import static org.junit.jupiter.api.Assertions.*; + +class SyslogBuildStartupLoggerTest { + @Test + void logsCodeRevisionAndEffectiveOverrides() { + MockEnvironment environment = new MockEnvironment() + .withProperty("syslog.raw-listener.max-poll-records", "700") + .withProperty("syslog.raw-listener.adaptive-max-records", "2500") + .withProperty("stats.projection-async.enabled", "false"); + environment.setActiveProfiles("prod"); + SyslogBuildStartupLogger reporter = new SyslogBuildStartupLogger(environment); + Logger logger = (Logger) LoggerFactory.getLogger(SyslogBuildStartupLogger.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + try { + reporter.onApplicationReady(); + assertEquals(1, appender.list.size()); + String message = appender.list.get(0).getFormattedMessage(); + assertTrue(message.contains("[SYSLOG_BUILD_READY] codeRevision=" + SyslogBuildStartupLogger.CODE_REVISION)); + assertTrue(message.contains("application=syslog-consumer-rule")); + assertTrue(message.contains("profiles=prod")); + assertTrue(message.contains("baseProcessingRecords=700")); + assertTrue(message.contains("adaptiveMaxRecords=2500")); + assertTrue(message.contains("statsPublisherEnabled=false")); + } finally { + logger.detachAppender(appender); + appender.stop(); + } + } + + @Test + void worksWithoutBuildMetadataAndIgnoresConfiguredRevision() { + String summary = new SyslogBuildStartupLogger(new MockEnvironment() + .withProperty("codeRevision", "fake")).startupSummary(); + assertTrue(summary.contains("codeRevision=" + SyslogBuildStartupLogger.CODE_REVISION)); + assertFalse(summary.contains("buildTime=")); + assertTrue(summary.contains("baseProcessingRecords=500")); + assertTrue(summary.contains("adaptiveMaxRecords=3000")); + } + +} diff --git a/haobang-security-dm/upgrade-doc/v4/README.md b/haobang-security-dm/upgrade-doc/v4/README.md index ae43e01..23f35d0 100644 --- a/haobang-security-dm/upgrade-doc/v4/README.md +++ b/haobang-security-dm/upgrade-doc/v4/README.md @@ -4,13 +4,16 @@ v4 只升级 `syslog-consumer-rule` 和达梦数据库，不修改 `syslog-serve`，也不新增 Kafka Topic。 +后续吞吐优化（独立入口统计、事实计数/审计/分钟桶异步化、RAW底层poll上限3000/业务基础500自适应、精细慢批次日志）见 +[吞吐优化升级配置与验证](./config/02-throughput-upgrade.md)。已完成v4表升级的现场应用此补丁无需新增DDL。 + 本次升级包含： - 根据消息 `receive_time` 与当前时间的差值，对积压消息分级采样； - `syslog_normal_alarm` 始终全量保留，并在 InfluxDB 原文、普通表和非标表之前单独提交； - 未被保留的 `syslog_normal_data`、`syslog_non_normal_message` 及对应原文不写入存储； - 按消息来源小时、Topic、Partition、策略版本和实际保留率记录采样审计； -- 审计表不可用时自动回退为全量保留，不执行有损采样； +- 最新吞吐补丁采用消费优先：审计后台尽力写入，失败不取消采样；分钟桶提交后后台重试，可能丢通知，详见上述文档； - InfluxDB 原文写入变为非关键旁路，启动检查或运行时写入失败只记录并继续主消费。 ## 文件与执行顺序 diff --git a/haobang-security-dm/upgrade-doc/v4/config/01-new-environment-variables.md b/haobang-security-dm/upgrade-doc/v4/config/01-new-environment-variables.md index 58ad7ba..0340594 100644 --- a/haobang-security-dm/upgrade-doc/v4/config/01-new-environment-variables.md +++ b/haobang-security-dm/upgrade-doc/v4/config/01-new-environment-variables.md @@ -18,7 +18,7 @@ syslog-consumer-rule/docker-compose/docker-compose-consumer-rule-prod.yaml - SYSLOG_BACKLOG_SAMPLING_DEGRADE_STEPS=10 - SYSLOG_BACKLOG_SAMPLING_POLICY_VERSION=v2 - # 小时审计最多等待 5 秒；失败后 60 秒内不重试，并回退全量保留。 + # 最新吞吐补丁：小时审计在后台独立事务执行；失败后60秒跳过新审计，不取消采样。 - SYSLOG_BACKLOG_SAMPLING_AUDIT_TIMEOUT_SECONDS=5 - SYSLOG_BACKLOG_SAMPLING_AUDIT_RETRY_BACKOFF_MS=60000 @@ -36,13 +36,14 @@ syslog-consumer-rule/docker-compose/docker-compose-consumer-rule-prod.yaml 每级4.5分钟； - `START_LAG_MS` 必须大于等于0并且小于 `LAG_THRESHOLD_MS`，配置非法时应用拒绝启动； - 修改起始延迟、结束阈值、最低保留率或级数时，必须同时修改 `POLICY_VERSION`，以便审计区分策略； -- `AUDIT_TIMEOUT_SECONDS` 是单次达梦审计事务超时；审计失败时本批次全量保留； +- `AUDIT_TIMEOUT_SECONDS` 是后台单次达梦审计事务超时；审计失败记录损失，业务仍按采样比例处理； - `INFLUXDB_RAW_WRITE_REQUEST_TIMEOUT` 是连接、写入、读取各网络阶段的上限，不是整个批次的总超时； - `INFLUXDB_RAW_FAILURE_BACKOFF_MS` 是确认 InfluxDB 依赖故障后的快速跳过时间。 这些默认值已经写在 JAR 的 `application.properties` 中。现场只替换 JAR 时同样会默认开启采样；显式写入 Compose 是为了便于后续核查和调整。启动新 JAR 前必须先创建 -`SYSLOG_BACKLOG_RETENTION_HOURLY`，否则应用会因审计失败回退全量保留，并周期性记录错误日志。 +`SYSLOG_BACKLOG_RETENTION_HOURLY`，否则审计会持续失败、丢失统计，但不会取消采样或阻止业务消费。 +最新普通RAW底层poll上限3000、业务基础500按保留率自适应，配置及精细日志说明见 [吞吐补丁](./02-throughput-upgrade.md)。 关闭采样只需设置： diff --git a/haobang-security-dm/upgrade-doc/v4/config/02-throughput-upgrade.md b/haobang-security-dm/upgrade-doc/v4/config/02-throughput-upgrade.md new file mode 100644 index 0000000..5107d5c --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v4/config/02-throughput-upgrade.md @@ -0,0 +1,199 @@ +# 吞吐优化：统计退出事实写库路径、普通业务批次自适应500～3000条 + +只升级 `syslog-consumer-rule`，保持 JDK 8。无需新增表或 Topic；不能删除现有 +`stats_projection_outbox`、`stats_projection_applied_event`：前者用于排空旧事件及兼容回滚，后者继续用于统计去重。 + +## 本次行为 + +- 普通 RAW 底层每次poll上限固定3000，业务块以500为基础按采样阶段动态扩大（默认最低档2500，最多3000）。 + 无积压时也可能一次poll返回3000条，随后分成500条业务块；Retry仍使用通用配置，大日志仍默认5条。 +- 普通业务处理按 Kafka value 序列化长度分成最多16MiB的块，单条不拆分；一个poll的全部块完成后ACK。 +- 每个业务块最多8个规则worker，按原顺序汇总结果。不会为2000条消息同时排队2000个任务。 +- 主处理方法不再调用入口统计写库，因此主批次日志 `ingressMs=0`。 +- 独立统计消费组读取原始普通/大日志Topic，解密后只读取设备头，不做规则或采样。 + 批内聚合后直接更新设备/采集任务MIN/MAX时间并提交达梦，再提交自己的offset；不写入口Outbox。 +- 独立统计数据库异常只重试统计组，不推进其offset。非法密文/头记录 + `INGRESS_STATS_INVALID` 和错误尝试计数后跳过，不阻塞其它入口状态、不重复投递业务Retry。 + 密钥配置错误也会导致非法记录，发现该标记必须检查；原始记录只能在Kafka保留期内重放。 +- 标准化告警优先提交；告警分钟桶改为提交后入队，后台独立事务按设备、分钟固定顺序写入。 + 通知失败只后台重试，不回滚已入库告警；队列满/进程退出可能丢通知，导致部分迟到数据漏重算。 +- 采样审计也改为后台独立事务：失败不再取消采样，不回退全量入库。审计可能缺失或提交结果不确定， + 对累加审计不盲目重试，避免重复计数；内存采样指标描述决策，不代表审计或业务数据已成功写库。 +- 生产默认 `stats.write-mode=DIRECT` 下，告警、普通、非标事实计数不再写Outbox，也不在事实事务内更新日统计表。 + 只做批内聚合并注册提交回调，事务成功提交后使用非等待式 `offer` 入有界内存队列。 + Kafka发送、等待确认和重试在专用后台线程执行，使用独立Producer缓冲区；事实回滚不会入队。 +- 统计构造异常、队列满、关闭发布等情况记录异常/计数后放行，不回滚事实、不重试主消费。 + Kafka发送失败暂存在有界队列内后台重试；饱和后丢弃新统计事件，不丢弃对应事实数据。 + 进程退出、崩溃及队列溢出可能造成统计缺失。这是本次明确选择的“消费连续性优先”，不保证统计精确。 +- 新统计事件标记 `outboxBacked=false`，独立投影消费者仅写统计及去重标记，不访问Outbox。 + 老事件默认 `true`，继续按原流程排空和标记Outbox。混合升级期间旧消费者仍可能执行无命中的Outbox UPDATE， + 应完成所有消费节点升级。旧入口Outbox的MIN/MAX重放安全。 + +## properties及变量 + +均位于 `syslog-consumer-rule/src/main/resources/application.properties`，prod继承这些配置。 + +| properties | 环境变量 | 默认值 | +| --- | --- | --- | +| syslog.raw-listener.max-poll-records | SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS | 500 | +| syslog.raw-listener.adaptive-max-records | SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS | 3000 | +| syslog.raw-listener.processing-batch-max-bytes | SYSLOG_RAW_PROCESSING_BATCH_MAX_BYTES | 16777216 | +| syslog.normalization.workers-per-batch | SYSLOG_NORMALIZATION_WORKERS_PER_BATCH | 8 | +| stats.ingress-metadata.group-id | STATS_INGRESS_GROUP_ID | 主消费组名加 `-ingress-v1` | +| stats.ingress-metadata.large-group-id | STATS_INGRESS_LARGE_GROUP_ID | 入口统计组名加 `-large` | +| stats.ingress-metadata.max-poll-records | STATS_INGRESS_MAX_POLL_RECORDS | 2000 | +| stats.ingress-metadata.auto-offset-reset | STATS_INGRESS_AUTO_OFFSET_RESET | earliest | +| stats.projection-async.memory-queue-capacity | STATS_PROJECTION_MEMORY_QUEUE_CAPACITY | 1000 | +| stats.projection-async.memory-max-pending-groups | STATS_PROJECTION_MEMORY_MAX_PENDING_GROUPS | 50000 | +| analysis.late-replay.notify-timeout-seconds | ANALYSIS_LATE_REPLAY_NOTIFY_TIMEOUT_SECONDS | 5 | + +只替换JAR也会使用上述默认值；已有 `SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS=2000` 时须删除或改成500。 +`spring.kafka.consumer.max-poll-records=500` 留给通用消费者，不再决定普通RAW批次上限。 +Kafka上限不保证每次拉满；业务块还受到16MiB限制。达梦每个小事务仍最多500行。 +保留旧配置名 `syslog.raw-listener.max-poll-records`，但其含义现在是**业务基础条数**。 +底层RAW的 `max.poll.records` 固定使用 `adaptive-max-records`，不重建consumer、不反射修改Kafka内部参数。 +基础值必须大于0且不超过上限，非法配置拒绝启动。独立入口统计默认2000条（不做规则计算），不采用该策略。 + +业务块目标条数 = min(上限, ceil(基础条数 × 10000 / 较高保留万分比))。 +普通最低1000、非标最低2000，取较高2000，500×10000÷2000=2500；两者都为1000时命中3000上限。 +两者都为0时也只到上限，避免除0；告警仍全部保留。各阶段自动沿用现有采样决策，不另配倍率。 + +实现采用**上一业务块的阶段反馈决定下一块大小**。每个RAW消费线程独立，首次或超过1分钟无有效反馈时从500探测。 +一个块内存在多个延迟阶段时取最高保留率，准备/规则异常保守回到500；恢复全量保存后的下一块回到500。 +例如首次poll3000条且均为默认最低档：先处理500，确认保留率后处理余下2500；后续poll可按2500+500处理。 +不会为了凑足目标条数等待新消息，也不会跳过已poll的尾部记录；每个poll所有块处理完才提交offset。 +启用连续性放行时失败块仍沿用既有DLT/记录策略，之后以基础批量继续，未增加新的业务丢弃规则。 +所有消息仍要解密、跑规则才能确认告警，扩大批量会增加整体poll处理耗时、内存和GC压力。 +整次poll的处理必须在max.poll.interval.ms内完成；16MiB仅限制单个业务块，不限制整个Kafka缓存。 + +内存队列容量按批次聚合事件计，不是原始日志条数；另有最多1个发送中事件。 +聚合分组上限包含排队和发送中事件，不储存原文。默认每200ms调度发送，失败按既有 +`stats.projection-async.retry-backoff-ms`（默认1000ms）后台重试，保留同一eventId以便下游去重。 +`stats.projection-async.enabled=false` 不会回退到同步统计SQL，而是跳过统计、记录损失；正常保持true。 +历史 `DIRECT_SHADOW` / `LEGACY_CHANGE_LOG` 模式保留旧语义，本次消除统计同步写库针对默认DIRECT。 + +保留 `stats.ingress-metadata.enabled=true`。此前建议用 `STATS_INGRESS_METADATA_ENABLED=false` +应急关闭不适用：当正式统计回写启用时，现有启动校验会拒绝此组合。 + +## 新消费组首次启动 + +三个节点需使用相同的入口统计组名；它们不能与业务组相同，程序会拒绝这种错误配置。 +新组需要读取对应原始Topic和提交offset的Kafka权限。 + +默认 `earliest` 从Kafka当前仍保留的最早记录开始，能够补齐保留期内入口时间；已有组offset时续跑。 +现场有大量历史数据时，新统计组也会回放这些记录，增加Kafka网络、解密CPU和数据库负载。 +只做MIN/MAX更新，不会根据原始消息再次增加成功入库条数。 + +若现场明确只需接管上线后的接收时间，可在首次启动前设置 `STATS_INGRESS_AUTO_OFFSET_RESET=latest`； +该选择不会回放启动前尚未更新的入口时间，且不影响已有offset。不能仅为了让LAG显示为零而选latest。 +Kafka保留期必须覆盖统计积压；过期记录无法由这个消费组恢复。 + +不需要重置业务组offset。混合版本滚动期间，旧版仍可能写入口Outbox，新版直接更新MIN/MAX， +这些时间字段允许重复；三个节点完成升级后主消费才全部移除入口统计等待。 + +## 现场验证 + +应用就绪时输出一条 `[SYSLOG_BUILD_READY]`，本次代码标识为 +`codeRevision=ASYNC_STATS_ADAPTIVE_RAW_V1`。同时打印应用名以及生效的基础条数、上限、并发和统计开关。 +标识固定在 Java 代码中，不依赖构建插件，无需修改 pom.xml，也不打印构建时间。 +代码标识证明包含本次功能修订，不代表唯一构建或未来永远是最新版本；后续功能修订需同步更新标识。 +只检查当前容器这次启动后的日志，避免把上次启动的记录当成本次证据： + +```bash +docker logs --since "$(docker inspect -f '{{.State.StartedAt}}' syslog-consumer-rule-dm)" syslog-consumer-rule-dm 2>&1 | grep 'SYSLOG_BUILD_READY' | tail -1 +``` + +该日志表示应用已就绪，不证明数据库写入正常或 Kafka 已分配分区，仍需检查以下运行指标。 + +在各消费节点执行（宿主机，单行命令）： + +```bash +docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'Raw Kafka batch configuration|Ingress stats consumer ready' | tail -10 +docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'SLOW_SYSLOG_BATCH|SLOW_DM_TRANSACTION|SLOW_DM_QUEUE' | tail -80 +docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'INGRESS_STATS_BATCH|INGRESS_STATS_INVALID|INGRESS_STATS_WRITE_FAILED' | tail -30 +docker exec syslog-consumer-rule-dm sh -c 'curl -fsS http://127.0.0.1:8289/xdrrule/api/syslog/stats/ingress/status' +docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'STATS_ASYNC_LOSS|STATS_ASYNC_PUBLISH_FAILED|STATS_ASYNC_SHUTDOWN_PENDING' | tail -30 +docker exec syslog-consumer-rule-dm sh -c 'curl -fsS http://127.0.0.1:8289/xdrrule/api/syslog/stats/direct/status' +``` + +确认普通配置输出 `maxPollRecords=3000, baseProcessingRecords=500, adaptiveMaxRecords=3000`， +主批次 `ingressMs=0`，入口状态模式为 `INDEPENDENT_KAFKA_DIRECT_MERGE`。 +`polledRecords`可大于500；`adaptiveTargetHistogram`和`actualChunkHistogram`分别表示业务目标与实际块大小。 +例如 `{500=1, 2500=1}` 表示对应大小各1块，不代表保留率。实际大小受剩余条数及字节上限限制。 +直接统计状态中 `daily_projection_write_mode=BEST_EFFORT_AFTER_COMMIT_KAFKA`， +`async_publisher` 下有排队事件/分组数、已发布、失败重试、丢弃计数。 +`dropped_fact_rows_total` 指受统计缺失影响的事实行数/尝试行数，并不表示事实数据被删除。 +这些内存指标会在重启后清零，不能覆盖崩溃时丢失的统计。损失日志最多每10秒一次，计数逐次累加。 +`published_total` 仅表示Kafka确认，不代表统计已落达梦；统计最终入库还需检查投影消费组积压。 +`SLOW_DM_TRANSACTION` 自动沿用500ms阈值，无须打开所有事务INFO日志： +`connectionBeginMs`包括取连接/事务初始化，`lateAlarmBucketMs`仅为分钟桶小键准备/注册提交回调， +`jdbcExecuteMs`是JDBC批执行，`statsProjectionStageMs`在DIRECT下是统计事件构造/注册回调阶段， +不含后台Kafka/统计写库；`commitMs`是事实事务提交阶段。状态中的统计成功指标表示聚合/注册完成，不保证投影落库。 +`SLOW_DM_QUEUE` 单独说明批写任务排队超过阈值。 + +在有Kafka脚本的现场节点执行（已确认bootstrap地址）： + +```bash +./kafka-consumer-groups.sh --bootstrap-server 10.141.121.35:9092 --group agent-01-syslog-group --describe +./kafka-consumer-groups.sh --bootstrap-server 10.141.121.35:9092 --group agent-01-syslog-group-ingress-v1 --describe +./kafka-consumer-groups.sh --bootstrap-server 10.141.121.35:9092 --group agent-01-syslog-group-ingress-v1-large --describe +``` + +如现场组名不同，使用启动日志中的真实组名。间隔60秒再查，比较所有分区CURRENT-OFFSET增量之和与 +LOG-END-OFFSET增量之和；前者持续大于后者才说明追赶积压。不要只比较筛选出的慢批次或单批毫秒数。 + +入口统计已不再阻塞主线程，但仍共享Kafka、JVM和数据库资源，不能保证固定倍数提速。 +本地测试不替代现场达梦压测。主消费仍可能受事实表执行、COMMIT、规则计算、InfluxDB及异常路由等待限制。 +采样审计和告警分钟桶不再在主消费/事实事务执行SQL，但后台仍共享数据库和连接池资源。 + +## 精细慢批次日志与后台状态 + +```bash +docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'SLOW_SYSLOG_BATCH|SLOW_DM_TRANSACTION|SLOW_DM_QUEUE|SLOW_NON_NORMAL_WRITE' | tail -80 +docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep -E 'SYSLOG_AUX_FAILURE|SYSLOG_AUX_LOSS|SYSLOG_AUX_SHUTDOWN_PENDING|SLOW_SYSLOG_AUX' | tail -40 +docker exec syslog-consumer-rule-dm sh -c 'curl -fsS http://127.0.0.1:8289/xdrrule/api/syslog/backlog-sampling/status' +``` + +拿到慢批次的batchId后，用 `docker logs --since 10m syslog-consumer-rule-dm 2>&1 | grep '实际batchId'` +关联该批次的达梦事务/队列、非标异步写入日志。日志不打印原文。 + +| 字段 | 如何判断 | +| --- | --- | +| batchId / offsetRanges | 本次监听调用的关联ID、各分区最小/最大offset（范围不代表中间全部连续） | +| adaptiveRaw / baseProcessingRecords / adaptiveMaxRecords | 是否RAW自适应通道、基础条数、硬上限；Retry/大日志adaptiveRaw=false | +| adaptiveTargetHistogram / actualChunkHistogram | 各业务目标条数/实际条数对应的块数，按整次poll累计 | +| observedRetainBps / nextTargetRecords | 最近完成块观测的较高保留万分比及下一块目标；依赖上一块反馈，可能滞后一块 | +| stageMs.ADAPTIVE_CHUNK_SELECTION | 计算RAW目标条数及按字节/条数选取业务块耗时 | +| listenerGapMs | 同一监听线程上次调用结束到本次调用的间隔；首次为-1，不能据此直接判定Kafka空闲或提交慢 | +| stageMs.PREPARE | 解密、元信息解析、原文点构造及准备失败路由 | +| stageMs.RULE_SNAPSHOT | 规则快照及批次上下文准备 | +| stageMs.RULE_EXECUTION_WAIT | 规则任务提交、排队和执行的整体墙钟时间 | +| ruleQueueMaxMs / ruleWorkerBusySumMs | 规则worker最大排队时间/worker执行累计时间；累计值可大于墙钟时间，不可与stageMs相加 | +| stageMs.SAMPLING_DECISION / AUDIT_ENQUEUE | 采样决策/审计聚合入队；没有审计SQL等待 | +| stageMs.ALARM_DM / NORMAL_DM | 告警/普通表批写等待；用相同batchId的SLOW_DM_QUEUE和SLOW_DM_TRANSACTION继续分解 | +| stageMs.DM_FAILURE_ROUTE / CONTINUITY_DLT | 业务坏行或整批异常的Retry/DLT处理耗时 | +| stageMs.NON_NORMAL_ENQUEUE | 只表示非标异步提交耗时，不表示非标落库；看SLOW_NON_NORMAL_WRITE的queueMs/writeMs | +| stageMs.INFLUX_AND_FAILURE_ROUTE | 仍然同步的原文写入及其失败处理耗时（此次未改成异步） | +| alarmRequested/alarmSaved/normalRequested/normalSaved | 计划提交行数及成功返回的写入报告行数；异常中断时不是数据库最终事实数 | +| rawSampledOut/normalSampledOut/nonNormalSampledOut | 已完成业务块的采样丢弃数，不包括异常丢弃 | +| auditAcceptedChunks/auditLostChunks | 采样审计入队成功/失败块数；入队成功不代表后台已入库 | +| failedChunks/failureStage/businessError | 中断的业务块数、最后失败阶段、异常类型；正常无失败时failureStage可为null | +| kafkaCommitStatus | SUCCESS、FAILED、NOT_ATTEMPTED；业务中断不得误判为Kafka提交成功 | + +`stageMs`各阶段为互斥墙钟时间，跨业务块累加；旧有normalizeMs等字段包含部分子阶段，不能再与stageMs相加。 +兼容字段ruleComputeWorkerMs实际是整批规则计算墙钟耗时；新增ruleWorkerBusySumMs才是worker耗时之和，二者不要混淆。 +业务异常和提交异常即使小于500ms也输出诊断。`processedRecords/prepared`等旧计数仅覆盖完成的业务块； +出现failedChunks时不能把它们理解为整次poll已成功入库。无限等待中的操作尚未结束，不会提前输出最终慢批次。 + +采样审计接口新增 `audit_async`，直接统计接口的 `dm_transaction.late_alarm_bucket_async` 查看通知队列。 +两个队列各固定最多256个等待任务、最多50000个小分组（含发送中），各有独立单线程；满时记录后放行。 +审计失败后按既有audit-retry-backoff-ms暂停后台执行并跳过新审计，不取消采样；分钟桶每1秒后台重试失败任务。 +辅助异常日志最多每10秒一次，损失计数逐次累加，重启清零。进程退出可能丢失队列任务；未实现补偿扫描。 +新审计指标 `audit_lost_or_unconfirmed_messages_total` 表示未记录或提交结果未知的消息数， +旧 `audit_fallback_messages_total` 本路径不再增加。 + +## 回滚 + +回滚旧JAR会恢复主消费同步入口统计、事实计数Outbox、同步审计及分钟桶保护，新独立listener随进程退出停止。 +新版本退出时尚未发送的内存统计可能丢失；回滚不自动恢复这些统计，后续需要另行校准。 +保留新增统计组offset、现有Outbox和审计表；不删除Topic、不重置主消费offset。