From 0e9e6656db7e845e2c954bd489c1643548cf7ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E7=BB=8D=E5=AE=8F?= <994233303@qq.com> Date: Sat, 12 Sep 2026 19:22:52 +0800 Subject: [PATCH] =?UTF-8?q?feat(syslog):=20=E4=BC=98=E5=8C=96=E5=A4=A7?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=80=9A=E9=81=93=E5=B9=B6=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E8=BF=9E=E7=BB=AD=E6=B6=88=E8=B4=B9=E5=85=9C?= =?UTF-8?q?=E5=BA=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 128 KiB 及以上日志默认进入大日志 Topic - InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存 - 截断内容追加 HB_XDR_TRUNCATED 标记 - InfluxDB 写入失败时继续执行规则解析和达梦写入 - 未知或系统性异常优先写入普通/大日志 DLT - DLT 写入失败时记录特殊告警并提交 offset，避免阻塞后续消息 - DLT 失败等待时间默认限制为 1 秒 - 增加连续性放行、DLT 成功及失败监控指标 - 更新 Docker Compose、v3 升级文档及相关测试 --- haobang-security-dm/.gitattributes | 1 + .../docker-compose-consumer-rule-prod.yaml | 16 + .../Modules/NormalData/SysLogProcessor.java | 131 +++++- .../NormalData/SyslogFailureRouter.java | 24 +- .../service/DirectStatsProjectionWriter.java | 40 +- .../common/service/DmLargeTextLimiter.java | 89 ++++ .../service/DmStandardizedBatchExecutor.java | 6 +- .../service/DmStandardizedBatchWriter.java | 14 +- .../service/IngressStatsProjectionWriter.java | 16 +- .../service/StatsProjectionDeltaConsumer.java | 386 ++++++++++++++++++ .../service/StatsProjectionOutboxEvent.java | 221 ++++++++++ .../service/StatsProjectionOutboxRelay.java | 151 +++++++ .../service/StatsProjectionOutboxWriter.java | 70 ++++ .../SyslogNonNormalMessageServiceImpl.java | 11 + .../com/common/util/Utf8TextTruncator.java | 88 ++++ .../com/config/KafkaReliabilityConfig.java | 46 +++ .../java/com/config/ThreadPoolConfig.java | 28 ++ .../controllers/DirectStatsController.java | 17 +- .../StorageLimitMetricsController.java | 26 ++ .../com/influx/raw/InfluxRawPointFactory.java | 31 +- .../java/com/influx/raw/RawInfluxMetrics.java | 11 + .../com/influx/raw/RawStorageProperties.java | 12 + .../src/main/resources/application.properties | 56 ++- .../SyslogFailureRouterReliabilityTest.java | 21 + .../NormalData/SyslogKafkaTopologyTest.java | 71 ++++ .../DirectStatsProjectionWriterTest.java | 19 +- .../service/DmLargeTextLimiterTest.java | 62 +++ .../IngressStatsProjectionWriterTest.java | 34 ++ .../service/StatsProjectionAsyncTest.java | 151 +++++++ .../common/util/Utf8TextTruncatorTest.java | 43 ++ .../influx/raw/InfluxRawV2ContractTest.java | 25 ++ .../docker-compose-serve-prod.yaml | 9 + .../com/haobang/apipull/ApiPullRunner.java | 2 + .../com/haobang/config/AppProperties.java | 21 +- .../haobang/dispatch/SyslogWalDispatcher.java | 99 ++++- .../java/com/haobang/wal/AdmissionStatus.java | 1 + .../java/com/haobang/wal/AppendAttempt.java | 2 +- .../com/haobang/wal/PendingWalAppend.java | 6 +- .../wal/SyslogLegacyArchiveImporter.java | 4 + .../java/com/haobang/wal/SyslogWalRecord.java | 10 + .../com/haobang/wal/SyslogWalService.java | 77 +++- .../wal/SyslogWalStatusController.java | 5 + .../main/java/com/kafka/kafkaProducer.java | 33 +- .../java/com/netty/SyslogMessageHandler.java | 23 ++ .../main/java/com/netty/SyslogServerBoth.java | 4 + .../main/resources/application-dev.properties | 2 +- .../resources/application-prod-zc.properties | 4 +- .../resources/application-prod.properties | 2 +- .../resources/application-test.properties | 2 +- .../src/main/resources/application.properties | 13 +- .../dispatch/SyslogWalDispatcherTest.java | 51 +++ .../com/haobang/wal/SyslogWalServiceTest.java | 52 +++ .../com/kafka/KafkaProducerEventIdTest.java | 14 + haobang-security-dm/upgrade-doc/README.md | 5 +- .../replace-syslog-consumer-rule.md | 9 +- haobang-security-dm/upgrade-doc/v3/README.md | 193 +++++++++ .../v3/config/01-new-environment-variables.md | 147 +++++++ .../01-stats-async-projection-precheck.sql | 40 ++ .../02-stats-async-projection-upgrade.sql | 41 ++ .../03-stats-async-projection-verify.sql | 44 ++ ...01-create-stats-projection-topic-docker.sh | 130 ++++++ .../kafka/01-create-stats-projection-topic.sh | 96 +++++ .../02-create-large-log-topics-docker.sh | 117 ++++++ .../v3/kafka/02-create-large-log-topics.sh | 77 ++++ 64 files changed, 3189 insertions(+), 63 deletions(-) create mode 100644 haobang-security-dm/.gitattributes create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmLargeTextLimiter.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionDeltaConsumer.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxEvent.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxRelay.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxWriter.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/Utf8TextTruncator.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/StorageLimitMetricsController.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmLargeTextLimiterTest.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncTest.java create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/util/Utf8TextTruncatorTest.java create mode 100644 haobang-security-dm/upgrade-doc/v3/README.md create mode 100644 haobang-security-dm/upgrade-doc/v3/config/01-new-environment-variables.md create mode 100644 haobang-security-dm/upgrade-doc/v3/database/01-stats-async-projection-precheck.sql create mode 100644 haobang-security-dm/upgrade-doc/v3/database/02-stats-async-projection-upgrade.sql create mode 100644 haobang-security-dm/upgrade-doc/v3/database/03-stats-async-projection-verify.sql create mode 100755 haobang-security-dm/upgrade-doc/v3/kafka/01-create-stats-projection-topic-docker.sh create mode 100755 haobang-security-dm/upgrade-doc/v3/kafka/01-create-stats-projection-topic.sh create mode 100755 haobang-security-dm/upgrade-doc/v3/kafka/02-create-large-log-topics-docker.sh create mode 100755 haobang-security-dm/upgrade-doc/v3/kafka/02-create-large-log-topics.sh diff --git a/haobang-security-dm/.gitattributes b/haobang-security-dm/.gitattributes new file mode 100644 index 0000000..779cd80 --- /dev/null +++ b/haobang-security-dm/.gitattributes @@ -0,0 +1 @@ +upgrade-doc/v3/kafka/*.sh text eol=lf diff --git a/haobang-security-dm/syslog-consumer-rule/docker-compose/docker-compose-consumer-rule-prod.yaml b/haobang-security-dm/syslog-consumer-rule/docker-compose/docker-compose-consumer-rule-prod.yaml index 005dfc5..62f5ae0 100644 --- a/haobang-security-dm/syslog-consumer-rule/docker-compose/docker-compose-consumer-rule-prod.yaml +++ b/haobang-security-dm/syslog-consumer-rule/docker-compose/docker-compose-consumer-rule-prod.yaml @@ -55,8 +55,20 @@ services: - SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest - SPRING_KAFKA_CONSUMER_ENABLE_AUTO_COMMIT=false - SPRING_KAFKA_CONSUMER_TOPIC=agent-01-syslog-topic + - KAFKA_CONSUMER_LARGE_TOPIC=agent-01-syslog-topic.large # consumer-rule 是活跃原始日志消费节点，多实例通过同一消费组分担分区。 - SYSLOG_RAW_CONSUMER_ENABLED=true + # 大日志的 poll、fetch、批次和规则计算线程池均独立且有界。 + - SYSLOG_LARGE_LISTENER_ENABLED=true + - SYSLOG_LARGE_LISTENER_CONCURRENCY=1 + - SYSLOG_LARGE_LISTENER_MAX_POLL_RECORDS=5 + - SYSLOG_LARGE_LISTENER_MAX_PARTITION_FETCH_BYTES=33554432 + - SYSLOG_LARGE_LISTENER_FETCH_MAX_BYTES=67108864 + - SYSLOG_LARGE_PROCESSING_BATCH_MAX_BYTES=8388608 + - SYSLOG_LARGE_PROCESSOR_CORE_POOL_SIZE=1 + - SYSLOG_LARGE_PROCESSOR_MAX_POOL_SIZE=2 + - SYSLOG_LARGE_PROCESSOR_QUEUE_CAPACITY=10 + - KAFKA_RELIABILITY_MAX_REQUEST_SIZE=33554432 # Retry/Replay 与 Raw 隔离，避免未到期重试占用实时消费线程。 - SYSLOG_RELIABILITY_CONSUMER_GROUP_ID=agent-01-syslog-group-dm-retry - SYSLOG_RELIABILITY_CONSUMER_ENABLED=true @@ -79,6 +91,10 @@ services: - INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED=true - INFLUXDB_RAW_QUERY_MAX_ROWS=10000 - INFLUXDB_RAW_QUERY_MAX_RESPONSE_BYTES=10485760 + - INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES=1048576 + - INFLUXDB_RAW_WRITE_BATCH_MAX_BYTES=8388608 + - DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES=65536 + - DM_NON_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES=262144 - INFLUXDB_BATCH_SIZE=1000 - INFLUXDB_FLUSH_INTERVAL=1000 - INFLUXDB_CONNECTION_TIMEOUT=30s diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java index 483239f..c090903 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java @@ -43,6 +43,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.time.LocalDateTime; +import java.nio.charset.StandardCharsets; import com.common.util.Sm4Util; @Slf4j @@ -53,12 +54,19 @@ public class SysLogProcessor { @Qualifier("logNormalProcessorExecutor") private Executor taskExecutor; + @Autowired + @Qualifier("largeLogProcessorExecutor") + private Executor largeTaskExecutor; + @Value("${app.processor.batch-size:100}") private int batchSize; @Value("${app.processor.process-timeout-ms:120000}") private long processTimeoutMs; + @Value("${syslog.large-listener.processing-batch-max-bytes:8388608}") + private long largeProcessingBatchMaxBytes; + @Autowired private AppProperties appProperties; @@ -132,7 +140,47 @@ public class SysLogProcessor { containerFactory = "rawKafkaListenerContainerFactory" ) public void consumeAndParallelProcess(List> records, Acknowledgment ack) { - consumeBatch(records, ack, false, "RAW"); + consumeBatch(records, ack, false, "RAW", taskExecutor); + } + + @KafkaListener( + id = "syslog-consumer-rule-raw-large", + groupId = "${syslog.large-listener.group-id:${spring.kafka.consumer.group-id}-large}", + autoStartup = "${syslog.large-listener.enabled:true}", + topics = "${spring.kafka.consumer.large-topic:agent-syslog-topic.large}", + containerFactory = "largeRawKafkaListenerContainerFactory" + ) + public void consumeLargeMessages(List> records, Acknowledgment ack) { + consumeLargeInByteChunks(records, ack, false, "RAW_LARGE"); + } + + @KafkaListener( + id = "syslog-consumer-rule-large-retry-replay", + groupId = "${syslog.large-reliability.consumer-group-id:${spring.kafka.consumer.group-id}-large-retry}", + autoStartup = "${syslog.large-reliability.consumer.enabled:true}", + topics = { + "${syslog.large-reliability.retry-1m-topic:${spring.kafka.consumer.large-topic}.RETRY.1M}", + "${syslog.large-reliability.retry-10m-topic:${spring.kafka.consumer.large-topic}.RETRY.10M}", + "${syslog.large-reliability.retry-1h-topic:${spring.kafka.consumer.large-topic}.RETRY.1H}", + "${syslog.large-reliability.replay-topic:${spring.kafka.consumer.large-topic}.REPLAY}" + }, + containerFactory = "largeRawKafkaListenerContainerFactory" + ) + public void consumeLargeRetryAndReplay(List> records, Acknowledgment ack) { + consumeLargeInByteChunks(records, ack, true, "RETRY_REPLAY_LARGE"); + } + + private void consumeLargeInByteChunks(List> records, + Acknowledgment ack, boolean enforceRetryDue, String lane) { + if (records.isEmpty()) { + ack.acknowledge(); + return; + } + for (List> chunk : partitionRecordsByBytes( + records, Math.max(1L, largeProcessingBatchMaxBytes))) { + processBatch(chunk, enforceRetryDue, lane, largeTaskExecutor); + } + ack.acknowledge(); } @KafkaListener( @@ -148,7 +196,7 @@ public class SysLogProcessor { concurrency = "${syslog.reliability.listener.concurrency:1}" ) public void consumeRetryAndReplay(List> records, Acknowledgment ack) { - consumeBatch(records, ack, true, "RETRY_REPLAY"); + consumeBatch(records, ack, true, "RETRY_REPLAY", taskExecutor); } /** @@ -157,11 +205,17 @@ public class SysLogProcessor { * 其写库结果不参与 Kafka ACK。 */ private void consumeBatch(List> records, Acknowledgment ack, - boolean enforceRetryDue, String lane) { + boolean enforceRetryDue, String lane, Executor normalizationExecutor) { if (records.isEmpty()) { ack.acknowledge(); return; } + processBatch(records, enforceRetryDue, lane, normalizationExecutor); + ack.acknowledge(); + } + + private void processBatch(List> records, + boolean enforceRetryDue, String lane, Executor normalizationExecutor) { long batchStartedAt = System.currentTimeMillis(); long prepareStartedAt = batchStartedAt; BatchMetrics metrics = new BatchMetrics(); @@ -189,10 +243,9 @@ public class SysLogProcessor { long influxMs = System.currentTimeMillis() - influxStartedAt; long normalizeStartedAt = System.currentTimeMillis(); - normalizePreparedMessages(persistedMessages, metrics); + normalizePreparedMessages(persistedMessages, metrics, normalizationExecutor); long normalizeMs = System.currentTimeMillis() - normalizeStartedAt; - ack.acknowledge(); totalProcessed.addAndGet(records.size()); log.info("Syslog批次完成: lane={}, topicPartitions={}, records={}, prepared={}, persisted={}, " + "success={}, filtered={}, retry={}, dlt={}, normalizedRows={}, dmSuccessfulRows={}, " @@ -218,6 +271,32 @@ public class SysLogProcessor { System.currentTimeMillis() - batchStartedAt); } + static List>> partitionRecordsByBytes( + List> records, long maxBytes) { + if (maxBytes <= 0L) throw new IllegalArgumentException("maxBytes 必须大于 0"); + List>> result = new ArrayList<>(); + List> current = new ArrayList<>(); + long currentBytes = 0L; + for (ConsumerRecord record : records) { + String value = record.value(); + long bytes = value == null ? 0L : value.getBytes(StandardCharsets.UTF_8).length; + if (!current.isEmpty() && bytes > maxBytes - currentBytes) { + result.add(current); + current = new ArrayList<>(); + currentBytes = 0L; + } + current.add(record); + currentBytes = Math.addExact(currentBytes, bytes); + if (currentBytes >= maxBytes) { + result.add(current); + current = new ArrayList<>(); + currentBytes = 0L; + } + } + if (!current.isEmpty()) result.add(current); + return result; + } + /** * 方案二：分批多线程处理 * 将大批次拆分为小批次进行并行处理 @@ -422,6 +501,16 @@ public class SysLogProcessor { if (messages.isEmpty()) { return messages; } + List persisted = new ArrayList<>(messages.size()); + long configuredBatchBytes = rawStorageProperties.getWriteBatchMaxBytes(); + long effectiveBatchBytes = configuredBatchBytes > 0L ? configuredBatchBytes : Long.MAX_VALUE; + for (List chunk : partitionPreparedByPointBytes(messages, effectiveBatchBytes)) { + persisted.addAll(persistRawChunk(chunk, metrics)); + } + return persisted; + } + + private List persistRawChunk(List messages, BatchMetrics metrics) { List points = new ArrayList<>(messages.size()); for (PreparedMessage message : messages) { points.add(message.point); @@ -452,6 +541,31 @@ public class SysLogProcessor { } } + static List> partitionPreparedByPointBytes( + List messages, long maxBytes) { + if (maxBytes <= 0L) throw new IllegalArgumentException("InfluxDB 批次字节上限必须大于 0"); + List> result = new ArrayList<>(); + List current = new ArrayList<>(); + long currentBytes = 0L; + for (PreparedMessage message : messages) { + long bytes = message.point.toLineProtocol().getBytes(StandardCharsets.UTF_8).length + 1L; + if (!current.isEmpty() && bytes > maxBytes - currentBytes) { + result.add(current); + current = new ArrayList<>(); + currentBytes = 0L; + } + current.add(message); + currentBytes = Math.addExact(currentBytes, bytes); + if (currentBytes >= maxBytes) { + result.add(current); + current = new ArrayList<>(); + currentBytes = 0L; + } + } + if (!current.isEmpty()) result.add(current); + return result; + } + /** * 历史单条兼容入口：原文已可靠持久化后，执行标准化并立即落库。 * 活动 Kafka 批处理链路使用 prepareNormalization()，避免在计算线程中逐条开启达梦事务。 @@ -484,6 +598,11 @@ public class SysLogProcessor { * 批内有界并行标准化。任务异常必须汇总回 listener；超时或系统性失败时整批不 ACK。 */ void normalizePreparedMessages(List messages, BatchMetrics metrics) { + normalizePreparedMessages(messages, metrics, taskExecutor); + } + + private void normalizePreparedMessages(List messages, BatchMetrics metrics, + Executor normalizationExecutor) { if (messages.isEmpty()) { return; } @@ -509,7 +628,7 @@ public class SysLogProcessor { return new PreparedNormalization(message, null, routeFailure(message.record, message.context, failure)); } - }, taskExecutor)); + }, normalizationExecutor)); } CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) .get(processTimeoutMs, TimeUnit.MILLISECONDS); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SyslogFailureRouter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SyslogFailureRouter.java index aec1153..44b4561 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SyslogFailureRouter.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SyslogFailureRouter.java @@ -53,6 +53,16 @@ public class SyslogFailureRouter { private String retry1hTopic; @Value("${syslog.reliability.dlt-topic:${spring.kafka.consumer.topic}.DLT}") private String dltTopic; + @Value("${spring.kafka.consumer.large-topic:${spring.kafka.consumer.topic}.large}") + private String largeOriginalTopic; + @Value("${syslog.large-reliability.retry-1m-topic:${spring.kafka.consumer.large-topic}.RETRY.1M}") + private String largeRetry1mTopic; + @Value("${syslog.large-reliability.retry-10m-topic:${spring.kafka.consumer.large-topic}.RETRY.10M}") + private String largeRetry10mTopic; + @Value("${syslog.large-reliability.retry-1h-topic:${spring.kafka.consumer.large-topic}.RETRY.1H}") + private String largeRetry1hTopic; + @Value("${syslog.large-reliability.dlt-topic:${spring.kafka.consumer.large-topic}.DLT}") + private String largeDltTopic; @Value("${syslog.reliability.publish-timeout-ms:30000}") private long publishTimeoutMs; @Value("${syslog.reliability.consumer-version:unknown}") @@ -129,7 +139,11 @@ public class SyslogFailureRouter { int nextAttempt = Math.max(0, context.getAttemptCount()) + 1; boolean permanent = category == FailureCategory.PERMANENT_DATA; boolean exhausted = nextAttempt > 3; - String targetTopic = (permanent || exhausted) ? dltTopic : retryTopic(nextAttempt); + boolean largeLane = largeOriginalTopic != null + && largeOriginalTopic.equals(context.getOriginalTopic()); + String targetTopic = (permanent || exhausted) + ? (largeLane ? largeDltTopic : dltTopic) + : retryTopic(nextAttempt, largeLane); Instant nextRetryAt = (permanent || exhausted) ? null : Instant.now().plusSeconds(delaySeconds(nextAttempt)); FailureEnvelope source = readEnvelope(record.value()); @@ -256,10 +270,10 @@ public class SyslogFailureRouter { return record.timestamp() > 0 ? Instant.ofEpochMilli(record.timestamp()) : Instant.now(); } - private String retryTopic(int attempt) { - if (attempt == 1) return retry1mTopic; - if (attempt == 2) return retry10mTopic; - return retry1hTopic; + private String retryTopic(int attempt, boolean largeLane) { + if (attempt == 1) return largeLane ? largeRetry1mTopic : retry1mTopic; + if (attempt == 2) return largeLane ? largeRetry10mTopic : retry10mTopic; + return largeLane ? largeRetry1hTopic : retry1hTopic; } private long delaySeconds(int attempt) { diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java index 759cf21..9b06959 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java @@ -23,9 +23,10 @@ import java.util.SortedMap; import java.util.TreeMap; /** - * 把当前事实事务的内存聚合结果直接写入小型统计表。 + * 把当前事实事务的内存聚合结果写入同步投影或可靠异步投影入口。 * - *

调用方必须已经开启事实事务；本类不会新开事务，也不会吞掉任何数据库异常。

+ *

DIRECT 默认只追加一条批次 Outbox；SHADOW/显式回退仍同步更新统计表。调用方必须 + * 已经开启事实事务，本类不会吞掉任何数据库异常。

*/ @Slf4j @Service @@ -63,6 +64,7 @@ public class DirectStatsProjectionWriter { private final StatsWriteModeProvider modeProvider; private final DirectStatsWriteMetrics metrics; private final StatsMaterializationPublisher materializationPublisher; + private final StatsProjectionOutboxWriter projectionOutboxWriter; @Value("${stats.incremental.read-enabled:false}") private boolean readEnabled; @@ -91,11 +93,13 @@ public class DirectStatsProjectionWriter { public DirectStatsProjectionWriter(JdbcTemplate jdbc, StatsWriteModeProvider modeProvider, DirectStatsWriteMetrics metrics, - StatsMaterializationPublisher materializationPublisher) { + StatsMaterializationPublisher materializationPublisher, + StatsProjectionOutboxWriter projectionOutboxWriter) { this.jdbc = jdbc; this.modeProvider = modeProvider; this.metrics = metrics; this.materializationPublisher = materializationPublisher; + this.projectionOutboxWriter = projectionOutboxWriter; } public WriteResult writeStandardizedRows(List> rows) { @@ -160,6 +164,8 @@ public class DirectStatsProjectionWriter { status.put("today_stats_date_enabled", todayStatsDateEnabled); status.put("business_materialization_mode", asynchronousMaterialization() ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS"); + status.put("daily_projection_write_mode", asynchronousProjection() + ? "DURABLE_OUTBOX" : "SYNCHRONOUS_MERGE"); return status; } @@ -194,6 +200,24 @@ public class DirectStatsProjectionWriter { } requireTransaction(); + if (asynchronousProjection()) { + try { + long outboxStarted = System.nanoTime(); + projectionOutboxWriter.append(StatsProjectionOutboxEvent.fromAccumulator(accumulator)); + long outboxMs = elapsedMs(outboxStarted); + int deviceGroups = effectiveDeviceEntries(accumulator).size(); + int ruleGroups = effectiveRuleEntries(accumulator).size(); + metrics.recordSuccess(accumulator.getInputRows(), deviceGroups, ruleGroups, + aggregateMs, outboxMs, 0L); + return WriteResult.direct(accumulator.getInputRows(), deviceGroups, ruleGroups, + aggregateMs, outboxMs, 0L); + } catch (RuntimeException failure) { + metrics.recordFailure(); + throw new DirectStatsWriteException( + "统计投影 Outbox 写入失败，事实事务必须整体回滚", failure); + } + } + List> devices = effectiveDeviceEntries(accumulator); List> rules = @@ -244,6 +268,11 @@ public class DirectStatsProjectionWriter { aggregateMs, 0L, 0L); } requireTransaction(); + // Additive positive/negative changes commute in the serialized projection consumer. This + // keeps non-normal UPDATE statistics outside the fact transaction as well. + if (asynchronousProjection()) { + return write(accumulator, aggregateMs); + } List keys = new ArrayList<>(accumulator.getDevices().keySet()); try { long mergeStarted = System.nanoTime(); @@ -408,6 +437,11 @@ public class DirectStatsProjectionWriter { && materializationPublisher.isEnabled(); } + private boolean asynchronousProjection() { + return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT + && projectionOutboxWriter != null && projectionOutboxWriter.isEnabled(); + } + private List materializationCommands( List> devices, List> rules) { diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmLargeTextLimiter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmLargeTextLimiter.java new file mode 100644 index 0000000..f21003f --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmLargeTextLimiter.java @@ -0,0 +1,89 @@ +package com.common.service; + +import com.common.entity.SyslogNonNormalMessage; +import com.common.util.Utf8TextTruncator; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.LongAdder; + +/** 达梦持久化边界的大字段保护；0 表示兼容模式下不限制。 */ +@Component +public class DmLargeTextLimiter { + private static final long MIN_ENABLED_LIMIT = 64L; + + @Value("${dm.normal.large-field-max-utf8-bytes:0}") + private long normalMaxBytes; + + @Value("${dm.normal.large-fields:http_req_body,http_resp_body,http_req_header,http_resp_header,detail,description,cmdline,shell_cmdline,parent_cmdline,mail_message}") + private String normalFieldsValue; + + @Value("${dm.non-normal.large-field-max-utf8-bytes:0}") + private long nonNormalMaxBytes; + + private volatile Set normalFields = Collections.emptySet(); + private final LongAdder normalFieldsTruncated = new LongAdder(); + private final LongAdder nonNormalFieldsTruncated = new LongAdder(); + + @PostConstruct + public void validate() { + validateLimit(normalMaxBytes, "dm.normal.large-field-max-utf8-bytes"); + validateLimit(nonNormalMaxBytes, "dm.non-normal.large-field-max-utf8-bytes"); + Set parsed = new LinkedHashSet<>(); + if (normalFieldsValue != null) { + Arrays.stream(normalFieldsValue.split(",")) + .map(String::trim).filter(value -> !value.isEmpty()).forEach(parsed::add); + } + normalFields = Collections.unmodifiableSet(parsed); + } + + private static void validateLimit(long value, String name) { + if (value < 0L || (value > 0L && value < MIN_ENABLED_LIMIT)) { + throw new IllegalArgumentException(name + " 必须为 0 或至少 " + MIN_ENABLED_LIMIT); + } + } + + public void truncateNormalRow(Map row) { + if (row == null || normalMaxBytes == 0L) return; + for (String field : normalFields) { + Object value = row.get(field); + if (!(value instanceof String)) continue; + Utf8TextTruncator.Result result = Utf8TextTruncator.truncate((String) value, normalMaxBytes); + if (result.isTruncated()) { + row.put(field, result.getValue()); + normalFieldsTruncated.increment(); + } + } + } + + public void truncateNonNormal(SyslogNonNormalMessage message) { + if (message == null || nonNormalMaxBytes == 0L) return; + message.setSyslogMessage(truncateNonNormalValue(message.getSyslogMessage())); + message.setHeaderMessage(truncateNonNormalValue(message.getHeaderMessage())); + message.setReasonDetail(truncateNonNormalValue(message.getReasonDetail())); + } + + private String truncateNonNormalValue(String value) { + Utf8TextTruncator.Result result = Utf8TextTruncator.truncate(value, nonNormalMaxBytes); + if (result.isTruncated()) nonNormalFieldsTruncated.increment(); + return result.getValue(); + } + + public long getNormalFieldsTruncated() { return normalFieldsTruncated.sum(); } + public long getNonNormalFieldsTruncated() { return nonNormalFieldsTruncated.sum(); } + + /** 测试和非 Spring 兼容入口。 */ + static DmLargeTextLimiter unlimited() { + DmLargeTextLimiter limiter = new DmLargeTextLimiter(); + limiter.normalMaxBytes = 0L; + limiter.nonNormalMaxBytes = 0L; + limiter.validate(); + return limiter; + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java index ea560a3..89cbdbf 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java @@ -174,8 +174,8 @@ public class DmStandardizedBatchExecutor { lateAlarmBucketWriter.record(rows); } - // DIRECT/SHADOW 模式在当前事务内按设备、规则聚合写统计；LEGACY 模式会直接返回。 - // 任一统计 SQL 失败都会回滚本事务的事实行，禁止产生无法定位的统计缺口。 + // DIRECT 异步投影模式只在当前事实事务追加一条聚合 Outbox；独立 Kafka consumer + // 再更新日统计。SHADOW/回退模式仍保留原同步 MERGE。 timing.statsResult = directStatsProjectionWriter.writeStandardizedRows(rows); return null; }); @@ -183,7 +183,7 @@ public class DmStandardizedBatchExecutor { transactionMetrics.recordSuccess(rows.size(), timing.factFlushMs, timing.commitMs, transactionMs); if (transactionLogEnabled && timing.statsResult != null && timing.statsResult.isDirect()) { log.info("DIRECT 达梦事务完成: factRows={}, deviceGroups={}, ruleGroups={}, aggregateMs={}, " + - "factFlushMs={}, statsMergeMs={}, materializeMs={}, commitMs={}, transactionMs={}, retry={}", + "factFlushMs={}, statsProjectionStageMs={}, materializeMs={}, commitMs={}, transactionMs={}, retry={}", rows.size(), timing.statsResult.getDeviceGroups(), timing.statsResult.getRuleGroups(), timing.statsResult.getAggregateMs(), timing.factFlushMs, timing.statsResult.getMergeMs(), timing.statsResult.getMaterializeMs(), diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java index 6b0a947..cae6c0b 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java @@ -60,6 +60,7 @@ public class DmStandardizedBatchWriter { private final SyslogNormalDataService normalDataService; private final DmStandardizedBatchExecutor batchExecutor; private final Executor writerExecutor; + private final DmLargeTextLimiter largeTextLimiter; // 未配置时直接启用本次改造的批写路径；SINGLE 只作为现场故障回退开关保留。 @Value("${dm.standardized-write.mode:BATCH}") @@ -89,12 +90,22 @@ public class DmStandardizedBatchWriter { private final AtomicInteger consecutiveSystemFailures = new AtomicInteger(); private volatile long circuitOpenUntilMs; + @org.springframework.beans.factory.annotation.Autowired public DmStandardizedBatchWriter(SyslogNormalDataService normalDataService, DmStandardizedBatchExecutor batchExecutor, - @Qualifier("dmStandardizedWriterExecutor") Executor writerExecutor) { + @Qualifier("dmStandardizedWriterExecutor") Executor writerExecutor, + DmLargeTextLimiter largeTextLimiter) { this.normalDataService = normalDataService; this.batchExecutor = batchExecutor; this.writerExecutor = writerExecutor; + this.largeTextLimiter = largeTextLimiter; + } + + /** 保留已有单元测试和离线工具入口，默认不改变历史存储内容。 */ + public DmStandardizedBatchWriter(SyslogNormalDataService normalDataService, + DmStandardizedBatchExecutor batchExecutor, + Executor writerExecutor) { + this(normalDataService, batchExecutor, writerExecutor, DmLargeTextLimiter.unlimited()); } /** @@ -148,6 +159,7 @@ public class DmStandardizedBatchWriter { List> prepared = new ArrayList<>(requests.size()); for (WriteRequest request : requests) { try { + largeTextLimiter.truncateNormalRow(request.getData()); normalDataService.prepareForInsert(request.getData()); prepared.add(request); } catch (RuntimeException invalidRow) { diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java index 93e873b..86c60cf 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsProjectionWriter.java @@ -29,6 +29,7 @@ public class IngressStatsProjectionWriter { private final PlatformTransactionManager transactionManager; private final StatsWriteModeProvider modeProvider; private final StatsMaterializationPublisher materializationPublisher; + private final StatsProjectionOutboxWriter projectionOutboxWriter; @Value("${stats.ingress-metadata.enabled:false}") private boolean enabled; @@ -50,11 +51,13 @@ public class IngressStatsProjectionWriter { public IngressStatsProjectionWriter(JdbcTemplate jdbc, PlatformTransactionManager transactionManager, StatsWriteModeProvider modeProvider, - StatsMaterializationPublisher materializationPublisher) { + StatsMaterializationPublisher materializationPublisher, + StatsProjectionOutboxWriter projectionOutboxWriter) { this.jdbc = jdbc; this.transactionManager = transactionManager; this.modeProvider = modeProvider; this.materializationPublisher = materializationPublisher; + this.projectionOutboxWriter = projectionOutboxWriter; } public int writeBatch(List events) { @@ -92,6 +95,8 @@ public class IngressStatsProjectionWriter { ? "device_daily_stats+" + shadowDeviceTable : "device_daily_stats"); status.put("business_materialization_mode", asynchronousMaterialization() ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS"); + status.put("daily_projection_write_mode", asynchronousProjection() + ? "DURABLE_OUTBOX" : "SYNCHRONOUS_MERGE"); status.put("successful_batches", successfulBatches.get()); status.put("received_events", receivedEvents.get()); status.put("updated_groups", updatedGroups.get()); @@ -109,6 +114,10 @@ public class IngressStatsProjectionWriter { TransactionTemplate transaction = new TransactionTemplate(transactionManager); transaction.setTimeout(Math.max(1, transactionTimeoutSeconds)); Integer updated = transaction.execute(status -> { + if (asynchronousProjection()) { + projectionOutboxWriter.append(StatsProjectionOutboxEvent.fromIngress(aggregation)); + return aggregation.devices.size() + aggregation.collects.size(); + } int groups = 0; boolean asyncMaterialization = asynchronousMaterialization(); List commands = asyncMaterialization @@ -237,6 +246,11 @@ public class IngressStatsProjectionWriter { && materializationPublisher.isEnabled(); } + private boolean asynchronousProjection() { + return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT + && projectionOutboxWriter != null && projectionOutboxWriter.isEnabled(); + } + private static String safeTable(String table) { if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) { throw new IllegalStateException("非法入口设备统计表名: " + table); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionDeltaConsumer.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionDeltaConsumer.java new file mode 100644 index 0000000..3ae37c0 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionDeltaConsumer.java @@ -0,0 +1,386 @@ +package com.common.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.Acknowledgment; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.sql.Date; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; + +/** Applies durable projection deltas outside the fact transaction. */ +@Slf4j +@Component +public class StatsProjectionDeltaConsumer { + private static final String DEVICE_MERGE = + "MERGE INTO %s t USING (SELECT ? stat_date,? device_id,? parse_delta," + + "? non_log_delta,? receive_time FROM DUAL) s " + + "ON (t.stat_date=s.stat_date AND t.device_id=s.device_id) " + + "WHEN MATCHED THEN UPDATE SET t.parse_count=t.parse_count+s.parse_delta," + + "t.non_log_count=t.non_log_count+s.non_log_delta," + + "t.last_receive_time=CASE WHEN s.receive_time IS NULL THEN t.last_receive_time " + + "WHEN t.last_receive_time IS NULL OR t.last_receive_times.first_success_time " + + "THEN s.first_success_time ELSE t.first_success_time END," + + "t.last_success_time=CASE WHEN s.last_success_time IS NULL THEN t.last_success_time " + + "WHEN t.last_success_time IS NULL OR t.last_success_time> records, Acknowledgment acknowledgment) { + if (records == null || records.isEmpty()) { + acknowledgment.acknowledge(); + return; + } + consumed.addAndGet(records.size()); + List events = parse(records); + long started = System.nanoTime(); + try { + ApplyResult result = applyWithRetry(events); + acknowledgment.acknowledge(); + appliedEvents.addAndGet(result.appliedEvents); + duplicates.addAndGet(result.duplicates); + deviceGroups.addAndGet(result.deviceGroups); + ruleGroups.addAndGet(result.ruleGroups); + collectGroups.addAndGet(result.collectGroups); + successfulBatches.incrementAndGet(); + lastSuccessTime = LocalDateTime.now(); + log.info("统计投影批次完成: received={}, appliedEvents={}, duplicates={}, " + + "deviceGroups={}, ruleGroups={}, collectGroups={}, durationMs={}", + records.size(), result.appliedEvents, result.duplicates, + result.deviceGroups, result.ruleGroups, result.collectGroups, + (System.nanoTime() - started) / 1_000_000L); + } catch (RuntimeException failure) { + failures.incrementAndGet(); + lastFailureTime = LocalDateTime.now(); + throw failure; + } + } + + private List parse(List> records) { + Map unique = new LinkedHashMap<>(); + for (ConsumerRecord record : records) { + try { + if (!StatsProjectionOutboxEvent.KAFKA_KEY.equals(record.key())) { + throw new IllegalArgumentException("Unexpected stats projection Kafka key: " + record.key()); + } + StatsProjectionOutboxEvent event = objectMapper.readValue( + record.value(), StatsProjectionOutboxEvent.class); + event.validate(); + unique.put(event.getEventId(), event); + } catch (Exception invalid) { + throw new IllegalStateException("Invalid internal stats projection event at " + + record.topic() + "-" + record.partition() + "@" + record.offset(), invalid); + } + } + return new ArrayList<>(unique.values()); + } + + ApplyResult applyWithRetry(List events) { + int attempt = 0; + while (true) { + try { + return applyOnce(events); + } catch (RuntimeException failure) { + if (!DirectStatsProjectionWriter.isConcurrentFailure(failure) + || attempt >= Math.max(0, maxConcurrencyRetries)) { + throw failure; + } + attempt++; + retries.incrementAndGet(); + sleepBeforeRetry(attempt); + } + } + } + + private ApplyResult applyOnce(List events) { + if (!enabled) throw new IllegalStateException("Stats projection consumer is disabled"); + TransactionTemplate transaction = new TransactionTemplate(transactionManager); + transaction.setTimeout(Math.max(1, transactionTimeoutSeconds)); + ApplyResult result = transaction.execute(status -> { + Set existing = existingEventIds(events); + List fresh = new ArrayList<>(); + for (StatsProjectionOutboxEvent event : events) { + if (!existing.contains(event.getEventId())) fresh.add(event); + } + ProjectionAggregation aggregation = aggregate(fresh); + if (!aggregation.devices.isEmpty()) { + jdbc.batchUpdate(String.format(DEVICE_MERGE, + StatsProjectionOutboxWriter.safeTable(deviceTable)), aggregation.deviceParams()); + } + if (!aggregation.rules.isEmpty()) { + jdbc.batchUpdate(String.format(RULE_MERGE, + StatsProjectionOutboxWriter.safeTable(ruleTable)), aggregation.ruleParams()); + } + if (!aggregation.collects.isEmpty()) { + jdbc.batchUpdate(COLLECT_MERGE, aggregation.collectParams()); + } + if (!fresh.isEmpty()) { + List markers = new ArrayList<>(fresh.size()); + for (StatsProjectionOutboxEvent event : fresh) { + markers.add(new Object[]{event.getEventId(), event.getSchema()}); + } + jdbc.batchUpdate("INSERT INTO " + StatsProjectionOutboxWriter.safeTable(appliedTable) + + " (event_id,event_schema,applied_at) VALUES (?,?,SYSDATE)", markers); + } + if (!events.isEmpty()) { + List acknowledged = new ArrayList<>(events.size()); + for (StatsProjectionOutboxEvent event : events) { + acknowledged.add(new Object[]{event.getEventId()}); + } + jdbc.batchUpdate("UPDATE " + StatsProjectionOutboxWriter.safeTable(outboxTable) + + " SET status='APPLIED',applied_at=SYSDATE,next_attempt_at=NULL," + + "updated_at=SYSDATE WHERE event_id=?", acknowledged); + } + Collection dirty = aggregation.materializationCommands(); + if (!dirty.isEmpty()) materializationPublisher.enqueueAfterCommit(dirty); + return new ApplyResult(fresh.size(), events.size() - fresh.size(), + aggregation.devices.size(), aggregation.rules.size(), aggregation.collects.size()); + }); + return result == null ? new ApplyResult(0, events.size(), 0, 0, 0) : result; + } + + private Set existingEventIds(List events) { + if (events.isEmpty()) return Collections.emptySet(); + Set result = new HashSet<>(); + int chunkSize = 900; + for (int start = 0; start < events.size(); start += chunkSize) { + int end = Math.min(events.size(), start + chunkSize); + StringBuilder placeholders = new StringBuilder(); + Object[] ids = new Object[end - start]; + for (int i = start; i < end; i++) { + if (placeholders.length() > 0) placeholders.append(','); + placeholders.append('?'); + ids[i - start] = events.get(i).getEventId(); + } + result.addAll(jdbc.queryForList("SELECT event_id FROM " + + StatsProjectionOutboxWriter.safeTable(appliedTable) + " WHERE event_id IN (" + + placeholders + ")", String.class, ids)); + } + return result; + } + + static ProjectionAggregation aggregate(List events) { + ProjectionAggregation result = new ProjectionAggregation(); + for (StatsProjectionOutboxEvent event : events) { + event.validate(); + for (StatsProjectionOutboxEvent.DeviceDelta delta : event.getDevices()) { + DeviceKey key = new DeviceKey(delta.date(), delta.getDeviceId()); + DeviceAggregate value = result.devices.computeIfAbsent(key, + ignored -> new DeviceAggregate()); + value.parseDelta += delta.getParseDelta(); + value.nonLogDelta += delta.getNonLogDelta(); + LocalDateTime receiveTime = delta.receiveTime(); + if (receiveTime != null && (value.lastReceiveTime == null + || receiveTime.isAfter(value.lastReceiveTime))) { + value.lastReceiveTime = receiveTime; + } + } + for (StatsProjectionOutboxEvent.RuleDelta delta : event.getRules()) { + RuleKey key = new RuleKey(delta.date(), delta.getNormalizeRuleId()); + LocalDateTime value = delta.time(); + LocalDateTime current = result.rules.get(key); + if (current == null || value.isAfter(current)) result.rules.put(key, value); + } + for (StatsProjectionOutboxEvent.CollectDelta delta : event.getCollects()) { + CollectAggregate value = result.collects.computeIfAbsent(delta.getDeviceCollectId(), + ignored -> new CollectAggregate()); + value.include(delta.firstTime()); + value.include(delta.lastTime()); + } + } + return result; + } + + private void sleepBeforeRetry(int attempt) { + try { + Thread.sleep(Math.max(0L, retryBackoffMs) * attempt); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Stats projection retry interrupted", interrupted); + } + } + + public Map status() { + Map result = new LinkedHashMap<>(); + result.put("enabled", enabled); + result.put("consumed_total", consumed.get()); + result.put("applied_events_total", appliedEvents.get()); + result.put("duplicate_events_total", duplicates.get()); + result.put("device_groups_total", deviceGroups.get()); + result.put("rule_groups_total", ruleGroups.get()); + result.put("collect_groups_total", collectGroups.get()); + result.put("successful_batches", successfulBatches.get()); + result.put("failures_total", failures.get()); + result.put("concurrency_retries_total", retries.get()); + result.put("last_success_time", lastSuccessTime); + result.put("last_failure_time", lastFailureTime); + return result; + } + + static final class ProjectionAggregation { + final Map devices = new TreeMap<>(); + final Map rules = new TreeMap<>(); + final Map collects = new TreeMap<>(); + + List deviceParams() { + List result = new ArrayList<>(devices.size()); + for (Map.Entry entry : devices.entrySet()) { + DeviceAggregate value = entry.getValue(); + result.add(new Object[]{Date.valueOf(entry.getKey().date), entry.getKey().id, + value.parseDelta, value.nonLogDelta, value.lastReceiveTime}); + } + return result; + } + + List ruleParams() { + List result = new ArrayList<>(rules.size()); + for (Map.Entry entry : rules.entrySet()) { + result.add(new Object[]{Date.valueOf(entry.getKey().date), entry.getKey().id, + entry.getValue()}); + } + return result; + } + + List collectParams() { + List result = new ArrayList<>(collects.size()); + for (Map.Entry entry : collects.entrySet()) { + result.add(new Object[]{entry.getKey(), entry.getValue().min, entry.getValue().max}); + } + return result; + } + + Collection materializationCommands() { + Set result = new LinkedHashSet<>(); + for (DeviceKey key : devices.keySet()) result.add(StatsMaterializationCommand.device(key.date, key.id)); + for (RuleKey key : rules.keySet()) result.add(StatsMaterializationCommand.rule(key.date, key.id)); + for (Long id : collects.keySet()) result.add(StatsMaterializationCommand.collect(id)); + return result; + } + } + + static class DimensionKey implements Comparable { + final LocalDate date; + final long id; + DimensionKey(LocalDate date, long id) { this.date = date; this.id = id; } + @Override public int compareTo(DimensionKey other) { + int dateOrder = date.compareTo(other.date); + return dateOrder != 0 ? dateOrder : Long.compare(id, other.id); + } + @Override public boolean equals(Object other) { + if (this == other) return true; + if (!(other instanceof DimensionKey)) return false; + DimensionKey key = (DimensionKey) other; + return id == key.id && date.equals(key.date); + } + @Override public int hashCode() { return 31 * date.hashCode() + Long.hashCode(id); } + } + static final class DeviceKey extends DimensionKey { DeviceKey(LocalDate date, long id) { super(date, id); } } + static final class RuleKey extends DimensionKey { RuleKey(LocalDate date, long id) { super(date, id); } } + static final class DeviceAggregate { long parseDelta; long nonLogDelta; LocalDateTime lastReceiveTime; } + static final class CollectAggregate { + LocalDateTime min; LocalDateTime max; + void include(LocalDateTime time) { + if (time == null) return; + if (min == null || time.isBefore(min)) min = time; + if (max == null || time.isAfter(max)) max = time; + } + } + + static final class ApplyResult { + final int appliedEvents; final int duplicates; + final int deviceGroups; final int ruleGroups; final int collectGroups; + ApplyResult(int appliedEvents, int duplicates, int deviceGroups, int ruleGroups, int collectGroups) { + this.appliedEvents = appliedEvents; this.duplicates = duplicates; + this.deviceGroups = deviceGroups; this.ruleGroups = ruleGroups; this.collectGroups = collectGroups; + } + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxEvent.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxEvent.java new file mode 100644 index 0000000..c6c22af --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxEvent.java @@ -0,0 +1,221 @@ +package com.common.service; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * A durable, batch-aggregated statistics delta written beside the fact rows. + * + *

The event is additive/max-min only, so events from different application nodes may be + * published in a different order without changing the final projection. Kafka duplicates are + * eliminated by {@code eventId} in the projection consumer transaction.

+ */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class StatsProjectionOutboxEvent { + public static final String SCHEMA = "SYSLOG_STATS_PROJECTION_V1"; + public static final String KAFKA_KEY = "SYSLOG_STATS_PROJECTION"; + + private String schema = SCHEMA; + private String eventId = UUID.randomUUID().toString(); + private List devices = new ArrayList<>(); + private List rules = new ArrayList<>(); + private List collects = new ArrayList<>(); + + public static StatsProjectionOutboxEvent fromAccumulator(DirectStatsAccumulator accumulator) { + StatsProjectionOutboxEvent event = new StatsProjectionOutboxEvent(); + for (Map.Entry entry + : accumulator.getDevices().entrySet()) { + DirectStatsAccumulator.DeviceDelta value = entry.getValue(); + if (value.getParseDelta() != 0L || value.getNonLogDelta() != 0L) { + event.devices.add(DeviceDelta.counts(entry.getKey().getDate(), + entry.getKey().getDimensionId(), value.getParseDelta(), value.getNonLogDelta())); + } + } + for (Map.Entry entry + : accumulator.getRules().entrySet()) { + if (entry.getValue().getMaxLogTime() != null) { + event.rules.add(RuleDelta.max(entry.getKey().getDate(), + entry.getKey().getDimensionId(), entry.getValue().getMaxLogTime())); + } + } + return event; + } + + static StatsProjectionOutboxEvent fromIngress(IngressStatsProjectionWriter.Aggregation aggregation) { + StatsProjectionOutboxEvent event = new StatsProjectionOutboxEvent(); + for (Map.Entry entry + : aggregation.devices.entrySet()) { + event.devices.add(DeviceDelta.received(entry.getKey().date, + entry.getKey().deviceId, entry.getValue())); + } + for (Map.Entry entry + : aggregation.collects.entrySet()) { + event.collects.add(CollectDelta.range(entry.getKey(), + entry.getValue().min, entry.getValue().max)); + } + return event; + } + + public void validate() { + if (!SCHEMA.equals(schema)) { + throw new IllegalArgumentException("Unsupported stats projection schema: " + schema); + } + if (eventId == null || eventId.trim().isEmpty()) { + throw new IllegalArgumentException("Stats projection event is missing eventId"); + } + if (devices == null || rules == null || collects == null) { + throw new IllegalArgumentException("Stats projection event contains null collections"); + } + if (devices.isEmpty() && rules.isEmpty() && collects.isEmpty()) { + throw new IllegalArgumentException("Stats projection event has no changes"); + } + for (DeviceDelta delta : devices) delta.validate(); + for (RuleDelta delta : rules) delta.validate(); + for (CollectDelta delta : collects) delta.validate(); + } + + public boolean isEmpty() { + return devices.isEmpty() && rules.isEmpty() && collects.isEmpty(); + } + + public String getSchema() { return schema; } + public void setSchema(String schema) { this.schema = schema; } + public String getEventId() { return eventId; } + public void setEventId(String eventId) { this.eventId = eventId; } + public List getDevices() { return devices; } + public void setDevices(List devices) { this.devices = devices; } + public List getRules() { return rules; } + public void setRules(List rules) { this.rules = rules; } + public List getCollects() { return collects; } + public void setCollects(List collects) { this.collects = collects; } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class DeviceDelta { + private String statDate; + private long deviceId; + private long parseDelta; + private long nonLogDelta; + private String lastReceiveTime; + + public DeviceDelta() {} + + static DeviceDelta counts(LocalDate date, long deviceId, long parseDelta, long nonLogDelta) { + DeviceDelta result = new DeviceDelta(); + result.statDate = date.toString(); + result.deviceId = deviceId; + result.parseDelta = parseDelta; + result.nonLogDelta = nonLogDelta; + return result; + } + + static DeviceDelta received(LocalDate date, long deviceId, LocalDateTime receiveTime) { + DeviceDelta result = counts(date, deviceId, 0L, 0L); + result.lastReceiveTime = receiveTime.toString(); + return result; + } + + void validate() { + date(); + if (deviceId <= 0L) throw new IllegalArgumentException("Invalid device projection id"); + if (parseDelta == 0L && nonLogDelta == 0L && receiveTime() == null) { + throw new IllegalArgumentException("Device projection delta is empty"); + } + } + + public LocalDate date() { return LocalDate.parse(statDate); } + public LocalDateTime receiveTime() { + return lastReceiveTime == null || lastReceiveTime.trim().isEmpty() + ? null : LocalDateTime.parse(lastReceiveTime); + } + public String getStatDate() { return statDate; } + public void setStatDate(String statDate) { this.statDate = statDate; } + public long getDeviceId() { return deviceId; } + public void setDeviceId(long deviceId) { this.deviceId = deviceId; } + public long getParseDelta() { return parseDelta; } + public void setParseDelta(long parseDelta) { this.parseDelta = parseDelta; } + public long getNonLogDelta() { return nonLogDelta; } + public void setNonLogDelta(long nonLogDelta) { this.nonLogDelta = nonLogDelta; } + public String getLastReceiveTime() { return lastReceiveTime; } + public void setLastReceiveTime(String lastReceiveTime) { this.lastReceiveTime = lastReceiveTime; } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class RuleDelta { + private String statDate; + private long normalizeRuleId; + private String maxLogTime; + + public RuleDelta() {} + + static RuleDelta max(LocalDate date, long ruleId, LocalDateTime maxLogTime) { + RuleDelta result = new RuleDelta(); + result.statDate = date.toString(); + result.normalizeRuleId = ruleId; + result.maxLogTime = maxLogTime.toString(); + return result; + } + + void validate() { + date(); + if (normalizeRuleId <= 0L || time() == null) { + throw new IllegalArgumentException("Invalid rule projection delta"); + } + } + + public LocalDate date() { return LocalDate.parse(statDate); } + public LocalDateTime time() { + return maxLogTime == null || maxLogTime.trim().isEmpty() + ? null : LocalDateTime.parse(maxLogTime); + } + public String getStatDate() { return statDate; } + public void setStatDate(String statDate) { this.statDate = statDate; } + public long getNormalizeRuleId() { return normalizeRuleId; } + public void setNormalizeRuleId(long normalizeRuleId) { this.normalizeRuleId = normalizeRuleId; } + public String getMaxLogTime() { return maxLogTime; } + public void setMaxLogTime(String maxLogTime) { this.maxLogTime = maxLogTime; } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class CollectDelta { + private long deviceCollectId; + private String firstSuccessTime; + private String lastSuccessTime; + + public CollectDelta() {} + + static CollectDelta range(long collectId, LocalDateTime first, LocalDateTime last) { + CollectDelta result = new CollectDelta(); + result.deviceCollectId = collectId; + result.firstSuccessTime = first == null ? null : first.toString(); + result.lastSuccessTime = last == null ? null : last.toString(); + return result; + } + + void validate() { + if (deviceCollectId <= 0L || (firstTime() == null && lastTime() == null)) { + throw new IllegalArgumentException("Invalid collector projection delta"); + } + } + + public LocalDateTime firstTime() { + return firstSuccessTime == null || firstSuccessTime.trim().isEmpty() + ? null : LocalDateTime.parse(firstSuccessTime); + } + public LocalDateTime lastTime() { + return lastSuccessTime == null || lastSuccessTime.trim().isEmpty() + ? null : LocalDateTime.parse(lastSuccessTime); + } + public long getDeviceCollectId() { return deviceCollectId; } + public void setDeviceCollectId(long deviceCollectId) { this.deviceCollectId = deviceCollectId; } + public String getFirstSuccessTime() { return firstSuccessTime; } + public void setFirstSuccessTime(String firstSuccessTime) { this.firstSuccessTime = firstSuccessTime; } + public String getLastSuccessTime() { return lastSuccessTime; } + public void setLastSuccessTime(String lastSuccessTime) { this.lastSuccessTime = lastSuccessTime; } + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxRelay.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxRelay.java new file mode 100644 index 0000000..477e36b --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxRelay.java @@ -0,0 +1,151 @@ +package com.common.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.SendResult; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.util.concurrent.ListenableFuture; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Relays durable outbox events to the dedicated projection topic. + * + *

Multiple application nodes may relay the same row during a race. The consumer's applied-event + * table makes that harmless, so the relay never needs a distributed lease that could stall the + * fact path.

+ */ +@Slf4j +@Component +public class StatsProjectionOutboxRelay { + private final JdbcTemplate jdbc; + private final KafkaTemplate kafkaTemplate; + private final AtomicBoolean running = new AtomicBoolean(); + private final AtomicLong published = new AtomicLong(); + private final AtomicLong failures = new AtomicLong(); + private final AtomicLong cleaned = new AtomicLong(); + private volatile long lastSuccessAt; + private volatile long lastFailureAt; + + @Value("${stats.projection-async.enabled:true}") private boolean enabled; + @Value("${stats.projection-async.outbox-table:stats_projection_outbox}") private String outboxTable; + @Value("${stats.projection-async.applied-table:stats_projection_applied_event}") private String appliedTable; + @Value("${stats.projection-async.topic:${spring.kafka.consumer.topic}.STATS.PROJECTION}") private String topic; + @Value("${stats.projection-async.relay-batch-size:100}") private int batchSize; + @Value("${stats.projection-async.publish-timeout-ms:10000}") private long publishTimeoutMs; + @Value("${stats.projection-async.retry-backoff-ms:1000}") private long retryBackoffMs; + @Value("${stats.projection-async.redelivery-interval-ms:60000}") private long redeliveryIntervalMs; + @Value("${stats.projection-async.retention-days:14}") private int retentionDays; + + public StatsProjectionOutboxRelay(JdbcTemplate jdbc, KafkaTemplate kafkaTemplate) { + this.jdbc = jdbc; + this.kafkaTemplate = kafkaTemplate; + } + + @Scheduled(fixedDelayString = "${stats.projection-async.relay-interval-ms:200}") + public void relay() { + if (!enabled || !running.compareAndSet(false, true)) return; + try { + for (OutboxRow row : loadDueRows()) { + publish(row); + } + } catch (RuntimeException failure) { + failures.incrementAndGet(); + lastFailureAt = System.currentTimeMillis(); + log.error("统计投影 Outbox 扫描失败，下轮自动重试", failure); + } finally { + running.set(false); + } + } + + private List loadDueRows() { + String table = StatsProjectionOutboxWriter.safeTable(outboxTable); + String sql = "SELECT event_id,payload FROM (SELECT event_id,payload FROM " + table + + " WHERE status IN ('PENDING','FAILED','PUBLISHED') AND (next_attempt_at IS NULL " + + "OR next_attempt_at<=SYSDATE) ORDER BY created_at,event_id) WHERE ROWNUM<=?"; + return jdbc.query(sql, ps -> ps.setInt(1, Math.max(1, batchSize)), + (rs, rowNum) -> new OutboxRow(rs.getString(1), rs.getString(2))); + } + + private void publish(OutboxRow row) { + try { + ListenableFuture> future = kafkaTemplate.send( + topic, StatsProjectionOutboxEvent.KAFKA_KEY, row.payload); + future.get(Math.max(1L, publishTimeoutMs), TimeUnit.MILLISECONDS); + jdbc.update("UPDATE " + StatsProjectionOutboxWriter.safeTable(outboxTable) + + " SET status='PUBLISHED',published_at=SYSDATE,next_attempt_at=?,last_error=NULL," + + "updated_at=SYSDATE WHERE event_id=? AND status<>'APPLIED'", + new Timestamp(System.currentTimeMillis() + Math.max(1000L, redeliveryIntervalMs)), + row.eventId); + published.incrementAndGet(); + lastSuccessAt = System.currentTimeMillis(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + markFailure(row, interrupted); + } catch (Exception failure) { + markFailure(row, failure); + } + } + + private void markFailure(OutboxRow row, Exception failure) { + failures.incrementAndGet(); + lastFailureAt = System.currentTimeMillis(); + String message = String.valueOf(failure.getMessage()); + if (message.length() > 1800) message = message.substring(0, 1800); + try { + jdbc.update("UPDATE " + StatsProjectionOutboxWriter.safeTable(outboxTable) + + " SET status='FAILED',attempt_count=attempt_count+1,next_attempt_at=?," + + "last_error=?,updated_at=SYSDATE WHERE event_id=? AND status<>'APPLIED'", + new Timestamp(System.currentTimeMillis() + Math.max(0L, retryBackoffMs)), + message, row.eventId); + } catch (RuntimeException markFailure) { + failure.addSuppressed(markFailure); + } + log.error("统计投影 Outbox 发布失败，将自动重试: eventId={}", row.eventId, failure); + } + + @Scheduled(initialDelayString = "${stats.projection-async.cleanup-initial-delay-ms:3600000}", + fixedDelayString = "${stats.projection-async.cleanup-interval-ms:3600000}") + public void cleanup() { + if (!enabled) return; + long keepMillis = TimeUnit.DAYS.toMillis(Math.max(1, retentionDays)); + Timestamp cutoff = new Timestamp(System.currentTimeMillis() - keepMillis); + int removed = jdbc.update("DELETE FROM " + StatsProjectionOutboxWriter.safeTable(outboxTable) + + " WHERE status='APPLIED' AND applied_at status() { + Map result = new LinkedHashMap<>(); + result.put("enabled", enabled); + result.put("topic", topic); + result.put("running", running.get()); + result.put("published_total", published.get()); + result.put("failures_total", failures.get()); + result.put("cleaned_rows_total", cleaned.get()); + result.put("last_success_epoch_ms", lastSuccessAt == 0L ? null : lastSuccessAt); + result.put("last_failure_epoch_ms", lastFailureAt == 0L ? null : lastFailureAt); + return result; + } + + private static final class OutboxRow { + private final String eventId; + private final String payload; + private OutboxRow(String eventId, String payload) { + this.eventId = eventId; + this.payload = payload; + } + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxWriter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxWriter.java new file mode 100644 index 0000000..9b09ba5 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/StatsProjectionOutboxWriter.java @@ -0,0 +1,70 @@ +package com.common.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** Writes one aggregated outbox row in the same transaction as the corresponding fact batch. */ +@Component +public class StatsProjectionOutboxWriter { + private final JdbcTemplate jdbc; + private final ObjectMapper objectMapper; + private final AtomicLong events = new AtomicLong(); + private final AtomicLong failures = new AtomicLong(); + + @Value("${stats.projection-async.enabled:true}") private boolean enabled; + @Value("${stats.projection-async.outbox-table:stats_projection_outbox}") private String outboxTable; + + public StatsProjectionOutboxWriter(JdbcTemplate jdbc, ObjectMapper objectMapper) { + this.jdbc = jdbc; + this.objectMapper = objectMapper.copy(); + this.objectMapper.deactivateDefaultTyping(); + } + + public boolean isEnabled() { return enabled; } + + public int append(StatsProjectionOutboxEvent event) { + if (!enabled || event == null || event.isEmpty()) return 0; + if (!TransactionSynchronizationManager.isActualTransactionActive()) { + throw new IllegalStateException("Stats projection outbox must be written inside a transaction"); + } + event.validate(); + try { + int inserted = jdbc.update("INSERT INTO " + safeTable(outboxTable) + + " (event_id,event_schema,payload,status,attempt_count,next_attempt_at," + + "created_at,updated_at) VALUES (?,?,?,'PENDING',0,SYSDATE,SYSDATE,SYSDATE)", + event.getEventId(), event.getSchema(), objectMapper.writeValueAsString(event)); + events.incrementAndGet(); + return inserted; + } catch (JsonProcessingException serializationFailure) { + failures.incrementAndGet(); + throw new IllegalStateException("Failed to serialize stats projection outbox event", serializationFailure); + } catch (RuntimeException databaseFailure) { + failures.incrementAndGet(); + throw databaseFailure; + } + } + + public Map status() { + Map result = new LinkedHashMap<>(); + result.put("enabled", enabled); + result.put("outbox_table", outboxTable); + result.put("events_written_total", events.get()); + result.put("write_failures_total", failures.get()); + return result; + } + + static String safeTable(String table) { + if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) { + throw new IllegalStateException("Invalid stats projection table: " + table); + } + return table; + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java index c49fc72..8c3d56b 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java @@ -9,6 +9,7 @@ import com.common.mapper.SyslogNonNormalMessageMapper; import com.common.service.SyslogNonNormalMessageService; import com.common.service.StatsChangeCaptureService; import com.common.service.DirectStatsProjectionWriter; +import com.common.service.DmLargeTextLimiter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -40,6 +41,9 @@ public class SyslogNonNormalMessageServiceImpl @Autowired private DirectStatsProjectionWriter directStatsProjectionWriter; + @Autowired(required = false) + private DmLargeTextLimiter largeTextLimiter; + @Value("${dm.non-normal-write.batch-size:500}") private int nonNormalBatchSize; @@ -47,6 +51,7 @@ public class SyslogNonNormalMessageServiceImpl @Transactional(rollbackFor = Exception.class) public boolean saveMessage(SyslogNonNormalMessage message) { try { + truncateLargeFields(message); // 设置ID（如果未提供） if (message.getId() == null || message.getId().isEmpty()) { message.setId(generateMessageId()); @@ -101,6 +106,7 @@ public class SyslogNonNormalMessageServiceImpl try { // 为每条记录设置默认值 for (SyslogNonNormalMessage message : messages) { + truncateLargeFields(message); if (message.getId() == null || message.getId().isEmpty()) { message.setId(generateMessageId()); } @@ -159,10 +165,15 @@ public class SyslogNonNormalMessageServiceImpl } } + private void truncateLargeFields(SyslogNonNormalMessage message) { + if (largeTextLimiter != null) largeTextLimiter.truncateNonNormal(message); + } + @Override @Transactional(rollbackFor = Exception.class) public boolean updateMessage(SyslogNonNormalMessage message) { try { + truncateLargeFields(message); SyslogNonNormalMessage oldValue = getMessageById(message.getId()); // 设置更新时间 message.setUpdateTime(LocalDateTime.now()); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/Utf8TextTruncator.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/Utf8TextTruncator.java new file mode 100644 index 0000000..c1a1d2b --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/Utf8TextTruncator.java @@ -0,0 +1,88 @@ +package com.common.util; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; + +/** UTF-8 字节预算截断器；标记属于最终存储值的一部分，且永不切断 Unicode 字符。 */ +public final class Utf8TextTruncator { + private static final String MARKER_FORMAT = "...[HB_XDR_TRUNCATED original_utf8_bytes=%d]"; + + private Utf8TextTruncator() { } + + public static Result truncate(String text, long maxUtf8Bytes) { + if (text == null) return new Result(null, false, 0L, 0L, null); + if (maxUtf8Bytes < 0L) throw new IllegalArgumentException("maxUtf8Bytes 不能小于 0"); + + byte[] original = text.getBytes(StandardCharsets.UTF_8); + long originalBytes = original.length; + if (maxUtf8Bytes == 0L || originalBytes <= maxUtf8Bytes) { + return new Result(text, false, originalBytes, originalBytes, null); + } + + String marker = String.format(MARKER_FORMAT, originalBytes); + int markerBytes = marker.getBytes(StandardCharsets.UTF_8).length; + if (maxUtf8Bytes < markerBytes) { + throw new IllegalArgumentException("UTF-8 截断上限小于截断标记长度: limit=" + + maxUtf8Bytes + ", markerBytes=" + markerBytes); + } + long prefixBudget = maxUtf8Bytes - markerBytes; + int end = prefixEnd(text, prefixBudget); + String stored = text.substring(0, end) + marker; + return new Result(stored, true, originalBytes, + stored.getBytes(StandardCharsets.UTF_8).length, sha256(original)); + } + + private static int prefixEnd(String text, long byteBudget) { + long used = 0L; + int index = 0; + while (index < text.length()) { + int codePoint = text.codePointAt(index); + int width = utf8Width(codePoint); + if (used + width > byteBudget) break; + used += width; + index += Character.charCount(codePoint); + } + return index; + } + + private static int utf8Width(int codePoint) { + if (codePoint <= 0x7f) return 1; + if (codePoint <= 0x7ff) return 2; + if (codePoint <= 0xffff) return 3; + return 4; + } + + private static String sha256(byte[] bytes) { + try { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(bytes); + StringBuilder hex = new StringBuilder(digest.length * 2); + for (byte item : digest) hex.append(String.format("%02x", item & 0xff)); + return hex.toString(); + } catch (Exception impossible) { + throw new IllegalStateException("JVM 不支持 SHA-256", impossible); + } + } + + public static final class Result { + private final String value; + private final boolean truncated; + private final long originalUtf8Bytes; + private final long storedUtf8Bytes; + private final String originalSha256; + + Result(String value, boolean truncated, long originalUtf8Bytes, + long storedUtf8Bytes, String originalSha256) { + this.value = value; + this.truncated = truncated; + this.originalUtf8Bytes = originalUtf8Bytes; + this.storedUtf8Bytes = storedUtf8Bytes; + this.originalSha256 = originalSha256; + } + + public String getValue() { return value; } + public boolean isTruncated() { return truncated; } + public long getOriginalUtf8Bytes() { return originalUtf8Bytes; } + public long getStoredUtf8Bytes() { return storedUtf8Bytes; } + public String getOriginalSha256() { return originalSha256; } + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/KafkaReliabilityConfig.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/KafkaReliabilityConfig.java index 243cf49..0347d36 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/KafkaReliabilityConfig.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/KafkaReliabilityConfig.java @@ -1,15 +1,20 @@ package com.config; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerConfig; import org.springframework.boot.autoconfigure.kafka.ConcurrentKafkaListenerContainerFactoryConfigurer; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.kafka.listener.DefaultErrorHandler; import org.springframework.util.backoff.FixedBackOff; +import java.util.HashMap; +import java.util.Map; + /** Kafka 消费失败边界：业务未确认的消息绝不能被框架默认恢复器静默跳过。 */ @Configuration public class KafkaReliabilityConfig { @@ -50,6 +55,32 @@ public class KafkaReliabilityConfig { return factory; } + /** 大日志使用独立 ConsumerFactory，限制单次 poll 条数并显式放开单条拉取上限。 */ + @Bean("largeRawKafkaListenerContainerFactory") + public ConcurrentKafkaListenerContainerFactory largeRawKafkaListenerContainerFactory( + ConcurrentKafkaListenerContainerFactoryConfigurer configurer, + ConsumerFactory consumerFactory, + DefaultErrorHandler syslogDefaultErrorHandler, + @Value("${syslog.large-listener.concurrency:1}") int concurrency, + @Value("${syslog.large-listener.max-poll-records:5}") int maxPollRecords, + @Value("${syslog.large-listener.max-partition-fetch-bytes:33554432}") int maxPartitionFetchBytes, + @Value("${syslog.large-listener.fetch-max-bytes:67108864}") int fetchMaxBytes) { + if (concurrency < 1 || maxPollRecords < 1 || maxPartitionFetchBytes < 1 + || fetchMaxBytes < maxPartitionFetchBytes) { + throw new IllegalArgumentException("大日志 Kafka 并发、poll 和 fetch 参数非法"); + } + Map properties = new HashMap<>(consumerFactory.getConfigurationProperties()); + properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); + properties.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, maxPartitionFetchBytes); + properties.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, fetchMaxBytes); + DefaultKafkaConsumerFactory largeConsumerFactory = + new DefaultKafkaConsumerFactory<>(properties); + ConcurrentKafkaListenerContainerFactory factory = + configuredFactory(configurer, largeConsumerFactory, syslogDefaultErrorHandler); + factory.setConcurrency(concurrency); + return factory; + } + /** 统计脏键与原始日志隔离；同一维度 key 由 Kafka 分区保证只被一个组成员处理。 */ @Bean("statsMaterializationKafkaListenerContainerFactory") public ConcurrentKafkaListenerContainerFactory statsMaterializationKafkaListenerContainerFactory( @@ -66,6 +97,21 @@ public class KafkaReliabilityConfig { return factory; } + /** + * Projection deltas intentionally use one consumer per instance. All events use one Kafka key, + * so only one group member owns the active partition and daily projection rows cannot deadlock. + */ + @Bean("statsProjectionKafkaListenerContainerFactory") + public ConcurrentKafkaListenerContainerFactory statsProjectionKafkaListenerContainerFactory( + ConcurrentKafkaListenerContainerFactoryConfigurer configurer, + ConsumerFactory consumerFactory, + DefaultErrorHandler syslogDefaultErrorHandler) { + ConcurrentKafkaListenerContainerFactory factory = + configuredFactory(configurer, consumerFactory, syslogDefaultErrorHandler); + factory.setConcurrency(1); + return factory; + } + private ConcurrentKafkaListenerContainerFactory configuredFactory( ConcurrentKafkaListenerContainerFactoryConfigurer configurer, ConsumerFactory consumerFactory, diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java index a57e542..9527b3a 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java @@ -25,6 +25,15 @@ public class ThreadPoolConfig { @Value("${app.processor.thread-pool.keep-alive-seconds:60}") private int keepAliveSeconds; + @Value("${syslog.large-processor.thread-pool.core-pool-size:1}") + private int largeCorePoolSize; + + @Value("${syslog.large-processor.thread-pool.max-pool-size:2}") + private int largeMaxPoolSize; + + @Value("${syslog.large-processor.thread-pool.queue-capacity:10}") + private int largeQueueCapacity; + @Value("${dm.standardized-write.writer-concurrency:2}") private int dmWriterConcurrency; @@ -54,6 +63,25 @@ public class ThreadPoolConfig { return executor; } + /** 大日志解析/规则计算独立线程池，避免占满普通日志的计算线程与任务队列。 */ + @Bean("largeLogProcessorExecutor") + public Executor largeLogProcessorExecutor() { + if (largeCorePoolSize < 1 || largeMaxPoolSize < largeCorePoolSize || largeQueueCapacity < 1) { + throw new IllegalArgumentException("大日志处理线程池参数非法"); + } + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(largeCorePoolSize); + executor.setMaxPoolSize(largeMaxPoolSize); + executor.setQueueCapacity(largeQueueCapacity); + executor.setKeepAliveSeconds(keepAliveSeconds); + executor.setThreadNamePrefix("large-log-processor-"); + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy()); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(60); + executor.initialize(); + return executor; + } + /** * 达梦批写使用独立的有界线程池，不能复用规则计算线程池。 * diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/DirectStatsController.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/DirectStatsController.java index 192a61f..b90fe5e 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/DirectStatsController.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/DirectStatsController.java @@ -7,6 +7,9 @@ import com.common.service.DmStandardizedBatchExecutor; import com.common.service.StatsMaterializationConsumer; import com.common.service.StatsMaterializationPublisher; import com.common.service.StatsMaterializationReconciler; +import com.common.service.StatsProjectionOutboxWriter; +import com.common.service.StatsProjectionOutboxRelay; +import com.common.service.StatsProjectionDeltaConsumer; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -25,6 +28,9 @@ public class DirectStatsController { private final StatsMaterializationPublisher materializationPublisher; private final StatsMaterializationConsumer materializationConsumer; private final StatsMaterializationReconciler materializationReconciler; + private final StatsProjectionOutboxWriter projectionOutboxWriter; + private final StatsProjectionOutboxRelay projectionOutboxRelay; + private final StatsProjectionDeltaConsumer projectionDeltaConsumer; public DirectStatsController(DirectStatsProjectionWriter writer, IngressStatsProjectionWriter ingressWriter, @@ -32,7 +38,10 @@ public class DirectStatsController { DmStandardizedBatchExecutor batchExecutor, StatsMaterializationPublisher materializationPublisher, StatsMaterializationConsumer materializationConsumer, - StatsMaterializationReconciler materializationReconciler) { + StatsMaterializationReconciler materializationReconciler, + StatsProjectionOutboxWriter projectionOutboxWriter, + StatsProjectionOutboxRelay projectionOutboxRelay, + StatsProjectionDeltaConsumer projectionDeltaConsumer) { this.writer = writer; this.ingressWriter = ingressWriter; this.dayRolloverService = dayRolloverService; @@ -40,6 +49,9 @@ public class DirectStatsController { this.materializationPublisher = materializationPublisher; this.materializationConsumer = materializationConsumer; this.materializationReconciler = materializationReconciler; + this.projectionOutboxWriter = projectionOutboxWriter; + this.projectionOutboxRelay = projectionOutboxRelay; + this.projectionDeltaConsumer = projectionDeltaConsumer; } @GetMapping("/status") @@ -51,6 +63,9 @@ public class DirectStatsController { status.put("materialization_publisher", materializationPublisher.status()); status.put("materialization_consumer", materializationConsumer.status()); status.put("materialization_reconciler", materializationReconciler.status()); + status.put("projection_outbox_writer", projectionOutboxWriter.status()); + status.put("projection_outbox_relay", projectionOutboxRelay.status()); + status.put("projection_delta_consumer", projectionDeltaConsumer.status()); return ResponseEntity.ok(status); } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/StorageLimitMetricsController.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/StorageLimitMetricsController.java new file mode 100644 index 0000000..8110c8f --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/controllers/StorageLimitMetricsController.java @@ -0,0 +1,26 @@ +package com.controllers; + +import com.common.service.DmLargeTextLimiter; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** 大字段截断观测，指标只包含计数，不暴露设备原文。 */ +@RestController +@RequestMapping("/api/syslog/storage-limits") +public class StorageLimitMetricsController { + private final DmLargeTextLimiter limiter; + + public StorageLimitMetricsController(DmLargeTextLimiter limiter) { this.limiter = limiter; } + + @GetMapping("/metrics") + public Map metrics() { + Map values = new LinkedHashMap<>(); + values.put("dm_normal_fields_truncated_total", limiter.getNormalFieldsTruncated()); + values.put("dm_non_normal_fields_truncated_total", limiter.getNonNormalFieldsTruncated()); + return values; + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/InfluxRawPointFactory.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/InfluxRawPointFactory.java index 7c1ce76..5575b83 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/InfluxRawPointFactory.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/InfluxRawPointFactory.java @@ -1,15 +1,27 @@ package com.influx.raw; +import com.common.util.Utf8TextTruncator; import com.influxdb.client.domain.WritePrecision; import com.influxdb.client.write.Point; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** 原始 Point 的唯一构造器；V2 的 uuid 只作为 field。 */ @Component public class InfluxRawPointFactory { private final RawStorageProperties properties; + private final RawInfluxMetrics metrics; - public InfluxRawPointFactory(RawStorageProperties properties) { this.properties = properties; } + @Autowired + public InfluxRawPointFactory(RawStorageProperties properties, RawInfluxMetrics metrics) { + this.properties = properties; + this.metrics = metrics; + } + + /** 保留已有单元测试及非 Spring 工具入口。 */ + public InfluxRawPointFactory(RawStorageProperties properties) { + this(properties, new RawInfluxMetrics()); + } public RawPoint create(String eventId, String message, String receiveTime, String deviceId, String deviceCollectId, String originalTopic, @@ -17,21 +29,34 @@ public class InfluxRawPointFactory { require(eventId, "event_id"); require(receiveTime, "receive_time"); require(deviceId, "device_id"); require(deviceCollectId, "device_collect_id"); require(originalTopic, "originalTopic"); + Utf8TextTruncator.Result storedMessage = Utf8TextTruncator.truncate( + message, properties.getMessageMaxUtf8Bytes()); + if (storedMessage.isTruncated()) { + metrics.messageTruncated(storedMessage.getOriginalUtf8Bytes(), storedMessage.getStoredUtf8Bytes()); + } String measurement = "syslog_security_" + parsed.getMeasurementDate(); if (properties.getWriteSchema() == RawStorageProperties.WriteSchema.V1) { Point point = Point.measurement(measurement) .addTag("device_id", deviceId).addTag("device_collect_id", deviceCollectId) .addTag("uuid", eventId).addTag("topic", originalTopic) - .addField("message", message).addField("receive_time", receiveTime) + .addField("message", storedMessage.getValue()).addField("receive_time", receiveTime) + .addField("message_truncated", storedMessage.isTruncated()) + .addField("message_original_bytes", storedMessage.getOriginalUtf8Bytes()) + .addField("message_stored_bytes", storedMessage.getStoredUtf8Bytes()) .addField("uuid", eventId).time(parsed.getEpochMillis(), WritePrecision.MS); + if (storedMessage.isTruncated()) point.addField("message_original_sha256", storedMessage.getOriginalSha256()); return new RawPoint(point, properties.getV1Bucket(), false); } if (pointTimeNs == null) throw new IllegalArgumentException("V2 Point 缺少 point_time_ns"); Point point = Point.measurement(measurement) .addTag("device_id", deviceId).addTag("device_collect_id", deviceCollectId) .addTag("topic", originalTopic).addTag("event_bucket", InfluxEventBucket.bucketOf(eventId)) - .addField("uuid", eventId).addField("message", message).addField("receive_time", receiveTime) + .addField("uuid", eventId).addField("message", storedMessage.getValue()).addField("receive_time", receiveTime) + .addField("message_truncated", storedMessage.isTruncated()) + .addField("message_original_bytes", storedMessage.getOriginalUtf8Bytes()) + .addField("message_stored_bytes", storedMessage.getStoredUtf8Bytes()) .time(pointTimeNs, WritePrecision.NS); + if (storedMessage.isTruncated()) point.addField("message_original_sha256", storedMessage.getOriginalSha256()); return new RawPoint(point, properties.getV2Bucket(), true); } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawInfluxMetrics.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawInfluxMetrics.java index af33fc2..aa052f2 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawInfluxMetrics.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawInfluxMetrics.java @@ -16,6 +16,9 @@ public class RawInfluxMetrics { private final LongAdder v2FailedBatches = new LongAdder(); private final LongAdder dualReadDuplicate = new LongAdder(); private final LongAdder dualReadConflict = new LongAdder(); + private final LongAdder messagesTruncated = new LongAdder(); + private final LongAdder originalBytesTruncated = new LongAdder(); + private final LongAdder storedBytesAfterTruncation = new LongAdder(); public void legacyMissing() { legacyMissing.increment(); } public void legacyAssigned() { legacyAssigned.increment(); } @@ -24,6 +27,11 @@ public class RawInfluxMetrics { public void v2FailedBatch() { v2FailedBatches.increment(); } public void dualReadDuplicate() { dualReadDuplicate.increment(); } public void dualReadConflict() { dualReadConflict.increment(); } + public void messageTruncated(long originalBytes, long storedBytes) { + messagesTruncated.increment(); + originalBytesTruncated.add(originalBytes); + storedBytesAfterTruncation.add(storedBytes); + } public Map snapshot() { Map values = new LinkedHashMap<>(); @@ -36,6 +44,9 @@ public class RawInfluxMetrics { values.put("raw_write_v2_failed_batches_total", v2FailedBatches.sum()); values.put("influx_dual_read_duplicate_total", dualReadDuplicate.sum()); values.put("influx_dual_read_conflict_total", dualReadConflict.sum()); + values.put("raw_message_truncated_total", messagesTruncated.sum()); + values.put("raw_message_truncated_original_bytes_total", originalBytesTruncated.sum()); + values.put("raw_message_truncated_stored_bytes_total", storedBytesAfterTruncation.sum()); return values; } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawStorageProperties.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawStorageProperties.java index c2f1b8e..5f3d6b6 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawStorageProperties.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/influx/raw/RawStorageProperties.java @@ -19,6 +19,10 @@ public class RawStorageProperties { @Value("${influxdb.raw.legacy-point-time-fallback-enabled:true}") private boolean legacyFallbackEnabled; @Value("${influxdb.raw.query-max-rows:10000}") private int queryMaxRows; @Value("${influxdb.raw.query-max-response-bytes:10485760}") private long queryMaxResponseBytes; + /** 原始 message 单字段的 UTF-8 存储上限；0 表示不限制。 */ + @Value("${influxdb.raw.message-max-utf8-bytes:0}") private long messageMaxUtf8Bytes; + /** 一次同步写请求的估算 Line Protocol 上限；按字节切批，不能只按条数。 */ + @Value("${influxdb.raw.write-batch-max-bytes:8388608}") private long writeBatchMaxBytes; private WriteSchema writeSchema; private ReadMode readMode; @@ -35,6 +39,12 @@ public class RawStorageProperties { if (queryMaxResponseBytes <= 0L) { throw new IllegalArgumentException("influxdb.raw.query-max-response-bytes 必须大于 0"); } + if (messageMaxUtf8Bytes < 0L || (messageMaxUtf8Bytes > 0L && messageMaxUtf8Bytes < 64L)) { + throw new IllegalArgumentException("influxdb.raw.message-max-utf8-bytes 必须为 0 或至少 64"); + } + if (writeBatchMaxBytes <= 0L) { + throw new IllegalArgumentException("influxdb.raw.write-batch-max-bytes 必须大于 0"); + } } private static String normalize(String value) { @@ -49,6 +59,8 @@ public class RawStorageProperties { public boolean isLegacyFallbackEnabled() { return legacyFallbackEnabled; } public int getQueryMaxRows() { return queryMaxRows; } public long getQueryMaxResponseBytes() { return queryMaxResponseBytes; } + public long getMessageMaxUtf8Bytes() { return messageMaxUtf8Bytes; } + public long getWriteBatchMaxBytes() { return writeBatchMaxBytes; } public enum WriteSchema { V1, V2 } public enum ReadMode { V1_ONLY, V1_V2, V2_ONLY } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties index 222e1d4..922b5f5 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties @@ -34,6 +34,10 @@ influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096} influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true} influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000} influxdb.raw.query-max-response-bytes=${INFLUXDB_RAW_QUERY_MAX_RESPONSE_BYTES:10485760} +# 0 表示不限制；启用后最终 message（含截断标记）不超过该 UTF-8 字节数。 +influxdb.raw.message-max-utf8-bytes=${INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES:1048576} +# Influx 同步写按 Line Protocol 估算字节切批，避免少量大日志形成超大 HTTP 请求。 +influxdb.raw.write-batch-max-bytes=${INFLUXDB_RAW_WRITE_BATCH_MAX_BYTES:8388608} # InfluxDB 2.7 连接超时配置 influxdb.connection.timeout=30s influxdb.connection.read-timeout=30s @@ -70,16 +74,31 @@ spring.kafka.consumer.auto-offset-reset=latest spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.auto-commit-interval=1000 spring.kafka.consumer.topic=test-topic +spring.kafka.consumer.large-topic=${KAFKA_CONSUMER_LARGE_TOPIC:${spring.kafka.consumer.topic}.large} # consumer-rule 是当前活跃的 Kafka 原始日志标准化节点，默认启动监听。 syslog.raw-consumer.enabled=true # Raw Topic 当前有 3 个分区，使用独立工厂显式创建 3 个 consumer；可通过环境变量灰度调整。 syslog.raw-listener.concurrency=${SYSLOG_RAW_LISTENER_CONCURRENCY:3} +# 大日志使用独立 Topic、消费组和 ConsumerFactory；建议生产环境独立 JVM 部署。 +syslog.large-listener.enabled=${SYSLOG_LARGE_LISTENER_ENABLED:true} +syslog.large-listener.group-id=${SYSLOG_LARGE_LISTENER_GROUP_ID:${spring.kafka.consumer.group-id}-large} +syslog.large-listener.concurrency=${SYSLOG_LARGE_LISTENER_CONCURRENCY:1} +syslog.large-listener.max-poll-records=${SYSLOG_LARGE_LISTENER_MAX_POLL_RECORDS:5} +syslog.large-listener.max-partition-fetch-bytes=${SYSLOG_LARGE_LISTENER_MAX_PARTITION_FETCH_BYTES:33554432} +syslog.large-listener.fetch-max-bytes=${SYSLOG_LARGE_LISTENER_FETCH_MAX_BYTES:67108864} +syslog.large-listener.processing-batch-max-bytes=${SYSLOG_LARGE_PROCESSING_BATCH_MAX_BYTES:8388608} +# 大日志规则计算不复用普通日志线程池，过载时由 Kafka offset 背压。 +syslog.large-processor.thread-pool.core-pool-size=${SYSLOG_LARGE_PROCESSOR_CORE_POOL_SIZE:1} +syslog.large-processor.thread-pool.max-pool-size=${SYSLOG_LARGE_PROCESSOR_MAX_POOL_SIZE:2} +syslog.large-processor.thread-pool.queue-capacity=${SYSLOG_LARGE_PROCESSOR_QUEUE_CAPACITY:10} # P0：Retry/DLT 发布与原始消费使用同一 Kafka 集群；Topic 名称全部可由环境变量覆盖。 spring.kafka.bootstrap-servers=${spring.kafka.consumer.bootstrap-servers} spring.kafka.producer.acks=all spring.kafka.producer.properties.enable.idempotence=true spring.kafka.producer.properties.delivery.timeout.ms=120000 spring.kafka.producer.properties.request.timeout.ms=30000 +# Retry/DLT 信封也可能携带大日志密文，不能保留 Kafka 默认 1MiB。 +spring.kafka.producer.properties.max.request.size=${KAFKA_RELIABILITY_MAX_REQUEST_SIZE:33554432} syslog.reliability.retry-1m-topic=${spring.kafka.consumer.topic}.RETRY.1M syslog.reliability.retry-10m-topic=${spring.kafka.consumer.topic}.RETRY.10M syslog.reliability.retry-1h-topic=${spring.kafka.consumer.topic}.RETRY.1H @@ -91,6 +110,14 @@ syslog.reliability.consumer-version=${APP_VERSION:1.0.0} syslog.reliability.consumer-group-id=${spring.kafka.consumer.group-id}-retry syslog.reliability.consumer.enabled=true syslog.reliability.listener.concurrency=1 +# 大日志 Retry/DLT 不回流普通重试通道，避免再次形成队头阻塞。 +syslog.large-reliability.retry-1m-topic=${spring.kafka.consumer.large-topic}.RETRY.1M +syslog.large-reliability.retry-10m-topic=${spring.kafka.consumer.large-topic}.RETRY.10M +syslog.large-reliability.retry-1h-topic=${spring.kafka.consumer.large-topic}.RETRY.1H +syslog.large-reliability.dlt-topic=${spring.kafka.consumer.large-topic}.DLT +syslog.large-reliability.replay-topic=${spring.kafka.consumer.large-topic}.REPLAY +syslog.large-reliability.consumer-group-id=${spring.kafka.consumer.group-id}-large-retry +syslog.large-reliability.consumer.enabled=true spring.kafka.consumer.max-poll-records=500 @@ -152,6 +179,10 @@ dm.standardized-write.mode=BATCH dm.standardized-write.batch-size=${DM_STANDARDIZED_WRITE_BATCH_SIZE:500} dm.standardized-write.writer-concurrency=2 dm.standardized-write.writer-queue-capacity=100 +# 标准化大字段和非标原文均在达梦持久化边界截断；0 表示不限制。 +dm.normal.large-field-max-utf8-bytes=${DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES:65536} +dm.normal.large-fields=${DM_NORMAL_LARGE_FIELDS:http_req_body,http_resp_body,http_req_header,http_resp_header,detail,description,cmdline,shell_cmdline,parent_cmdline,mail_message} +dm.non-normal.large-field-max-utf8-bytes=${DM_NON_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES:262144} dm.standardized-write.transaction-timeout-seconds=45 dm.standardized-write.kafka-batch-deadline-ms=90000 dm.standardized-write.max-split-depth=10 @@ -181,7 +212,7 @@ spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule spring.datasource.hikari.auto-commit=false -# 统计写入采用事实同事务直写，不再为每条结果写 change_log + pending。 +# DIRECT 不再写逐事实 change_log；默认只在事实事务追加一条批次 Outbox，日统计异步投影。 # 首次部署 DIRECT 前必须先执行 08/09/10/11 脚本并按设计文档排空旧 pending。 stats.write-mode=DIRECT stats.incremental.capture-enabled=false @@ -197,11 +228,11 @@ stats.direct.control-table=stats_direct_control stats.direct.today-stats-date-enabled=true stats.direct.max-concurrency-retries=${STATS_DIRECT_MAX_CONCURRENCY_RETRIES:3} stats.direct.concurrency-retry-backoff-ms=${STATS_DIRECT_CONCURRENCY_RETRY_BACKOFF_MS:50} -stats.direct.transaction-log-enabled=false +stats.direct.transaction-log-enabled=${STATS_DIRECT_TRANSACTION_LOG_ENABLED:false} stats.direct.day-rollover-check-cron=5 * * * * ? stats.business-zone=Asia/Shanghai stats.fact-only.confirm-rebuild-required=false -# 事实事务只维护小型精确统计表；提交后按维度脏键异步刷新业务展示字段。 +# 日统计投影完成后按维度脏键异步刷新业务展示字段。 # 三个实例必须配置相同 topic/group-id，Topic 分区数建议与原始 Topic 相同。 stats.materialization.enabled=${STATS_MATERIALIZATION_ENABLED:true} stats.materialization.topic=${STATS_MATERIALIZATION_TOPIC:${spring.kafka.consumer.topic}.STATS.MATERIALIZE} @@ -216,6 +247,25 @@ stats.materialization.concurrency-retry-backoff-ms=${STATS_MATERIALIZATION_CONCU stats.materialization.reconcile-initial-delay-ms=${STATS_MATERIALIZATION_RECONCILE_INITIAL_DELAY_MS:30000} stats.materialization.reconcile-interval-ms=${STATS_MATERIALIZATION_RECONCILE_INTERVAL_MS:60000} +# Fact transactions append one aggregated durable event instead of locking the daily projection rows. +# A dedicated single-lane Kafka consumer applies device/rule/collector projection deltas and then +# emits the existing idempotent business-materialization dirty keys. +stats.projection-async.enabled=${STATS_PROJECTION_ASYNC_ENABLED:true} +stats.projection-async.outbox-table=${STATS_PROJECTION_OUTBOX_TABLE:stats_projection_outbox} +stats.projection-async.applied-table=${STATS_PROJECTION_APPLIED_TABLE:stats_projection_applied_event} +stats.projection-async.topic=${STATS_PROJECTION_TOPIC:${spring.kafka.consumer.topic}.STATS.PROJECTION} +stats.projection-async.group-id=${STATS_PROJECTION_GROUP_ID:${spring.kafka.consumer.group-id}-stats-projection} +stats.projection-async.relay-interval-ms=${STATS_PROJECTION_RELAY_INTERVAL_MS:200} +stats.projection-async.relay-batch-size=${STATS_PROJECTION_RELAY_BATCH_SIZE:100} +stats.projection-async.publish-timeout-ms=${STATS_PROJECTION_PUBLISH_TIMEOUT_MS:10000} +stats.projection-async.retry-backoff-ms=${STATS_PROJECTION_RETRY_BACKOFF_MS:1000} +stats.projection-async.redelivery-interval-ms=${STATS_PROJECTION_REDELIVERY_INTERVAL_MS:60000} +stats.projection-async.max-concurrency-retries=${STATS_PROJECTION_MAX_CONCURRENCY_RETRIES:3} +stats.projection-async.concurrency-retry-backoff-ms=${STATS_PROJECTION_CONCURRENCY_RETRY_BACKOFF_MS:50} +stats.projection-async.retention-days=${STATS_PROJECTION_RETENTION_DAYS:14} +stats.projection-async.cleanup-initial-delay-ms=${STATS_PROJECTION_CLEANUP_INITIAL_DELAY_MS:3600000} +stats.projection-async.cleanup-interval-ms=${STATS_PROJECTION_CLEANUP_INTERVAL_MS:3600000} + # 标准化规则编译与批次缓存灰度开关 normalization.jetcache.enabled=true normalization.rule-source-cache.enabled=true diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SyslogFailureRouterReliabilityTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SyslogFailureRouterReliabilityTest.java index c1400f1..105fc66 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SyslogFailureRouterReliabilityTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SyslogFailureRouterReliabilityTest.java @@ -42,6 +42,11 @@ class SyslogFailureRouterReliabilityTest { ReflectionTestUtils.setField(router, "retry10mTopic", "raw.RETRY.10M"); ReflectionTestUtils.setField(router, "retry1hTopic", "raw.RETRY.1H"); ReflectionTestUtils.setField(router, "dltTopic", "raw.DLT"); + ReflectionTestUtils.setField(router, "largeOriginalTopic", "raw.large"); + ReflectionTestUtils.setField(router, "largeRetry1mTopic", "raw.large.RETRY.1M"); + ReflectionTestUtils.setField(router, "largeRetry10mTopic", "raw.large.RETRY.10M"); + ReflectionTestUtils.setField(router, "largeRetry1hTopic", "raw.large.RETRY.1H"); + ReflectionTestUtils.setField(router, "largeDltTopic", "raw.large.DLT"); ReflectionTestUtils.setField(router, "publishTimeoutMs", 1000L); ReflectionTestUtils.setField(router, "consumerVersion", "test"); record = new ConsumerRecord<>("raw", 2, 42L, "device-1", "ciphertext"); @@ -86,6 +91,22 @@ class SyslogFailureRouterReliabilityTest { assertFalse(body.getValue().contains("schema_version")); } + @Test + void largeMessageFailureNeverFallsBackIntoNormalRetryTopic() { + ConsumerRecord largeRecord = new ConsumerRecord<>( + "raw.large", 1, 7L, "event", "large-ciphertext"); + SettableListenableFuture> acknowledged = new SettableListenableFuture<>(); + acknowledged.set(null); + when(template.send(eq("raw.large.RETRY.1M"), anyString(), anyString())) + .thenReturn(acknowledged); + + SyslogFailureRouter.TerminalState state = router.route( + largeRecord, router.resolve(largeRecord), new SocketException("reset")); + + assertEquals(SyslogFailureRouter.TerminalState.RETRY_PUBLISHED, state); + verify(template).send(eq("raw.large.RETRY.1M"), anyString(), anyString()); + } + @Test void publishFailureNeverBecomesTerminal() { SettableListenableFuture> failed = new SettableListenableFuture<>(); diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SyslogKafkaTopologyTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SyslogKafkaTopologyTest.java index 5fa2659..7fd46ca 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SyslogKafkaTopologyTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SyslogKafkaTopologyTest.java @@ -4,12 +4,17 @@ import org.junit.jupiter.api.Test; import org.springframework.kafka.annotation.KafkaListener; import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.List; +import org.apache.kafka.clients.consumer.ConsumerRecord; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import com.common.service.StatsMaterializationConsumer; +import com.common.service.StatsProjectionDeltaConsumer; /** 防止多节点 consumer-rule 的监听器 ID 或显式消费组在后续合并中回退。 */ class SyslogKafkaTopologyTest { @@ -50,6 +55,56 @@ class SyslogKafkaTopologyTest { }, listener.topics()); } + @Test + void largeRawListenerUsesDedicatedTopicGroupAndFactory() throws Exception { + Method method = SysLogProcessor.class.getMethod( + "consumeLargeMessages", List.class, + org.springframework.kafka.support.Acknowledgment.class); + KafkaListener listener = method.getAnnotation(KafkaListener.class); + + assertNotNull(listener); + assertEquals("syslog-consumer-rule-raw-large", listener.id()); + assertEquals("${syslog.large-listener.group-id:${spring.kafka.consumer.group-id}-large}", + listener.groupId()); + assertEquals("largeRawKafkaListenerContainerFactory", listener.containerFactory()); + assertArrayEquals(new String[]{"${spring.kafka.consumer.large-topic:agent-syslog-topic.large}"}, + listener.topics()); + } + + @Test + void largeRetryReplayListenerIsAlsoIsolated() throws Exception { + Method method = SysLogProcessor.class.getMethod( + "consumeLargeRetryAndReplay", List.class, + org.springframework.kafka.support.Acknowledgment.class); + KafkaListener listener = method.getAnnotation(KafkaListener.class); + + assertNotNull(listener); + assertEquals("syslog-consumer-rule-large-retry-replay", listener.id()); + assertEquals("largeRawKafkaListenerContainerFactory", listener.containerFactory()); + assertArrayEquals(new String[]{ + "${syslog.large-reliability.retry-1m-topic:${spring.kafka.consumer.large-topic}.RETRY.1M}", + "${syslog.large-reliability.retry-10m-topic:${spring.kafka.consumer.large-topic}.RETRY.10M}", + "${syslog.large-reliability.retry-1h-topic:${spring.kafka.consumer.large-topic}.RETRY.1H}", + "${syslog.large-reliability.replay-topic:${spring.kafka.consumer.large-topic}.REPLAY}" + }, listener.topics()); + } + + @Test + void largePollIsPartitionedByUtf8BytesInsteadOfOnlyRecordCount() { + List> records = Arrays.asList( + new ConsumerRecord<>("large", 0, 0L, "k1", "中文"), + new ConsumerRecord<>("large", 0, 1L, "k2", "12"), + new ConsumerRecord<>("large", 0, 2L, "k3", "🙂")); + + List>> chunks = + SysLogProcessor.partitionRecordsByBytes(records, 7L); + + assertEquals(2, chunks.size()); + assertEquals(1, chunks.get(0).size()); + assertEquals(2, chunks.get(1).size()); + assertEquals(6, chunks.get(0).get(0).value().getBytes(StandardCharsets.UTF_8).length); + } + @Test void statsMaterializerUsesDedicatedTopicAndGroup() throws Exception { Method method = StatsMaterializationConsumer.class.getMethod( @@ -65,4 +120,20 @@ class SyslogKafkaTopologyTest { "${stats.materialization.topic:${spring.kafka.consumer.topic}.STATS.MATERIALIZE}" }, listener.topics()); } + + @Test + void statsProjectionUsesSeparateSingleLaneTopicAndGroup() throws Exception { + Method method = StatsProjectionDeltaConsumer.class.getMethod( + "consume", List.class, org.springframework.kafka.support.Acknowledgment.class); + KafkaListener listener = method.getAnnotation(KafkaListener.class); + + assertNotNull(listener); + assertEquals("syslog-stats-projection", listener.id()); + assertEquals("${stats.projection-async.group-id:${spring.kafka.consumer.group-id}-stats-projection}", + listener.groupId()); + assertEquals("statsProjectionKafkaListenerContainerFactory", listener.containerFactory()); + assertArrayEquals(new String[]{ + "${stats.projection-async.topic:${spring.kafka.consumer.topic}.STATS.PROJECTION}" + }, listener.topics()); + } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DirectStatsProjectionWriterTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DirectStatsProjectionWriterTest.java index efc5b60..837a704 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DirectStatsProjectionWriterTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DirectStatsProjectionWriterTest.java @@ -119,12 +119,14 @@ class DirectStatsProjectionWriterTest { } @Test - void kafkaMaterializationRemovesBusinessTableUpdatesFromFactTransaction() { + void durableOutboxRemovesDailyAndBusinessProjectionUpdatesFromFactTransaction() { JdbcTemplate jdbc = mock(JdbcTemplate.class); StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class); when(publisher.isEnabled()).thenReturn(true); + StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class); + when(outbox.isEnabled()).thenReturn(true); DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter( - jdbc, new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), publisher); + jdbc, new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), publisher, outbox); configure(writer, true); TransactionSynchronizationManager.setActualTransactionActive(true); @@ -135,20 +137,19 @@ class DirectStatsProjectionWriterTest { row.put("normalize_rule_id", 11L); writer.writeStandardizedRow(row); - ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); - verify(jdbc, times(2)).batchUpdate(sql.capture(), anyList()); - assertTrue(sql.getAllValues().get(0).contains("MERGE INTO device_daily_stats")); - assertTrue(sql.getAllValues().get(1).contains("MERGE INTO normalize_rule_daily_stats")); - verify(publisher).enqueueAfterCommit(anyCollection()); - assertTrue("KAFKA_DIRTY_KEY".equals(writer.status().get("business_materialization_mode"))); + verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class)); + verifyNoInteractions(jdbc); + assertTrue("DURABLE_OUTBOX".equals(writer.status().get("daily_projection_write_mode"))); } private DirectStatsProjectionWriter writer(JdbcTemplate jdbc, String mode, boolean readEnabled) { StatsWriteModeProvider modeProvider = new StatsWriteModeProvider(mode); StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class); when(publisher.isEnabled()).thenReturn(false); + StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class); + when(outbox.isEnabled()).thenReturn(false); DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter( - jdbc, modeProvider, new DirectStatsWriteMetrics(), publisher); + jdbc, modeProvider, new DirectStatsWriteMetrics(), publisher, outbox); configure(writer, readEnabled); return writer; } diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmLargeTextLimiterTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmLargeTextLimiterTest.java new file mode 100644 index 0000000..8123f89 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmLargeTextLimiterTest.java @@ -0,0 +1,62 @@ +package com.common.service; + +import com.common.entity.SyslogNonNormalMessage; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DmLargeTextLimiterTest { + + @Test + void normalConfiguredFieldsAreTruncatedButOtherFieldsRemainUntouched() { + DmLargeTextLimiter limiter = limiter(96L, "http_req_body,description", 0L); + Map row = new HashMap<>(); + row.put("http_req_body", repeat("中文🙂", 100)); + row.put("other", repeat("x", 200)); + + limiter.truncateNormalRow(row); + + String body = (String) row.get("http_req_body"); + assertTrue(body.contains("HB_XDR_TRUNCATED")); + assertTrue(body.getBytes(StandardCharsets.UTF_8).length <= 96); + assertEquals(repeat("x", 200), row.get("other")); + assertEquals(1L, limiter.getNormalFieldsTruncated()); + } + + @Test + void allNonNormalLargeColumnsUseTheSameVisibleMarker() { + DmLargeTextLimiter limiter = limiter(0L, "", 96L); + SyslogNonNormalMessage message = new SyslogNonNormalMessage(); + message.setSyslogMessage(repeat("原文", 100)); + message.setHeaderMessage(repeat("头", 100)); + message.setReasonDetail(repeat("原因", 100)); + + limiter.truncateNonNormal(message); + + assertTrue(message.getSyslogMessage().contains("HB_XDR_TRUNCATED")); + assertTrue(message.getHeaderMessage().contains("HB_XDR_TRUNCATED")); + assertTrue(message.getReasonDetail().contains("HB_XDR_TRUNCATED")); + assertEquals(3L, limiter.getNonNormalFieldsTruncated()); + } + + private DmLargeTextLimiter limiter(long normal, String fields, long nonNormal) { + DmLargeTextLimiter limiter = new DmLargeTextLimiter(); + ReflectionTestUtils.setField(limiter, "normalMaxBytes", normal); + ReflectionTestUtils.setField(limiter, "normalFieldsValue", fields); + ReflectionTestUtils.setField(limiter, "nonNormalMaxBytes", nonNormal); + limiter.validate(); + return limiter; + } + + private static String repeat(String value, int count) { + StringBuilder result = new StringBuilder(value.length() * count); + for (int i = 0; i < count; i++) result.append(value); + return result.toString(); + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsProjectionWriterTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsProjectionWriterTest.java index fa08f06..e846156 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsProjectionWriterTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/IngressStatsProjectionWriterTest.java @@ -2,6 +2,11 @@ package com.common.service; import com.common.entity.IngressStatsEvent; import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.SimpleTransactionStatus; +import org.springframework.jdbc.core.JdbcTemplate; import java.time.LocalDateTime; import java.util.Arrays; @@ -9,6 +14,11 @@ import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; class IngressStatsProjectionWriterTest { @Test @@ -41,4 +51,28 @@ class IngressStatsProjectionWriterTest { assertEquals(Arrays.asList(8L, 2L, 9L), devices); assertEquals(Arrays.asList(3L, 4L, 7L), new ArrayList<>(result.collects.keySet())); } + + @Test + void directAsyncModeWritesOnlyOneOutboxEvent() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + PlatformTransactionManager manager = mock(PlatformTransactionManager.class); + when(manager.getTransaction(any(TransactionDefinition.class))) + .thenReturn(new SimpleTransactionStatus()); + StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class); + when(outbox.isEnabled()).thenReturn(true); + IngressStatsProjectionWriter writer = new IngressStatsProjectionWriter( + jdbc, manager, new StatsWriteModeProvider("DIRECT"), + mock(StatsMaterializationPublisher.class), outbox); + ReflectionTestUtils.setField(writer, "enabled", true); + ReflectionTestUtils.setField(writer, "transactionTimeoutSeconds", 45); + + int groups = writer.writeBatch(Arrays.asList( + new IngressStatsEvent("a", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 0)), + new IngressStatsEvent("b", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 1)))); + + assertEquals(2, groups); + verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class)); + verifyNoInteractions(jdbc); + verify(manager).commit(any()); + } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncTest.java new file mode 100644 index 0000000..1b30807 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/StatsProjectionAsyncTest.java @@ -0,0 +1,151 @@ +package com.common.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.SimpleTransactionStatus; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class StatsProjectionAsyncTest { + @AfterEach + void clearTransactionState() { + TransactionSynchronizationManager.clear(); + } + + @Test + void outboxEventAggregatesOneFactBatchByDimension() { + LocalDateTime time = LocalDateTime.of(2026, 9, 12, 8, 0); + List> rows = new ArrayList<>(); + for (long ruleId : Arrays.asList(11L, 12L, 11L)) { + Map row = new HashMap<>(); + row.put("created_at", time); + row.put("log_time", time.plusMinutes(ruleId)); + row.put("device_id", 7L); + row.put("normalize_rule_id", ruleId); + rows.add(row); + } + + StatsProjectionOutboxEvent event = StatsProjectionOutboxEvent.fromAccumulator( + new DirectStatsAccumulator().addStandardizedRows(rows)); + + event.validate(); + assertEquals(1, event.getDevices().size()); + assertEquals(3L, event.getDevices().get(0).getParseDelta()); + assertEquals(2, event.getRules().size()); + } + + @Test + void outboxWriterRequiresAndUsesFactTransaction() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + StatsProjectionOutboxWriter writer = new StatsProjectionOutboxWriter(jdbc, new ObjectMapper()); + ReflectionTestUtils.setField(writer, "enabled", true); + ReflectionTestUtils.setField(writer, "outboxTable", "stats_projection_outbox"); + StatsProjectionOutboxEvent event = oneDeviceEvent("event-1"); + + assertThrows(IllegalStateException.class, () -> writer.append(event)); + + TransactionSynchronizationManager.setActualTransactionActive(true); + writer.append(event); + verify(jdbc).update(org.mockito.ArgumentMatchers.contains("INSERT INTO stats_projection_outbox"), + org.mockito.ArgumentMatchers.eq("event-1"), + org.mockito.ArgumentMatchers.eq(StatsProjectionOutboxEvent.SCHEMA), + org.mockito.ArgumentMatchers.anyString()); + } + + @Test + void projectionConsumerAppliesFreshEventAndRecordsIdempotencyMarker() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + when(jdbc.queryForList(org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.eq(String.class), org.mockito.ArgumentMatchers.any())) + .thenReturn(Collections.emptyList()); + PlatformTransactionManager manager = mock(PlatformTransactionManager.class); + when(manager.getTransaction(any(TransactionDefinition.class))).thenReturn(new SimpleTransactionStatus()); + StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class); + StatsProjectionDeltaConsumer consumer = new StatsProjectionDeltaConsumer( + jdbc, manager, new ObjectMapper(), publisher); + configure(consumer); + + StatsProjectionDeltaConsumer.ApplyResult result = consumer.applyWithRetry( + Collections.singletonList(oneDeviceEvent("event-1"))); + + assertEquals(1, result.appliedEvents); + assertEquals(1, result.deviceGroups); + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(jdbc, times(3)).batchUpdate(sql.capture(), anyList()); + assertTrue(sql.getAllValues().get(0).contains("MERGE INTO device_daily_stats")); + assertTrue(sql.getAllValues().get(1).contains("stats_projection_applied_event")); + assertTrue(sql.getAllValues().get(2).contains("SET status='APPLIED'")); + verify(publisher).enqueueAfterCommit(org.mockito.ArgumentMatchers.anyCollection()); + verify(manager).commit(any()); + } + + @Test + void projectionConsumerDoesNotApplyDuplicateDeltaTwice() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + when(jdbc.queryForList(org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.eq(String.class), org.mockito.ArgumentMatchers.any())) + .thenReturn(Collections.singletonList("event-1")); + PlatformTransactionManager manager = mock(PlatformTransactionManager.class); + when(manager.getTransaction(any(TransactionDefinition.class))).thenReturn(new SimpleTransactionStatus()); + StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class); + StatsProjectionDeltaConsumer consumer = new StatsProjectionDeltaConsumer( + jdbc, manager, new ObjectMapper(), publisher); + configure(consumer); + + StatsProjectionDeltaConsumer.ApplyResult result = consumer.applyWithRetry( + Collections.singletonList(oneDeviceEvent("event-1"))); + + assertEquals(0, result.appliedEvents); + assertEquals(1, result.duplicates); + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(jdbc).batchUpdate(sql.capture(), anyList()); + assertTrue(sql.getValue().contains("SET status='APPLIED'")); + } + + private StatsProjectionOutboxEvent oneDeviceEvent(String eventId) { + DirectStatsAccumulator accumulator = new DirectStatsAccumulator(); + Map row = new HashMap<>(); + row.put("created_at", LocalDate.of(2026, 9, 12)); + row.put("log_time", LocalDateTime.of(2026, 9, 12, 8, 0)); + row.put("device_id", 7L); + StatsProjectionOutboxEvent event = StatsProjectionOutboxEvent.fromAccumulator( + accumulator.addStandardizedRow(row)); + event.setEventId(eventId); + return event; + } + + private void configure(StatsProjectionDeltaConsumer consumer) { + ReflectionTestUtils.setField(consumer, "enabled", true); + ReflectionTestUtils.setField(consumer, "appliedTable", "stats_projection_applied_event"); + ReflectionTestUtils.setField(consumer, "outboxTable", "stats_projection_outbox"); + ReflectionTestUtils.setField(consumer, "deviceTable", "device_daily_stats"); + ReflectionTestUtils.setField(consumer, "ruleTable", "normalize_rule_daily_stats"); + ReflectionTestUtils.setField(consumer, "transactionTimeoutSeconds", 45); + ReflectionTestUtils.setField(consumer, "maxConcurrencyRetries", 3); + ReflectionTestUtils.setField(consumer, "retryBackoffMs", 0L); + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/util/Utf8TextTruncatorTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/util/Utf8TextTruncatorTest.java new file mode 100644 index 0000000..6a0a03e --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/util/Utf8TextTruncatorTest.java @@ -0,0 +1,43 @@ +package com.common.util; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class Utf8TextTruncatorTest { + @Test + void truncatesWithinBudgetWithoutSplittingChineseOrEmoji() { + String source = repeat("中文🙂", 100); + + Utf8TextTruncator.Result result = Utf8TextTruncator.truncate(source, 96L); + + assertTrue(result.isTruncated()); + assertTrue(result.getValue().endsWith("]")); + assertTrue(result.getValue().contains("HB_XDR_TRUNCATED")); + assertTrue(result.getValue().getBytes(StandardCharsets.UTF_8).length <= 96); + assertEquals(source.getBytes(StandardCharsets.UTF_8).length, result.getOriginalUtf8Bytes()); + assertNotNull(result.getOriginalSha256()); + assertFalse(result.getValue().contains("�")); + } + + @Test + void zeroMeansUnlimitedAndSmallEnabledLimitFailsClearly() { + Utf8TextTruncator.Result result = Utf8TextTruncator.truncate("中文🙂", 0L); + assertFalse(result.isTruncated()); + assertEquals("中文🙂", result.getValue()); + assertThrows(IllegalArgumentException.class, + () -> Utf8TextTruncator.truncate(repeat("x", 100), 8L)); + } + + private static String repeat(String value, int count) { + StringBuilder result = new StringBuilder(value.length() * count); + for (int i = 0; i < count; i++) result.append(value); + return result.toString(); + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java index 4bdce65..4a3c1b9 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/influx/raw/InfluxRawV2ContractTest.java @@ -67,6 +67,25 @@ class InfluxRawV2ContractTest { String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket_v2",raw.getBucket()); } + @Test void oversizedRawMessageIsMarkedAndCarriesStructuredTruncationMetadata() { + RawStorageProperties properties=mock(RawStorageProperties.class); + when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2); + when(properties.getV2Bucket()).thenReturn("yelangbucket_v2"); + when(properties.getMessageMaxUtf8Bytes()).thenReturn(128L); + RawInfluxMetrics metrics=new RawInfluxMetrics(); + InfluxRawPointFactory factory=new InfluxRawPointFactory(properties,metrics); + + String line=factory.create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5", + repeat("中文🙂",100),"20260814000000000","126","1","raw.large", + ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L) + .getPoint().toLineProtocol(); + + assertTrue(line.contains("HB_XDR_TRUNCATED")); + assertTrue(line.contains("message_truncated=true")); + assertTrue(line.contains("message_original_sha256=")); + assertEquals(1L,metrics.snapshot().get("raw_message_truncated_total")); + } + @Test void onlyMissingPointTimeFallsBack() { RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); long millis=ReceiveTimeParser.parse("20260814000000000").getEpochMillis(); assertNotNull(resolver.resolve(false,null,millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,"-1",millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,"not-number",millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,Long.toString(millis*1_000_000L+1_000_000L),millis,true)); assertEquals(1L,metrics.snapshot().get("legacy_missing_point_time_total")); @@ -108,4 +127,10 @@ class InfluxRawV2ContractTest { assertTrue(query.indexOf("|> group()") < query.indexOf("|> sort(")); assertTrue(query.indexOf("|> sort(") < query.indexOf("|> limit(n: 2)")); } + + private static String repeat(String value, int count) { + StringBuilder result = new StringBuilder(value.length() * count); + for (int i=0;i= deadline) diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java index 1bdb226..41072a0 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java @@ -19,9 +19,17 @@ public class AppProperties { @Value("${syslog.udp.port:515}") private int syslogUdpPort; - @Value("${syslog.max.frame.length:65536}") + @Value("${syslog.max.frame.length:6291456}") private int syslogMaxFrameLength; + /** 所有接入方式共用的单条原文字节硬上限；0 表示不限制。 */ + @Value("${syslog.message.max-bytes:6291456}") + private long syslogMessageMaxBytes; + + /** 超过该值投递大日志 Topic；0 表示关闭分流。 */ + @Value("${syslog.large-message.threshold-bytes:262144}") + private long syslogLargeMessageThresholdBytes; + @Value("${syslog.buffer.size:1024}") private int syslogBufferSize; @@ -50,6 +58,10 @@ public class AppProperties { @Value("${syslog.wal.queue-capacity:10000}") private int syslogWalQueueCapacity; + /** 队列与待 force 记录共用的内存字节预算；0 表示不限制。 */ + @Value("${syslog.wal.max-pending-bytes:268435456}") + private long syslogWalMaxPendingBytes; + @Value("${syslog.wal.high-watermark-percent:70}") private int syslogWalHighWatermarkPercent; @@ -114,6 +126,10 @@ public class AppProperties { @Value("${syslog.dispatch.max-inflight:5000}") private int syslogDispatchMaxInflight; + /** Dispatcher 正文在途字节预算；0 表示不限制。 */ + @Value("${syslog.dispatch.max-inflight-bytes:268435456}") + private long syslogDispatchMaxInflightBytes; + @Value("${syslog.dispatch.retry-backoff-ms:1000}") private long syslogDispatchRetryBackoffMs; @@ -166,6 +182,9 @@ public class AppProperties { @Value("${spring.kafka.producer.topic:agent-syslog-topic}") private String kafkaProducerTopic; + @Value("${spring.kafka.producer.large-topic:agent-syslog-topic.large}") + private String kafkaProducerLargeTopic; + @Value("${spring.kafka.producer.ack-timeout-ms:130000}") private long kafkaProducerAckTimeoutMs; diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/SyslogWalDispatcher.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/SyslogWalDispatcher.java index 9417e14..969bb4b 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/SyslogWalDispatcher.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/dispatch/SyslogWalDispatcher.java @@ -48,9 +48,11 @@ public class SyslogWalDispatcher { private final LongAdder kafkaAcked = new LongAdder(); private final LongAdder terminalFiltered = new LongAdder(); private final LongAdder retryCount = new LongAdder(); + private final LongAdder largeDispatched = new LongAdder(); private final AtomicLong inflight = new AtomicLong(); private Semaphore permits; + private ByteBudget byteBudget; private ExecutorService workers; private ScheduledExecutorService scheduler; private Thread readerThread; @@ -72,6 +74,7 @@ public class SyslogWalDispatcher { public void start() throws Exception { validateConfiguration(); permits = new Semaphore(properties.getSyslogDispatchMaxInflight()); + byteBudget = new ByteBudget(properties.getSyslogDispatchMaxInflightBytes()); workers = Executors.newFixedThreadPool(properties.getSyslogDispatchWorkerThreads(), namedFactory("syslog-dispatch-worker")); scheduler = Executors.newScheduledThreadPool(2, namedFactory("syslog-dispatch-scheduler")); @@ -85,8 +88,9 @@ public class SyslogWalDispatcher { scheduler.scheduleWithFixedDelay(this::logStatus, properties.getSyslogStatusLogIntervalMs(), properties.getSyslogStatusLogIntervalMs(), TimeUnit.MILLISECONDS); - log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, checkpointSequence={}", + log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, maxInflightBytes={}, checkpointSequence={}", properties.getSyslogDispatchWorkerThreads(), properties.getSyslogDispatchMaxInflight(), + properties.getSyslogDispatchMaxInflightBytes(), walService.getCheckpointSequence()); } @@ -99,7 +103,11 @@ public class SyslogWalDispatcher { permits.release(); continue; } + record.setDispatchPayloadBytes(Math.max(1L, + Math.max(record.getRawLength(), utf8Length(record.getRawMessage())))); + byteBudget.acquire(record.getDispatchPayloadBytes()); if (record.getSequence() != nextReadSequence) { + byteBudget.release(record.getDispatchPayloadBytes()); permits.release(); throw new IllegalStateException("WAL sequence 不连续，拒绝越过: expected=" + nextReadSequence + ", actual=" + record.getSequence()); @@ -158,8 +166,13 @@ public class SyslogWalDispatcher { eventTime = LocalDateTime.parse(walEventTime, WAL_TIME).format(HEADER_TIME); // v2/v3 继续调用旧重载，确保升级前已落盘记录产生的 Kafka 消息完全不变。 - java.util.concurrent.CompletableFuture deliveryFuture = - record.getSchemaVersion() >= 4 + boolean large = properties.getSyslogLargeMessageThresholdBytes() > 0L + && record.getDispatchPayloadBytes() > properties.getSyslogLargeMessageThresholdBytes(); + java.util.concurrent.CompletableFuture deliveryFuture = large + ? producer.messagePushLarge( + resolution.getDevice(), record.getRawMessage(), receiveTime, eventTime, + record.getEventId(), record.getPointTimeNs()) + : record.getSchemaVersion() >= 4 ? producer.messagePush( resolution.getDevice(), record.getRawMessage(), @@ -177,6 +190,7 @@ public class SyslogWalDispatcher { .whenComplete((delivery, failure) -> { if (failure == null) { kafkaAcked.increment(); + if (large) largeDispatched.increment(); markComplete(record); } else { scheduleRetry(record, attempt, "KAFKA_ACK", unwrap(failure)); @@ -206,7 +220,9 @@ public class SyslogWalDispatcher { private void markComplete(SyslogWalRecord record) { synchronized (completionMonitor) { if (record.getSequence() < nextCommitSequence) return; - completedOutOfOrder.put(record.getSequence(), record); + if (completedOutOfOrder.putIfAbsent(record.getSequence(), checkpointOnly(record)) != null) return; + // 已 ACK 的正文不再占字节预算；条数许可仍要等连续 checkpoint，限制乱序完成表大小。 + byteBudget.release(record.getDispatchPayloadBytes()); SyslogWalRecord latest = null; int released = 0; while (true) { @@ -224,16 +240,26 @@ public class SyslogWalDispatcher { } } + private SyslogWalRecord checkpointOnly(SyslogWalRecord source) { + SyslogWalRecord value = new SyslogWalRecord(); + value.setSequence(source.getSequence()); + value.setSegment(source.getSegment()); + value.setStartOffset(source.getStartOffset()); + value.setEndOffset(source.getEndOffset()); + return value; + } + private void logStatus() { if (!running.get()) return; - log.info("Syslog WAL状态: networkReceived={}, appended={}, walPendingApprox={}, walQueue={}/{}, durableSequence={}, " - + "checkpointSequence={}, kafkaInflight={}, kafkaAcked={}, terminalFiltered={}, " + log.info("Syslog WAL状态: networkReceived={}, appended={}, admissionOversized={}, walPendingApprox={}, walPendingBytes={}, walQueue={}/{}, durableSequence={}, " + + "checkpointSequence={}, kafkaInflight={}, kafkaInflightBytes={}, kafkaAcked={}, largeDispatched={}, terminalFiltered={}, " + "retry={}, tcpPauseTotal={}, udpDropped={}, deviceCacheSize={}, deviceCacheHits={}, " + "deviceCacheMisses={}", SyslogMessageHandler.getNetworkReceivedCount(), walService.getAppendedCount(), - walService.getPendingRecordsApprox(), + walService.getOversizedCount(), walService.getPendingRecordsApprox(), walService.getPendingAppendBytes(), walService.getQueueSize(), walService.getQueueCapacity(), walService.getDurableSequence(), - walService.getCheckpointSequence(), inflight.get(), kafkaAcked.sum(), terminalFiltered.sum(), + walService.getCheckpointSequence(), inflight.get(), byteBudget == null ? 0L : byteBudget.used(), + kafkaAcked.sum(), largeDispatched.sum(), terminalFiltered.sum(), retryCount.sum(), SyslogMessageHandler.getTcpPauseCount(), SyslogMessageHandler.getUdpDroppedCount(), deviceResolver.getCacheSize(), deviceResolver.getHitCount(), deviceResolver.getMissCount()); @@ -246,6 +272,23 @@ public class SyslogWalDispatcher { return failure; } + /** 不分配临时 byte[] 的 UTF-8 长度估算；非法孤立代理项按 3 字节保守计算。 */ + static long utf8Length(String value) { + if (value == null || value.isEmpty()) return 0L; + long bytes = 0L; + for (int index = 0; index < value.length(); index++) { + char current = value.charAt(index); + if (current <= 0x7f) bytes++; + else if (current <= 0x7ff) bytes += 2L; + else if (Character.isHighSurrogate(current) && index + 1 < value.length() + && Character.isLowSurrogate(value.charAt(index + 1))) { + bytes += 4L; + index++; + } else bytes += 3L; + } + return bytes; + } + private ThreadFactory namedFactory(String prefix) { AtomicLong sequence = new AtomicLong(); return runnable -> { @@ -258,18 +301,58 @@ public class SyslogWalDispatcher { private void validateConfiguration() { if (properties.getSyslogDispatchWorkerThreads() <= 0 || properties.getSyslogDispatchMaxInflight() <= 0 + || properties.getSyslogDispatchMaxInflightBytes() < 0L || properties.getSyslogDispatchRetryBackoffMs() <= 0 || properties.getSyslogDispatchShutdownTimeoutMs() <= 0 || properties.getSyslogDeviceContextCacheTtlMs() <= 0 || properties.getSyslogStatusLogIntervalMs() <= 0) { throw new IllegalArgumentException("Dispatcher 线程、在途、重试、缓存和状态日志参数必须大于 0"); } + if (properties.getSyslogMessageMaxBytes() > 0L + && properties.getSyslogDispatchMaxInflightBytes() > 0L + && properties.getSyslogMessageMaxBytes() > properties.getSyslogDispatchMaxInflightBytes()) { + throw new IllegalArgumentException("单条消息上限不能大于 Dispatcher 在途字节预算"); + } } public long getInflight() { return inflight.get(); } public long getKafkaAcked() { return kafkaAcked.sum(); } public long getTerminalFiltered() { return terminalFiltered.sum(); } public long getRetryCount() { return retryCount.sum(); } + public long getLargeDispatched() { return largeDispatched.sum(); } + public long getInflightBytes() { return byteBudget == null ? 0L : byteBudget.used(); } + + /** 同时保留条数 Semaphore 和字节预算，避免少量大消息占满堆。 */ + static final class ByteBudget { + private final long capacity; + private long used; + + ByteBudget(long capacity) { + if (capacity < 0L) throw new IllegalArgumentException("capacity 不能小于 0"); + this.capacity = capacity; + } + + synchronized void acquire(long bytes) throws InterruptedException { + if (bytes <= 0L) throw new IllegalArgumentException("bytes 必须大于 0"); + if (capacity == 0L) { used += bytes; return; } + if (bytes > capacity) { + // 硬上限关闭时仍保证进展：超预算单条只能独占，不允许与其他正文并存。 + while (used != 0L) wait(); + used = bytes; + return; + } + while (used > capacity - bytes) wait(); + used += bytes; + } + + synchronized void release(long bytes) { + used -= bytes; + if (used < 0L) throw new IllegalStateException("Dispatcher 字节预算释放越界"); + notifyAll(); + } + + synchronized long used() { return used; } + } @PreDestroy public void shutdown() { diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AdmissionStatus.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AdmissionStatus.java index d4757e7..3d071cb 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AdmissionStatus.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AdmissionStatus.java @@ -4,5 +4,6 @@ package com.haobang.wal; public enum AdmissionStatus { ACCEPTED, BACKPRESSURE, + OVERSIZED, UNAVAILABLE } diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AppendAttempt.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AppendAttempt.java index bcbcb70..faa0a58 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AppendAttempt.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/AppendAttempt.java @@ -24,7 +24,7 @@ public final class AppendAttempt { public static AppendAttempt rejected(AdmissionStatus status) { if (status == null || status == AdmissionStatus.ACCEPTED) { - throw new IllegalArgumentException("rejected 只能使用 BACKPRESSURE 或 UNAVAILABLE"); + throw new IllegalArgumentException("rejected 不能使用 ACCEPTED"); } return new AppendAttempt(status, null); } diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/PendingWalAppend.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/PendingWalAppend.java index 90df38d..0ac7b85 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/PendingWalAppend.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/PendingWalAppend.java @@ -5,18 +5,22 @@ import java.util.concurrent.CompletableFuture; /** WAL 写线程队列中的事件及其待完成 durable future。 sequence/schemaVersion 由唯一写线程赋值，生产者线程不得提前推断 WAL 位置。 */ final class PendingWalAppend { private final IngressEvent event; + private final int rawBytes; private final CompletableFuture durableFuture = new CompletableFuture<>(); private long sequence; private int schemaVersion; - PendingWalAppend(IngressEvent event) { + PendingWalAppend(IngressEvent event, int rawBytes) { this.event = event; + this.rawBytes = rawBytes; } IngressEvent getEvent() { return event; } + int getRawBytes() { return rawBytes; } + CompletableFuture getDurableFuture() { return durableFuture; } diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogLegacyArchiveImporter.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogLegacyArchiveImporter.java index 80d2857..77d204b 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogLegacyArchiveImporter.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogLegacyArchiveImporter.java @@ -112,6 +112,10 @@ public class SyslogLegacyArchiveImporter { while (true) { SyslogWalService.AdmissionResult result = walService.tryAppend(event); if (result == SyslogWalService.AdmissionResult.ACCEPTED) return; + if (result == SyslogWalService.AdmissionResult.OVERSIZED) { + throw new IllegalStateException("旧归档事件超过当前单条消息硬上限，拒绝永久重试: eventId=" + + event.getEventId()); + } if (result == SyslogWalService.AdmissionResult.UNAVAILABLE) { throw new IllegalStateException("WAL 不可用，无法导入旧归档 eventId=" + event.getEventId(), walService.getFatalFailure()); diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecord.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecord.java index 379fdbf..7cefefc 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecord.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalRecord.java @@ -87,6 +87,8 @@ public class SyslogWalRecord { @JsonIgnore private long endOffset; @JsonIgnore private boolean quarantinedInvalid; @JsonIgnore private String validationError; + /** Dispatcher 对正文使用的 UTF-8/源编码较大值；只存在内存中，不改变 WAL 协议。 */ + @JsonIgnore private long dispatchPayloadBytes; public int getSchemaVersion() { return schemaVersion; @@ -232,6 +234,14 @@ public class SyslogWalRecord { this.rawLength = rawLength; } + public long getDispatchPayloadBytes() { + return dispatchPayloadBytes; + } + + public void setDispatchPayloadBytes(long dispatchPayloadBytes) { + this.dispatchPayloadBytes = dispatchPayloadBytes; + } + public String getRawSha256() { return rawSha256; } diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalService.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalService.java index fd8ab4e..bf7918c 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalService.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalService.java @@ -64,7 +64,7 @@ public class SyslogWalService { private static final String SEGMENT_GLOB = "syslog-wal-*.jsonl"; static final long COMPLETED_CLEANUP_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(30L); - public enum AdmissionResult { ACCEPTED, BACKPRESSURE, UNAVAILABLE } + public enum AdmissionResult { ACCEPTED, BACKPRESSURE, OVERSIZED, UNAVAILABLE } private final AppProperties properties; private final ObjectMapper objectMapper; @@ -76,6 +76,8 @@ public class SyslogWalService { private final AtomicLong durableSequence = new AtomicLong(); private final LongAdder appended = new LongAdder(); private final LongAdder admissionBackpressure = new LongAdder(); + private final LongAdder admissionOversized = new LongAdder(); + private final AtomicLong pendingAppendBytes = new AtomicLong(); private final LongAdder quarantined = new LongAdder(); private final LongAdder terminalRecorded = new LongAdder(); private final Object durableMonitor = new Object(); @@ -211,6 +213,7 @@ public class SyslogWalService { AppendAttempt attempt = tryAppendUnified(toUnified(event)); if (attempt.getStatus() == AdmissionStatus.ACCEPTED) return AdmissionResult.ACCEPTED; if (attempt.getStatus() == AdmissionStatus.BACKPRESSURE) return AdmissionResult.BACKPRESSURE; + if (attempt.getStatus() == AdmissionStatus.OVERSIZED) return AdmissionResult.OVERSIZED; return AdmissionResult.UNAVAILABLE; } @@ -231,8 +234,19 @@ public class SyslogWalService { if (!running || fatalFailure != null || diskState == DiskState.CRITICAL) { return AppendAttempt.rejected(AdmissionStatus.UNAVAILABLE); } - PendingWalAppend pending = new PendingWalAppend(event); + int rawBytes = event.getRawMessage().getBytes(event.getCharset()).length; + if (properties.getSyslogMessageMaxBytes() > 0L + && rawBytes > properties.getSyslogMessageMaxBytes()) { + admissionOversized.increment(); + return AppendAttempt.rejected(AdmissionStatus.OVERSIZED); + } + if (!reservePendingBytes(rawBytes)) { + admissionBackpressure.increment(); + return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE); + } + PendingWalAppend pending = new PendingWalAppend(event, rawBytes); if (!appendQueue.offer(pending)) { + releasePendingBytes(rawBytes); admissionBackpressure.increment(); return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE); } @@ -243,14 +257,18 @@ public class SyslogWalService { refreshDiskState(false); if (fatalFailure != null || diskState != DiskState.NORMAL) return true; int high = percentageCount(properties.getSyslogWalHighWatermarkPercent()); - return appendQueue != null && appendQueue.size() >= high; + long byteHigh = percentageBytes(properties.getSyslogWalHighWatermarkPercent()); + return appendQueue != null && (appendQueue.size() >= high + || (byteHigh > 0L && pendingAppendBytes.get() >= byteHigh)); } public boolean canResumeTcp() { refreshDiskState(false); if (fatalFailure != null || diskState != DiskState.NORMAL) return false; int low = percentageCount(properties.getSyslogWalLowWatermarkPercent()); - return appendQueue != null && appendQueue.size() <= low; + long byteLow = percentageBytes(properties.getSyslogWalLowWatermarkPercent()); + return appendQueue != null && appendQueue.size() <= low + && (byteLow == 0L || pendingAppendBytes.get() <= byteLow); } public long getRetryIntervalMs() { @@ -263,6 +281,8 @@ public class SyslogWalService { public int getQueueCapacity() { return properties.getSyslogWalQueueCapacity(); } public long getAppendedCount() { return appended.sum(); } public long getBackpressureCount() { return admissionBackpressure.sum(); } + public long getOversizedCount() { return admissionOversized.sum(); } + public long getPendingAppendBytes() { return pendingAppendBytes.get(); } public long getQuarantinedCount() { return quarantined.sum(); } public long getTerminalRecordedCount() { return terminalRecorded.sum(); } public long getIngressPointTimeAssignedCount() { return pointTimeAllocator.getAssignedCount(); } @@ -495,6 +515,7 @@ public class SyslogWalService { pending.getSequence(), pending.getSchemaVersion(), durableAt); + releasePendingBytes(pending.getRawBytes()); pending.getDurableFuture().complete(receipt); } pendingForce.clear(); @@ -538,12 +559,14 @@ public class SyslogWalService { private void completePendingExceptionally(Throwable failure) { for (PendingWalAppend pending : pendingForce) { + releasePendingBytes(pending.getRawBytes()); pending.getDurableFuture().completeExceptionally(failure); } pendingForce.clear(); if (appendQueue == null) return; PendingWalAppend pending; while ((pending = appendQueue.poll()) != null) { + releasePendingBytes(pending.getRawBytes()); pending.getDurableFuture().completeExceptionally(failure); } } @@ -552,6 +575,38 @@ public class SyslogWalService { return Math.max(1, (int) Math.floor(properties.getSyslogWalQueueCapacity() * percent / 100.0)); } + private long percentageBytes(int percent) { + long capacity = properties.getSyslogWalMaxPendingBytes(); + return capacity <= 0L ? 0L : Math.max(1L, capacity * percent / 100L); + } + + private boolean reservePendingBytes(long bytes) { + long capacity = properties.getSyslogWalMaxPendingBytes(); + if (capacity <= 0L) { + pendingAppendBytes.addAndGet(bytes); + return true; + } + while (true) { + long current = pendingAppendBytes.get(); + // 单条硬上限显式关闭时，允许一个超过预算的事件独占通道，避免永久 BACKPRESSURE。 + if (bytes > capacity) { + if (current != 0L) return false; + if (pendingAppendBytes.compareAndSet(0L, bytes)) return true; + continue; + } + if (bytes > capacity - current) return false; + if (pendingAppendBytes.compareAndSet(current, current + bytes)) return true; + } + } + + private void releasePendingBytes(long bytes) { + long remaining = pendingAppendBytes.addAndGet(-bytes); + if (remaining < 0L) { + pendingAppendBytes.set(0L); + throw new IllegalStateException("WAL pending 字节计数出现负值"); + } + } + private void validateConfiguration() { if (!properties.isSyslogWalEnabled()) { throw new IllegalArgumentException("syslog.wal.enabled=false 会破坏可靠接入边界，正式链路禁止关闭 WAL"); @@ -563,6 +618,20 @@ public class SyslogWalService { || properties.getSyslogWalCheckpointFlushIntervalMs() <= 0) { throw new IllegalArgumentException("WAL 队列、刷盘、分段、游标和停机参数必须大于 0"); } + if (properties.getSyslogMessageMaxBytes() < 0L + || properties.getSyslogLargeMessageThresholdBytes() < 0L + || properties.getSyslogWalMaxPendingBytes() < 0L) { + throw new IllegalArgumentException("消息上限、大日志阈值和 WAL 字节预算不能小于 0"); + } + if (properties.getSyslogMessageMaxBytes() > 0L + && properties.getSyslogWalMaxPendingBytes() > 0L + && properties.getSyslogMessageMaxBytes() > properties.getSyslogWalMaxPendingBytes()) { + throw new IllegalArgumentException("单条消息上限不能大于 WAL pending 字节预算"); + } + if (properties.getSyslogMessageMaxBytes() > 0L + && properties.getSyslogLargeMessageThresholdBytes() > properties.getSyslogMessageMaxBytes()) { + throw new IllegalArgumentException("大日志分流阈值不能大于单条消息硬上限"); + } if (properties.getSyslogWalRetryIntervalMs() <= 0 || properties.getSyslogWalDiskRetryIntervalMs() <= 0) { throw new IllegalArgumentException("WAL 队列和磁盘重试间隔必须大于 0"); diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalStatusController.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalStatusController.java index c30c884..ce238b8 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalStatusController.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/wal/SyslogWalStatusController.java @@ -29,13 +29,16 @@ public class SyslogWalStatusController { public Map status() { Map value = new LinkedHashMap<>(); value.put("network_received", SyslogMessageHandler.getNetworkReceivedCount()); + value.put("network_oversized_dropped", SyslogMessageHandler.getOversizedDroppedCount()); value.put("wal_appended", wal.getAppendedCount()); value.put("wal_queue_size", wal.getQueueSize()); value.put("wal_queue_capacity", wal.getQueueCapacity()); + value.put("wal_pending_bytes", wal.getPendingAppendBytes()); value.put("wal_durable_sequence", wal.getDurableSequence()); value.put("wal_checkpoint_sequence", wal.getCheckpointSequence()); value.put("wal_pending_approx", wal.getPendingRecordsApprox()); value.put("wal_admission_backpressure", wal.getBackpressureCount()); + value.put("wal_admission_oversized", wal.getOversizedCount()); value.put("wal_terminal_recorded", wal.getTerminalRecordedCount()); value.put("wal_quarantined", wal.getQuarantinedCount()); Map pointTimeAssigned = new LinkedHashMap<>(); @@ -49,7 +52,9 @@ public class SyslogWalStatusController { value.put("wal_fatal_error", wal.getFatalFailure() == null ? null : wal.getFatalFailure().toString()); value.put("dispatch_inflight", dispatcher.getInflight()); + value.put("dispatch_inflight_bytes", dispatcher.getInflightBytes()); value.put("kafka_acked", dispatcher.getKafkaAcked()); + value.put("kafka_large_dispatched", dispatcher.getLargeDispatched()); value.put("dispatch_terminal_filtered", dispatcher.getTerminalFiltered()); value.put("dispatch_retry", dispatcher.getRetryCount()); value.put("tcp_pause_total", SyslogMessageHandler.getTcpPauseCount()); diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java b/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java index d14e355..da5e658 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java @@ -53,7 +53,7 @@ public class kafkaProducer { @Value("${spring.kafka.producer.compression-type:lz4}") String compressionType, @Value("${spring.kafka.producer.properties.max.block.ms:10000}") int maxBlockMs, @Value("${spring.kafka.producer.properties.buffer.memory:134217728}") long bufferMemory, - @Value("${spring.kafka.producer.properties.max.request.size:8388608}") int maxRequestSize, + @Value("${spring.kafka.producer.properties.max.request.size:33554432}") int maxRequestSize, @Value("${syslog.kafka.partition-key-strategy:event_id}") String partitionKeyStrategy) { this.appProperties = appProperties; this.partitionKeyStrategy = PartitionKeyStrategy.parse(partitionKeyStrategy); @@ -130,12 +130,34 @@ public class kafkaProducer { String eventTime, String eventId, Long pointTimeNs) { + return messagePushToTopic(appProperties.getKafkaProducerTopic(), deviceDevice, syslog, + receiveTime, eventTime, eventId, pointTimeNs); + } + + /** 大日志只改变目标 Topic，载荷协议、event_id 和 ACK 语义与普通通道完全一致。 */ + public CompletableFuture messagePushLarge(DeviceDevice deviceDevice, + String syslog, + String receiveTime, + String eventTime, + String eventId, + Long pointTimeNs) { + return messagePushToTopic(appProperties.getKafkaProducerLargeTopic(), deviceDevice, syslog, + receiveTime, eventTime, eventId, pointTimeNs); + } + + private CompletableFuture messagePushToTopic(String topic, + DeviceDevice deviceDevice, + String syslog, + String receiveTime, + String eventTime, + String eventId, + Long pointTimeNs) { // [流程节点3.1] event_id 在接入边界生成并写入加密载荷，Retry/Replay 不再重新生成。 String fullMessage = DeviceInfoUtil.getFullLogString( deviceDevice, syslog, receiveTime, eventTime, eventId, pointTimeNs, appProperties); // [流程节点3.2] Kafka 中保存携带 event_id 的完整密文，原始正文不写普通应用日志。 String encrypted = Sm4Util.encryptCbc(fullMessage, appProperties.getSm4Key()); - ProducerRecord record = buildRecord(deviceDevice.getId(), eventId, encrypted); + ProducerRecord record = buildRecord(topic, deviceDevice.getId(), eventId, encrypted); CompletableFuture result = new CompletableFuture<>(); // [SYSLOG-ALARM][KPROD][已修复] 回调成功才表示 Broker 接管；send() 返回本身不再记录 PRODUCED。 @@ -150,8 +172,13 @@ public class kafkaProducer { } ProducerRecord buildRecord(Integer deviceId, String eventId, String encrypted) { + return buildRecord(appProperties.getKafkaProducerTopic(), deviceId, eventId, encrypted); + } + + ProducerRecord buildRecord(String topic, Integer deviceId, String eventId, String encrypted) { + if (topic == null || topic.trim().isEmpty()) throw new IllegalArgumentException("Kafka Topic 不能为空"); String partitionKey = partitionKeyStrategy.partitionKey(deviceId, eventId); - return new ProducerRecord<>(appProperties.getKafkaProducerTopic(), partitionKey, encrypted); + return new ProducerRecord<>(topic, partitionKey, encrypted); } public static String newEventId() { diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java b/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java index 3ae1317..59d7180 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java @@ -33,6 +33,7 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler { private static final LongAdder NETWORK_RECEIVED = new LongAdder(); private static final LongAdder TCP_PAUSE_COUNT = new LongAdder(); private static final LongAdder UDP_DROPPED = new LongAdder(); + private static final LongAdder OVERSIZED_DROPPED = new LongAdder(); /** 每个 TCP Channel 独立 Handler，pending 只保存关闭 AUTO_READ 前已进入 Pipeline 的帧。 */ private final Deque pendingTcp = new ArrayDeque<>(); @@ -93,6 +94,10 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler { private void acceptUdp(SyslogWalService wal, SyslogIngressEvent event) { SyslogWalService.AdmissionResult result = wal.tryAppend(event); if (result == SyslogWalService.AdmissionResult.ACCEPTED) return; + if (result == SyslogWalService.AdmissionResult.OVERSIZED) { + recordOversized(event, "UDP"); + return; + } UDP_DROPPED.increment(); long dropped = UDP_DROPPED.sum(); if (dropped == 1 || dropped % 100 == 0) { @@ -110,6 +115,11 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler { return; } SyslogWalService.AdmissionResult result = wal.tryAppend(event); + if (result == SyslogWalService.AdmissionResult.OVERSIZED) { + recordOversized(event, "TCP"); + ctx.close(); + return; + } if (result != SyslogWalService.AdmissionResult.ACCEPTED) { pendingTcp.addLast(event); pauseAndSchedule(ctx, wal); @@ -144,6 +154,11 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler { while (!pendingTcp.isEmpty()) { SyslogIngressEvent event = pendingTcp.peekFirst(); SyslogWalService.AdmissionResult result = wal.tryAppend(event); + if (result == SyslogWalService.AdmissionResult.OVERSIZED) { + pendingTcp.removeFirst(); + recordOversized(event, "TCP"); + continue; + } if (result != SyslogWalService.AdmissionResult.ACCEPTED) { scheduleDrain(ctx, wal); return; @@ -172,4 +187,12 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler { public static long getNetworkReceivedCount() { return NETWORK_RECEIVED.sum(); } public static long getTcpPauseCount() { return TCP_PAUSE_COUNT.sum(); } public static long getUdpDroppedCount() { return UDP_DROPPED.sum(); } + public static long getOversizedDroppedCount() { return OVERSIZED_DROPPED.sum(); } + + private void recordOversized(SyslogIngressEvent event, String protocol) { + OVERSIZED_DROPPED.increment(); + log.error("Syslog 超过单条消息硬上限，已拒绝且不记录原文: eventId={}, protocol={}, sourceIp={}, chars={}, rejectedTotal={}", + event.getEventId(), protocol, event.getSourceIp(), event.getRawMessage().length(), + OVERSIZED_DROPPED.sum()); + } } diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogServerBoth.java b/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogServerBoth.java index bea9222..add6901 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogServerBoth.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogServerBoth.java @@ -47,6 +47,10 @@ public class SyslogServerBoth { this.syslogCharset = java.util.Objects.requireNonNull(syslogCharset, "syslogCharset"); this.tcpPort = properties.getSyslogTcpPort(); this.udpPort = properties.getSyslogUdpPort(); + if (properties.getSyslogMessageMaxBytes() > 0L + && properties.getSyslogMaxFrameLength() < properties.getSyslogMessageMaxBytes()) { + throw new IllegalArgumentException("syslog.max.frame.length 不能小于 syslog.message.max-bytes"); + } } public void start() throws InterruptedException { diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties index 00f0459..892d16a 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties @@ -9,7 +9,7 @@ server.error.include-binding-errors=always # Syslog Server Configuration syslog.tcp.port=514 syslog.udp.port=514 -syslog.max.frame.length=262144 +syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456} syslog.buffer.size=1000 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 syslog.charset=GBK diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties index 40e4cce..50ce97c 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties @@ -9,7 +9,7 @@ server.error.include-binding-errors=always # Syslog Server Configuration syslog.tcp.port=514 syslog.udp.port=514 -syslog.max.frame.length=262144 +syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456} syslog.buffer.size=1000 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 syslog.charset=GBK @@ -56,4 +56,4 @@ spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 # 生产环境缓存时间较长 -spring.cache.redis.time-to-live=3600000 \ No newline at end of file +spring.cache.redis.time-to-live=3600000 diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties index a3146be..98e7ea7 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties @@ -9,7 +9,7 @@ server.error.include-binding-errors=always # Syslog Server Configuration syslog.tcp.port=514 syslog.udp.port=514 -syslog.max.frame.length=2621440 +syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456} syslog.buffer.size=10000 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 syslog.charset=UTF-8 diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-test.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-test.properties index d7c4daf..f923a5b 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-test.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-test.properties @@ -9,7 +9,7 @@ server.error.include-binding-errors=always # Syslog Server Configuration syslog.tcp.port=514 syslog.udp.port=514 -syslog.max.frame.length=262144 +syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456} syslog.buffer.size=1000 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 syslog.charset=GBK diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application.properties b/haobang-security-dm/syslog-serve/src/main/resources/application.properties index c00a8dc..78db766 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application.properties @@ -20,7 +20,11 @@ spring.kafka.producer.properties.linger.ms=5 spring.kafka.producer.properties.batch.size=65536 spring.kafka.producer.compression-type=lz4 spring.kafka.producer.ack-timeout-ms=130000 -syslog.max.frame.length=262144 +syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456} +# 单条原文硬上限；0 表示不限制。超过阈值的消息在进入 WAL 前被明确拒绝。 +syslog.message.max-bytes=${SYSLOG_MESSAGE_MAX_BYTES:6291456} +# 超过 256KiB 的已接纳消息投递独立大日志 Topic；0 表示关闭分流。 +syslog.large-message.threshold-bytes=${SYSLOG_LARGE_MESSAGE_THRESHOLD_BYTES:262144} syslog.buffer.size=1000 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 syslog.charset=GBK @@ -31,6 +35,8 @@ syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a syslog.wal.enabled=${SYSLOG_WAL_ENABLED:true} syslog.wal.directory=${SYSLOG_WAL_DIRECTORY:${SYSLOG_RAW_ARCHIVE_DIRECTORY:/data/syslog-wal}} syslog.wal.queue-capacity=${SYSLOG_WAL_QUEUE_CAPACITY:10000} +# 队列和待 force 记录共享的内存字节预算，防止大日志把“有界条数”放大成无界堆占用。 +syslog.wal.max-pending-bytes=${SYSLOG_WAL_MAX_PENDING_BYTES:268435456} syslog.wal.high-watermark-percent=${SYSLOG_WAL_HIGH_WATERMARK_PERCENT:70} syslog.wal.low-watermark-percent=${SYSLOG_WAL_LOW_WATERMARK_PERCENT:40} syslog.wal.retry-interval-ms=${SYSLOG_WAL_RETRY_INTERVAL_MS:10} @@ -52,6 +58,7 @@ syslog.point-time.write-enabled=${SYSLOG_POINT_TIME_WRITE_ENABLED:true} # WAL 后台投递 Kafka：内存只保留有限在途消息，失败记录留在 WAL 自动重试。 syslog.dispatch.worker-threads=${SYSLOG_DISPATCH_WORKER_THREADS:4} syslog.dispatch.max-inflight=${SYSLOG_DISPATCH_MAX_INFLIGHT:5000} +syslog.dispatch.max-inflight-bytes=${SYSLOG_DISPATCH_MAX_INFLIGHT_BYTES:268435456} syslog.dispatch.retry-backoff-ms=${SYSLOG_DISPATCH_RETRY_BACKOFF_MS:1000} syslog.dispatch.shutdown-timeout-ms=${SYSLOG_DISPATCH_SHUTDOWN_TIMEOUT_MS:30000} syslog.device-context.cache-ttl-ms=${SYSLOG_DEVICE_CONTEXT_CACHE_TTL_MS:300000} @@ -77,6 +84,7 @@ app.service.data_type=json # kafka Configuration spring.kafka.producer.bootstrap-servers=192.168.4.99:9092 spring.kafka.producer.topic =${KAFKA_PRODUCER_TOPIC:test-topic} +spring.kafka.producer.large-topic=${KAFKA_PRODUCER_LARGE_TOPIC:${spring.kafka.producer.topic}.large} # kafka Configuration 新增优化配置 spring.kafka.producer.properties.retries=10 spring.kafka.producer.properties.retry.backoff.ms=500 @@ -87,7 +95,8 @@ spring.kafka.producer.properties.delivery.timeout.ms=120000 spring.kafka.producer.properties.max.block.ms=10000 spring.kafka.producer.properties.buffer.memory=134217728 # 允许投递加密后超过 Kafka 默认 1 MiB 的 Syslog；Broker/Topic 上限必须不小于此值。 -spring.kafka.producer.properties.max.request.size=${KAFKA_PRODUCER_MAX_REQUEST_SIZE:8388608} +# 6MiB GBK 原文转 UTF-8 后再做 Base64 加密可能明显膨胀，按 32MiB 留出确定余量。 +spring.kafka.producer.properties.max.request.size=${KAFKA_PRODUCER_MAX_REQUEST_SIZE:33554432} # event_id 使高流量设备的独立事件均匀分布到多个分区；device_id 仅用于兼容回滚。 syslog.kafka.partition-key-strategy=${SYSLOG_KAFKA_PARTITION_KEY_STRATEGY:event_id} diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/dispatch/SyslogWalDispatcherTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/dispatch/SyslogWalDispatcherTest.java index 260fb3c..d561b3d 100644 --- a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/dispatch/SyslogWalDispatcherTest.java +++ b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/dispatch/SyslogWalDispatcherTest.java @@ -121,6 +121,56 @@ class SyslogWalDispatcherTest { verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any()); } + @Test + void largeRecordUsesDedicatedKafkaLaneAndReleasesByteBudgetAfterAck() throws Exception { + AppProperties properties = properties(tempDirectory); + properties.setSyslogLargeMessageThresholdBytes(4L); + properties.setSyslogMessageMaxBytes(32L); + properties.setSyslogWalMaxPendingBytes(64L); + properties.setSyslogDispatchMaxInflightBytes(64L); + wal = new SyslogWalService(properties, new ObjectMapper()); + wal.initialize(); + wal.tryAppend(event("20260811-large", "12345")); + waitUntil(() -> wal.getAppendedCount() == 1L); + + DeviceDevice device = new DeviceDevice(); + device.setId(126); + DeviceContextResolver resolver = mock(DeviceContextResolver.class); + when(resolver.resolve(anyString(), anyString())) + .thenReturn(DeviceContextResolver.Resolution.registered(device)); + kafkaProducer producer = mock(kafkaProducer.class); + when(producer.messagePushLarge(any(DeviceDevice.class), anyString(), anyString(), + anyString(), anyString(), any())) + .thenReturn(CompletableFuture.completedFuture(null)); + + dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties); + dispatcher.start(); + waitUntil(() -> wal.getCheckpointSequence() == 1L); + + verify(producer).messagePushLarge(eq(device), eq("12345"), + eq("20260811200000000"), eq("20260811200000000"), + eq("20260811-large"), eq(null)); + verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any()); + assertEquals(1L, dispatcher.getLargeDispatched()); + assertEquals(0L, dispatcher.getInflightBytes()); + } + + @Test + void singleRecordLargerThanByteBudgetCanRunExclusivelyWhenHardLimitIsDisabled() throws Exception { + SyslogWalDispatcher.ByteBudget budget = new SyslogWalDispatcher.ByteBudget(4L); + + budget.acquire(5L); + + assertEquals(5L, budget.used()); + budget.release(5L); + assertEquals(0L, budget.used()); + } + + @Test + void utf8BudgetCountsChineseAndEmojiWithoutAllocatingEncodedPayload() { + assertEquals(11L, SyslogWalDispatcher.utf8Length("中文🙂x")); + } + private SyslogIngressEvent event(String id, String raw) { return new SyslogIngressEvent(id, LocalDateTime.of(2026, 8, 11, 20, 0), "192.168.4.19", 514, "TCP", raw); @@ -147,6 +197,7 @@ class SyslogWalDispatcherTest { value.setSyslogCharset("UTF-8"); value.setSyslogDispatchWorkerThreads(2); value.setSyslogDispatchMaxInflight(8); + value.setSyslogDispatchMaxInflightBytes(0L); value.setSyslogDispatchRetryBackoffMs(10); value.setSyslogDispatchShutdownTimeoutMs(2000); value.setSyslogDeviceContextCacheTtlMs(60000); diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/wal/SyslogWalServiceTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/wal/SyslogWalServiceTest.java index 6b4d421..1f58d5a 100644 --- a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/wal/SyslogWalServiceTest.java +++ b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/wal/SyslogWalServiceTest.java @@ -98,6 +98,58 @@ class SyslogWalServiceTest { assertEquals(null, attempt.getDurableStage()); } + @Test + void oversizedMessageIsRejectedBeforeEnteringWal() { + AppProperties properties = properties(tempDirectory); + properties.setSyslogMessageMaxBytes(4L); + properties.setSyslogWalMaxPendingBytes(16L); + service = new SyslogWalService(properties, new ObjectMapper()); + service.initialize(); + + SyslogWalService.AdmissionResult result = service.tryAppend( + event("20260811-too-large", "12345")); + + assertEquals(SyslogWalService.AdmissionResult.OVERSIZED, result); + assertEquals(1L, service.getOversizedCount()); + assertEquals(0, service.getQueueSize()); + assertEquals(0L, service.getPendingAppendBytes()); + } + + @Test + void pendingByteBudgetBackpressuresEvenWhenRecordQueueHasCapacity() { + AppProperties properties = properties(tempDirectory); + properties.setSyslogMessageMaxBytes(5L); + properties.setSyslogWalMaxPendingBytes(5L); + properties.setSyslogWalFlushRecords(100); + properties.setSyslogWalFlushIntervalMs(10000L); + service = new SyslogWalService(properties, new ObjectMapper()); + service.initialize(); + + assertEquals(SyslogWalService.AdmissionResult.ACCEPTED, + service.tryAppend(event("20260811-first", "12345"))); + assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE, + service.tryAppend(event("20260811-second", "x"))); + assertEquals(5L, service.getPendingAppendBytes()); + assertEquals(1L, service.getBackpressureCount()); + } + + @Test + void disabledHardLimitLetsOneOverBudgetRecordMakeProgressExclusively() { + AppProperties properties = properties(tempDirectory); + properties.setSyslogMessageMaxBytes(0L); + properties.setSyslogWalMaxPendingBytes(4L); + properties.setSyslogWalFlushRecords(100); + properties.setSyslogWalFlushIntervalMs(10000L); + service = new SyslogWalService(properties, new ObjectMapper()); + service.initialize(); + + assertEquals(SyslogWalService.AdmissionResult.ACCEPTED, + service.tryAppend(event("20260811-over-budget", "12345"))); + assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE, + service.tryAppend(event("20260811-waits", "x"))); + assertEquals(5L, service.getPendingAppendBytes()); + } + @Test void forceBeforeCursorAndPersistSchemaV2() throws Exception { service = new SyslogWalService(properties(tempDirectory), new ObjectMapper()); diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java index 8e1153b..afcf96b 100644 --- a/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java +++ b/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java @@ -63,6 +63,20 @@ class KafkaProducerEventIdTest { assertEquals("126", record.key()); } + @Test + void largeLaneUsesDedicatedTopicWithoutChangingPartitionKey() { + AppProperties properties = new AppProperties(); + properties.setKafkaProducerTopic("raw"); + properties.setKafkaProducerLargeTopic("raw.large"); + kafkaProducer producer = new kafkaProducer(properties, mock(Producer.class), "event_id"); + + ProducerRecord record = producer.buildRecord( + properties.getKafkaProducerLargeTopic(), 126, "20260811-event", "ciphertext"); + + assertEquals("raw.large", record.topic()); + assertEquals("20260811-event", record.key()); + } + @Test void invalidPartitionKeyStrategyFailsFast() { assertThrows(IllegalArgumentException.class, () -> producer("random")); diff --git a/haobang-security-dm/upgrade-doc/README.md b/haobang-security-dm/upgrade-doc/README.md index b71b003..b05f2be 100644 --- a/haobang-security-dm/upgrade-doc/README.md +++ b/haobang-security-dm/upgrade-doc/README.md @@ -6,9 +6,12 @@ | --- | --- | | `v1/` | 原有升级资料 | | `v2/` | 从提交 `b61ec2e` 升级到当前源码所需的增量资料 | +| `v3/` | 日统计异步投影，以及大日志独立 Kafka 通道、字节背压和存储截断升级资料 | -现场如果已经执行过 v1，只执行 v2；新环境按 v1、v2 的顺序执行。 +新环境按 v1、v2、v3 的顺序执行；已经完成 v2 的现场只执行 v3。已经执行过早期 v3 日统计升级的 +现场，只需补做 v3 中的大日志 Topic、应用升级和验证步骤，不要重复执行数据库升级脚本。 数据库脚本不要重复执行。执行过程中有报错时，先确认已经落库的对象，再决定从哪里继续。 v2 的执行步骤见 [v2/README.md](./v2/README.md)。 +v3 的执行步骤见 [v3/README.md](./v3/README.md)。 diff --git a/haobang-security-dm/upgrade-doc/replace-syslog-consumer-rule.md b/haobang-security-dm/upgrade-doc/replace-syslog-consumer-rule.md index 4e9b6bc..4b03b48 100644 --- a/haobang-security-dm/upgrade-doc/replace-syslog-consumer-rule.md +++ b/haobang-security-dm/upgrade-doc/replace-syslog-consumer-rule.md @@ -1,4 +1,4 @@ - +先执行这条查看启动目录和启动命令： docker inspect syslog-consumer-rule-dm \ --format 'WorkingDir={{.Config.WorkingDir}}{{println}}Entrypoint={{json .Config.Entrypoint}}{{println}}Cmd={{json .Config.Cmd}}' @@ -7,4 +7,9 @@ docker inspect syslog-consumer-rule-dm \ 再确认文件确实存在： docker exec syslog-consumer-rule-dm \ - sh -c 'ls -lh /app/*.jar 2>/dev/null' \ No newline at end of file + sh -c 'ls -lh /app/*.jar 2>/dev/null' + + + 最后执行 + + ./upgrade-doc/replace-syslog-consumer-rule-jar.sh /实际路径/syslog-consumer-rule-1.0.0.jar \ No newline at end of file diff --git a/haobang-security-dm/upgrade-doc/v3/README.md b/haobang-security-dm/upgrade-doc/v3/README.md new file mode 100644 index 0000000..28d5d5e --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v3/README.md @@ -0,0 +1,193 @@ +# v3 日统计异步投影与大日志通道升级说明 + +## 升级内容 + +v3 将以下统计表的热点更新移出 `syslog-consumer-rule` 事实事务： + +- `device_daily_stats` +- `normalize_rule_daily_stats` +- `device_collect_stats` + +事实事务只批量写事实表并追加一条聚合 Outbox。独立的 Kafka 投影消费者串行更新日统计，之后继续通过 +已有的 `STATS.MATERIALIZE` Topic 刷新 `device_device`、`dm_normalize_rule` 和 +`device_collect_task`。这样三台 `syslog-consumer-rule` 可以并行写事实，不再同时争抢相同日统计行锁。 + +v3 同时增加大日志独立处理链路： + +- `syslog-serve` 在进入 WAL 前执行单条硬上限检查；超过上限明确拒绝，不把原文写入应用日志； +- 接纳后的消息仍先进入 `syslog-serve` 本地 WAL，WAL 入队和 Kafka 投递同时受条数、字节数约束； +- 超过分流阈值的消息进入独立 `.large` Topic，其 Retry、DLT、Replay 也使用独立 Topic； +- `syslog-consumer-rule` 使用独立的大日志 listener、消费组、ConsumerFactory、poll 批量和规则计算线程池； +- 规则解析使用完整内容，写 InfluxDB 和达梦前才按 UTF-8 字节数截断，结尾带 + `HB_XDR_TRUNCATED` 标记；InfluxDB 另存原长度、保存长度和原文 SHA-256。 + +## 文件与执行顺序 + +| 顺序 | 文件 | 作用 | +| --- | --- | --- | +| 1 | `database/01-stats-async-projection-precheck.sql` | 只读预检 | +| 2 | `database/02-stats-async-projection-upgrade.sql` | 创建 Outbox 和消费幂等表 | +| 3 | `database/03-stats-async-projection-verify.sql` | 只读验证表、约束和索引 | +| 4 | `kafka/01-create-stats-projection-topic.sh` | 创建独立统计投影 Topic | +| 5 | `kafka/02-create-large-log-topics.sh` | 创建大日志主 Topic 及独立 Retry、DLT、Replay Topic | +| 6 | `config/01-new-environment-variables.md` | 升级应用前配置新增环境变量，并核对 Kafka Broker 参数 | + +数据库脚本必须使用 `syslog-consumer-rule` 实际连接的达梦用户执行。 + +如果现场已经执行过早期 v3 日统计异步投影升级，只执行 `kafka/02-create-large-log-topics.sh` 和本页的 +大日志应用配置、验证步骤；不要重复执行数据库升级脚本。 + +## 第一步：达梦 + +依次执行： + +```text +01-stats-async-projection-precheck.sql +02-stats-async-projection-upgrade.sql +03-stats-async-projection-verify.sql +``` + +预检判断： + +- 第一组应返回 `DEVICE_DAILY_STATS`、`NORMALIZE_RULE_DAILY_STATS`、`DEVICE_COLLECT_STATS` 三行； +- 后面三组必须为空； +- 如果目标表、约束或索引已经存在，不要执行 `02`，先执行 `03` 判断是否已经升级完成。 + +验证判断： + +- 返回两张新表； +- 三个约束状态为 `ENABLED`； +- 两个索引状态为 `VALID`； +- 首次建表时两张表的行数都是 0。 + +## 第二步：Kafka + +### 统计投影 Topic + +在 Kafka 主机的 `kafka_2.13-3.6.1/bin` 目录执行： + +```bash +bash /脚本所在路径/01-create-stats-projection-topic.sh +``` + +默认创建： + +```text +agent-01-syslog-topic.STATS.PROJECTION +``` + +新建时为 1 个分区，副本数沿用 `agent-01-syslog-topic`。如果现场此前已经建成 6 个分区，脚本只告警， +不会修改或删除；应用使用固定 Kafka key，投影事件仍然只落入其中一个稳定分区。 + +必须看到： + +```text +PASS: agent-01-syslog-topic.STATS.PROJECTION 已创建/确认。 +``` + +`STATS.PROJECTION` 不能与已有的 `STATS.MATERIALIZE` 配成同一个 Topic。 + +### 大日志 Topic + +先确认所有 Kafka Broker 的 `replica.fetch.max.bytes` 不小于 `33554432`，并按现场 Kafka 版本要求使 +配置生效。然后执行： + +```bash +bash /脚本所在路径/02-create-large-log-topics.sh +``` + +脚本默认从 `agent-01-syslog-topic` 读取分区数和副本数，并创建： + +```text +agent-01-syslog-topic.large +agent-01-syslog-topic.large.RETRY.1M +agent-01-syslog-topic.large.RETRY.10M +agent-01-syslog-topic.large.RETRY.1H +agent-01-syslog-topic.large.DLT +agent-01-syslog-topic.large.REPLAY +``` + +必须看到： + +```text +PASS: 大日志 Topic 已创建并统一配置。 +``` + +## 第三步：升级应用 + +升级 `syslog-serve` 和三台 `syslog-consumer-rule`；`syslog-consumer` 不需要因为本次改造升级。 + +升级前先按 [新增变量配置清单](./config/01-new-environment-variables.md) 修改现场实际使用的 Compose 文件。 +三台 `syslog-consumer-rule` 的 Topic、消费组和截断上限必须保持一致。 + +统计投影默认已经启用。确认现场没有显式配置： + +```text +STATS_PROJECTION_ASYNC_ENABLED=false +``` + +先逐台升级 `syslog-consumer-rule`，可以启动一台观察后再升级另外两台；确认大日志 listener 正常后再 +升级 `syslog-serve`。新旧版本滚动期间，同一条原始 Kafka 消息只会被消费组中的一个实例处理，不会因为 +滚动升级重复统计。 + +## 升级后验证 + +查看处理日志： + +```bash +docker logs --since 2m syslog-consumer-rule-dm 2>&1 \ +| grep -E 'Syslog批次完成|统计投影批次完成|统计物化批次完成|Outbox|Deadlock' \ +| tail -100 +``` + +查看 Outbox 状态： + +```sql +SELECT status, COUNT(*) +FROM stats_projection_outbox +GROUP BY status +ORDER BY status; +``` + +正常表现： + +- 日志持续出现 `统计投影批次完成`； +- `APPLIED` 持续增长； +- `PENDING`、`PUBLISHED` 只短暂停留，不持续增长； +- `FAILED` 不持续增长； +- `Syslog批次完成` 中 `dmWriteMs` 明显下降； +- 不再出现 daily stats 相关死锁。 + +查看新消费组积压： + +```bash +./kafka-consumer-groups.sh \ + --bootstrap-server 10.141.121.35:9092,10.141.121.50:9092,10.141.121.59:9092 \ + --group agent-01-syslog-group-dm-stats-projection \ + --describe +``` + +验证大日志链路： + +- `GET /syslogserve/api/syslog/wal/status`：检查 `wal_admission_oversized`、`wal_pending_bytes`、 + `dispatch_inflight_bytes`、`kafka_large_dispatched`； +- `GET /xdrrule/api/syslog/raw-storage/metrics`：检查 `raw_message_truncated_total` 及截断前后字节累计值； +- `GET /xdrrule/api/syslog/storage-limits/metrics`：检查达梦正常/非标字段截断计数； +- 发送一条略大于 256 KiB、但小于 6 MiB 的测试消息，确认 `.large` Topic 有数据且普通 Topic 无该事件； +- 查询 InfluxDB/达梦，确认被截断值以 `...[HB_XDR_TRUNCATED original_utf8_bytes=...]` 结尾。 + +## 回滚 + +回滚最后一台新版本前，先确认 Outbox 没有未应用事件： + +```sql +SELECT status, COUNT(*) +FROM stats_projection_outbox +WHERE status <> 'APPLIED' +GROUP BY status; +``` + +查询为空后再回滚最后一台。两张新表、Kafka Topic 和消费组 offset 全部保留，不要删除或重置。 + +回滚旧版本前，还要先停止新生产者并排空 `.large` 主 Topic；大日志 Topic 和消费组 offset 同样保留， +不要删除或重置。 diff --git a/haobang-security-dm/upgrade-doc/v3/config/01-new-environment-variables.md b/haobang-security-dm/upgrade-doc/v3/config/01-new-environment-variables.md new file mode 100644 index 0000000..727c85a --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v3/config/01-new-environment-variables.md @@ -0,0 +1,147 @@ +# v3 新增变量配置 + +应用侧只需要修改两个 Docker Compose 文件。变量放到对应服务的 `environment:` 下，不要修改 JAR 中的 +`application.properties`；Kafka Broker 参数单独按第 4 节核对。 + +## 1. syslog-serve + +配置文件： + +```text +syslog-serve/docker-compose/docker-compose-serve-prod.yaml +``` + +在 `services.syslog-serve.environment` 中增加： + +```yaml + # 开启 Syslog TCP/UDP 监听，字符集按现场设备实际编码调整。 + - SYSLOG_SERVER_ENABLED=true + - SYSLOG_CHARSET=UTF-8 + + # 单条最多接收 6 MiB，超过 256 KiB 时发送到大日志 Topic。 + - SYSLOG_MAX_FRAME_LENGTH=6291456 + - SYSLOG_MESSAGE_MAX_BYTES=6291456 + - SYSLOG_LARGE_MESSAGE_THRESHOLD_BYTES=262144 + + # WAL 和 Kafka 投递最多各占用 256 MiB 在途内存。 + - SYSLOG_WAL_MAX_PENDING_BYTES=268435456 + - SYSLOG_DISPATCH_MAX_INFLIGHT_BYTES=268435456 + + # 大日志 Kafka Topic 及 Producer 请求上限。 + - KAFKA_PRODUCER_LARGE_TOPIC=agent-01-syslog-topic.large + - KAFKA_PRODUCER_MAX_REQUEST_SIZE=33554432 +``` + +其中： + +- `SYSLOG_MAX_FRAME_LENGTH`：TCP 单帧最大长度； +- `SYSLOG_MESSAGE_MAX_BYTES`：所有来源的单条日志硬上限，超过后拒绝； +- `SYSLOG_LARGE_MESSAGE_THRESHOLD_BYTES`：超过该值后走大日志 Topic； +- `SYSLOG_WAL_MAX_PENDING_BYTES`、`SYSLOG_DISPATCH_MAX_INFLIGHT_BYTES`：限制 JVM 在途数据量。 + +`SYSLOG_MAX_FRAME_LENGTH` 必须大于或等于 `SYSLOG_MESSAGE_MAX_BYTES`。如果设备发送 GBK，应将 +`SYSLOG_CHARSET` 改为 `GBK`。 + +## 2. syslog-consumer-rule + +配置文件： + +```text +syslog-consumer-rule/docker-compose/docker-compose-consumer-rule-prod.yaml +``` + +在 `services.syslog-consumer-rule.environment` 中增加，三台实例保持一致： + +```yaml + # 大日志独立消费通道。 + - KAFKA_CONSUMER_LARGE_TOPIC=agent-01-syslog-topic.large + - SYSLOG_LARGE_LISTENER_ENABLED=true + - SYSLOG_LARGE_LISTENER_CONCURRENCY=1 + - SYSLOG_LARGE_LISTENER_MAX_POLL_RECORDS=5 + - SYSLOG_LARGE_LISTENER_MAX_PARTITION_FETCH_BYTES=33554432 + - SYSLOG_LARGE_LISTENER_FETCH_MAX_BYTES=67108864 + + # 大日志规则处理批次和独立线程池。 + - SYSLOG_LARGE_PROCESSING_BATCH_MAX_BYTES=8388608 + - SYSLOG_LARGE_PROCESSOR_CORE_POOL_SIZE=1 + - SYSLOG_LARGE_PROCESSOR_MAX_POOL_SIZE=2 + - SYSLOG_LARGE_PROCESSOR_QUEUE_CAPACITY=10 + + # Retry、DLT、Replay 也允许传输大日志。 + - KAFKA_RELIABILITY_MAX_REQUEST_SIZE=33554432 + + # InfluxDB 原文最多保存 1 MiB，单批写入最多 8 MiB。 + - INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES=1048576 + - INFLUXDB_RAW_WRITE_BATCH_MAX_BYTES=8388608 + + # 达梦正常日志大字段最多保存 64 KiB，非标日志原文最多保存 256 KiB。 + - DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES=65536 + - DM_NON_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES=262144 +``` + +被截断的数据会以以下标记结尾： + +```text +...[HB_XDR_TRUNCATED original_utf8_bytes=原始字节数] +``` + +以下三个存储上限配置为 `0` 时表示不截断： + +```text +INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES +DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES +DM_NON_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES +``` + +规则解析始终使用完整原文，只在写入 InfluxDB 或达梦时截断。 + +## 3. 日统计投影 + +日统计投影默认开启，一般不需要新增配置。只需确认现场没有配置： + +```text +STATS_PROJECTION_ASYNC_ENABLED=false +``` + +如果现场需要显式写明，可以在 `syslog-consumer-rule` 的 `environment` 中配置： + +```yaml + - STATS_PROJECTION_ASYNC_ENABLED=true + - STATS_PROJECTION_TOPIC=agent-01-syslog-topic.STATS.PROJECTION + - STATS_PROJECTION_GROUP_ID=agent-01-syslog-group-dm-stats-projection +``` + +## 4. Kafka 配置 + +每个 Kafka Broker 的 `replica.fetch.max.bytes` 必须不小于 32 MiB： + +```properties +replica.fetch.max.bytes=33554432 +``` + +然后执行： + +```bash +bash upgrade-doc/v3/kafka/02-create-large-log-topics.sh +``` + +生产者和消费者的大日志 Topic 名称必须完全相同： + +```text +KAFKA_PRODUCER_LARGE_TOPIC=agent-01-syslog-topic.large +KAFKA_CONSUMER_LARGE_TOPIC=agent-01-syslog-topic.large +``` + +## 5. 使配置生效 + +在现场对应的部署目录先检查 Compose，再重建容器： + +```bash +docker compose -f docker-compose-serve-prod.yaml config +docker compose -f docker-compose-serve-prod.yaml up -d --force-recreate syslog-serve + +docker compose -f docker-compose-consumer-rule-prod.yaml config +docker compose -f docker-compose-consumer-rule-prod.yaml up -d --force-recreate syslog-consumer-rule +``` + +建议先升级三台 `syslog-consumer-rule`，确认大日志消费者正常后，再升级 `syslog-serve`。 diff --git a/haobang-security-dm/upgrade-doc/v3/database/01-stats-async-projection-precheck.sql b/haobang-security-dm/upgrade-doc/v3/database/01-stats-async-projection-precheck.sql new file mode 100644 index 0000000..173824c --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v3/database/01-stats-async-projection-precheck.sql @@ -0,0 +1,40 @@ +-- v3 达梦升级预检：只读，不修改任何数据。 +-- 第一组应返回 3 行；第二、三、四组应全部为空。 + +-- v1 的三张 DIRECT 统计表必须已经存在。 +SELECT table_name +FROM user_tables +WHERE table_name IN ( + 'DEVICE_DAILY_STATS', + 'NORMALIZE_RULE_DAILY_STATS', + 'DEVICE_COLLECT_STATS' +) +ORDER BY table_name; + +-- v3 目标表必须尚未创建。若有返回，停止执行 02，先确认是否执行过 v3 或曾中途失败。 +SELECT table_name +FROM user_tables +WHERE table_name IN ( + 'STATS_PROJECTION_OUTBOX', + 'STATS_PROJECTION_APPLIED_EVENT' +) +ORDER BY table_name; + +-- 约束名不得与现有对象冲突。 +SELECT constraint_name, table_name +FROM user_constraints +WHERE constraint_name IN ( + 'PK_STATS_PROJECTION_OUTBOX', + 'CK_STATS_PROJECTION_OUTBOX_STATUS', + 'PK_STATS_PROJECTION_APPLIED' +) +ORDER BY constraint_name; + +-- 索引名不得与现有对象冲突。 +SELECT index_name, table_name +FROM user_indexes +WHERE index_name IN ( + 'IDX_STATS_PROJECTION_OUTBOX_DUE', + 'IDX_STATS_PROJECTION_APPLIED_TIME' +) +ORDER BY index_name; diff --git a/haobang-security-dm/upgrade-doc/v3/database/02-stats-async-projection-upgrade.sql b/haobang-security-dm/upgrade-doc/v3/database/02-stats-async-projection-upgrade.sql new file mode 100644 index 0000000..fadd707 --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v3/database/02-stats-async-projection-upgrade.sql @@ -0,0 +1,41 @@ +-- v3 达梦升级：为事实事务外的可靠日统计投影建立 Outbox 和消费幂等表。 +-- 仅在 01 预检通过后执行一次；不要重复执行。 + +CREATE TABLE stats_projection_outbox ( + event_id VARCHAR2(64) NOT NULL, + event_schema VARCHAR2(64) NOT NULL, + payload CLOB NOT NULL, + status VARCHAR2(16) DEFAULT 'PENDING' NOT NULL, + attempt_count INT DEFAULT 0 NOT NULL, + next_attempt_at TIMESTAMP DEFAULT SYSDATE, + published_at TIMESTAMP, + applied_at TIMESTAMP, + last_error VARCHAR2(2000), + created_at TIMESTAMP DEFAULT SYSDATE NOT NULL, + updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL, + CONSTRAINT pk_stats_projection_outbox PRIMARY KEY(event_id), + CONSTRAINT ck_stats_projection_outbox_status + CHECK(status IN ('PENDING','FAILED','PUBLISHED','APPLIED')) +); + +CREATE INDEX idx_stats_projection_outbox_due +ON stats_projection_outbox(status,next_attempt_at,created_at); + +CREATE TABLE stats_projection_applied_event ( + event_id VARCHAR2(64) NOT NULL, + event_schema VARCHAR2(64) NOT NULL, + applied_at TIMESTAMP DEFAULT SYSDATE NOT NULL, + CONSTRAINT pk_stats_projection_applied PRIMARY KEY(event_id) +); + +CREATE INDEX idx_stats_projection_applied_time +ON stats_projection_applied_event(applied_at); + +COMMENT ON TABLE stats_projection_outbox IS +'事实批次统计 Outbox；Kafka 发布或消费失败时可持续恢复'; +COMMENT ON COLUMN stats_projection_outbox.status IS +'PENDING/FAILED/PUBLISHED/APPLIED；只有 APPLIED 才允许到期清理'; +COMMENT ON TABLE stats_projection_applied_event IS +'统计投影消费幂等标记；保留时间必须长于 STATS.PROJECTION Topic'; + +COMMIT; diff --git a/haobang-security-dm/upgrade-doc/v3/database/03-stats-async-projection-verify.sql b/haobang-security-dm/upgrade-doc/v3/database/03-stats-async-projection-verify.sql new file mode 100644 index 0000000..1aa58d8 --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v3/database/03-stats-async-projection-verify.sql @@ -0,0 +1,44 @@ +-- v3 达梦升级验证：只读。 + +-- 应返回 2 行。 +SELECT table_name +FROM user_tables +WHERE table_name IN ( + 'STATS_PROJECTION_OUTBOX', + 'STATS_PROJECTION_APPLIED_EVENT' +) +ORDER BY table_name; + +-- 核对关键字段；OUTBOX 应包含 11 列，APPLIED_EVENT 应包含 3 列。 +SELECT table_name, column_id, column_name, data_type, nullable +FROM user_tab_columns +WHERE table_name IN ( + 'STATS_PROJECTION_OUTBOX', + 'STATS_PROJECTION_APPLIED_EVENT' +) +ORDER BY table_name, column_id; + +-- 三个约束都应为 ENABLED。 +SELECT table_name, constraint_name, constraint_type, status +FROM user_constraints +WHERE constraint_name IN ( + 'PK_STATS_PROJECTION_OUTBOX', + 'CK_STATS_PROJECTION_OUTBOX_STATUS', + 'PK_STATS_PROJECTION_APPLIED' +) +ORDER BY constraint_name; + +-- 两个索引都应为 VALID。 +SELECT table_name, index_name, uniqueness, status +FROM user_indexes +WHERE index_name IN ( + 'IDX_STATS_PROJECTION_OUTBOX_DUE', + 'IDX_STATS_PROJECTION_APPLIED_TIME' +) +ORDER BY index_name; + +-- 刚建表时应返回 0、0；应用启动后数字会正常增长。 +SELECT + (SELECT COUNT(*) FROM stats_projection_outbox) AS outbox_rows, + (SELECT COUNT(*) FROM stats_projection_applied_event) AS applied_rows +FROM dual; diff --git a/haobang-security-dm/upgrade-doc/v3/kafka/01-create-stats-projection-topic-docker.sh b/haobang-security-dm/upgrade-doc/v3/kafka/01-create-stats-projection-topic-docker.sh new file mode 100755 index 0000000..993e52e --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v3/kafka/01-create-stats-projection-topic-docker.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Docker 单文件版：直接在 Kafka 容器内创建并校验统计投影 Topic。 +# 默认值适配测试环境；如容器名、Kafka 地址或安装目录不同，可通过环境变量覆盖。 + +DOCKER_BIN="${DOCKER_BIN:-docker}" +KAFKA_CONTAINER="${KAFKA_CONTAINER:-kafka_ycgy-bt-kafka-1}" +BOOTSTRAP="${BOOTSTRAP:-localhost:9092}" +KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}" +RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}" +PROJECTION_TOPIC="${PROJECTION_TOPIC:-${RAW_TOPIC}.STATS.PROJECTION}" +RETENTION_MS="${RETENTION_MS:-604800000}" +COMMAND_CONFIG="${COMMAND_CONFIG:-}" + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +command -v "${DOCKER_BIN}" >/dev/null 2>&1 || die "未找到 Docker CLI: ${DOCKER_BIN}" + +RUNNING="$("${DOCKER_BIN}" inspect -f '{{.State.Running}}' "${KAFKA_CONTAINER}" 2>/dev/null || true)" +[[ "${RUNNING}" == "true" ]] || die "Kafka 容器不存在或未运行: ${KAFKA_CONTAINER}" + +"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-topics.sh" \ + || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-topics.sh" +"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-configs.sh" \ + || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-configs.sh" +"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" bash -c 'exit 0' \ + || die "容器内缺少 bash，无法执行原始升级脚本" + +DOCKER_ENV=( + -e "BOOTSTRAP=${BOOTSTRAP}" + -e "KAFKA_BIN=${KAFKA_BIN}" + -e "RAW_TOPIC=${RAW_TOPIC}" + -e "PROJECTION_TOPIC=${PROJECTION_TOPIC}" + -e "RETENTION_MS=${RETENTION_MS}" +) +KAFKA_COMMON=(--bootstrap-server "${BOOTSTRAP}") + +if [[ -n "${COMMAND_CONFIG}" ]]; then + "${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -r "${COMMAND_CONFIG}" \ + || die "容器内无法读取 COMMAND_CONFIG: ${COMMAND_CONFIG}" + DOCKER_ENV+=(-e "COMMAND_CONFIG=${COMMAND_CONFIG}") + KAFKA_COMMON+=(--command-config "${COMMAND_CONFIG}") +fi + +echo "检查 Kafka 连接及原始 topic ..." +"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" \ + "${KAFKA_BIN}/kafka-topics.sh" \ + "${KAFKA_COMMON[@]}" \ + --describe \ + --topic "${RAW_TOPIC}" >/dev/null \ + || die "无法连接 Kafka 或原始 topic 不存在: ${RAW_TOPIC}" + +echo "Kafka 容器: ${KAFKA_CONTAINER}" +echo "Bootstrap: ${BOOTSTRAP}" +echo "原始 topic: ${RAW_TOPIC}" +echo "投影 topic: ${PROJECTION_TOPIC}" +echo "开始执行 Docker 版升级 ..." + +exec "${DOCKER_BIN}" exec -i "${DOCKER_ENV[@]}" "${KAFKA_CONTAINER}" \ + bash -s <<'KAFKA_UPGRADE_SCRIPT' +set -euo pipefail + +if [[ "${PROJECTION_TOPIC}" == "${RAW_TOPIC}.STATS.MATERIALIZE" ]]; then + echo "ERROR: STATS.PROJECTION 不能与 STATS.MATERIALIZE 使用同一个 Topic" >&2 + exit 1 +fi + +TOPICS="${KAFKA_BIN}/kafka-topics.sh" +CONFIGS="${KAFKA_BIN}/kafka-configs.sh" +COMMON=(--bootstrap-server "${BOOTSTRAP}") +if [[ -n "${COMMAND_CONFIG:-}" ]]; then + [[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; } + COMMON+=(--command-config "${COMMAND_CONFIG}") +fi + +echo "Kafka=${BOOTSTRAP}" +echo "原始 Topic=${RAW_TOPIC}" +echo "统计投影 Topic=${PROJECTION_TOPIC}" + +RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}") +REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk ' + /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}} +') +if [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: 无法从 ${RAW_TOPIC} 读取副本数" >&2 + exit 1 +fi + +echo "[1/3] 创建 Topic（已经存在时不重复创建）" +"${TOPICS}" "${COMMON[@]}" --create --if-not-exists \ + --topic "${PROJECTION_TOPIC}" \ + --partitions 1 \ + --replication-factor "${REPLICAS}" \ + --config cleanup.policy=delete \ + --config "retention.ms=${RETENTION_MS}" + +echo "[2/3] 更新保留策略" +if ! "${CONFIGS}" "${COMMON[@]}" --alter \ + --entity-type topics --entity-name "${PROJECTION_TOPIC}" \ + --add-config "cleanup.policy=delete,retention.ms=${RETENTION_MS}"; then + echo "WARN: Kafka 配置更新超时；新建命令已携带相同策略，继续验证 Topic" >&2 +fi + +echo "[3/3] 验证 Topic" +INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${PROJECTION_TOPIC}") +printf '%s\n' "${INFO}" +PARTITIONS=$(printf '%s\n' "${INFO}" | awk ' + /PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}} +') +ACTUAL_REPLICAS=$(printf '%s\n' "${INFO}" | awk ' + /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}} +') +if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: ${PROJECTION_TOPIC} 没有有效分区" >&2 + exit 1 +fi +if [[ "${PARTITIONS}" != "1" ]]; then + echo "WARN: Topic 已有 ${PARTITIONS} 个分区；不会修改分区数，固定 key 仍只使用其中一个分区" >&2 +fi +if [[ "${ACTUAL_REPLICAS}" != "${REPLICAS}" ]]; then + echo "WARN: Topic 副本数=${ACTUAL_REPLICAS}，原始 Topic 副本数=${REPLICAS}" >&2 +fi + +echo "PASS: ${PROJECTION_TOPIC} 已创建/确认。分区数=${PARTITIONS}，保留时间=${RETENTION_MS}ms。" +echo "脚本未删除消息，也未修改任何消费组 offset。" +KAFKA_UPGRADE_SCRIPT diff --git a/haobang-security-dm/upgrade-doc/v3/kafka/01-create-stats-projection-topic.sh b/haobang-security-dm/upgrade-doc/v3/kafka/01-create-stats-projection-topic.sh new file mode 100755 index 0000000..8428696 --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v3/kafka/01-create-stats-projection-topic.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +# v3 统计投影 Topic。新建时使用 1 个分区；如果现场已经建成多个分区则保留现状， +# 应用使用固定 Kafka key，投影消息仍只会落入其中一个稳定分区。 +BOOTSTRAP="${BOOTSTRAP:-10.141.121.35:9092,10.141.121.50:9092,10.141.121.59:9092}" +RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}" +PROJECTION_TOPIC="${PROJECTION_TOPIC:-${RAW_TOPIC}.STATS.PROJECTION}" +RETENTION_MS="${RETENTION_MS:-604800000}" + +if [[ "${PROJECTION_TOPIC}" == "${RAW_TOPIC}.STATS.MATERIALIZE" ]]; then + echo "ERROR: STATS.PROJECTION 不能与 STATS.MATERIALIZE 使用同一个 Topic" >&2 + exit 1 +fi + +if [[ -z "${KAFKA_BIN:-}" ]]; then + for candidate in \ + "$(pwd)" \ + "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" \ + "/opt/kafka/kafka_2.13-3.6.1/bin" \ + "/opt/kafka/bin" \ + "/usr/local/kafka/bin"; do + if [[ -x "${candidate}/kafka-topics.sh" ]] \ + && [[ -x "${candidate}/kafka-configs.sh" ]]; then + KAFKA_BIN="${candidate}" + break + fi + done +fi + +if [[ -z "${KAFKA_BIN:-}" ]] && command -v kafka-topics.sh >/dev/null 2>&1; then + KAFKA_BIN=$(dirname -- "$(command -v kafka-topics.sh)") +fi +if [[ -z "${KAFKA_BIN:-}" ]]; then + echo "ERROR: 未找到 kafka-topics.sh；请在 Kafka bin 目录执行，或设置 KAFKA_BIN" >&2 + exit 1 +fi + +TOPICS="${KAFKA_BIN}/kafka-topics.sh" +CONFIGS="${KAFKA_BIN}/kafka-configs.sh" +COMMON=(--bootstrap-server "${BOOTSTRAP}") +if [[ -n "${COMMAND_CONFIG:-}" ]]; then + [[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; } + COMMON+=(--command-config "${COMMAND_CONFIG}") +fi + +echo "Kafka=${BOOTSTRAP}" +echo "原始 Topic=${RAW_TOPIC}" +echo "统计投影 Topic=${PROJECTION_TOPIC}" + +RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}") +REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk ' + /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}} +') +if [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: 无法从 ${RAW_TOPIC} 读取副本数" >&2 + exit 1 +fi + +echo "[1/3] 创建 Topic（已经存在时不重复创建）" +"${TOPICS}" "${COMMON[@]}" --create --if-not-exists \ + --topic "${PROJECTION_TOPIC}" \ + --partitions 1 \ + --replication-factor "${REPLICAS}" \ + --config cleanup.policy=delete \ + --config "retention.ms=${RETENTION_MS}" + +echo "[2/3] 更新保留策略" +if ! "${CONFIGS}" "${COMMON[@]}" --alter \ + --entity-type topics --entity-name "${PROJECTION_TOPIC}" \ + --add-config "cleanup.policy=delete,retention.ms=${RETENTION_MS}"; then + echo "WARN: Kafka 配置更新超时；新建命令已携带相同策略，继续验证 Topic" >&2 +fi + +echo "[3/3] 验证 Topic" +INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${PROJECTION_TOPIC}") +printf '%s\n' "${INFO}" +PARTITIONS=$(printf '%s\n' "${INFO}" | awk ' + /PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}} +') +ACTUAL_REPLICAS=$(printf '%s\n' "${INFO}" | awk ' + /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}} +') +if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: ${PROJECTION_TOPIC} 没有有效分区" >&2 + exit 1 +fi +if [[ "${PARTITIONS}" != "1" ]]; then + echo "WARN: Topic 已有 ${PARTITIONS} 个分区；不会修改分区数，固定 key 仍只使用其中一个分区" >&2 +fi +if [[ "${ACTUAL_REPLICAS}" != "${REPLICAS}" ]]; then + echo "WARN: Topic 副本数=${ACTUAL_REPLICAS}，原始 Topic 副本数=${REPLICAS}" >&2 +fi + +echo "PASS: ${PROJECTION_TOPIC} 已创建/确认。分区数=${PARTITIONS}，保留时间=${RETENTION_MS}ms。" +echo "脚本未删除消息，也未修改任何消费组 offset。" diff --git a/haobang-security-dm/upgrade-doc/v3/kafka/02-create-large-log-topics-docker.sh b/haobang-security-dm/upgrade-doc/v3/kafka/02-create-large-log-topics-docker.sh new file mode 100755 index 0000000..fe64cdf --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v3/kafka/02-create-large-log-topics-docker.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Docker 单文件版：直接在 Kafka 容器内创建并校验大日志相关 Topic。 +# 默认值适配测试环境；如容器名、Kafka 地址或安装目录不同，可通过环境变量覆盖。 + +DOCKER_BIN="${DOCKER_BIN:-docker}" +KAFKA_CONTAINER="${KAFKA_CONTAINER:-kafka_ycgy-bt-kafka-1}" +BOOTSTRAP="${BOOTSTRAP:-localhost:9092}" +KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}" +RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}" +LARGE_TOPIC="${LARGE_TOPIC:-${RAW_TOPIC}.large}" +MAX_MESSAGE_BYTES="${MAX_MESSAGE_BYTES:-33554432}" +RETENTION_MS="${RETENTION_MS:-604800000}" +DLT_RETENTION_MS="${DLT_RETENTION_MS:-2592000000}" +COMMAND_CONFIG="${COMMAND_CONFIG:-}" + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +command -v "${DOCKER_BIN}" >/dev/null 2>&1 || die "未找到 Docker CLI: ${DOCKER_BIN}" + +RUNNING="$("${DOCKER_BIN}" inspect -f '{{.State.Running}}' "${KAFKA_CONTAINER}" 2>/dev/null || true)" +[[ "${RUNNING}" == "true" ]] || die "Kafka 容器不存在或未运行: ${KAFKA_CONTAINER}" + +"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-topics.sh" \ + || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-topics.sh" +"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-configs.sh" \ + || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-configs.sh" +"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" bash -c 'exit 0' \ + || die "容器内缺少 bash，无法执行原始升级脚本" + +DOCKER_ENV=( + -e "BOOTSTRAP=${BOOTSTRAP}" + -e "KAFKA_BIN=${KAFKA_BIN}" + -e "RAW_TOPIC=${RAW_TOPIC}" + -e "LARGE_TOPIC=${LARGE_TOPIC}" + -e "MAX_MESSAGE_BYTES=${MAX_MESSAGE_BYTES}" + -e "RETENTION_MS=${RETENTION_MS}" + -e "DLT_RETENTION_MS=${DLT_RETENTION_MS}" +) +KAFKA_COMMON=(--bootstrap-server "${BOOTSTRAP}") + +if [[ -n "${COMMAND_CONFIG}" ]]; then + "${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -r "${COMMAND_CONFIG}" \ + || die "容器内无法读取 COMMAND_CONFIG: ${COMMAND_CONFIG}" + DOCKER_ENV+=(-e "COMMAND_CONFIG=${COMMAND_CONFIG}") + KAFKA_COMMON+=(--command-config "${COMMAND_CONFIG}") +fi + +echo "检查 Kafka 连接及原始 topic ..." +"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" \ + "${KAFKA_BIN}/kafka-topics.sh" \ + "${KAFKA_COMMON[@]}" \ + --describe \ + --topic "${RAW_TOPIC}" >/dev/null \ + || die "无法连接 Kafka 或原始 topic 不存在: ${RAW_TOPIC}" + +echo "Kafka 容器: ${KAFKA_CONTAINER}" +echo "Bootstrap: ${BOOTSTRAP}" +echo "原始 topic: ${RAW_TOPIC}" +echo "大日志 topic: ${LARGE_TOPIC}" +echo "开始执行 Docker 版升级 ..." + +exec "${DOCKER_BIN}" exec -i "${DOCKER_ENV[@]}" "${KAFKA_CONTAINER}" \ + bash -s <<'KAFKA_UPGRADE_SCRIPT' +set -euo pipefail + +TOPICS="${KAFKA_BIN}/kafka-topics.sh" +CONFIGS="${KAFKA_BIN}/kafka-configs.sh" +COMMON=(--bootstrap-server "${BOOTSTRAP}") +if [[ -n "${COMMAND_CONFIG:-}" ]]; then + [[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; } + COMMON+=(--command-config "${COMMAND_CONFIG}") +fi + +RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}") +PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk ' + /PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}} +') +REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk ' + /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}} +') +if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: 无法读取 ${RAW_TOPIC} 的分区数或副本数" >&2 + exit 1 +fi + +declare -a TOPIC_NAMES=( + "${LARGE_TOPIC}" + "${LARGE_TOPIC}.RETRY.1M" + "${LARGE_TOPIC}.RETRY.10M" + "${LARGE_TOPIC}.RETRY.1H" + "${LARGE_TOPIC}.DLT" + "${LARGE_TOPIC}.REPLAY" +) + +echo "Kafka=${BOOTSTRAP}，分区=${PARTITIONS}，副本=${REPLICAS}，单条上限=${MAX_MESSAGE_BYTES}" +for topic in "${TOPIC_NAMES[@]}"; do + topic_retention="${RETENTION_MS}" + [[ "${topic}" == "${LARGE_TOPIC}.DLT" ]] && topic_retention="${DLT_RETENTION_MS}" + "${TOPICS}" "${COMMON[@]}" --create --if-not-exists \ + --topic "${topic}" --partitions "${PARTITIONS}" --replication-factor "${REPLICAS}" \ + --config cleanup.policy=delete \ + --config "retention.ms=${topic_retention}" \ + --config "max.message.bytes=${MAX_MESSAGE_BYTES}" + "${CONFIGS}" "${COMMON[@]}" --alter --entity-type topics --entity-name "${topic}" \ + --add-config "cleanup.policy=delete,retention.ms=${topic_retention},max.message.bytes=${MAX_MESSAGE_BYTES}" + "${TOPICS}" "${COMMON[@]}" --describe --topic "${topic}" +done + +echo "PASS: 大日志 Topic 已创建并统一配置。" +echo "注意：Broker 的 replica.fetch.max.bytes 也必须不小于 ${MAX_MESSAGE_BYTES}。" +echo "脚本未删除消息，也未修改任何消费组 offset。" +KAFKA_UPGRADE_SCRIPT diff --git a/haobang-security-dm/upgrade-doc/v3/kafka/02-create-large-log-topics.sh b/haobang-security-dm/upgrade-doc/v3/kafka/02-create-large-log-topics.sh new file mode 100755 index 0000000..70bd882 --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v3/kafka/02-create-large-log-topics.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +# 创建大日志主 Topic 及独立 Retry/DLT/Replay Topic。不删除消息、不修改消费组 offset。 +BOOTSTRAP="${BOOTSTRAP:-10.141.121.35:9092,10.141.121.50:9092,10.141.121.59:9092}" +RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}" +LARGE_TOPIC="${LARGE_TOPIC:-${RAW_TOPIC}.large}" +MAX_MESSAGE_BYTES="${MAX_MESSAGE_BYTES:-33554432}" +RETENTION_MS="${RETENTION_MS:-604800000}" +DLT_RETENTION_MS="${DLT_RETENTION_MS:-2592000000}" + +if [[ -z "${KAFKA_BIN:-}" ]]; then + for candidate in \ + "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" \ + "/opt/kafka/kafka_2.13-3.6.1/bin" \ + "/opt/kafka/bin" \ + "/usr/local/kafka/bin"; do + if [[ -x "${candidate}/kafka-topics.sh" ]] && [[ -x "${candidate}/kafka-configs.sh" ]]; then + KAFKA_BIN="${candidate}" + break + fi + done +fi +if [[ -z "${KAFKA_BIN:-}" ]] && command -v kafka-topics.sh >/dev/null 2>&1; then + KAFKA_BIN=$(dirname -- "$(command -v kafka-topics.sh)") +fi +if [[ -z "${KAFKA_BIN:-}" ]]; then + echo "ERROR: 未找到 Kafka CLI；请设置 KAFKA_BIN" >&2 + exit 1 +fi + +TOPICS="${KAFKA_BIN}/kafka-topics.sh" +CONFIGS="${KAFKA_BIN}/kafka-configs.sh" +COMMON=(--bootstrap-server "${BOOTSTRAP}") +if [[ -n "${COMMAND_CONFIG:-}" ]]; then + [[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; } + COMMON+=(--command-config "${COMMAND_CONFIG}") +fi + +RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}") +PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk ' + /PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}} +') +REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk ' + /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}} +') +if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: 无法读取 ${RAW_TOPIC} 的分区数或副本数" >&2 + exit 1 +fi + +declare -a TOPIC_NAMES=( + "${LARGE_TOPIC}" + "${LARGE_TOPIC}.RETRY.1M" + "${LARGE_TOPIC}.RETRY.10M" + "${LARGE_TOPIC}.RETRY.1H" + "${LARGE_TOPIC}.DLT" + "${LARGE_TOPIC}.REPLAY" +) + +echo "Kafka=${BOOTSTRAP}，分区=${PARTITIONS}，副本=${REPLICAS}，单条上限=${MAX_MESSAGE_BYTES}" +for topic in "${TOPIC_NAMES[@]}"; do + topic_retention="${RETENTION_MS}" + [[ "${topic}" == "${LARGE_TOPIC}.DLT" ]] && topic_retention="${DLT_RETENTION_MS}" + "${TOPICS}" "${COMMON[@]}" --create --if-not-exists \ + --topic "${topic}" --partitions "${PARTITIONS}" --replication-factor "${REPLICAS}" \ + --config cleanup.policy=delete \ + --config "retention.ms=${topic_retention}" \ + --config "max.message.bytes=${MAX_MESSAGE_BYTES}" + "${CONFIGS}" "${COMMON[@]}" --alter --entity-type topics --entity-name "${topic}" \ + --add-config "cleanup.policy=delete,retention.ms=${topic_retention},max.message.bytes=${MAX_MESSAGE_BYTES}" + "${TOPICS}" "${COMMON[@]}" --describe --topic "${topic}" +done + +echo "PASS: 大日志 Topic 已创建并统一配置。" +echo "注意：Broker 的 replica.fetch.max.bytes 也必须不小于 ${MAX_MESSAGE_BYTES}。" +echo "脚本未删除消息，也未修改任何消费组 offset。"