feat: 新增设备 API 拉取采集及配套处理能力

- 接入 NDRServer 告警和流量日志，支持凭据与任务管理 - 支持分页拉取、窗口拆分、调度优先级及 WAL 持久化后推进断点 - 增加迟到告警规则重放，优化多条泛化规则选择和事件时间处理 - 补充回归测试、数据库升级脚本及部署文档
2026-09-06 16:46:21 +08:00
parent b61ec2e37c
commit 00c0a1fb7f
141 changed files with 11318 additions and 467 deletions
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+: "${KAFKA_BIN:?请设置 KAFKA_BIN，例如 /opt/kafka/bin}"
+: "${BOOTSTRAP:?请设置 BOOTSTRAP，例如 127.0.0.1:9092}"
+
+RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
+TOPICS_CMD="${KAFKA_BIN}/kafka-topics.sh"
+CONFIGS_CMD="${KAFKA_BIN}/kafka-configs.sh"
+
+if [[ ! -x "${TOPICS_CMD}" ]] || [[ ! -x "${CONFIGS_CMD}" ]]; then
+  echo "ERROR: ${KAFKA_BIN} 下缺少 kafka-topics.sh 或 kafka-configs.sh" >&2
+  exit 1
+fi
+
+COMMON_ARGS=(--bootstrap-server "${BOOTSTRAP}")
+if [[ -n "${COMMAND_CONFIG:-}" ]]; then
+  COMMON_ARGS+=(--command-config "${COMMAND_CONFIG}")
+fi
+
+echo "检查原始 Topic：${RAW_TOPIC}"
+RAW_INFO=$("${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}")
+printf '%s\n' "${RAW_INFO}"
+
+PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk '
+  /PartitionCount:/ {
+    for (i=1; i<=NF; i++) if ($i=="PartitionCount:") { print $(i+1); exit }
+  }
+')
+REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
+  /ReplicationFactor:/ {
+    for (i=1; i<=NF; i++) if ($i=="ReplicationFactor:") { print $(i+1); exit }
+  }
+')
+
+if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
+  echo "ERROR: 无法读取原始 Topic 的分区数或副本数" >&2
+  exit 1
+fi
+
+SUFFIXES=("RETRY.1M" "RETRY.10M" "RETRY.1H" "DLT" "REPLAY")
+RETENTIONS=(259200000 604800000 604800000 7776000000 604800000)
+
+for i in 0 1 2 3 4; do
+  TOPIC="${RAW_TOPIC}.${SUFFIXES[$i]}"
+  echo "创建或确认 Topic：${TOPIC}"
+  "${TOPICS_CMD}" "${COMMON_ARGS[@]}" \
+    --create --if-not-exists \
+    --topic "${TOPIC}" \
+    --partitions "${PARTITIONS}" \
+    --replication-factor "${REPLICAS}" \
+    --config "retention.ms=${RETENTIONS[$i]}"
+
+  "${CONFIGS_CMD}" "${COMMON_ARGS[@]}" \
+    --alter --entity-type topics --entity-name "${TOPIC}" \
+    --add-config "retention.ms=${RETENTIONS[$i]}"
+done
+
+echo "检查升级结果"
+for SUFFIX in "${SUFFIXES[@]}"; do
+  "${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}.${SUFFIX}"
+done
+
+echo "PASS: Kafka Topic 已创建/确认；脚本未删除 Topic，也未修改消费组 offset。"