ai-security-xdr/haobang-security-dm/upgrade-doc/v3/kafka/01-create-stats-projection-topic-docker.sh

#!/usr/bin/env bash
set -euo pipefail

# Docker 单文件版：直接在 Kafka 容器内创建并校验统计投影 Topic。
# 默认值适配测试环境；如容器名、Kafka 地址或安装目录不同，可通过环境变量覆盖。

DOCKER_BIN="${DOCKER_BIN:-docker}"
KAFKA_CONTAINER="${KAFKA_CONTAINER:-kafka_ycgy-bt-kafka-1}"
BOOTSTRAP="${BOOTSTRAP:-localhost:9092}"
KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}"
RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
PROJECTION_TOPIC="${PROJECTION_TOPIC:-${RAW_TOPIC}.STATS.PROJECTION}"
RETENTION_MS="${RETENTION_MS:-604800000}"
COMMAND_CONFIG="${COMMAND_CONFIG:-}"

die() {
  echo "ERROR: $*" >&2
  exit 1
}

command -v "${DOCKER_BIN}" >/dev/null 2>&1 || die "未找到 Docker CLI: ${DOCKER_BIN}"

RUNNING="$("${DOCKER_BIN}" inspect -f '{{.State.Running}}' "${KAFKA_CONTAINER}" 2>/dev/null || true)"
[[ "${RUNNING}" == "true" ]] || die "Kafka 容器不存在或未运行: ${KAFKA_CONTAINER}"

"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-topics.sh" \
  || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-topics.sh"
"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-configs.sh" \
  || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-configs.sh"
"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" bash -c 'exit 0' \
  || die "容器内缺少 bash，无法执行原始升级脚本"

DOCKER_ENV=(
  -e "BOOTSTRAP=${BOOTSTRAP}"
  -e "KAFKA_BIN=${KAFKA_BIN}"
  -e "RAW_TOPIC=${RAW_TOPIC}"
  -e "PROJECTION_TOPIC=${PROJECTION_TOPIC}"
  -e "RETENTION_MS=${RETENTION_MS}"
)
KAFKA_COMMON=(--bootstrap-server "${BOOTSTRAP}")

if [[ -n "${COMMAND_CONFIG}" ]]; then
  "${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -r "${COMMAND_CONFIG}" \
    || die "容器内无法读取 COMMAND_CONFIG: ${COMMAND_CONFIG}"
  DOCKER_ENV+=(-e "COMMAND_CONFIG=${COMMAND_CONFIG}")
  KAFKA_COMMON+=(--command-config "${COMMAND_CONFIG}")
fi

echo "检查 Kafka 连接及原始 topic ..."
"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" \
  "${KAFKA_BIN}/kafka-topics.sh" \
  "${KAFKA_COMMON[@]}" \
  --describe \
  --topic "${RAW_TOPIC}" >/dev/null \
  || die "无法连接 Kafka 或原始 topic 不存在: ${RAW_TOPIC}"

echo "Kafka 容器: ${KAFKA_CONTAINER}"
echo "Bootstrap: ${BOOTSTRAP}"
echo "原始 topic: ${RAW_TOPIC}"
echo "投影 topic: ${PROJECTION_TOPIC}"
echo "开始执行 Docker 版升级 ..."

exec "${DOCKER_BIN}" exec -i "${DOCKER_ENV[@]}" "${KAFKA_CONTAINER}" \
  bash -s <<'KAFKA_UPGRADE_SCRIPT'
set -euo pipefail

if [[ "${PROJECTION_TOPIC}" == "${RAW_TOPIC}.STATS.MATERIALIZE" ]]; then
  echo "ERROR: STATS.PROJECTION 不能与 STATS.MATERIALIZE 使用同一个 Topic" >&2
  exit 1
fi

TOPICS="${KAFKA_BIN}/kafka-topics.sh"
CONFIGS="${KAFKA_BIN}/kafka-configs.sh"
COMMON=(--bootstrap-server "${BOOTSTRAP}")
if [[ -n "${COMMAND_CONFIG:-}" ]]; then
  [[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; }
  COMMON+=(--command-config "${COMMAND_CONFIG}")
fi

echo "Kafka=${BOOTSTRAP}"
echo "原始 Topic=${RAW_TOPIC}"
echo "统计投影 Topic=${PROJECTION_TOPIC}"

RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}")
REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
  /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}}
')
if [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
  echo "ERROR: 无法从 ${RAW_TOPIC} 读取副本数" >&2
  exit 1
fi

echo "[1/3] 创建 Topic（已经存在时不重复创建）"
"${TOPICS}" "${COMMON[@]}" --create --if-not-exists \
  --topic "${PROJECTION_TOPIC}" \
  --partitions 1 \
  --replication-factor "${REPLICAS}" \
  --config cleanup.policy=delete \
  --config "retention.ms=${RETENTION_MS}"

echo "[2/3] 更新保留策略"
if ! "${CONFIGS}" "${COMMON[@]}" --alter \
  --entity-type topics --entity-name "${PROJECTION_TOPIC}" \
  --add-config "cleanup.policy=delete,retention.ms=${RETENTION_MS}"; then
  echo "WARN: Kafka 配置更新超时；新建命令已携带相同策略，继续验证 Topic" >&2
fi

echo "[3/3] 验证 Topic"
INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${PROJECTION_TOPIC}")
printf '%s\n' "${INFO}"
PARTITIONS=$(printf '%s\n' "${INFO}" | awk '
  /PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}}
')
ACTUAL_REPLICAS=$(printf '%s\n' "${INFO}" | awk '
  /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}}
')
if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]]; then
  echo "ERROR: ${PROJECTION_TOPIC} 没有有效分区" >&2
  exit 1
fi
if [[ "${PARTITIONS}" != "1" ]]; then
  echo "WARN: Topic 已有 ${PARTITIONS} 个分区；不会修改分区数，固定 key 仍只使用其中一个分区" >&2
fi
if [[ "${ACTUAL_REPLICAS}" != "${REPLICAS}" ]]; then
  echo "WARN: Topic 副本数=${ACTUAL_REPLICAS}，原始 Topic 副本数=${REPLICAS}" >&2
fi

echo "PASS: ${PROJECTION_TOPIC} 已创建/确认。分区数=${PARTITIONS}，保留时间=${RETENTION_MS}ms。"
echo "脚本未删除消息，也未修改任何消费组 offset。"
KAFKA_UPGRADE_SCRIPT