#!/usr/bin/env bash set -euo pipefail # Docker 单文件版:直接在 Kafka 容器内创建并校验统计投影 Topic。 # 默认值适配测试环境;如容器名、Kafka 地址或安装目录不同,可通过环境变量覆盖。 DOCKER_BIN="${DOCKER_BIN:-docker}" KAFKA_CONTAINER="${KAFKA_CONTAINER:-kafka_ycgy-bt-kafka-1}" BOOTSTRAP="${BOOTSTRAP:-localhost:9092}" KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}" RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}" PROJECTION_TOPIC="${PROJECTION_TOPIC:-${RAW_TOPIC}.STATS.PROJECTION}" RETENTION_MS="${RETENTION_MS:-604800000}" COMMAND_CONFIG="${COMMAND_CONFIG:-}" die() { echo "ERROR: $*" >&2 exit 1 } command -v "${DOCKER_BIN}" >/dev/null 2>&1 || die "未找到 Docker CLI: ${DOCKER_BIN}" RUNNING="$("${DOCKER_BIN}" inspect -f '{{.State.Running}}' "${KAFKA_CONTAINER}" 2>/dev/null || true)" [[ "${RUNNING}" == "true" ]] || die "Kafka 容器不存在或未运行: ${KAFKA_CONTAINER}" "${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-topics.sh" \ || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-topics.sh" "${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-configs.sh" \ || die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-configs.sh" "${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" bash -c 'exit 0' \ || die "容器内缺少 bash,无法执行原始升级脚本" DOCKER_ENV=( -e "BOOTSTRAP=${BOOTSTRAP}" -e "KAFKA_BIN=${KAFKA_BIN}" -e "RAW_TOPIC=${RAW_TOPIC}" -e "PROJECTION_TOPIC=${PROJECTION_TOPIC}" -e "RETENTION_MS=${RETENTION_MS}" ) KAFKA_COMMON=(--bootstrap-server "${BOOTSTRAP}") if [[ -n "${COMMAND_CONFIG}" ]]; then "${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -r "${COMMAND_CONFIG}" \ || die "容器内无法读取 COMMAND_CONFIG: ${COMMAND_CONFIG}" DOCKER_ENV+=(-e "COMMAND_CONFIG=${COMMAND_CONFIG}") KAFKA_COMMON+=(--command-config "${COMMAND_CONFIG}") fi echo "检查 Kafka 连接及原始 topic ..." "${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" \ "${KAFKA_BIN}/kafka-topics.sh" \ "${KAFKA_COMMON[@]}" \ --describe \ --topic "${RAW_TOPIC}" >/dev/null \ || die "无法连接 Kafka 或原始 topic 不存在: ${RAW_TOPIC}" echo "Kafka 容器: ${KAFKA_CONTAINER}" echo "Bootstrap: ${BOOTSTRAP}" echo "原始 topic: ${RAW_TOPIC}" echo "投影 topic: ${PROJECTION_TOPIC}" echo "开始执行 Docker 版升级 ..." exec "${DOCKER_BIN}" exec -i "${DOCKER_ENV[@]}" "${KAFKA_CONTAINER}" \ bash -s <<'KAFKA_UPGRADE_SCRIPT' set -euo pipefail if [[ "${PROJECTION_TOPIC}" == "${RAW_TOPIC}.STATS.MATERIALIZE" ]]; then echo "ERROR: STATS.PROJECTION 不能与 STATS.MATERIALIZE 使用同一个 Topic" >&2 exit 1 fi TOPICS="${KAFKA_BIN}/kafka-topics.sh" CONFIGS="${KAFKA_BIN}/kafka-configs.sh" COMMON=(--bootstrap-server "${BOOTSTRAP}") if [[ -n "${COMMAND_CONFIG:-}" ]]; then [[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; } COMMON+=(--command-config "${COMMAND_CONFIG}") fi echo "Kafka=${BOOTSTRAP}" echo "原始 Topic=${RAW_TOPIC}" echo "统计投影 Topic=${PROJECTION_TOPIC}" RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}") REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk ' /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}} ') if [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: 无法从 ${RAW_TOPIC} 读取副本数" >&2 exit 1 fi echo "[1/3] 创建 Topic(已经存在时不重复创建)" "${TOPICS}" "${COMMON[@]}" --create --if-not-exists \ --topic "${PROJECTION_TOPIC}" \ --partitions 1 \ --replication-factor "${REPLICAS}" \ --config cleanup.policy=delete \ --config "retention.ms=${RETENTION_MS}" echo "[2/3] 更新保留策略" if ! "${CONFIGS}" "${COMMON[@]}" --alter \ --entity-type topics --entity-name "${PROJECTION_TOPIC}" \ --add-config "cleanup.policy=delete,retention.ms=${RETENTION_MS}"; then echo "WARN: Kafka 配置更新超时;新建命令已携带相同策略,继续验证 Topic" >&2 fi echo "[3/3] 验证 Topic" INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${PROJECTION_TOPIC}") printf '%s\n' "${INFO}" PARTITIONS=$(printf '%s\n' "${INFO}" | awk ' /PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}} ') ACTUAL_REPLICAS=$(printf '%s\n' "${INFO}" | awk ' /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}} ') if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: ${PROJECTION_TOPIC} 没有有效分区" >&2 exit 1 fi if [[ "${PARTITIONS}" != "1" ]]; then echo "WARN: Topic 已有 ${PARTITIONS} 个分区;不会修改分区数,固定 key 仍只使用其中一个分区" >&2 fi if [[ "${ACTUAL_REPLICAS}" != "${REPLICAS}" ]]; then echo "WARN: Topic 副本数=${ACTUAL_REPLICAS},原始 Topic 副本数=${REPLICAS}" >&2 fi echo "PASS: ${PROJECTION_TOPIC} 已创建/确认。分区数=${PARTITIONS},保留时间=${RETENTION_MS}ms。" echo "脚本未删除消息,也未修改任何消费组 offset。" KAFKA_UPGRADE_SCRIPT