#!/usr/bin/env bash set -euo pipefail # 创建大日志主 Topic 及独立 Retry/DLT/Replay Topic。不删除消息、不修改消费组 offset。 BOOTSTRAP="${BOOTSTRAP:-10.141.121.35:9092,10.141.121.50:9092,10.141.121.59:9092}" RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}" LARGE_TOPIC="${LARGE_TOPIC:-${RAW_TOPIC}.large}" MAX_MESSAGE_BYTES="${MAX_MESSAGE_BYTES:-33554432}" RETENTION_MS="${RETENTION_MS:-604800000}" DLT_RETENTION_MS="${DLT_RETENTION_MS:-2592000000}" if [[ -z "${KAFKA_BIN:-}" ]]; then for candidate in \ "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" \ "/opt/kafka/kafka_2.13-3.6.1/bin" \ "/opt/kafka/bin" \ "/usr/local/kafka/bin"; do if [[ -x "${candidate}/kafka-topics.sh" ]] && [[ -x "${candidate}/kafka-configs.sh" ]]; then KAFKA_BIN="${candidate}" break fi done fi if [[ -z "${KAFKA_BIN:-}" ]] && command -v kafka-topics.sh >/dev/null 2>&1; then KAFKA_BIN=$(dirname -- "$(command -v kafka-topics.sh)") fi if [[ -z "${KAFKA_BIN:-}" ]]; then echo "ERROR: 未找到 Kafka CLI;请设置 KAFKA_BIN" >&2 exit 1 fi TOPICS="${KAFKA_BIN}/kafka-topics.sh" CONFIGS="${KAFKA_BIN}/kafka-configs.sh" COMMON=(--bootstrap-server "${BOOTSTRAP}") if [[ -n "${COMMAND_CONFIG:-}" ]]; then [[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; } COMMON+=(--command-config "${COMMAND_CONFIG}") fi RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}") PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk ' /PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}} ') REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk ' /ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}} ') if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: 无法读取 ${RAW_TOPIC} 的分区数或副本数" >&2 exit 1 fi declare -a TOPIC_NAMES=( "${LARGE_TOPIC}" "${LARGE_TOPIC}.RETRY.1M" "${LARGE_TOPIC}.RETRY.10M" "${LARGE_TOPIC}.RETRY.1H" "${LARGE_TOPIC}.DLT" "${LARGE_TOPIC}.REPLAY" ) echo "Kafka=${BOOTSTRAP},分区=${PARTITIONS},副本=${REPLICAS},单条上限=${MAX_MESSAGE_BYTES}" for topic in "${TOPIC_NAMES[@]}"; do topic_retention="${RETENTION_MS}" [[ "${topic}" == "${LARGE_TOPIC}.DLT" ]] && topic_retention="${DLT_RETENTION_MS}" "${TOPICS}" "${COMMON[@]}" --create --if-not-exists \ --topic "${topic}" --partitions "${PARTITIONS}" --replication-factor "${REPLICAS}" \ --config cleanup.policy=delete \ --config "retention.ms=${topic_retention}" \ --config "max.message.bytes=${MAX_MESSAGE_BYTES}" "${CONFIGS}" "${COMMON[@]}" --alter --entity-type topics --entity-name "${topic}" \ --add-config "cleanup.policy=delete,retention.ms=${topic_retention},max.message.bytes=${MAX_MESSAGE_BYTES}" "${TOPICS}" "${COMMON[@]}" --describe --topic "${topic}" done echo "PASS: 大日志 Topic 已创建并统一配置。" echo "注意:Broker 的 replica.fetch.max.bytes 也必须不小于 ${MAX_MESSAGE_BYTES}。" echo "脚本未删除消息,也未修改任何消费组 offset。"