78 lines
3.1 KiB
Bash
78 lines
3.1 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
# 创建大日志主 Topic 及独立 Retry/DLT/Replay Topic。不删除消息、不修改消费组 offset。
|
||
|
|
BOOTSTRAP="${BOOTSTRAP:-10.141.121.35:9092,10.141.121.50:9092,10.141.121.59:9092}"
|
||
|
|
RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
|
||
|
|
LARGE_TOPIC="${LARGE_TOPIC:-${RAW_TOPIC}.large}"
|
||
|
|
MAX_MESSAGE_BYTES="${MAX_MESSAGE_BYTES:-33554432}"
|
||
|
|
RETENTION_MS="${RETENTION_MS:-604800000}"
|
||
|
|
DLT_RETENTION_MS="${DLT_RETENTION_MS:-2592000000}"
|
||
|
|
|
||
|
|
if [[ -z "${KAFKA_BIN:-}" ]]; then
|
||
|
|
for candidate in \
|
||
|
|
"$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" \
|
||
|
|
"/opt/kafka/kafka_2.13-3.6.1/bin" \
|
||
|
|
"/opt/kafka/bin" \
|
||
|
|
"/usr/local/kafka/bin"; do
|
||
|
|
if [[ -x "${candidate}/kafka-topics.sh" ]] && [[ -x "${candidate}/kafka-configs.sh" ]]; then
|
||
|
|
KAFKA_BIN="${candidate}"
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
fi
|
||
|
|
if [[ -z "${KAFKA_BIN:-}" ]] && command -v kafka-topics.sh >/dev/null 2>&1; then
|
||
|
|
KAFKA_BIN=$(dirname -- "$(command -v kafka-topics.sh)")
|
||
|
|
fi
|
||
|
|
if [[ -z "${KAFKA_BIN:-}" ]]; then
|
||
|
|
echo "ERROR: 未找到 Kafka CLI；请设置 KAFKA_BIN" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
TOPICS="${KAFKA_BIN}/kafka-topics.sh"
|
||
|
|
CONFIGS="${KAFKA_BIN}/kafka-configs.sh"
|
||
|
|
COMMON=(--bootstrap-server "${BOOTSTRAP}")
|
||
|
|
if [[ -n "${COMMAND_CONFIG:-}" ]]; then
|
||
|
|
[[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; }
|
||
|
|
COMMON+=(--command-config "${COMMAND_CONFIG}")
|
||
|
|
fi
|
||
|
|
|
||
|
|
RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}")
|
||
|
|
PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk '
|
||
|
|
/PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}}
|
||
|
|
')
|
||
|
|
REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
|
||
|
|
/ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}}
|
||
|
|
')
|
||
|
|
if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
|
||
|
|
echo "ERROR: 无法读取 ${RAW_TOPIC} 的分区数或副本数" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
declare -a TOPIC_NAMES=(
|
||
|
|
"${LARGE_TOPIC}"
|
||
|
|
"${LARGE_TOPIC}.RETRY.1M"
|
||
|
|
"${LARGE_TOPIC}.RETRY.10M"
|
||
|
|
"${LARGE_TOPIC}.RETRY.1H"
|
||
|
|
"${LARGE_TOPIC}.DLT"
|
||
|
|
"${LARGE_TOPIC}.REPLAY"
|
||
|
|
)
|
||
|
|
|
||
|
|
echo "Kafka=${BOOTSTRAP}，分区=${PARTITIONS}，副本=${REPLICAS}，单条上限=${MAX_MESSAGE_BYTES}"
|
||
|
|
for topic in "${TOPIC_NAMES[@]}"; do
|
||
|
|
topic_retention="${RETENTION_MS}"
|
||
|
|
[[ "${topic}" == "${LARGE_TOPIC}.DLT" ]] && topic_retention="${DLT_RETENTION_MS}"
|
||
|
|
"${TOPICS}" "${COMMON[@]}" --create --if-not-exists \
|
||
|
|
--topic "${topic}" --partitions "${PARTITIONS}" --replication-factor "${REPLICAS}" \
|
||
|
|
--config cleanup.policy=delete \
|
||
|
|
--config "retention.ms=${topic_retention}" \
|
||
|
|
--config "max.message.bytes=${MAX_MESSAGE_BYTES}"
|
||
|
|
"${CONFIGS}" "${COMMON[@]}" --alter --entity-type topics --entity-name "${topic}" \
|
||
|
|
--add-config "cleanup.policy=delete,retention.ms=${topic_retention},max.message.bytes=${MAX_MESSAGE_BYTES}"
|
||
|
|
"${TOPICS}" "${COMMON[@]}" --describe --topic "${topic}"
|
||
|
|
done
|
||
|
|
|
||
|
|
echo "PASS: 大日志 Topic 已创建并统一配置。"
|
||
|
|
echo "注意：Broker 的 replica.fetch.max.bytes 也必须不小于 ${MAX_MESSAGE_BYTES}。"
|
||
|
|
echo "脚本未删除消息，也未修改任何消费组 offset。"
|