王绍宏
0e9e6656db
- 128 KiB 及以上日志默认进入大日志 Topic - InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存 - 截断内容追加 HB_XDR_TRUNCATED 标记 - InfluxDB 写入失败时继续执行规则解析和达梦写入 - 未知或系统性异常优先写入普通/大日志 DLT - DLT 写入失败时记录特殊告警并提交 offset，避免阻塞后续消息 - DLT 失败等待时间默认限制为 1 秒 - 增加连续性放行、DLT 成功及失败监控指标 - 更新 Docker Compose、v3 升级文档及相关测试
118 lines
4.6 KiB
Bash
Executable File
118 lines
4.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Docker 单文件版：直接在 Kafka 容器内创建并校验大日志相关 Topic。
|
|
# 默认值适配测试环境；如容器名、Kafka 地址或安装目录不同，可通过环境变量覆盖。
|
|
|
|
DOCKER_BIN="${DOCKER_BIN:-docker}"
|
|
KAFKA_CONTAINER="${KAFKA_CONTAINER:-kafka_ycgy-bt-kafka-1}"
|
|
BOOTSTRAP="${BOOTSTRAP:-localhost:9092}"
|
|
KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}"
|
|
RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
|
|
LARGE_TOPIC="${LARGE_TOPIC:-${RAW_TOPIC}.large}"
|
|
MAX_MESSAGE_BYTES="${MAX_MESSAGE_BYTES:-33554432}"
|
|
RETENTION_MS="${RETENTION_MS:-604800000}"
|
|
DLT_RETENTION_MS="${DLT_RETENTION_MS:-2592000000}"
|
|
COMMAND_CONFIG="${COMMAND_CONFIG:-}"
|
|
|
|
die() {
|
|
echo "ERROR: $*" >&2
|
|
exit 1
|
|
}
|
|
|
|
command -v "${DOCKER_BIN}" >/dev/null 2>&1 || die "未找到 Docker CLI: ${DOCKER_BIN}"
|
|
|
|
RUNNING="$("${DOCKER_BIN}" inspect -f '{{.State.Running}}' "${KAFKA_CONTAINER}" 2>/dev/null || true)"
|
|
[[ "${RUNNING}" == "true" ]] || die "Kafka 容器不存在或未运行: ${KAFKA_CONTAINER}"
|
|
|
|
"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-topics.sh" \
|
|
|| die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-topics.sh"
|
|
"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -x "${KAFKA_BIN}/kafka-configs.sh" \
|
|
|| die "容器内找不到可执行文件: ${KAFKA_BIN}/kafka-configs.sh"
|
|
"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" bash -c 'exit 0' \
|
|
|| die "容器内缺少 bash，无法执行原始升级脚本"
|
|
|
|
DOCKER_ENV=(
|
|
-e "BOOTSTRAP=${BOOTSTRAP}"
|
|
-e "KAFKA_BIN=${KAFKA_BIN}"
|
|
-e "RAW_TOPIC=${RAW_TOPIC}"
|
|
-e "LARGE_TOPIC=${LARGE_TOPIC}"
|
|
-e "MAX_MESSAGE_BYTES=${MAX_MESSAGE_BYTES}"
|
|
-e "RETENTION_MS=${RETENTION_MS}"
|
|
-e "DLT_RETENTION_MS=${DLT_RETENTION_MS}"
|
|
)
|
|
KAFKA_COMMON=(--bootstrap-server "${BOOTSTRAP}")
|
|
|
|
if [[ -n "${COMMAND_CONFIG}" ]]; then
|
|
"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" test -r "${COMMAND_CONFIG}" \
|
|
|| die "容器内无法读取 COMMAND_CONFIG: ${COMMAND_CONFIG}"
|
|
DOCKER_ENV+=(-e "COMMAND_CONFIG=${COMMAND_CONFIG}")
|
|
KAFKA_COMMON+=(--command-config "${COMMAND_CONFIG}")
|
|
fi
|
|
|
|
echo "检查 Kafka 连接及原始 topic ..."
|
|
"${DOCKER_BIN}" exec "${KAFKA_CONTAINER}" \
|
|
"${KAFKA_BIN}/kafka-topics.sh" \
|
|
"${KAFKA_COMMON[@]}" \
|
|
--describe \
|
|
--topic "${RAW_TOPIC}" >/dev/null \
|
|
|| die "无法连接 Kafka 或原始 topic 不存在: ${RAW_TOPIC}"
|
|
|
|
echo "Kafka 容器: ${KAFKA_CONTAINER}"
|
|
echo "Bootstrap: ${BOOTSTRAP}"
|
|
echo "原始 topic: ${RAW_TOPIC}"
|
|
echo "大日志 topic: ${LARGE_TOPIC}"
|
|
echo "开始执行 Docker 版升级 ..."
|
|
|
|
exec "${DOCKER_BIN}" exec -i "${DOCKER_ENV[@]}" "${KAFKA_CONTAINER}" \
|
|
bash -s <<'KAFKA_UPGRADE_SCRIPT'
|
|
set -euo pipefail
|
|
|
|
TOPICS="${KAFKA_BIN}/kafka-topics.sh"
|
|
CONFIGS="${KAFKA_BIN}/kafka-configs.sh"
|
|
COMMON=(--bootstrap-server "${BOOTSTRAP}")
|
|
if [[ -n "${COMMAND_CONFIG:-}" ]]; then
|
|
[[ -r "${COMMAND_CONFIG}" ]] || { echo "ERROR: 无法读取 ${COMMAND_CONFIG}" >&2; exit 1; }
|
|
COMMON+=(--command-config "${COMMAND_CONFIG}")
|
|
fi
|
|
|
|
RAW_INFO=$("${TOPICS}" "${COMMON[@]}" --describe --topic "${RAW_TOPIC}")
|
|
PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk '
|
|
/PartitionCount:/ {for(i=1;i<=NF;i++) if($i=="PartitionCount:"){print $(i+1);exit}}
|
|
')
|
|
REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
|
|
/ReplicationFactor:/ {for(i=1;i<=NF;i++) if($i=="ReplicationFactor:"){print $(i+1);exit}}
|
|
')
|
|
if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "ERROR: 无法读取 ${RAW_TOPIC} 的分区数或副本数" >&2
|
|
exit 1
|
|
fi
|
|
|
|
declare -a TOPIC_NAMES=(
|
|
"${LARGE_TOPIC}"
|
|
"${LARGE_TOPIC}.RETRY.1M"
|
|
"${LARGE_TOPIC}.RETRY.10M"
|
|
"${LARGE_TOPIC}.RETRY.1H"
|
|
"${LARGE_TOPIC}.DLT"
|
|
"${LARGE_TOPIC}.REPLAY"
|
|
)
|
|
|
|
echo "Kafka=${BOOTSTRAP}，分区=${PARTITIONS}，副本=${REPLICAS}，单条上限=${MAX_MESSAGE_BYTES}"
|
|
for topic in "${TOPIC_NAMES[@]}"; do
|
|
topic_retention="${RETENTION_MS}"
|
|
[[ "${topic}" == "${LARGE_TOPIC}.DLT" ]] && topic_retention="${DLT_RETENTION_MS}"
|
|
"${TOPICS}" "${COMMON[@]}" --create --if-not-exists \
|
|
--topic "${topic}" --partitions "${PARTITIONS}" --replication-factor "${REPLICAS}" \
|
|
--config cleanup.policy=delete \
|
|
--config "retention.ms=${topic_retention}" \
|
|
--config "max.message.bytes=${MAX_MESSAGE_BYTES}"
|
|
"${CONFIGS}" "${COMMON[@]}" --alter --entity-type topics --entity-name "${topic}" \
|
|
--add-config "cleanup.policy=delete,retention.ms=${topic_retention},max.message.bytes=${MAX_MESSAGE_BYTES}"
|
|
"${TOPICS}" "${COMMON[@]}" --describe --topic "${topic}"
|
|
done
|
|
|
|
echo "PASS: 大日志 Topic 已创建并统一配置。"
|
|
echo "注意：Broker 的 replica.fetch.max.bytes 也必须不小于 ${MAX_MESSAGE_BYTES}。"
|
|
echo "脚本未删除消息，也未修改任何消费组 offset。"
|
|
KAFKA_UPGRADE_SCRIPT
|