王绍宏
c1472b9a95
- 将设备、规则和采集任务业务统计改为专用 Kafka Topic 异步物化 - 使用维度 ID 作为 Kafka key，保证同一统计维度串行更新 - 物化过程读取小型统计表精确值，确保重复消费不会重复计数 - 增加提交后发布、失败重试及定时小表对账补偿 - 固定日统计聚合键写入顺序，完善达梦死锁识别和事务重试 - JSON 解析失败直接转为非标日志，避免进入 DLT 重试链路 - 非标日志使用独立异步队列，不再阻塞正常消息 Kafka ACK - Kafka 升级脚本新增 STATS.MATERIALIZE Topic - 增加统计物化状态指标、配置示例及单元测试
66 lines
2.1 KiB
Bash
Executable File
66 lines
2.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
|
||
set -euo pipefail
|
||
|
||
: "${KAFKA_BIN:?请设置 KAFKA_BIN，例如 /opt/kafka/bin}"
|
||
: "${BOOTSTRAP:?请设置 BOOTSTRAP，例如 127.0.0.1:9092}"
|
||
|
||
RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
|
||
TOPICS_CMD="${KAFKA_BIN}/kafka-topics.sh"
|
||
CONFIGS_CMD="${KAFKA_BIN}/kafka-configs.sh"
|
||
|
||
if [[ ! -x "${TOPICS_CMD}" ]] || [[ ! -x "${CONFIGS_CMD}" ]]; then
|
||
echo "ERROR: ${KAFKA_BIN} 下缺少 kafka-topics.sh 或 kafka-configs.sh" >&2
|
||
exit 1
|
||
fi
|
||
|
||
COMMON_ARGS=(--bootstrap-server "${BOOTSTRAP}")
|
||
if [[ -n "${COMMAND_CONFIG:-}" ]]; then
|
||
COMMON_ARGS+=(--command-config "${COMMAND_CONFIG}")
|
||
fi
|
||
|
||
echo "检查原始 Topic：${RAW_TOPIC}"
|
||
RAW_INFO=$("${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}")
|
||
printf '%s\n' "${RAW_INFO}"
|
||
|
||
PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk '
|
||
/PartitionCount:/ {
|
||
for (i=1; i<=NF; i++) if ($i=="PartitionCount:") { print $(i+1); exit }
|
||
}
|
||
')
|
||
REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
|
||
/ReplicationFactor:/ {
|
||
for (i=1; i<=NF; i++) if ($i=="ReplicationFactor:") { print $(i+1); exit }
|
||
}
|
||
')
|
||
|
||
if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
|
||
echo "ERROR: 无法读取原始 Topic 的分区数或副本数" >&2
|
||
exit 1
|
||
fi
|
||
|
||
SUFFIXES=("RETRY.1M" "RETRY.10M" "RETRY.1H" "DLT" "REPLAY" "STATS.MATERIALIZE")
|
||
RETENTIONS=(259200000 604800000 604800000 7776000000 604800000 604800000)
|
||
|
||
for i in 0 1 2 3 4 5; do
|
||
TOPIC="${RAW_TOPIC}.${SUFFIXES[$i]}"
|
||
echo "创建或确认 Topic：${TOPIC}"
|
||
"${TOPICS_CMD}" "${COMMON_ARGS[@]}" \
|
||
--create --if-not-exists \
|
||
--topic "${TOPIC}" \
|
||
--partitions "${PARTITIONS}" \
|
||
--replication-factor "${REPLICAS}" \
|
||
--config "retention.ms=${RETENTIONS[$i]}"
|
||
|
||
"${CONFIGS_CMD}" "${COMMON_ARGS[@]}" \
|
||
--alter --entity-type topics --entity-name "${TOPIC}" \
|
||
--add-config "retention.ms=${RETENTIONS[$i]}"
|
||
done
|
||
|
||
echo "检查升级结果"
|
||
for SUFFIX in "${SUFFIXES[@]}"; do
|
||
"${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}.${SUFFIX}"
|
||
done
|
||
|
||
echo "PASS: Kafka Topic 已创建/确认；脚本未删除 Topic，也未修改消费组 offset。"
|