perf(syslog-consumer-rule): 非标日志改为单线程异步批量写入
- 非标写库结果不再参与 Kafka ACK - 使用独立单线程和有界队列处理非标日志 - 非标数据按固定大小分批写入达梦 - 增加异步成功、失败和队列拒绝指标 - 补充异步解耦及批量写入测试 perf(syslog-consumer-rule): 按批次字段并集优化达梦批量写入 - 普通日志和告警日志按非空字段并集生成批写模板 - 为同批记录补齐缺失字段并绑定 NULL - 使用 MyBatis ExecutorType.BATCH 复用 PreparedStatement - 避免固定写入全部 373 个候选字段 - 非标日志保留固定 24 列多行批量写入 - 批量大小和非标队列容量默认调整为 500 - 补充字段并集与动态 SQL 一致性测试
This commit is contained in:
王绍宏
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# 将指定消费组在原始 Topic 上的 offset 重置到指定 Kafka 消息时间。
|
||||
# 适用于“丢弃截止时间之前的历史积压，保留截止时间之后消息”的现场操作。
|
||||
#
|
||||
# 本脚本不会停止/启动应用，也不会删除 Topic 或消息：
|
||||
# - 执行前必须由运维停止该消费组的全部 syslog-consumer-rule 实例；
|
||||
# - 自动导出原 offset，Kafka 数据仍在 retention 内时可用于人工回退；
|
||||
# - 先执行 dry-run，只有输入精确确认串才执行真正的 offset 重置。
|
||||
|
||||
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
||||
BOOTSTRAP="${BOOTSTRAP:-10.141.121.35:9092,10.141.121.50:9092,10.141.121.59:9092}"
|
||||
GROUP_ID="${GROUP_ID:-agent-01-syslog-group}"
|
||||
RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
|
||||
CUTOFF_DATETIME="${CUTOFF_DATETIME:-2026-09-11T00:00:00.000}"
|
||||
KAFKA_TIMEZONE="${KAFKA_TIMEZONE:-Asia/Shanghai}"
|
||||
|
||||
# 优先使用调用方显式指定的目录；否则自动识别脚本同目录、现场 Kafka 安装目录或 PATH。
|
||||
if [[ -z "${KAFKA_BIN:-}" ]]; then
|
||||
for CANDIDATE_BIN in \
|
||||
"${SCRIPT_DIR}" \
|
||||
"/opt/kafka/kafka_2.13-3.6.1/bin"; do
|
||||
if [[ -x "${CANDIDATE_BIN}/kafka-consumer-groups.sh" ]] \
|
||||
&& [[ -x "${CANDIDATE_BIN}/kafka-topics.sh" ]]; then
|
||||
KAFKA_BIN="${CANDIDATE_BIN}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -z "${KAFKA_BIN:-}" ]] && command -v kafka-consumer-groups.sh >/dev/null 2>&1; then
|
||||
KAFKA_BIN=$(dirname -- "$(command -v kafka-consumer-groups.sh)")
|
||||
fi
|
||||
|
||||
if [[ -z "${KAFKA_BIN:-}" ]]; then
|
||||
echo "ERROR: 未找到 Kafka CLI；现场预期路径为 /opt/kafka/kafka_2.13-3.6.1/bin" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CONSUMER_GROUPS_CMD="${KAFKA_BIN}/kafka-consumer-groups.sh"
|
||||
TOPICS_CMD="${KAFKA_BIN}/kafka-topics.sh"
|
||||
|
||||
if [[ ! -x "${CONSUMER_GROUPS_CMD}" ]] || [[ ! -x "${TOPICS_CMD}" ]]; then
|
||||
echo "ERROR: ${KAFKA_BIN} 下缺少可执行的 kafka-consumer-groups.sh 或 kafka-topics.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "${CUTOFF_DATETIME}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}$ ]]; then
|
||||
echo "ERROR: CUTOFF_DATETIME 格式必须是 YYYY-MM-DDTHH:mm:ss.SSS" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
COMMON_ARGS=(--bootstrap-server "${BOOTSTRAP}")
|
||||
if [[ -n "${COMMAND_CONFIG:-}" ]]; then
|
||||
if [[ ! -r "${COMMAND_CONFIG}" ]]; then
|
||||
echo "ERROR: 无法读取认证配置 ${COMMAND_CONFIG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
COMMON_ARGS+=(--command-config "${COMMAND_CONFIG}")
|
||||
fi
|
||||
|
||||
# Kafka 3.6 的 --to-datetime 不携带时区，显式固定 JVM 时区，避免宿主机差异造成 8 小时偏移。
|
||||
export KAFKA_OPTS="${KAFKA_OPTS:-} -Duser.timezone=${KAFKA_TIMEZONE}"
|
||||
|
||||
RUN_ID=$(date '+%Y%m%d_%H%M%S')
|
||||
AUDIT_DIR="${OFFSET_BACKUP_DIR:-./kafka-offset-reset-${RUN_ID}}"
|
||||
mkdir -p "${AUDIT_DIR}"
|
||||
|
||||
BEFORE_FILE="${AUDIT_DIR}/before-describe.txt"
|
||||
RESTORE_FILE="${AUDIT_DIR}/restore-current-offsets.csv"
|
||||
DRY_RUN_FILE="${AUDIT_DIR}/reset-dry-run.txt"
|
||||
EXECUTE_FILE="${AUDIT_DIR}/reset-execute.txt"
|
||||
AFTER_FILE="${AUDIT_DIR}/after-describe.txt"
|
||||
|
||||
echo "目标 Kafka：${BOOTSTRAP}"
|
||||
echo "目标消费组：${GROUP_ID}"
|
||||
echo "目标 Topic：${RAW_TOPIC}"
|
||||
echo "保留起点：${CUTOFF_DATETIME} (${KAFKA_TIMEZONE})"
|
||||
echo "审计目录：${AUDIT_DIR}"
|
||||
|
||||
echo "[1/6] 检查 Topic"
|
||||
TOPIC_INFO=$("${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}")
|
||||
printf '%s\n' "${TOPIC_INFO}"
|
||||
|
||||
PARTITION_COUNT=$(printf '%s\n' "${TOPIC_INFO}" | awk '
|
||||
/PartitionCount:/ {
|
||||
for (i=1; i<=NF; i++) if ($i=="PartitionCount:") { print $(i+1); exit }
|
||||
}
|
||||
')
|
||||
if [[ ! "${PARTITION_COUNT}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "ERROR: 无法确认 ${RAW_TOPIC} 的分区数" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[2/6] 检查消费组是否已停止，并保存执行前快照"
|
||||
"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \
|
||||
--describe --group "${GROUP_ID}" 2>&1 | tee "${BEFORE_FILE}"
|
||||
|
||||
MATCHED_ROWS=$(awk -v topic="${RAW_TOPIC}" '
|
||||
$2==topic && $3~/^[0-9]+$/ && $4~/^[0-9]+$/ && $5~/^[0-9]+$/ {count++}
|
||||
END {print count+0}
|
||||
' "${BEFORE_FILE}")
|
||||
if [[ "${MATCHED_ROWS}" -ne "${PARTITION_COUNT}" ]]; then
|
||||
echo "ERROR: 快照只识别到 ${MATCHED_ROWS}/${PARTITION_COUNT} 个目标分区，拒绝继续" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if awk -v topic="${RAW_TOPIC}" '
|
||||
$2==topic && $3~/^[0-9]+$/ && $7!="-" {active=1}
|
||||
END {exit(active ? 0 : 1)}
|
||||
' "${BEFORE_FILE}"; then
|
||||
echo "ERROR: 消费组 ${GROUP_ID} 仍有在线成员。请先停止全部 syslog-consumer-rule 实例。" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CURRENT_LAG=$(awk -v topic="${RAW_TOPIC}" '
|
||||
$2==topic && $6~/^[0-9]+$/ {sum+=$6}
|
||||
END {printf "%.0f", sum+0}
|
||||
' "${BEFORE_FILE}")
|
||||
echo "当前积压：${CURRENT_LAG} 条"
|
||||
|
||||
echo "[3/6] 导出原 offset 回退文件"
|
||||
"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \
|
||||
--group "${GROUP_ID}" \
|
||||
--topic "${RAW_TOPIC}" \
|
||||
--reset-offsets \
|
||||
--to-current \
|
||||
--dry-run \
|
||||
--export > "${RESTORE_FILE}"
|
||||
|
||||
RESTORE_ROWS=$(awk -F, -v topic="${RAW_TOPIC}" '
|
||||
$1==topic && $2~/^[0-9]+$/ && $3~/^[0-9]+$/ {count++}
|
||||
END {print count+0}
|
||||
' "${RESTORE_FILE}")
|
||||
if [[ "${RESTORE_ROWS}" -ne "${PARTITION_COUNT}" ]]; then
|
||||
echo "ERROR: 回退文件只包含 ${RESTORE_ROWS}/${PARTITION_COUNT} 个分区，拒绝继续" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "原 offset 已保存：${RESTORE_FILE}"
|
||||
|
||||
echo "[4/6] 预演按时间重置"
|
||||
"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \
|
||||
--group "${GROUP_ID}" \
|
||||
--topic "${RAW_TOPIC}" \
|
||||
--reset-offsets \
|
||||
--to-datetime "${CUTOFF_DATETIME}" \
|
||||
--dry-run | tee "${DRY_RUN_FILE}"
|
||||
|
||||
PLAN_ROWS=$(awk -v topic="${RAW_TOPIC}" '
|
||||
$1==topic && $2~/^[0-9]+$/ && $3~/^[0-9]+$/ {count++; next}
|
||||
$2==topic && $3~/^[0-9]+$/ && $4~/^[0-9]+$/ {count++}
|
||||
END {print count+0}
|
||||
' "${DRY_RUN_FILE}")
|
||||
if [[ "${PLAN_ROWS}" -ne "${PARTITION_COUNT}" ]]; then
|
||||
echo "ERROR: dry-run 只生成 ${PLAN_ROWS}/${PARTITION_COUNT} 个分区，拒绝执行" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
EXPECTED_CONFIRM="RESET:${GROUP_ID}:${RAW_TOPIC}:${CUTOFF_DATETIME}"
|
||||
echo "[5/6] 等待人工确认"
|
||||
echo "即将永久跳过 ${CUTOFF_DATETIME} (${KAFKA_TIMEZONE}) 之前尚未消费的消息。"
|
||||
echo "原 offset 回退文件：${RESTORE_FILE}"
|
||||
echo "如确认执行，请完整输入：${EXPECTED_CONFIRM}"
|
||||
|
||||
if [[ -n "${CONFIRM_RESET:-}" ]]; then
|
||||
CONFIRM_INPUT="${CONFIRM_RESET}"
|
||||
elif [[ -t 0 ]]; then
|
||||
read -r CONFIRM_INPUT
|
||||
else
|
||||
echo "ERROR: 当前不是交互终端；必须通过 CONFIRM_RESET 提供精确确认串" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${CONFIRM_INPUT}" != "${EXPECTED_CONFIRM}" ]]; then
|
||||
echo "CANCELLED: 确认串不匹配，未修改任何 offset。" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[6/6] 执行 offset 重置"
|
||||
"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \
|
||||
--group "${GROUP_ID}" \
|
||||
--topic "${RAW_TOPIC}" \
|
||||
--reset-offsets \
|
||||
--to-datetime "${CUTOFF_DATETIME}" \
|
||||
--execute | tee "${EXECUTE_FILE}"
|
||||
|
||||
echo "保存执行后快照"
|
||||
"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \
|
||||
--describe --group "${GROUP_ID}" 2>&1 | tee "${AFTER_FILE}"
|
||||
|
||||
echo "PASS: 已将 ${GROUP_ID}/${RAW_TOPIC} 重置到 ${CUTOFF_DATETIME} (${KAFKA_TIMEZONE})。"
|
||||
echo "未删除 Topic；现在可以启动 syslog-consumer-rule 并检查 LAG。"
|
||||
echo "回退文件：${RESTORE_FILE}（仅在 Kafka 原消息仍处于 retention 内时有效）。"
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Kafka 集群只需在一台装有 Kafka CLI、能够访问集群的服务器上执行一次。
|
||||
# 示例：
|
||||
# export KAFKA_BIN=<现场Kafka客户端bin目录>
|
||||
# export BOOTSTRAP=<现场broker1地址>:<端口>,<现场broker2地址>:<端口>
|
||||
# export RAW_TOPIC=<现场原始Topic名称>
|
||||
# export COMMAND_CONFIG=<现场client.properties路径> # 无认证时不设置
|
||||
# ./kafka-cluster-upgrade.sh
|
||||
# BOOTSTRAP 必须使用客户 Kafka 运维提供的地址；集群建议填写两个以上 broker。
|
||||
|
||||
: "${KAFKA_BIN:?请根据现场环境设置 KAFKA_BIN}"
|
||||
: "${BOOTSTRAP:?请填写客户 Kafka 运维提供的 broker 地址和端口}"
|
||||
: "${RAW_TOPIC:?请填写现场实际使用的原始 Topic 名称}"
|
||||
|
||||
TOPICS_CMD="${KAFKA_BIN}/kafka-topics.sh"
|
||||
CONFIGS_CMD="${KAFKA_BIN}/kafka-configs.sh"
|
||||
|
||||
if [[ ! -x "${TOPICS_CMD}" ]] || [[ ! -x "${CONFIGS_CMD}" ]]; then
|
||||
echo "ERROR: ${KAFKA_BIN} 下缺少 kafka-topics.sh 或 kafka-configs.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
COMMON_ARGS=(--bootstrap-server "${BOOTSTRAP}")
|
||||
if [[ -n "${COMMAND_CONFIG:-}" ]]; then
|
||||
if [[ ! -r "${COMMAND_CONFIG}" ]]; then
|
||||
echo "ERROR: 无法读取认证配置 ${COMMAND_CONFIG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
COMMON_ARGS+=(--command-config "${COMMAND_CONFIG}")
|
||||
fi
|
||||
|
||||
echo "[1/4] 测试 Kafka 集群连接：${BOOTSTRAP}"
|
||||
"${TOPICS_CMD}" "${COMMON_ARGS[@]}" --list >/dev/null
|
||||
|
||||
echo "[2/4] 读取原始 Topic：${RAW_TOPIC}"
|
||||
RAW_INFO=$("${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}")
|
||||
printf '%s\n' "${RAW_INFO}"
|
||||
|
||||
PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk '
|
||||
/PartitionCount:/ {
|
||||
for (i = 1; i <= NF; i++) if ($i == "PartitionCount:") { print $(i + 1); exit }
|
||||
}
|
||||
')
|
||||
REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
|
||||
/ReplicationFactor:/ {
|
||||
for (i = 1; i <= NF; i++) if ($i == "ReplicationFactor:") { print $(i + 1); exit }
|
||||
}
|
||||
')
|
||||
|
||||
if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "ERROR: 无法读取原始 Topic 的分区数或副本数" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "新 Topic 使用分区数=${PARTITIONS}，副本数=${REPLICAS}"
|
||||
|
||||
SUFFIXES=("RETRY.1M" "RETRY.10M" "RETRY.1H" "DLT" "REPLAY")
|
||||
RETENTIONS=(259200000 604800000 604800000 7776000000 604800000)
|
||||
|
||||
echo "[3/4] 创建或更新升级 Topic"
|
||||
for i in 0 1 2 3 4; do
|
||||
TOPIC="${RAW_TOPIC}.${SUFFIXES[$i]}"
|
||||
RETENTION_MS="${RETENTIONS[$i]}"
|
||||
|
||||
echo "处理 ${TOPIC}"
|
||||
"${TOPICS_CMD}" "${COMMON_ARGS[@]}" \
|
||||
--create --if-not-exists \
|
||||
--topic "${TOPIC}" \
|
||||
--partitions "${PARTITIONS}" \
|
||||
--replication-factor "${REPLICAS}" \
|
||||
--config cleanup.policy=delete \
|
||||
--config "retention.ms=${RETENTION_MS}"
|
||||
|
||||
"${CONFIGS_CMD}" "${COMMON_ARGS[@]}" \
|
||||
--alter --entity-type topics --entity-name "${TOPIC}" \
|
||||
--add-config "cleanup.policy=delete,retention.ms=${RETENTION_MS}"
|
||||
done
|
||||
|
||||
echo "[4/4] 显示升级结果"
|
||||
for SUFFIX in "${SUFFIXES[@]}"; do
|
||||
"${TOPICS_CMD}" "${COMMON_ARGS[@]}" \
|
||||
--describe --topic "${RAW_TOPIC}.${SUFFIX}"
|
||||
done
|
||||
|
||||
echo "PASS: Kafka 集群 Topic 已创建或更新；未删除 Topic，未修改消费组 offset。"
|
||||
@@ -7,7 +7,8 @@ set -euo pipefail
|
||||
|
||||
INFLUX_CONFIG_PATH="${INFLUX_CONFIG_PATH:-/etc/influxdb2/influx-configs}"
|
||||
V2_BUCKET="${V2_BUCKET:-yelangbucket_v2}"
|
||||
RETENTION="${RETENTION:-720h}"
|
||||
# 现场要求原始日志保留 180 天；仍可通过 RETENTION 环境变量覆盖。
|
||||
RETENTION="${RETENTION:-180d}"
|
||||
SHARD_GROUP_DURATION="${SHARD_GROUP_DURATION:-24h}"
|
||||
|
||||
command -v docker >/dev/null 2>&1 || {
|
||||
|
||||
Reference in New Issue
Block a user