feat(syslog): 优化大日志通道并增加异常连续消费兜底
- 128 KiB 及以上日志默认进入大日志 Topic - InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存 - 截断内容追加 HB_XDR_TRUNCATED 标记 - InfluxDB 写入失败时继续执行规则解析和达梦写入 - 未知或系统性异常优先写入普通/大日志 DLT - DLT 写入失败时记录特殊告警并提交 offset,避免阻塞后续消息 - DLT 失败等待时间默认限制为 1 秒 - 增加连续性放行、DLT 成功及失败监控指标 - 更新 Docker Compose、v3 升级文档及相关测试
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
-- v3 达梦升级预检:只读,不修改任何数据。
|
||||
-- 第一组应返回 3 行;第二、三、四组应全部为空。
|
||||
|
||||
-- v1 的三张 DIRECT 统计表必须已经存在。
|
||||
SELECT table_name
|
||||
FROM user_tables
|
||||
WHERE table_name IN (
|
||||
'DEVICE_DAILY_STATS',
|
||||
'NORMALIZE_RULE_DAILY_STATS',
|
||||
'DEVICE_COLLECT_STATS'
|
||||
)
|
||||
ORDER BY table_name;
|
||||
|
||||
-- v3 目标表必须尚未创建。若有返回,停止执行 02,先确认是否执行过 v3 或曾中途失败。
|
||||
SELECT table_name
|
||||
FROM user_tables
|
||||
WHERE table_name IN (
|
||||
'STATS_PROJECTION_OUTBOX',
|
||||
'STATS_PROJECTION_APPLIED_EVENT'
|
||||
)
|
||||
ORDER BY table_name;
|
||||
|
||||
-- 约束名不得与现有对象冲突。
|
||||
SELECT constraint_name, table_name
|
||||
FROM user_constraints
|
||||
WHERE constraint_name IN (
|
||||
'PK_STATS_PROJECTION_OUTBOX',
|
||||
'CK_STATS_PROJECTION_OUTBOX_STATUS',
|
||||
'PK_STATS_PROJECTION_APPLIED'
|
||||
)
|
||||
ORDER BY constraint_name;
|
||||
|
||||
-- 索引名不得与现有对象冲突。
|
||||
SELECT index_name, table_name
|
||||
FROM user_indexes
|
||||
WHERE index_name IN (
|
||||
'IDX_STATS_PROJECTION_OUTBOX_DUE',
|
||||
'IDX_STATS_PROJECTION_APPLIED_TIME'
|
||||
)
|
||||
ORDER BY index_name;
|
||||
@@ -0,0 +1,41 @@
|
||||
-- v3 达梦升级:为事实事务外的可靠日统计投影建立 Outbox 和消费幂等表。
|
||||
-- 仅在 01 预检通过后执行一次;不要重复执行。
|
||||
|
||||
CREATE TABLE stats_projection_outbox (
|
||||
event_id VARCHAR2(64) NOT NULL,
|
||||
event_schema VARCHAR2(64) NOT NULL,
|
||||
payload CLOB NOT NULL,
|
||||
status VARCHAR2(16) DEFAULT 'PENDING' NOT NULL,
|
||||
attempt_count INT DEFAULT 0 NOT NULL,
|
||||
next_attempt_at TIMESTAMP DEFAULT SYSDATE,
|
||||
published_at TIMESTAMP,
|
||||
applied_at TIMESTAMP,
|
||||
last_error VARCHAR2(2000),
|
||||
created_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||
updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||
CONSTRAINT pk_stats_projection_outbox PRIMARY KEY(event_id),
|
||||
CONSTRAINT ck_stats_projection_outbox_status
|
||||
CHECK(status IN ('PENDING','FAILED','PUBLISHED','APPLIED'))
|
||||
);
|
||||
|
||||
CREATE INDEX idx_stats_projection_outbox_due
|
||||
ON stats_projection_outbox(status,next_attempt_at,created_at);
|
||||
|
||||
CREATE TABLE stats_projection_applied_event (
|
||||
event_id VARCHAR2(64) NOT NULL,
|
||||
event_schema VARCHAR2(64) NOT NULL,
|
||||
applied_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
|
||||
CONSTRAINT pk_stats_projection_applied PRIMARY KEY(event_id)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_stats_projection_applied_time
|
||||
ON stats_projection_applied_event(applied_at);
|
||||
|
||||
COMMENT ON TABLE stats_projection_outbox IS
|
||||
'事实批次统计 Outbox;Kafka 发布或消费失败时可持续恢复';
|
||||
COMMENT ON COLUMN stats_projection_outbox.status IS
|
||||
'PENDING/FAILED/PUBLISHED/APPLIED;只有 APPLIED 才允许到期清理';
|
||||
COMMENT ON TABLE stats_projection_applied_event IS
|
||||
'统计投影消费幂等标记;保留时间必须长于 STATS.PROJECTION Topic';
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,44 @@
|
||||
-- v3 达梦升级验证:只读。
|
||||
|
||||
-- 应返回 2 行。
|
||||
SELECT table_name
|
||||
FROM user_tables
|
||||
WHERE table_name IN (
|
||||
'STATS_PROJECTION_OUTBOX',
|
||||
'STATS_PROJECTION_APPLIED_EVENT'
|
||||
)
|
||||
ORDER BY table_name;
|
||||
|
||||
-- 核对关键字段;OUTBOX 应包含 11 列,APPLIED_EVENT 应包含 3 列。
|
||||
SELECT table_name, column_id, column_name, data_type, nullable
|
||||
FROM user_tab_columns
|
||||
WHERE table_name IN (
|
||||
'STATS_PROJECTION_OUTBOX',
|
||||
'STATS_PROJECTION_APPLIED_EVENT'
|
||||
)
|
||||
ORDER BY table_name, column_id;
|
||||
|
||||
-- 三个约束都应为 ENABLED。
|
||||
SELECT table_name, constraint_name, constraint_type, status
|
||||
FROM user_constraints
|
||||
WHERE constraint_name IN (
|
||||
'PK_STATS_PROJECTION_OUTBOX',
|
||||
'CK_STATS_PROJECTION_OUTBOX_STATUS',
|
||||
'PK_STATS_PROJECTION_APPLIED'
|
||||
)
|
||||
ORDER BY constraint_name;
|
||||
|
||||
-- 两个索引都应为 VALID。
|
||||
SELECT table_name, index_name, uniqueness, status
|
||||
FROM user_indexes
|
||||
WHERE index_name IN (
|
||||
'IDX_STATS_PROJECTION_OUTBOX_DUE',
|
||||
'IDX_STATS_PROJECTION_APPLIED_TIME'
|
||||
)
|
||||
ORDER BY index_name;
|
||||
|
||||
-- 刚建表时应返回 0、0;应用启动后数字会正常增长。
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM stats_projection_outbox) AS outbox_rows,
|
||||
(SELECT COUNT(*) FROM stats_projection_applied_event) AS applied_rows
|
||||
FROM dual;
|
||||
Reference in New Issue
Block a user