feat(syslog): 增加积压采样免采样窗口

- 新增 start-lag-ms 配置，默认15分钟内全量保留 - 在15分钟至1小时之间按10级逐步降低保留率 - 延迟达到1小时后使用普通10%、非标20%的最低保留率 - 校验起始延迟必须小于最低保留率阈值 - 状态接口新增 start_lag_ms 配置展示 - 将采样策略版本升级为 v2 - 更新 Docker Compose 和 v4 升级配置说明 - 补充采样时间边界及非法配置单元测试
2026-09-12 23:43:56 +08:00
parent c868e3befd
commit 87fee6fe69
27 changed files with 1777 additions and 94 deletions
@@ -0,0 +1,21 @@
+-- v4 达梦升级预检：只读，不修改任何对象。
+-- 必须使用 syslog-consumer-rule 实际连接的达梦用户执行。
+
+-- 首次升级应返回 0 行；如果返回 1 行，不要重复执行 02，直接执行 03 验证。
+SELECT table_name
+FROM user_tables
+WHERE table_name = 'SYSLOG_BACKLOG_RETENTION_HOURLY';
+
+-- 首次升级应返回 0 行。
+SELECT constraint_name, constraint_type, status, table_name
+FROM user_constraints
+WHERE constraint_name IN (
+  'PK_SYSLOG_BACKLOG_RET_HOURLY',
+  'CK_SYSLOG_BACKLOG_RET_BPS'
+)
+ORDER BY constraint_name;
+
+-- 首次升级应返回 0 行。
+SELECT index_name, table_name, uniqueness, status
+FROM user_indexes
+WHERE index_name = 'IDX_SYSLOG_BACKLOG_RET_UPDATED';
@@ -0,0 +1,49 @@
+-- v4 达梦升级：创建 Kafka 积压分级采样的小时审计表。
+-- 仅在 01 预检确认目标对象均不存在时执行一次，不要重复执行。
+
+CREATE TABLE syslog_backlog_retention_hourly (
+  source_hour TIMESTAMP NOT NULL,
+  source_topic VARCHAR2(255) NOT NULL,
+  kafka_partition INT NOT NULL,
+  policy_version VARCHAR2(64) NOT NULL,
+  normal_retain_bps INT NOT NULL,
+  non_normal_retain_bps INT NOT NULL,
+  message_total BIGINT DEFAULT 0 NOT NULL,
+  alarm_rows_total BIGINT DEFAULT 0 NOT NULL,
+  alarm_rows_saved BIGINT DEFAULT 0 NOT NULL,
+  normal_rows_total BIGINT DEFAULT 0 NOT NULL,
+  normal_rows_saved BIGINT DEFAULT 0 NOT NULL,
+  normal_rows_dropped BIGINT DEFAULT 0 NOT NULL,
+  non_normal_rows_total BIGINT DEFAULT 0 NOT NULL,
+  non_normal_rows_saved BIGINT DEFAULT 0 NOT NULL,
+  non_normal_rows_dropped BIGINT DEFAULT 0 NOT NULL,
+  raw_total BIGINT DEFAULT 0 NOT NULL,
+  raw_saved BIGINT DEFAULT 0 NOT NULL,
+  raw_dropped BIGINT DEFAULT 0 NOT NULL,
+  min_lag_ms BIGINT DEFAULT 0 NOT NULL,
+  max_lag_ms BIGINT DEFAULT 0 NOT NULL,
+  created_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
+  updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
+  CONSTRAINT pk_syslog_backlog_ret_hourly PRIMARY KEY (
+    source_hour, source_topic, kafka_partition, policy_version,
+    normal_retain_bps, non_normal_retain_bps
+  ),
+  CONSTRAINT ck_syslog_backlog_ret_bps CHECK (
+    normal_retain_bps BETWEEN 0 AND 10000
+    AND non_normal_retain_bps BETWEEN 0 AND 10000
+  )
+);
+
+CREATE INDEX idx_syslog_backlog_ret_updated
+ON syslog_backlog_retention_hourly(updated_at);
+
+COMMENT ON TABLE syslog_backlog_retention_hourly IS
+'Kafka积压分级采样小时审计；按来源小时、Topic、Partition和策略聚合';
+COMMENT ON COLUMN syslog_backlog_retention_hourly.source_hour IS
+'syslog-serve生成的receive_time所在小时（Asia/Shanghai）';
+COMMENT ON COLUMN syslog_backlog_retention_hourly.normal_retain_bps IS
+'普通标准化记录该级实际保留万分比，10000表示100%';
+COMMENT ON COLUMN syslog_backlog_retention_hourly.non_normal_retain_bps IS
+'非标记录该级实际保留万分比，10000表示100%';
+
+COMMIT;
@@ -0,0 +1,56 @@
+-- v4 Kafka 积压分级采样升级验证：只读。
+
+-- 应返回 SYSLOG_BACKLOG_RETENTION_HOURLY 一行。
+SELECT table_name
+FROM user_tables
+WHERE table_name = 'SYSLOG_BACKLOG_RETENTION_HOURLY';
+
+-- 应返回 22 个字段。
+SELECT column_id, column_name, data_type, nullable
+FROM user_tab_columns
+WHERE table_name = 'SYSLOG_BACKLOG_RETENTION_HOURLY'
+ORDER BY column_id;
+
+-- 两个约束都应为 ENABLED。
+SELECT constraint_name, constraint_type, status
+FROM user_constraints
+WHERE constraint_name IN (
+  'PK_SYSLOG_BACKLOG_RET_HOURLY',
+  'CK_SYSLOG_BACKLOG_RET_BPS'
+)
+ORDER BY constraint_name;
+
+-- 索引应为 VALID。
+SELECT index_name, uniqueness, status
+FROM user_indexes
+WHERE index_name = 'IDX_SYSLOG_BACKLOG_RET_UPDATED';
+
+-- 结构验证后应能执行；首次建表且还未发生积压采样时返回 0。
+SELECT COUNT(*) AS audit_row_count
+FROM syslog_backlog_retention_hourly;
+
+-- 查看每个来源小时的实际选择率。该值反映采样选择，不代表后续存储一定写入成功。
+-- 没有发生积压采样时，本查询返回空属于正常。
+SELECT
+  source_hour,
+  SUM(message_total) AS message_total,
+  SUM(alarm_rows_total) AS alarm_rows_total,
+  SUM(alarm_rows_saved) AS alarm_rows_saved,
+  SUM(normal_rows_total) AS normal_rows_total,
+  SUM(normal_rows_saved) AS normal_rows_saved,
+  SUM(normal_rows_dropped) AS normal_rows_dropped,
+  CASE WHEN SUM(normal_rows_total) = 0 THEN 1
+       ELSE SUM(normal_rows_saved) * 1.0 / SUM(normal_rows_total) END AS normal_actual_rate,
+  SUM(non_normal_rows_total) AS non_normal_rows_total,
+  SUM(non_normal_rows_saved) AS non_normal_rows_saved,
+  SUM(non_normal_rows_dropped) AS non_normal_rows_dropped,
+  CASE WHEN SUM(non_normal_rows_total) = 0 THEN 1
+       ELSE SUM(non_normal_rows_saved) * 1.0 / SUM(non_normal_rows_total) END AS non_normal_actual_rate,
+  SUM(raw_total) AS raw_total,
+  SUM(raw_saved) AS raw_saved,
+  SUM(raw_dropped) AS raw_dropped,
+  MIN(min_lag_ms) AS min_lag_ms,
+  MAX(max_lag_ms) AS max_lag_ms
+FROM syslog_backlog_retention_hourly
+GROUP BY source_hour
+ORDER BY source_hour DESC;