feat: 新增设备 API 拉取采集及配套处理能力

- 接入 NDRServer 告警和流量日志，支持凭据与任务管理 - 支持分页拉取、窗口拆分、调度优先级及 WAL 持久化后推进断点 - 增加迟到告警规则重放，优化多条泛化规则选择和事件时间处理 - 补充回归测试、数据库升级脚本及部署文档
2026-09-06 16:46:21 +08:00
parent b61ec2e37c
commit 00c0a1fb7f
141 changed files with 11318 additions and 467 deletions
@@ -0,0 +1,142 @@
+-- DIRECT 统计初始化（仅执行一次）。
+-- 执行前：停止 syslog-consumer-rule、syslog-consumer，完成 01、02；
+-- 将 BUSINESS_DATE 替换为升级日（如 2026-08-24），YYYYMMDD 替换为 20260824。
+-- 任一语句报错立即停止并 ROLLBACK；全部成功后再 COMMIT。
+
+ROLLBACK;
+SET TRANSACTION ISOLATION LEVEL SERIALIZABLE;
+
+-- 1. 初始化当天设备统计。
+INSERT INTO device_daily_stats(
+  stat_date,device_id,parse_count,non_log_count,
+  last_receive_time,projection_seq,updated_at
+)
+SELECT TO_DATE('BUSINESS_DATE','YYYY-MM-DD'),device_id,
+       SUM(parse_count),SUM(non_log_count),MAX(last_receive_time),0,SYSDATE
+FROM (
+  SELECT device_id,COUNT(*) parse_count,0 non_log_count,
+         CAST(NULL AS TIMESTAMP) last_receive_time
+  FROM syslog_normal_data_p_YYYYMMDD
+  WHERE created_at>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+    AND created_at<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
+    AND device_id IS NOT NULL
+  GROUP BY device_id
+
+  UNION ALL
+
+  SELECT device_id,COUNT(*),0,CAST(NULL AS TIMESTAMP)
+  FROM syslog_normal_alarm_p_YYYYMMDD
+  WHERE created_at>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+    AND created_at<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
+    AND device_id IS NOT NULL
+  GROUP BY device_id
+
+  UNION ALL
+
+  SELECT device_id,0,COUNT(*),CAST(NULL AS TIMESTAMP)
+  FROM syslog_non_normal_message_p_YYYYMMDD
+  WHERE created_at>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+    AND created_at<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
+    AND device_id IS NOT NULL
+  GROUP BY device_id
+
+  UNION ALL
+
+  SELECT id,0,0,last_receive_time
+  FROM device_device
+  WHERE id IS NOT NULL
+    AND last_receive_time>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+    AND last_receive_time<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
+) facts
+GROUP BY device_id;
+
+-- 2. 初始化当天规则统计。
+INSERT INTO normalize_rule_daily_stats(
+  stat_date,normalize_rule_id,max_log_time,projection_seq,updated_at
+)
+SELECT TO_DATE('BUSINESS_DATE','YYYY-MM-DD'),normalize_rule_id,
+       MAX(log_time),0,SYSDATE
+FROM (
+  SELECT normalize_rule_id,log_time
+  FROM syslog_normal_data_p_YYYYMMDD
+  WHERE created_at>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+    AND created_at<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
+    AND normalize_rule_id IS NOT NULL
+
+  UNION ALL
+
+  SELECT normalize_rule_id,log_time
+  FROM syslog_normal_alarm_p_YYYYMMDD
+  WHERE created_at>=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+    AND created_at<TO_DATE('BUSINESS_DATE','YYYY-MM-DD')+1
+    AND normalize_rule_id IS NOT NULL
+) facts
+GROUP BY normalize_rule_id;
+
+-- 3. 初始化采集任务统计。
+INSERT INTO device_collect_stats(
+  device_collect_id,first_success_time,last_success_time,
+  last_fail_time,projection_seq,updated_at
+)
+SELECT id,first_time,last_success_time,last_failed_time,0,SYSDATE
+FROM device_collect_task;
+
+-- 4. 回填设备今日计数。
+UPDATE device_device
+SET today_parse_count=0,
+    today_non_log_count=0,
+    today_stats_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD'),
+    updated_at=SYSDATE;
+
+UPDATE device_device d
+SET today_parse_count=(
+      SELECT s.parse_count
+      FROM device_daily_stats s
+      WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+        AND s.device_id=d.id
+    ),
+    today_non_log_count=(
+      SELECT s.non_log_count
+      FROM device_daily_stats s
+      WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+        AND s.device_id=d.id
+    ),
+    updated_at=SYSDATE
+WHERE EXISTS (
+  SELECT 1
+  FROM device_daily_stats s
+  WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+    AND s.device_id=d.id
+);
+
+-- 5. 回填规则最后命中时间，只允许时间向前推进。
+UPDATE dm_normalize_rule r
+SET first_data_saved_at=CASE
+      WHEN first_data_saved_at IS NULL OR first_data_saved_at<(
+        SELECT s.max_log_time
+        FROM normalize_rule_daily_stats s
+        WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+          AND s.normalize_rule_id=r.id
+      ) THEN (
+        SELECT s.max_log_time
+        FROM normalize_rule_daily_stats s
+        WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+          AND s.normalize_rule_id=r.id
+      )
+      ELSE first_data_saved_at
+    END,
+    updated_at=SYSDATE
+WHERE r.del_flag='0'
+  AND EXISTS (
+    SELECT 1
+    FROM normalize_rule_daily_stats s
+    WHERE s.stat_date=TO_DATE('BUSINESS_DATE','YYYY-MM-DD')
+      AND s.normalize_rule_id=r.id
+      AND s.max_log_time IS NOT NULL
+  );
+
+UPDATE stats_direct_control
+SET rebuild_required=0,updated_at=SYSDATE
+WHERE control_name='SYSLOG_STATS';
+
+COMMIT;