关联分析规则-数据降噪

2026-03-18 18:00:25 +08:00
parent cf6b89ea94
commit c0063a5a44
64 changed files with 6642 additions and 2007 deletions
@@ -33,8 +33,9 @@ public class ETLOrchestrator {
    private NormalizeRuleHitTimeService normalizeRuleHitTimeService;
    /**
     * 定时任务 - 从每小时第1分钟开始，5分钟间隔执行
+     * 20260317：暂定硬规则关联分析
     */
-    @Scheduled(cron = "0 1/5 * * * ?")
+    //@Scheduled(cron = "0 1/5 * * * ?")
    public void scheduledETL() {

        long startTime = System.currentTimeMillis();
@@ -46,7 +47,7 @@ public class ETLOrchestrator {
        //泛化标准数据告警降噪任务
        try {
            //retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcess24HoursGroupedData());
-            retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcessQueryHoursGroupedData(strStartTime,strEndTime ));
+            //retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcessQueryHoursGroupedData(strStartTime,strEndTime ));
            retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcessQueryHoursAlarm(strStartTime,strEndTime ));
            long endTime = System.currentTimeMillis();
            long duration = (endTime - startTime) / 1000;
@@ -0,0 +1,62 @@
+package com.common.schedule;
+
+import com.common.service.AnalysisRuleService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 离线分析定时任务
+ */
+@Slf4j
+@Component
+public class OfflineAnalysisScheduler {
+
+    @Autowired
+    private AnalysisRuleService analysisRuleService;
+
+    @Value("${analysis.offline.enabled:true}")
+    private boolean offlineEnabled;
+
+    /**
+     * 定时执行离线分析（使用cron表达式，默认每小时执行一次）
+     * 具体分析规则运行需要根据配置运行时间周期进行，离线暂停
+     */
+   // @Scheduled(cron = "${analysis.offline.cron-expression:0 0 */1 * * ?}")
+    public void executeOfflineAnalysis() {
+        if (!offlineEnabled) {
+            log.debug("离线分析引擎已禁用，跳过执行");
+            return;
+        }
+
+        try {
+            log.info("========== 开始执行离线分析任务 ==========");
+            long startTime = System.currentTimeMillis();
+
+            List<Map<String, Object>> results = analysisRuleService.executeOfflineAnalysis();
+
+            long endTime = System.currentTimeMillis();
+            long duration = endTime - startTime;
+
+            log.info("========== 离线分析任务完成，耗时: {} ms，处理规则数: {} ==========",
+                    duration, results.size());
+
+            // 输出执行结果摘要
+            for (Map<String, Object> result : results) {
+                log.info("规则: {}, 状态: {}, 处理记录数: {}, 生成告警数: {}",
+                        result.get("ruleName"),
+                        result.get("status"),
+                        result.get("processedCount"),
+                        result.get("alarmCount"));
+            }
+
+        } catch (Exception e) {
+            log.error("执行离线分析任务失败", e);
+        }
+    }
+}
@@ -0,0 +1,196 @@
+package com.common.schedule;
+
+import com.common.entity.AnalysisAnalysisRule;
+import com.common.entity.AnalysisGroupBy;
+import com.common.entity.AnalysisGroupByWindow;
+import com.common.mapper.AnalysisAnalysisRuleMapper;
+import com.common.mapper.AnalysisGroupByMapper;
+import com.common.mapper.AnalysisGroupByWindowMapper;
+import com.common.service.AnalysisRuleService;
+import com.common.service.RuleExecutionTimeService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 实时分析定时任务调度器
+ * 每个规则根据窗口类型（滚动、滑动、会话）独立管理下次运行时间
+ */
+@Slf4j
+@Component
+public class RealtimeAnalysisScheduler {
+
+    @Autowired
+    private AnalysisRuleService analysisRuleService;
+
+    @Autowired
+    private RuleExecutionTimeService ruleExecutionTimeService;
+
+    @Autowired
+    private AnalysisAnalysisRuleMapper ruleMapper;
+
+    @Autowired
+    private AnalysisGroupByMapper groupByMapper;
+
+    @Autowired
+    private AnalysisGroupByWindowMapper groupByWindowMapper;
+
+    @Value("${analysis.realtime.enabled:true}")
+    private boolean realtimeEnabled;
+
+    @Value("${analysis.realtime.check-interval-seconds:10}")
+    private int checkIntervalSeconds;
+
+    /**
+     * 应用启动时初始化所有规则的执行时间
+     */
+    @PostConstruct
+    public void init() {
+        if (!realtimeEnabled) {
+            log.info("实时分析引擎已禁用，跳过初始化");
+            return;
+        }
+
+        log.info("========== 初始化实时分析调度器 ==========");
+        try {
+            initAllRules();
+            log.info("========== 实时分析调度器初始化完成 ==========");
+        } catch (Exception e) {
+            log.error("初始化实时分析调度器失败", e);
+        }
+    }
+
+    /**
+     * 定时检查规则是否需要执行（默认每10秒检查一次）
+     * 根据窗口类型（滚动、滑动、会话）独立计算下次执行时间
+     */
+    @Scheduled(fixedDelayString = "${analysis.realtime.check-interval-seconds:10}000")
+    public void checkAndExecuteRules() {
+        if (!realtimeEnabled) {
+            return;
+        }
+
+        try {
+            LocalDateTime now = LocalDateTime.now();
+
+            // 查询所有启用的实时规则
+            List<AnalysisAnalysisRule> rules = ruleMapper.selectActiveRulesByRunMode("realtime");
+
+            if (rules.isEmpty()) {
+                return;
+            }
+
+            int executedCount = 0;
+            int skippedCount = 0;
+
+            for (AnalysisAnalysisRule rule : rules) {
+                try {
+                    // 获取规则下次执行时间
+                    LocalDateTime nextTime = ruleExecutionTimeService.getNextExecuteTime(rule.getRuleId());
+
+                    // 如果未初始化或已到执行时间
+                    if (nextTime == null || !nextTime.isAfter(now)) {
+                        log.info("执行规则: ruleId={}, ruleName={}, nextTime={}, now={}",
+                                rule.getRuleId(), rule.getRuleName(),
+                                nextTime, now);
+
+                        // 执行规则
+                        Map<String, Object> result = analysisRuleService.executeRealtimeRule(rule.getRuleId());
+
+                        // 计算下次执行时间
+                        updateNextExecuteTime(rule);
+                        executedCount++;
+                    } else {
+                        skippedCount++;
+                        log.debug("规则未到执行时间: ruleId={}, nextTime={}, now={}, diff={}",
+                                rule.getRuleId(), nextTime, now,
+                                java.time.Duration.between(now, nextTime).getSeconds());
+                    }
+                } catch (Exception e) {
+                    log.error("检查和执行规则失败: ruleId={}", rule.getRuleId(), e);
+                }
+            }
+
+            if (executedCount > 0) {
+                log.info("本次调度执行规则数: {}, 跳过规则数: {}", executedCount, skippedCount);
+            }
+
+        } catch (Exception e) {
+            log.error("检查和执行规则失败", e);
+        }
+    }
+
+    /**
+     * 初始化所有规则的执行时间
+     */
+    private void initAllRules() {
+        List<AnalysisAnalysisRule> rules = ruleMapper.selectActiveRulesByRunMode("realtime");
+        log.info("查询到 {} 个实时分析规则", rules.size());
+
+        for (AnalysisAnalysisRule rule : rules) {
+            try {
+                // 加载窗口配置
+                AnalysisGroupByWindow groupByWindow = loadGroupByWindow(rule.getRuleId());
+
+                // 初始化下次执行时间
+                ruleExecutionTimeService.initRuleExecuteTime(rule, groupByWindow);
+
+                log.info("初始化规则: ruleId={}, ruleName={}, windowType={}",
+                        rule.getRuleId(), rule.getRuleName(),
+                        groupByWindow != null ? groupByWindow.getWindowType() : "NONE");
+            } catch (Exception e) {
+                log.error("初始化规则执行时间失败: ruleId={}", rule.getRuleId(), e);
+            }
+        }
+    }
+
+    /**
+     * 加载规则的窗口配置
+     */
+    private AnalysisGroupByWindow loadGroupByWindow(String ruleId) {
+        try {
+            List<AnalysisGroupBy> groupByList = groupByMapper.selectByRuleId(ruleId);
+            if (groupByList != null && !groupByList.isEmpty()) {
+                AnalysisGroupBy groupBy = groupByList.get(0);
+                if (groupBy.getId() != null) {
+                    return groupByWindowMapper.selectByGroupById(groupBy.getId().intValue());
+                }
+            }
+            return null;
+        } catch (Exception e) {
+            log.error("加载窗口配置失败: ruleId={}", ruleId, e);
+            return null;
+        }
+    }
+
+    /**
+     * 更新规则下次执行时间
+     */
+    private void updateNextExecuteTime(AnalysisAnalysisRule rule) {
+        try {
+            // 加载窗口配置
+            AnalysisGroupByWindow groupByWindow = loadGroupByWindow(rule.getRuleId());
+
+            // 更新下次执行时间
+            ruleExecutionTimeService.updateNextExecuteTime(rule, groupByWindow);
+        } catch (Exception e) {
+            log.error("更新规则下次执行时间失败: ruleId={}", rule.getRuleId(), e);
+        }
+    }
+
+    /**
+     * 应用关闭时清理
+     */
+    @PreDestroy
+    public void destroy() {
+        log.info("========== 关闭实时分析调度器 ==========");
+    }
+}