初次提交代码

2026-01-11 15:33:22 +08:00
commit 6603c6f4a1
455 changed files with 32175 additions and 0 deletions
@@ -0,0 +1,115 @@
+package com.common.schedule;
+
+
+import com.common.service.DataExtractor;
+import  com.Modules.etl.handler.ETLRetryHandler;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import com.Modules.etl.TimeWindowCalculator;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import com.common.service.DeviceReceiveLogService;
+import lombok.RequiredArgsConstructor;
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class ETLOrchestrator {
+
+    @Autowired
+    private DataExtractor dataExtractor;
+
+    @Autowired
+    private ETLRetryHandler retryHandler;
+
+    @Autowired
+    private  DeviceReceiveLogService deviceReceiveLogService;
+    /**
+     * 定时任务 - 从每小时第1分钟开始，5分钟间隔执行
+     */
+    @Scheduled(cron = "0 1/5 * * * ?")
+    public void scheduledETL() {
+        //log.info("开始定时ETL任务");
+        long startTime = System.currentTimeMillis();
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+        LocalDateTime[] currentWindow=TimeWindowCalculator.getPrevious5MinuteWindow();
+        String strStartTime= currentWindow[0].format(formatter);
+        String strEndTime= currentWindow[1].format(formatter);
+        log.info("ETL任务开始执行，开始时间：{},结束时间：{}",strStartTime,strEndTime );
+        try {
+            //retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcess24HoursGroupedData());
+            retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcessQueryHoursGroupedData(strStartTime,strEndTime ));
+            retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcessQueryHoursAlarm(strStartTime,strEndTime ));
+            long endTime = System.currentTimeMillis();
+            long duration = (endTime - startTime) / 1000;
+            log.info("定时ETL任务执行完成，耗时: {} 秒", duration);
+
+        } catch (Exception e) {
+            log.error("定时ETL任务执行失败", e);
+        }
+    }
+
+    /**
+     * 手动触发ETL任务
+     */
+    public void triggerETLManually() {
+        log.info("手动触发ETL任务");
+        try {
+            //retryHandler.executeWithRetry(() ->
+                    //dataExtractor.extractAndProcess24HoursGroupedData());
+            retryHandler.executeWithRetry(() ->
+                    dataExtractor.extractAndProcessQueryHoursGroupedData("2025-11-01 00:00:00","2025-11-15 00:00:00"));
+            log.info("手动ETL任务执行完成");
+        } catch (Exception e) {
+            log.error("手动ETL任务执行失败", e);
+            throw new RuntimeException("ETL任务执行失败", e);
+        }
+    }
+
+    /**
+     * 手动触发ETL任务,参数开始时间，结束时间
+     * @param startTime
+     * @param endTime
+     */
+    public void triggerETLManuallyByTime(String startTime ,String endTime) {
+        log.info("手动触发ETL任务");
+        try {
+            retryHandler.executeWithRetry(() ->
+                    dataExtractor.extractAndProcessQueryHoursGroupedData( startTime ,  endTime ));
+            retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcessQueryHoursAlarm(startTime,endTime ));
+            log.info("手动ETL任务执行完成");
+        } catch (Exception e) {
+            log.error("手动ETL任务执行失败", e);
+            throw new RuntimeException("ETL任务执行失败", e);
+        }
+    }
+
+    /**
+     * 每天凌晨3点清理2天前的数据
+     */
+    //@Scheduled(cron = "0 0 3 * * ?")
+    @Scheduled(cron = "0 * * * * ?")
+    public void cleanupOldLogs() {
+        try {
+            LocalDateTime cutoffTime = LocalDateTime.now().minusDays(2);
+            int deleted = deviceReceiveLogService.deleteOldLogs(cutoffTime);
+            log.info("定时清理任务完成，删除{}条2天前的日志", deleted);
+        } catch (Exception e) {
+            log.error("定时清理日志失败", e);
+        }
+    }
+
+
+    public static void main(String[] args) {
+
+        LocalDateTime befor= LocalDateTime.of(2025, 11, 1, 0, 0  ,0); // 默认2025年11月
+        LocalDateTime cutoffTime = LocalDateTime.now().minusDays(2);
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+        System.out.println(befor.format(formatter).toString());
+        System.out.println(cutoffTime.format(formatter).toString());
+    }
+
+
+}
@@ -0,0 +1,122 @@
+package com.common.schedule;
+
+import com.common.service.PartitionTableService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import com.common.service.PartitionTableService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import java.util.List;
+@Component
+public class PartitionTableSchedule {
+
+    private static final Logger logger = LoggerFactory.getLogger(PartitionTableSchedule.class);
+
+    @Autowired
+    private PartitionTableService partitionTableService;
+
+    /**
+     * 每月15日凌晨1点执行
+     * cron表达式: 秒 分 时 日 月 周
+     */
+    @Scheduled(cron = "0 0 1 15 * ?")
+    public void createNextMonthPartitionTables() {
+        logger.info("开始执行创建下个月分区表任务...");
+
+        try {
+            int partitionCount = partitionTableService.getNextMonthPartitionCount();
+            logger.info("需要创建 {} 个分区表", partitionCount);
+
+            partitionTableService.createNextMonthPartitionTable();
+
+            logger.info("创建下个月分区表任务完成");
+        } catch (Exception e) {
+            logger.error("定时创建分区表任务失败: {}", e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 测试任务 - 每分钟执行一次（开发环境使用）
+     * 生产环境注释或删除此方法
+     */
+    //@Scheduled(cron = "0 * * * * ?")
+    public void testCreatePartitionTables() {
+        logger.info("测试任务: 开始创建分区表...");
+        partitionTableService.createNextMonthPartitionTable();
+        logger.info("测试任务: 分区表创建完成");
+    }
+
+    /**
+     * 每天检查一次分区表状态（可选）
+     */
+    @Scheduled(cron = "0 0 2 * * ?")
+    public void checkPartitionTableStatus() {
+        logger.info("开始检查分区表状态...");
+        // 这里可以添加分区表状态检查逻辑
+        logger.info("分区表状态检查完成");
+    }
+
+
+    /**
+     * 每天检查第二天的分区表状态 - 凌晨2点执行
+     */
+    @Scheduled(cron = "0 0 2 * * ?")
+    public void checkTomorrowPartitionTable() {
+        logger.info("开始检查第二天的分区表状态...");
+
+        try {
+            PartitionTableService.PartitionTableStatus status =
+                    partitionTableService.checkTomorrowPartitionTable();
+
+            if (status.isExists()) {
+                logger.info("第二天分区表已存在: {}", status.getTableName());
+            } else if (status.isCreated()) {
+                logger.info("第二天分区表创建成功: {}", status.getTableName());
+            } else {
+                logger.warn("第二天分区表检查异常: {}", status.getMessage());
+                // 这里添加告警通知，比如发送邮件、短信等
+                //sendAlertNotification(status);
+            }
+
+        } catch (Exception e) {
+            logger.error("检查第二天分区表失败: {}", e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 每周一检查未来7天的分区表状态
+     */
+    @Scheduled(cron = "0 0 3 * * MON")
+    public void checkNextWeekPartitionTables() {
+        logger.info("开始检查未来7天的分区表状态...");
+
+        try {
+            List<PartitionTableService.PartitionTableStatus> statusList = partitionTableService.checkFuturePartitionTables(7);
+
+            int missingCount = 0;
+            for (PartitionTableService.PartitionTableStatus status : statusList) {
+                if (!status.isExists()) {
+                    missingCount++;
+                    logger.warn("未来分区表缺失: {}", status.getTableName());
+                }
+            }
+
+            if (missingCount > 0) {
+                logger.warn("发现 {} 个未来分区表缺失", missingCount);
+                // 可以触发自动创建或发送告警
+            } else {
+                logger.info("未来7天分区表状态正常");
+            }
+
+        } catch (Exception e) {
+            logger.error("检查未来分区表失败: {}", e.getMessage(), e);
+        }
+    }
+
+}
@@ -0,0 +1,58 @@
+package com.common.schedule;
+
+
+
+
+import com.common.service.EsToDbSyncService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+
+//众诚运维日志数据计划任务
+@Slf4j
+//@Component
+public class ScheduledTask {
+
+    @Autowired
+    private EsToDbSyncService esToDbSyncService;
+
+    private static final DateTimeFormatter INDEX_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
+
+    /**
+     * 每天1点同步昨天的数据
+     */
+   //c
+    public void syncYesterdayData() {
+        log.info("开始执行昨天数据同步任务");
+        LocalDate yesterday = LocalDate.now().minusDays(1);
+        String indexName = "applog-" + yesterday.format(INDEX_FORMATTER);
+        esToDbSyncService.syncEsLogsByIndexToDatabase(indexName);
+        log.info("昨天数据同步任务完成");
+    }
+
+    /**
+     * 每天1,5,9,13,17,21点同步今天的数据
+     */
+   // @Scheduled(cron = "0 0 1,5,9,13,17,21 * * ?")
+    public void syncTodayData() {
+        log.info("开始执行今天数据同步任务");
+        LocalDate today = LocalDate.now();
+        String indexName = "applog-" + today.format(INDEX_FORMATTER);
+        esToDbSyncService.syncEsLogsByIndexToDatabase(indexName);
+        log.info("今天数据同步任务完成");
+    }
+
+    /**
+     * 手动同步指定日期的数据（可用于测试）
+     */
+    public void syncSpecificDateData(LocalDate date) {
+        log.info("开始执行指定日期数据同步任务: {}", date);
+        String indexName = "applog-" + date.format(INDEX_FORMATTER);
+        esToDbSyncService.syncEsLogsByIndexToDatabase(indexName);
+        log.info("指定日期数据同步任务完成: {}", date);
+    }
+}