From 5e73c1c8f6e0816a3931a9caa5569ba53b0d5278 Mon Sep 17 00:00:00 2001 From: nanChen Date: Wed, 6 May 2026 17:30:21 +0800 Subject: [PATCH] =?UTF-8?q?1=E3=80=81=E5=AE=8C=E5=96=84kafka=20=E6=8E=A5?= =?UTF-8?q?=E6=94=B6=E6=B6=88=E6=81=AF=E8=BF=9B=E8=A1=8Csm4=20=E8=A7=A3?= =?UTF-8?q?=E5=AF=86=202=E3=80=81=E6=96=B0=E5=A2=9EIP=E8=81=94=E5=8A=A8?= =?UTF-8?q?=E5=B0=81=E7=A6=81=E7=9B=B8=E5=85=B3=E7=9A=84API=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=8C=E4=BE=9B=E6=8E=A2=E9=92=88=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=E8=BF=9B=E8=A1=8C=E8=B0=83=E7=94=A8=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../NormalData/LogNormalProcessor.java | 16 +++-- .../Modules/NormalData/SysLogProcessor.java | 20 ++++-- .../com/common/schedule/ETLOrchestrator.java | 38 +++++++---- .../schedule/PartitionTableSchedule.java | 10 +-- .../service/DeviceStatsUpdateService.java | 4 +- .../src/main/java/com/config/AppConfig.java | 4 +- .../SyslogNonNormalMessageController.java | 3 +- .../main/resources/application-dev.properties | 24 ++++++- .../resources/application-prod-zc.properties | 12 +++- .../resources/application-prod.properties | 21 +++++- .../src/main/resources/application.properties | 21 +++++- .../mapper/AnalysisAnalysisRuleMapper.xml | 1 - .../mapper/SyslogNormalAlarmMapper.xml | 68 +++++++++---------- .../mapper/SyslogNormalDataMapper.xml | 66 +++++++++--------- 14 files changed, 197 insertions(+), 111 deletions(-) diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java index cd0f502..f00d35e 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java @@ -79,9 +79,9 @@ public class LogNormalProcessor { @Autowired DmNormalizeRuleMapper dmNormalizeRuleMapper; - private static List> dmNormalizeRuleList; - private static List> dmColumnList; - private static LinkedHashMap OrginalColumnMap ; + private List> dmNormalizeRuleList; + private List> dmColumnList; + private LinkedHashMap OrginalColumnMap ; public LogNormalProcessor( String LogMsg, String syslogUUID,String syslogTopic) { @@ -489,7 +489,15 @@ public class LogNormalProcessor { { Map columnMap= new HashMap<>(); for (Map map : normalColumnList) { - columnMap.put(map.get("dest_field").toString(),map.get("dest_field_value")); + + Object destFieldValue = map.get("dest_field_value"); + // 判断 dest_field_value 是否为 String 且包含 "\u0000" + if (destFieldValue instanceof String && ((String) destFieldValue).contains("\u0000")) { + // 替换掉所有 "\u0000" 字符 + destFieldValue = ((String) destFieldValue).replace("\u0000", ""); + } + columnMap.put(map.get("dest_field").toString(), destFieldValue); + //columnMap.put(map.get("dest_field").toString(),map.get("dest_field_value")); } return columnMap; } diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java index f98bdbf..4337fe7 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java @@ -26,6 +26,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.time.LocalDate; import java.time.LocalDate; import java.time.format.DateTimeFormatter; + +import com.common.util.Sm4Util; +import com.config.AppConfig; @Slf4j @Component public class SysLogProcessor { @@ -40,10 +43,14 @@ public class SysLogProcessor { @Value("${app.processor.process-timeout-ms:30000}") private long processTimeoutMs; + private static String strhexKey=AppConfig.getSM4Key(); private final AtomicInteger totalProcessed = new AtomicInteger(0); private final AtomicInteger currentBatchCount = new AtomicInteger(0); // 初始化 InfluxDB 客户端 private final com.influx.InfluxDBClient influxClient = new InfluxDBClient(); + + + /** * 方案一：直接多线程并行处理（推荐） * 单线程消费，每条消息独立提交给线程池处理 @@ -80,7 +87,7 @@ public class SysLogProcessor { CompletableFuture future = CompletableFuture.runAsync(() -> { try { // 异步处理单条消息 - log.info("收到syslogmessage："+ record.value()); + log.info("收到syslogmessage："+ Sm4Util.decryptCbc(record.value(), strhexKey)); processSingleMessageAsync(record); } catch (Exception e) { log.error("处理消息失败, topic: {}, partition: {}, offset: {}", @@ -251,8 +258,13 @@ public class SysLogProcessor { // 模拟业务处理 //processBusinessLogic(message); + //Message进行SM4解密 + String Sm4message=Sm4Util.decryptCbc(record.value(), strhexKey); + System.out.println("Sm4message:"+Sm4message); + + String sysLogUUID =getSysLogUUID(); - String strDeviceInfo= SyslogParser.substringBeforeFirstChar(record.value(),']'); + String strDeviceInfo= SyslogParser.substringBeforeFirstChar(Sm4message,']'); Map mapdev =SyslogParser.parseKeyValuePairs(strDeviceInfo); // 初始化 InfluxDB 客户端 @@ -261,7 +273,7 @@ public class SysLogProcessor { .addTag("device_collect_id", mapdev.get("device_collect_id")) // 添加探针ID标签 .addTag("uuid", sysLogUUID) //syslog uuid .addTag("topic", AppConfig.getTopic()) //kafka topic - .addField("message", record.value()) // 添加字段 + .addField("message", Sm4message) // 添加字段 .addField("receive_time", mapdev.get("receive_time")) // 添加字段 .addField("uuid", sysLogUUID) .time(System.currentTimeMillis(), WritePrecision.MS) ;// 毫秒级时间戳 @@ -272,7 +284,7 @@ public class SysLogProcessor { //insertSingleRecord( record.value()); //String syslogMessage= AppConfig.geRunEnvironment‌().equals("test")? record.value().substring(34) : record.value(); - String syslogMessage= record.value(); + String syslogMessage= Sm4message; //剔除测试环境本机syslog新增的头部信息 LogNormalProcessor logNormalProcessor = new LogNormalProcessor(syslogMessage,sysLogUUID,AppConfig.getTopic()); //LogNormalProcessor logNormalProcessor =new LogNormalProcessor(record.value()); diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/schedule/ETLOrchestrator.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/schedule/ETLOrchestrator.java index 460f172..cd7a23c 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/schedule/ETLOrchestrator.java +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/schedule/ETLOrchestrator.java @@ -33,11 +33,27 @@ public class ETLOrchestrator { private NormalizeRuleHitTimeService normalizeRuleHitTimeService; /** * 定时任务 - 从每小时第1分钟开始，5分钟间隔执行 - * 20260317：暂定硬规则关联分析 + * 20260317：暂停硬规则关联分析，由可配置关联分析规则取代 + * 泛化规则最新命中时间更新任务保留 */ - //@Scheduled(cron = "0 1/5 * * * ?") + @Scheduled(cron = "0 1/5 * * * ?") public void scheduledETL() { + //暂停ETL数据降噪任务(关联分析） + //RunETL(); + + //泛化规则最新命中时间更新任务 + try { + normalizeRuleHitTimeService.updateRuleHitTimeTask(); + } catch (Exception e) { + log.error("泛化规则最新命中时间更新任务执行失败", e); + } + + } + + //ETL数据降噪任务处理 + private void RunETL() + { long startTime = System.currentTimeMillis(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); LocalDateTime[] currentWindow=TimeWindowCalculator.getPrevious5MinuteWindow(); @@ -48,7 +64,7 @@ public class ETLOrchestrator { try { //retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcess24HoursGroupedData()); //retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcessQueryHoursGroupedData(strStartTime,strEndTime )); - retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcessQueryHoursAlarm(strStartTime,strEndTime )); + //retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcessQueryHoursAlarm(strStartTime,strEndTime )); long endTime = System.currentTimeMillis(); long duration = (endTime - startTime) / 1000; log.info("定时ETL任务执行完成，耗时: {} 秒", duration); @@ -56,14 +72,6 @@ public class ETLOrchestrator { } catch (Exception e) { log.error("定时ETL任务执行失败", e); } - - //泛化规则最新命中时间更新任务 - try { - normalizeRuleHitTimeService.updateRuleHitTimeTask(); - } catch (Exception e) { - log.error("泛化规则最新命中时间更新任务执行失败", e); - } - } /** @@ -105,12 +113,12 @@ public class ETLOrchestrator { * 每天凌晨3点清理2天前的数据 */ @Scheduled(cron = "0 0 3 * * ?") - //@Scheduled(cron = "0 * * * * ?") - public void cleanupOldLogs() { + public void cleanupOldLogs() { try { - LocalDateTime cutoffTime = LocalDateTime.now().minusDays(2); + //默认删除7天内接收日志记录 + LocalDateTime cutoffTime = LocalDateTime.now().minusDays(7); int deleted = deviceReceiveLogService.deleteOldLogs(cutoffTime); - log.info("定时清理任务完成，删除{}条2天前的日志", deleted); + log.info("定时清理任务完成，删除{}条7天前的日志", deleted); } catch (Exception e) { log.error("定时清理日志失败", e); } diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java index 709b1b7..f048f14 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java @@ -52,15 +52,7 @@ public class PartitionTableSchedule { logger.info("测试任务: 分区表创建完成"); } - /** - * 每天检查一次分区表状态（可选） - */ - @Scheduled(cron = "0 0 2 * * ?") - public void checkPartitionTableStatus() { - logger.info("开始检查分区表状态..."); - // 这里可以添加分区表状态检查逻辑 - logger.info("分区表状态检查完成"); - } + /** diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java index 185cccf..f3713dc 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java @@ -85,9 +85,9 @@ public class DeviceStatsUpdateService { " updated_at = NOW() " ; /** - * 每分钟执行一次统计更新（秒：0，分：*，时：*） + * 每5分钟执行一次设备统计更新（秒：0，分：*，时：*） */ - @Scheduled(cron = "0 * * * * ?") + @Scheduled(cron = "0 */5 * * * ?") @Transactional public void updateDeviceStats() { long startTime = System.currentTimeMillis(); diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/config/AppConfig.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/config/AppConfig.java index 59c5e2c..fcd3485 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/java/com/config/AppConfig.java +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/config/AppConfig.java @@ -101,5 +101,7 @@ public class AppConfig { public static String geRunEnvironment‌() { return config.getString("server.run.environment"); } - + public static String getSM4Key() { + return config.getString("syslog.sm4.generateKey"); + } } diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/controllers/SyslogNonNormalMessageController.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/controllers/SyslogNonNormalMessageController.java index 4ca8203..8a9d7eb 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/java/com/controllers/SyslogNonNormalMessageController.java +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/controllers/SyslogNonNormalMessageController.java @@ -95,8 +95,7 @@ public class SyslogNonNormalMessageController { /** * 删除非标日志 */ - @DeleteMapping("/delete/{id}") - + //@DeleteMapping("/delete/{id}") public ResponseEntity> deleteMessage( @PathVariable String id) { Map result = new HashMap<>(); diff --git a/haobang-security-xdr/syslog-consumer/src/main/resources/application-dev.properties b/haobang-security-xdr/syslog-consumer/src/main/resources/application-dev.properties index 16ee85e..43185a8 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/resources/application-dev.properties +++ b/haobang-security-xdr/syslog-consumer/src/main/resources/application-dev.properties @@ -15,7 +15,7 @@ syslog.tcp.port=514 syslog.udp.port=514 syslog.max.frame.length=65536 syslog.buffer.size=1000 - +syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # InfluxDB 2.7 Configuration influxdb.url=http://192.168.222.131:8086 influxdb.token=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA== @@ -146,4 +146,24 @@ spring.datasource.hikari.schema=public # analysis.realtime.enabled= true # 룩 - Ĭ10 -analysis.realtime.check-interval-seconds: 10 \ No newline at end of file +analysis.realtime.check-interval-seconds: 10 + + + +# ============================================ +# ̽API +# ============================================ +# API-KEY֤32λʹɵԿ +interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 +# APIӿڻURLsyslog-serveã +interlocking.api.base-url=http://localhost:8089/xdrservice/interlocking + +# ============================================ +# 澯 +# ============================================ +# 澯ֵСʱ +alarm.health-check.alarm-hours=2 +# 澯־ֵСʱ +alarm.health-check.alarm-visit-hours=4 +# ǷöʱѲ +alarm.health-check.enabled=true \ No newline at end of file diff --git a/haobang-security-xdr/syslog-consumer/src/main/resources/application-prod-zc.properties b/haobang-security-xdr/syslog-consumer/src/main/resources/application-prod-zc.properties index 80372c2..597fdfa 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/resources/application-prod-zc.properties +++ b/haobang-security-xdr/syslog-consumer/src/main/resources/application-prod-zc.properties @@ -57,7 +57,7 @@ mybatis-plus.type-handlers-package=com.Modules.etl.handler spring.kafka.consumer.bootstrap-servers=10.11.2.142:9092 spring.kafka.consumer.group-id=agent-syslog-group spring.kafka.consumer.auto-offset-reset=latest -spring.kafka.consumer.enable-auto-commit=true +spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.auto-commit-interval=1000 spring.kafka.consumer.topic=agent-syslog-topic @@ -121,6 +121,8 @@ spring.elasticsearch.password=t2NZCiajmdazxBrF spring.elasticsearch.connection-timeout=10s # Socket ʱʱ spring.elasticsearch.socket-timeout=30s + + # ETL etl.batch.page-size=1000 etl.batch.insert-batch-size=500 @@ -140,4 +142,10 @@ spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer spring.datasource.hikari.auto-commit=false -spring.datasource.hikari.schema=public \ No newline at end of file +spring.datasource.hikari.schema=public + + +# +analysis.realtime.enabled= true +# 룩 - Ĭ10 +analysis.realtime.check-interval-seconds: 10 \ No newline at end of file diff --git a/haobang-security-xdr/syslog-consumer/src/main/resources/application-prod.properties b/haobang-security-xdr/syslog-consumer/src/main/resources/application-prod.properties index e4703f5..ad0b25b 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/resources/application-prod.properties +++ b/haobang-security-xdr/syslog-consumer/src/main/resources/application-prod.properties @@ -15,6 +15,7 @@ syslog.tcp.port=514 syslog.udp.port=514 syslog.max.frame.length=65536 syslog.buffer.size=1000 +syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # InfluxDB 2.7 Configuration influxdb.url=http://192.168.4.26:8087 @@ -148,4 +149,22 @@ spring.datasource.hikari.schema=public # analysis.realtime.enabled= true # 룩 - Ĭ10 -analysis.realtime.check-interval-seconds: 10 \ No newline at end of file +analysis.realtime.check-interval-seconds: 10 + +# ============================================ +# ̽API +# ============================================ +# API-KEY֤32λʹɵԿ +interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 +# APIӿڻURLsyslog-serveã +interlocking.api.base-url=http://localhost:8089/xdrservice/interlocking + +# ============================================ +# 澯 +# ============================================ +# 澯ֵСʱ +alarm.health-check.alarm-hours=2 +# 澯־ֵСʱ +alarm.health-check.alarm-visit-hours=4 +# ǷöʱѲ +alarm.health-check.enabled=true \ No newline at end of file diff --git a/haobang-security-xdr/syslog-consumer/src/main/resources/application.properties b/haobang-security-xdr/syslog-consumer/src/main/resources/application.properties index e4703f5..ad0b25b 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/resources/application.properties +++ b/haobang-security-xdr/syslog-consumer/src/main/resources/application.properties @@ -15,6 +15,7 @@ syslog.tcp.port=514 syslog.udp.port=514 syslog.max.frame.length=65536 syslog.buffer.size=1000 +syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # InfluxDB 2.7 Configuration influxdb.url=http://192.168.4.26:8087 @@ -148,4 +149,22 @@ spring.datasource.hikari.schema=public # analysis.realtime.enabled= true # 룩 - Ĭ10 -analysis.realtime.check-interval-seconds: 10 \ No newline at end of file +analysis.realtime.check-interval-seconds: 10 + +# ============================================ +# ̽API +# ============================================ +# API-KEY֤32λʹɵԿ +interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 +# APIӿڻURLsyslog-serveã +interlocking.api.base-url=http://localhost:8089/xdrservice/interlocking + +# ============================================ +# 澯 +# ============================================ +# 澯ֵСʱ +alarm.health-check.alarm-hours=2 +# 澯־ֵСʱ +alarm.health-check.alarm-visit-hours=4 +# ǷöʱѲ +alarm.health-check.enabled=true \ No newline at end of file diff --git a/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml b/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml index fbe747b..9402d7e 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml +++ b/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml @@ -44,7 +44,6 @@ FROM analysis_analysis_rule WHERE run_mode = #{runMode} AND del_flag = '0' - AND task_status IN ('stopped', 'waiting', 'STOPPED') AND rule_status =1 ORDER BY priority DESC, create_time ASC diff --git a/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/SyslogNormalAlarmMapper.xml b/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/SyslogNormalAlarmMapper.xml index 33fe265..004a2b3 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/SyslogNormalAlarmMapper.xml +++ b/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/SyslogNormalAlarmMapper.xml @@ -3,6 +3,7 @@ "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> + INSERT INTO syslog_normal_alarm @@ -405,7 +406,7 @@ #{dataMap.container_name}, #{dataMap.container_id}, #{dataMap.http_resp_server}, - #{dataMap.srcip_id}, + #{dataMap.srcip_id}::int8, #{dataMap.cdnip}::inet, #{dataMap.natip}::inet, #{dataMap.mail_sender}, @@ -438,8 +439,8 @@ #{dataMap.print_time}, #{dataMap.printer}, #{dataMap.printer_type}, - #{dataMap.print_pages}, - #{dataMap.print_copies}, + #{dataMap.print_pages}::int8, + #{dataMap.print_copies}::int8, #{dataMap.src_device}, #{dataMap.dst_device}, #{dataMap.src_file}, @@ -456,7 +457,7 @@ #{dataMap.env}, #{dataMap.brute_force_service}, #{dataMap.vuirs_name}, - #{dataMap.http_req_length}, + #{dataMap.http_req_length}::int8, #{dataMap.http_req_content_type}, #{dataMap.tc_scan_port}::inet, #{dataMap.tc_labels}::inet, @@ -487,25 +488,25 @@ #{dataMap.src_ip_apt}, #{dataMap.srcip_name}, #{dataMap.tc_client}, - #{dataMap.srcip_organization_id}, + #{dataMap.srcip_organization_id}::int8, #{dataMap.dest_ip_intranetip}, #{dataMap.dest_ip_ioc}, - #{dataMap.desip_id}, + #{dataMap.desip_id}::int8, #{dataMap.desip_name}, #{dataMap.tc_hostip}::inet, - #{dataMap.desip_organization_id}, + #{dataMap.desip_organization_id}::int8, #{dataMap.origin_confidence}, #{dataMap.origin_malscore}, #{dataMap.attacker_icampaign}, - #{dataMap.attacker_host_asset_id}, - #{dataMap.attacker_organization_id}, - #{dataMap.victim_host_asset_id}, - #{dataMap.victim_organization_id}, + #{dataMap.attacker_host_asset_id}::int8, + #{dataMap.attacker_organization_id}::int8, + #{dataMap.victim_host_asset_id}::int8, + #{dataMap.victim_organization_id}::int8, #{dataMap.logout_time}, #{dataMap.http_req_line}, #{dataMap.desip_security_scope_id}, #{dataMap.srcip_security_scope_id}, - #{dataMap.http_resp_length}, + #{dataMap.http_resp_length}::int8, #{dataMap.tc_attack_type}, #{dataMap.tc_realip}::inet, #{dataMap.attacker_ip_lists}, @@ -529,7 +530,7 @@ #{dataMap.tc_client_ip}::inet, #{dataMap.tc_server_ip}::inet, #{dataMap.tc_externalip}::inet, - #{dataMap.http_status_code}, + #{dataMap.http_status_code}::int8, #{dataMap.device_domian}, #{dataMap.src_ip_str}, #{dataMap.src_port_str}, @@ -575,12 +576,12 @@ #{dataMap.origin_agent_name}, #{dataMap.origin_work_group}, #{dataMap.origin_asset_group}, - #{dataMap.origin_local_port}, + #{dataMap.origin_local_port}::int8, #{dataMap.origin_agent_ip}::inet, #{dataMap.origin_internal_ip}::inet, #{dataMap.origin_external_ip}::inet, #{dataMap.origin_local_addr}::inet, - #{dataMap.agent_id}, + #{dataMap.agent_id}::int8, #{dataMap.agent_name}, #{dataMap.tc_title}, #{dataMap.log_id}, @@ -596,7 +597,7 @@ #{dataMap.src_mac}, #{dataMap.dest_mac}, #{dataMap.proto}, - #{dataMap.dev_id}, + #{dataMap.dev_id}::int8, #{dataMap.created_time}, #{dataMap.src_country}, #{dataMap.src_country_code}, @@ -631,9 +632,9 @@ #{dataMap.check_item}, #{dataMap.check_type}, #{dataMap.attacker_ip}::inet, - #{dataMap.attacker_port}, + #{dataMap.attacker_port}::int8, #{dataMap.victim_ip}::inet, - #{dataMap.victim_port}, + #{dataMap.victim_port}::int8, #{dataMap.attacker_city}, #{dataMap.attacker_lon}, #{dataMap.attacker_lat}, @@ -686,15 +687,15 @@ #{dataMap.dest_city}, #{dataMap.dest_lon}, #{dataMap.dest_lat}, - #{dataMap.event_category}, - #{dataMap.attack_result}::int, + #{dataMap.event_category}::int4, + #{dataMap.attack_result}::int4, #{dataMap.probe_ip}::inet, #{dataMap.device_ip}::inet, #{dataMap.device_manufacturer}, #{dataMap.device_name}, #{dataMap.product_name}, #{dataMap.__id}, - #{dataMap.__count}, + #{dataMap.__count}::int8, #{dataMap.__count_reason}, #{dataMap.event_type}::int, #{dataMap.protocol}, @@ -702,19 +703,19 @@ #{dataMap.parent_name}, #{dataMap.host_file_path}, #{dataMap.uid}, - #{dataMap.fall}, + #{dataMap.fall}::int4, #{dataMap.tc_miguan_server_ip}::inet, - #{dataMap.dev_type}, - #{dataMap.collect_method}, - #{dataMap.field_cate_id}, - #{dataMap.device_type}, + #{dataMap.dev_type}::int4, + #{dataMap.collect_method}::int4, + #{dataMap.field_cate_id}::int4, + #{dataMap.device_type}::int4, #{dataMap.tc_miguan_client_ip}::inet, #{dataMap.tc_miguan_name}::inet, - #{dataMap.origin_total_packages}, - #{dataMap.origin_total_bytes}, - #{dataMap.origin_peak_packages_rate}, - #{dataMap.origin_peak_bytes_rate}, - #{dataMap.origin_peak_flows_rate}, + #{dataMap.origin_total_packages}::int8, + #{dataMap.origin_total_bytes}::int8, + #{dataMap.origin_peak_packages_rate}::int8, + #{dataMap.origin_peak_bytes_rate}::int8, + #{dataMap.origin_peak_flows_rate}::int8, #{dataMap.apt_orgname}, #{dataMap.apt_orgmsg}, #{dataMap.mail_message_id}, @@ -731,11 +732,11 @@ #{dataMap.origin_source_servername}, #{dataMap.mail_filename}, #{dataMap.dst_upload_appname}, - #{dataMap.target_port}, + #{dataMap.target_port}::int8, #{dataMap.gid}, #{dataMap.origin_uid}, #{dataMap.origin_gid}, - #{dataMap.target_ports}, + #{dataMap.target_ports}::int8, #{dataMap.tc_miguan_name1}, #{dataMap.tc_miguan_class1}, #{dataMap.etl_time}, @@ -755,7 +756,6 @@ #{dataMap.syslog_topic}, - INSERT INTO syslog_normal_alarm diff --git a/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml b/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml index d1bb7a9..9d60d7c 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml +++ b/haobang-security-xdr/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml @@ -541,7 +541,7 @@ #{dataMap.container_name}, #{dataMap.container_id}, #{dataMap.http_resp_server}, - #{dataMap.srcip_id}, + #{dataMap.srcip_id}::int8, #{dataMap.cdnip}::inet, #{dataMap.natip}::inet, #{dataMap.mail_sender}, @@ -574,8 +574,8 @@ #{dataMap.print_time}, #{dataMap.printer}, #{dataMap.printer_type}, - #{dataMap.print_pages}, - #{dataMap.print_copies}, + #{dataMap.print_pages}::int8, + #{dataMap.print_copies}::int8, #{dataMap.src_device}, #{dataMap.dst_device}, #{dataMap.src_file}, @@ -592,7 +592,7 @@ #{dataMap.env}, #{dataMap.brute_force_service}, #{dataMap.vuirs_name}, - #{dataMap.http_req_length}, + #{dataMap.http_req_length}::int8, #{dataMap.http_req_content_type}, #{dataMap.tc_scan_port}::inet, #{dataMap.tc_labels}::inet, @@ -623,25 +623,25 @@ #{dataMap.src_ip_apt}, #{dataMap.srcip_name}, #{dataMap.tc_client}, - #{dataMap.srcip_organization_id}, + #{dataMap.srcip_organization_id}::int8, #{dataMap.dest_ip_intranetip}, #{dataMap.dest_ip_ioc}, - #{dataMap.desip_id}, + #{dataMap.desip_id}::int8, #{dataMap.desip_name}, #{dataMap.tc_hostip}::inet, - #{dataMap.desip_organization_id}, + #{dataMap.desip_organization_id}::int8, #{dataMap.origin_confidence}, #{dataMap.origin_malscore}, #{dataMap.attacker_icampaign}, - #{dataMap.attacker_host_asset_id}, - #{dataMap.attacker_organization_id}, - #{dataMap.victim_host_asset_id}, - #{dataMap.victim_organization_id}, + #{dataMap.attacker_host_asset_id}::int8, + #{dataMap.attacker_organization_id}::int8, + #{dataMap.victim_host_asset_id}::int8, + #{dataMap.victim_organization_id}::int8, #{dataMap.logout_time}, #{dataMap.http_req_line}, #{dataMap.desip_security_scope_id}, #{dataMap.srcip_security_scope_id}, - #{dataMap.http_resp_length}, + #{dataMap.http_resp_length}::int8, #{dataMap.tc_attack_type}, #{dataMap.tc_realip}::inet, #{dataMap.attacker_ip_lists}, @@ -665,7 +665,7 @@ #{dataMap.tc_client_ip}::inet, #{dataMap.tc_server_ip}::inet, #{dataMap.tc_externalip}::inet, - #{dataMap.http_status_code}, + #{dataMap.http_status_code}::int8, #{dataMap.device_domian}, #{dataMap.src_ip_str}, #{dataMap.src_port_str}, @@ -711,12 +711,12 @@ #{dataMap.origin_agent_name}, #{dataMap.origin_work_group}, #{dataMap.origin_asset_group}, - #{dataMap.origin_local_port}, + #{dataMap.origin_local_port}::int8, #{dataMap.origin_agent_ip}::inet, #{dataMap.origin_internal_ip}::inet, #{dataMap.origin_external_ip}::inet, #{dataMap.origin_local_addr}::inet, - #{dataMap.agent_id}, + #{dataMap.agent_id}::int8, #{dataMap.agent_name}, #{dataMap.tc_title}, #{dataMap.log_id}, @@ -732,7 +732,7 @@ #{dataMap.src_mac}, #{dataMap.dest_mac}, #{dataMap.proto}, - #{dataMap.dev_id}, + #{dataMap.dev_id}::int8, #{dataMap.created_time}, #{dataMap.src_country}, #{dataMap.src_country_code}, @@ -767,9 +767,9 @@ #{dataMap.check_item}, #{dataMap.check_type}, #{dataMap.attacker_ip}::inet, - #{dataMap.attacker_port}, + #{dataMap.attacker_port}::int8, #{dataMap.victim_ip}::inet, - #{dataMap.victim_port}, + #{dataMap.victim_port}::int8, #{dataMap.attacker_city}, #{dataMap.attacker_lon}, #{dataMap.attacker_lat}, @@ -822,15 +822,15 @@ #{dataMap.dest_city}, #{dataMap.dest_lon}, #{dataMap.dest_lat}, - #{dataMap.event_category}, - #{dataMap.attack_result}, + #{dataMap.event_category}::int4, + #{dataMap.attack_result}::int4, #{dataMap.probe_ip}::inet, #{dataMap.device_ip}::inet, #{dataMap.device_manufacturer}, #{dataMap.device_name}, #{dataMap.product_name}, #{dataMap.__id}, - #{dataMap.__count}, + #{dataMap.__count}::int8, #{dataMap.__count_reason}, #{dataMap.event_type}::int, #{dataMap.protocol}, @@ -838,19 +838,19 @@ #{dataMap.parent_name}, #{dataMap.host_file_path}, #{dataMap.uid}, - #{dataMap.fall}, + #{dataMap.fall}::int4, #{dataMap.tc_miguan_server_ip}::inet, - #{dataMap.dev_type}, - #{dataMap.collect_method}, - #{dataMap.field_cate_id}, - #{dataMap.device_type}, + #{dataMap.dev_type}::int4, + #{dataMap.collect_method}::int4, + #{dataMap.field_cate_id}::int4, + #{dataMap.device_type}::int4, #{dataMap.tc_miguan_client_ip}::inet, #{dataMap.tc_miguan_name}::inet, - #{dataMap.origin_total_packages}, - #{dataMap.origin_total_bytes}, - #{dataMap.origin_peak_packages_rate}, - #{dataMap.origin_peak_bytes_rate}, - #{dataMap.origin_peak_flows_rate}, + #{dataMap.origin_total_packages}::int8, + #{dataMap.origin_total_bytes}::int8, + #{dataMap.origin_peak_packages_rate}::int8, + #{dataMap.origin_peak_bytes_rate}::int8, + #{dataMap.origin_peak_flows_rate}::int8, #{dataMap.apt_orgname}, #{dataMap.apt_orgmsg}, #{dataMap.mail_message_id}, @@ -867,11 +867,11 @@ #{dataMap.origin_source_servername}, #{dataMap.mail_filename}, #{dataMap.dst_upload_appname}, - #{dataMap.target_port}, + #{dataMap.target_port}::int8, #{dataMap.gid}, #{dataMap.origin_uid}, #{dataMap.origin_gid}, - #{dataMap.target_ports}, + #{dataMap.target_ports}::int8, #{dataMap.tc_miguan_name1}, #{dataMap.tc_miguan_class1}, #{dataMap.etl_time},