From 037cca32dd27eb12cf9e2482134bbfe5e54e864c Mon Sep 17 00:00:00 2001 From: nanChen Date: Mon, 7 Sep 2026 15:34:17 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9Esyslog-collector-api=20?= =?UTF-8?q?=E9=87=87=E9=9B=86=E6=A8=A1=E5=9D=97=EF=BC=8C=E9=80=9A=E8=BF=87?= =?UTF-8?q?api=20=E6=96=B9=E5=BC=8F=E9=87=87=E9=9B=86=E5=A4=A9=E7=9C=BC?= =?UTF-8?q?=E3=80=81=E5=BE=AE=E6=AD=A5=E7=9A=84=E8=AE=BE=E5=A4=87=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- haobang-security-dm/.idea/workspace.xml | 662 ++++++++--------- .../syslog-collector-api/Dockerfile | 15 + .../syslog-collector-api/doc/doc_v08.md | 694 ++++++++++++++++++ .../docker-compose-collector-api-dm-prod.yaml | 99 +++ .../syslog-collector-api/pom.xml | 131 ++++ .../sql/device_api_config_test_data.sql | 158 ++++ .../sql/mock_server_test.sql | 93 +++ .../main/java/com/CollectorApiMainApp.java | 17 + .../main/java/com/collector/AuthManager.java | 111 +++ .../java/com/collector/CollectResult.java | 122 +++ .../java/com/collector/CollectScheduler.java | 532 ++++++++++++++ .../com/collector/CollectorException.java | 19 + .../com/collector/DeviceDataCollector.java | 38 + .../main/java/com/collector/FieldMapper.java | 120 +++ .../com/collector/TianyanDataCollector.java | 682 +++++++++++++++++ .../com/collector/WeibuDataCollector.java | 459 ++++++++++++ .../java/com/collector/XdrDataCollector.java | 100 +++ .../common/entity/DeviceApiCollectLog.java | 170 +++++ .../com/common/entity/DeviceApiConfig.java | 167 +++++ .../mapper/DeviceApiCollectLogMapper.java | 43 ++ .../common/mapper/DeviceApiConfigMapper.java | 54 ++ .../service/DeviceApiCollectLogService.java | 43 ++ .../service/DeviceApiConfigService.java | 78 ++ .../main/java/com/config/AppProperties.java | 103 +++ .../src/main/java/com/config/CacheConfig.java | 38 + .../src/main/java/com/config/RedisConfig.java | 41 ++ .../java/com/config/RestTemplateConfig.java | 40 + .../DeviceApiCollectController.java | 183 +++++ .../java/com/kafka/KafkaProducerUtil.java | 129 ++++ .../main/java/com/mock/MockDataGenerator.java | 166 +++++ .../java/com/mock/TianyanMockController.java | 216 ++++++ .../java/com/mock/WeibuMockController.java | 137 ++++ .../src/main/java/com/util/HttpUtil.java | 130 ++++ .../src/main/java/com/util/Sm4Util.java | 114 +++ .../main/java/com/util/SpringContextUtil.java | 40 + .../src/main/java/com/util/TimeUtils.java | 80 ++ .../resources/application-prod.properties | 114 +++ .../src/main/resources/application.properties | 114 +++ .../src/main/resources/logback.xml | 52 ++ .../mapper/DeviceApiCollectLogMapper.xml | 61 ++ .../mapper/DeviceApiConfigMapper.xml | 88 +++ .../docker-compose-consumer-prod.yaml | 4 +- .../service/impl/OfflineAnalysisEngine.java | 7 +- .../service/impl/RealtimeAnalysisEngine.java | 7 +- .../src/main/resources/application.properties | 6 +- .../src/main/resources/logback.xml | 6 +- .../src/main/resources/application.properties | 44 +- .../src/main/resources/logback.xml | 6 +- .../docker-compose-serve-prod.yaml | 3 +- .../com/Modules/Device/DeviceProcess.java | 2 +- .../main/java/com/kafka/kafkaProducer.java | 294 +++++--- .../resources/application-prod.properties | 6 +- .../src/main/resources/logback.xml | 8 +- 53 files changed, 6358 insertions(+), 488 deletions(-) create mode 100644 haobang-security-dm/syslog-collector-api/Dockerfile create mode 100644 haobang-security-dm/syslog-collector-api/doc/doc_v08.md create mode 100644 haobang-security-dm/syslog-collector-api/docker-compose/docker-compose-collector-api-dm-prod.yaml create mode 100644 haobang-security-dm/syslog-collector-api/pom.xml create mode 100644 haobang-security-dm/syslog-collector-api/sql/device_api_config_test_data.sql create mode 100644 haobang-security-dm/syslog-collector-api/sql/mock_server_test.sql create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/CollectorApiMainApp.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/collector/AuthManager.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectResult.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectScheduler.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectorException.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/collector/DeviceDataCollector.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/collector/FieldMapper.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/collector/TianyanDataCollector.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/collector/WeibuDataCollector.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/collector/XdrDataCollector.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/common/entity/DeviceApiCollectLog.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/common/entity/DeviceApiConfig.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/common/mapper/DeviceApiCollectLogMapper.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/common/mapper/DeviceApiConfigMapper.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/common/service/DeviceApiCollectLogService.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/common/service/DeviceApiConfigService.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/config/AppProperties.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/config/CacheConfig.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/config/RedisConfig.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/config/RestTemplateConfig.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/controller/DeviceApiCollectController.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/kafka/KafkaProducerUtil.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/mock/MockDataGenerator.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/mock/TianyanMockController.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/mock/WeibuMockController.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/util/HttpUtil.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/util/Sm4Util.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/util/SpringContextUtil.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/java/com/util/TimeUtils.java create mode 100644 haobang-security-dm/syslog-collector-api/src/main/resources/application-prod.properties create mode 100644 haobang-security-dm/syslog-collector-api/src/main/resources/application.properties create mode 100644 haobang-security-dm/syslog-collector-api/src/main/resources/logback.xml create mode 100644 haobang-security-dm/syslog-collector-api/src/main/resources/mapper/DeviceApiCollectLogMapper.xml create mode 100644 haobang-security-dm/syslog-collector-api/src/main/resources/mapper/DeviceApiConfigMapper.xml diff --git a/haobang-security-dm/.idea/workspace.xml b/haobang-security-dm/.idea/workspace.xml index 1a787ff..512416c 100644 --- a/haobang-security-dm/.idea/workspace.xml +++ b/haobang-security-dm/.idea/workspace.xml @@ -15,14 +15,18 @@ - - - - - + + + + + + + + + @@ -185,7 +189,7 @@ - + @@ -196,27 +200,28 @@ - + - + - + + - + - + - - + + - + @@ -224,10 +229,10 @@ - + - - + + @@ -255,93 +260,74 @@ - - + + - - + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - + + @@ -357,7 +343,6 @@ - tenant_id syslog-consumer @Scheduled 配置 Elasticsearch @@ -387,6 +372,7 @@ init findRequiredFieldsAfterTime KafkaConsumerProperties + alarmb com.common. @@ -511,7 +497,7 @@ - @@ -763,7 +754,7 @@ - + @@ -794,7 +785,7 @@ - + - @@ -1119,10 +1116,10 @@ - + - + @@ -1132,8 +1129,8 @@ - - + + @@ -1246,148 +1243,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1426,26 +1281,12 @@ - - - - - - - - - - - - - - @@ -1474,13 +1315,6 @@ - - - - - - - @@ -1512,10 +1346,66 @@ - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1536,43 +1426,139 @@ - + - - + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - + + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/haobang-security-dm/syslog-collector-api/Dockerfile b/haobang-security-dm/syslog-collector-api/Dockerfile new file mode 100644 index 0000000..6428e78 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/Dockerfile @@ -0,0 +1,15 @@ +FROM openjdk:8 + +MAINTAINER chenchunan@139.com + +# 创建程序运行和日志目录 +RUN mkdir -p /app +RUN mkdir -p /app/logs +WORKDIR /app +# 挂载日志运行目录 +VOLUME /app/logs + +ADD syslog-collector-api-1.0.0.jar syslog-collector-api.jar +EXPOSE 8290 + +CMD java ${JAVA_OPTS} -jar syslog-collector-api.jar diff --git a/haobang-security-dm/syslog-collector-api/doc/doc_v08.md b/haobang-security-dm/syslog-collector-api/doc/doc_v08.md new file mode 100644 index 0000000..2f0efc3 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/doc/doc_v08.md @@ -0,0 +1,694 @@ +智能安全运营平台 +设备API接口数据采集功能模块 +开发设计说明 +文档版本：V1.0.0 +文档密级：内部 +2026年08月24日 +文档修订记录 + +[TABLE] +| 版本号 | 修订日期 | 修订内容 | 修订人 | +| V1.0.0 | 2026-08-24 | 初始版本，创建文档 | | +| | | | | +| | | | | +[/TABLE] + + +# 1 概述 +1.1 编写目的 +本文档描述智能安全运营平台"设备API接口数据采集功能模块"的需求与开发设计，涵盖从奇安信天眼分析平台、微步在线威胁感知平台、深信服XDR平台三类安全设备通过API接口定时采集安全数据（Web访问日志、告警日志、攻击流量、日志流量）的功能需求、非功能性需求、接口规范及详细设计方案。 +本文档的预期读者为：项目开发人员、系统测试人员、项目管理人员及运维人员。 +1.2 系统架构 +设备API接口数据采集功能模块部署于智能安全运营平台侧，采用"定时拉取—字段映射—标准化数据格式—写入Kafka"的处理模式，整体数据流如下： +┌─────────┐ ┌─────────┐ ┌─────────┐ +│ 天眼平台 │ │ 微步平台 │ │ 深信服XDR│ +└────┬────┘ └────┬────┘ └─── ─┬────┘ + │ HTTPS API │ │ + └──────┬───────┴──────┬──────┘ + ▼ ▼ + ┌────────────────────────────┐ + │ 设备API数据采集模块 │ + │ (定时调度/认证/分页/映射) │ + └────────────┬───────────────┘ + ▼ + ┌────────────┐ ┌──────┐ + │ Kafka │───▶│ Redis │(断点/Token缓存) + └─────┬──────┘ └──────┘ + ▼ + syslog-consumer-rule 消费处理(泛化规则) +采集源汇总：共3个设备平台、4种数据类型、2种认证机制（免密登录/HMAC签名）、3种API协议。 +1.3 适用范围 +本文档适用于智能安全运营平台设备API接口数据采集功能模块的设计、开发、测试与验收工作。 +1.4 术语与缩略语 + +[TABLE] +| 术语/缩略语 | 全称 | 说明 | +| 天眼 | 奇安信威胁监测与分析系统 | 奇安信天眼分析平台，提供日志检索与告警API | +| 微步 | 微步在线威胁感知平台(TDP) | 提供攻击流量与日志流量查询API | +| XDR | Extended Detection and Response | 深信服扩展检测与响应平台，提供开放API | +| csrf_token | Cross-Site Request Forgery Token | 天眼免密登录后获取的业务请求令牌 | +| AK/SK | Access Key / Secret Key | XDR联动码解码获得的访问密钥对 | +| HMAC-SHA256 | Hash-based Message Authentication Code | 微步与XDR采用的请求签名算法 | +| IOC | Indicators of Compromise | 失陷威胁指标 | +| ATT&CK | Adversarial Tactics, Techniques & Common Knowledge | 攻击战术技术知识库 | +[/TABLE] + +1.5 参考文档 + +[TABLE] +| 序号 | 文档名称 | 版本 | +| 1 | 奇安信网神威胁监测与分析系统（天眼-分析平台）V4.0.12.0-API调用手册V1.0 | V1.0.0 | +| 2 | 微步在线API配置文档 | 最新 | +| 3 | 深信服XDR开放API认证方案及接口开放列表 | 最新 | +| 4 | 智能安全运营平台系统架构设计文档 | V1.0.0 | +[/TABLE] + + +# 2 功能需求 +2.1 数据采集 +设备API接口数据采集模块需要从以下安全设备平台定时拉取安全数据，采集的数据类型主要包括Web访问日志、告警日志、攻击流量和日志流量。 +2.1.1 天眼分析平台API接口 + +#### 2.1.1.1 功能需求 +通过天眼分析平台API接口，定时采集以下两类数据： +• Web访问日志（skyeye-weblog）：记录外部对内网的HTTP/HTTPS访问行为，包括源/目的IP、端口、URI、Host、Method、状态码、Cookie、Referer等。 +• 告警日志（alarm_collection等）：记录各类安全告警事件，包括规则告警、威胁情报告警、蜜罐告警、WebShell检测、Web攻击检测、IDS入侵检测等。 + +#### 2.1.1.2 认证机制 +天眼分析平台采用免密登录（OAuth式）认证，流程如下： +1. 获取login_key（系统管理-账号管理） +2. 生成client_id = SHA256("mNSLP9UJCtBHtegjDPJnK3v" + "|" + login_key) +3. 生成client_secret = SHA256("3460681205014671737" + "|" + login_key) +4. 生成timestamp（10位Unix时间戳，与平台时差不能超过10分钟） +5. 生成X-Authorization = SHA256(JSON({"client_id":..,"username":"tapadmin"}) + timestamp + client_secret) +6. POST /skyeye/v1/admin/auth 获取access_token +7. GET /skyeye/v1/admin/auth?token=... 提取csrf_token和cookies +8. 携带csrf_token + cookies请求业务数据接口 +注意：csrf_token有效期约30分钟，需在每次采集周期前重新认证；cookies需在后续请求Header中携带，格式为 key1=value1;key2=value2;。 + +#### 2.1.1.3 相关API接口说明 +（1）Web访问日志查询接口 + +[TABLE] +| 属性 | 说明 | +| 接口URL | /skyeye/v1/analysis/log-search/list | +| 请求协议 | HTTPS | +| 请求方法 | GET | +| 认证方式 | csrf_token + cookies（免密登录获取） | +| 数据格式 | JSON | +[/TABLE] + +请求参数： + +[TABLE] +| 参数名 | 类型 | 必填 | 说明 | 示例 | +| branch_id | string | 是 | 级联单位ID | QbJK/sLHi | +| start_time | number | 是 | 开始时间（毫秒时间戳） | 1787534456117 | +| end_time | number | 是 | 结束时间（毫秒时间戳） | 1787534756117 | +| index | string | 是 | 索引名称（web日志固定为skyeye-weblog） | skyeye-weblog | +| category | string | 是 | 索引类别 | skyeye | +| offset | int | 是 | 页码，从1开始 | 1 | +| limit | int | 是 | 每页条数 | 100 | +| mode | string | 否 | 查询模式（fast_model快速查询） | fast_model | +| keyword | string | 否 | 关键词检索 | | +| csrf_token | string | 是 | 认证令牌（免密登录获取） | 8696f4a3f08fe4c48b310f269bd4adad | +| r | string | 否 | 随机数（防缓存） | 0.8929389461785662 | +[/TABLE] + +响应字段（Web访问日志核心数据字段）： + +[TABLE] +| 字段名 | 说明 | +| serial_num | 设备序列号 | +| access_time | 访问时间 | +| sip / dip | 源IP / 目的IP | +| sport / dport | 源端口 / 目的端口 | +| uri / uri.raw | 请求URI / 原始URI | +| host / host_raw | 主机域名 | +| method | HTTP方法（GET/POST等） | +| status | HTTP状态码 | +| agent | User-Agent | +| xff | X-Forwarded-For | +| data | 请求体数据 | +| content_type | Content-Type | +| setcookie / cookie / referer | Set-Cookie / Cookie / Referer | +| authorization | Authorization头 | +| device_ip | 设备IP | +| src_mac / dst_mac | 源MAC / 目的MAC | +| vlan_id | VLAN ID | +| sess_key | 会话Key | +| priv_info | 私有信息（产品类型等） | +| geo_sip.* / geo_dip.* | 源/目的IP地理位置（城市、国家、经纬度） | +| @timestamp | ES时间戳 | +| sipv6 / dipv6 | IPv6源/目的地址 | +| origin / accept_language | Origin / Accept-Language | +[/TABLE] + +（2）告警日志查询接口 + +[TABLE] +| 属性 | 说明 | +| 接口URL | /skyeye/v1/analysis/log-search/list | +| 请求方法 | GET | +| 索引名称 | alarm_collection, skyeye-sandbox, sandbox_mail, alarm_honeypot, alarm_yunsuo, webids-webshell_dolog, webids-webattack_dolog, webids-ids_dolog, webids-ioc_dolog | +| 查询模式 | advance_model（高级模式） | +[/TABLE] + +告警日志数据字段（核心字段）： + +[TABLE] +| 字段名 | 说明 | +| rule_id / rule_name | 规则ID / 规则名称 | +| dolog_count | 告警次数 | +| severity | 严重级别 | +| rule_version | 规则版本 | +| sip / dip / sport / dport | 源/目的IP、端口 | +| method / host / uri | HTTP方法 / 主机 / URI | +| file_name / referer / agent / cookie / parameter | 文件名 / Referer / UA / Cookie / 参数 | +| req_header / req_body | 请求头 / 请求体 | +| rsp_status / rsp_content_length / rsp_content_type / rsp_header / rsp_body / rsp_body_len | 响应状态/长度/类型/头/体 | +| victim_type / attack_flag | 受害类型 / 攻击标识 | +| attacker / victim | 攻击者 / 受害者 | +| attack_type / attack_result | 攻击类型 / 攻击结果 | +| confidence | 置信度 | +| detail_info / solution | 详细信息 / 解决方案 | +| vuln_desc / vuln_harm / vuln_name / vuln_type | 漏洞描述/危害/名称/类型 | +| kill_chain / att_ck_all | 杀伤链 / ATT&CK | +| cve_id / bid_id / cwe_id / msb_id | CVE/BID/CWE/MSB编号 | +| webrules_tag / rule_labels | Web规则标签 | +| code_language / site_app | 代码语言 / 站点应用 | +| device_ip / vendor_id | 设备IP / 厂商ID | +[/TABLE] + +（3）告警列表查询接口 + +[TABLE] +| 属性 | 说明 | +| 接口URL | /alarm/alarm/list | +| 请求方法 | GET | +| 认证方式 | csrf_token + cookies | +[/TABLE] + +核心请求参数：start_time、end_time、threat_type、hazard_level（0低危/1中危/2高危/3危急）、host_state（攻击结果）、status（处理状态）、offset（页码）、limit（每页条数）等。 +2.1.2 微步API接口 + +#### 2.1.2.1 功能需求 +通过微步在线威胁感知平台API接口，定时采集以下两类数据： +• 攻击流量（net_data_type: ["attack"]）：检测到的攻击行为日志，包括威胁名称、攻击结果、严重级别、攻击阶段、源/目的IP端口、HTTP详情、IOC等。 +• 日志流量（net_data_type: ["info"]）：网络流量日志记录，用于行为分析和审计。 + +#### 2.1.2.2 认证机制 +微步平台采用HMAC-SHA256签名认证，流程如下： +1. 获取当前时间戳 Timestamp（10位Unix秒级时间戳） +2. 拼接签名数据：SignData = ApiKey + Timestamp +3. 计算签名：Sign = Base64.encodeBase64URLSafeString(HmacSHA256(SignData, Secret)) +4. 构造请求URL： + POST https://domain/api/v1/log/searchBySql?api_key=&auth_timestamp=&sign= +说明：Base64 URL Safe编码将"+"替换为"-"、"/"替换为"_"并去除末尾"="，防止URL参数解析不完整；请求方时间与服务端时间差须在10分钟以内，否则签名过期。 + +#### 2.1.2.3 相关API接口说明 + +[TABLE] +| 属性 | 说明 | +| 接口URL | /api/v1/log/searchBySql | +| 请求协议 | HTTPS | +| 请求方法 | POST | +| 频率限制 | 50次/秒 | +| 认证方式 | api_key + auth_timestamp + sign（HMAC-SHA256） | +| Content-Type | application/json | +[/TABLE] + +请求Body参数： + +[TABLE] +| 参数名 | 类型 | 必填 | 说明 | 示例 | +| net_data_type | Array | 是 | 数据类型：attack-攻击流量，info-日志流量 | ["attack"] | +| time_from | Integer | 是 | 开始时间戳（秒级） | 1787500800 | +| time_to | Integer | 是 | 截止时间戳（秒级） | 1787587199 | +| assets_group | Array | 否 | 业务组ID列表 | [] | +| search_connection | Boolean | 否 | 是否搜索关联连接 | false | +| sql | String | 否 | 查询SQL条件 | (threat.level IN ('attack','action')) | +| columns | Array | 否 | 查询字段列表（label+value对） | [{"label":"告警时间","value":"time"}] | +| asc | Boolean | 否 | 是否升序排列 | false | +| page | Object | 否 | 分页参数：cur_page, page_size, sort | {"cur_page":1,"page_size":100} | +[/TABLE] + +查询字段列表（columns）： + +[TABLE] +| 字段（value） | 中文名（label） | 说明 | +| time | 告警时间 | 事件发生时间（秒级时间戳） | +| threat.name | 威胁名称 | 如"路径穿越攻击"、"疑似冰蝎Webshell" | +| threat.result | 攻击结果 | success/failed/unknown | +| threat.severity | 严重级别 | 0:信息, 1:低, 2:中, 3:高, 4:严重 | +| threat.phase | 攻击阶段 | 如exploit, post_exploit | +| threat.type | 威胁类型 | 如exploit, virus, c2 | +| threat.level | 威胁分类 | attack-攻击, action-事件, risk-风险 | +| threat.ioc | IOC | 威胁指标 | +| threat.is_apt | 是否APT | 0:否, 1:是 | +| threat.tag | 威胁标签 | Mitre ATT&CK标签等 | +| threat.status | 处理状态 | 处理状态标识 | +| machine | 告警主机 | 受影响主机IP | +| net.src_ip / net.src_port | 源IP / 源端口 | 攻击源地址与端口 | +| net.dest_ip / net.dest_port | 目的IP / 目的端口 | 目标地址与端口 | +| net.type | 协议 | 应用层协议（http等） | +| net.http.method / status | HTTP方法 / 状态码 | GET/POST等 / 响应状态码 | +| net.http.reqs_header / reqs_body | 请求头 / 请求体 | HTTP请求信息 | +| net.http.resp_header / resp_body | 响应头 / 响应体 | HTTP响应信息 | +| data | URL/域名 | 告警主体数据 | +| assets.name / assets.group_name | 资产名称 / 业务组 | 受影响资产信息 | +| direction | 告警方向 | in:外部攻击, out:失陷破坏, lateral:内网渗透 | +| src_tag / dest_tag | 来源/目的标签 | 内/外 | +| ip_reputation / ip_reputation_zh | IP信誉 | IP信誉标签（英文/中文） | +| hit_whitelist | 命中白名单 | 0:未命中, 1:命中 | +[/TABLE] + +响应数据结构（示例）： +{ + "data": { + "data": [ + { + "data": "URL或域名", + "index": "tdp-2026-08-24", + "src_tag": "外", "dest_tag": "内", + "ip_reputation_zh": ["网关"], + "assets": {"group_id":1, "group_name":"互联网应用", "name":["蜜罐系统"], "id":36}, + "machine": "172.20.34.33", + "time": 1787545680, + "threat": { + "severity": 1, "result": "unknown", "phase": "exploit", + "level": "attack", "name": "路径穿越攻击", + "tag": ["Mitre: T1190..."], "type": "exploit", + "is_apt": 0, "status": 0 + }, + "id": "J7UGMqABp3egYciwfG0Z", + "net": { + "src_ip": "119.132.173.119", "src_port": 55105, + "dest_ip": "172.20.34.33", "dest_port": 80, + "type": "http", + "http": {"method":"POST", "reqs_header":"Host: ...", + "reqs_body":"...", "resp_header":"...", "resp_body":"..."} + }, + "hit_whitelist": 0, "direction": "in" + } + ] + }, + "response_code": 0, + "verbose_msg": "OK" +} +2.1.3 XDR OpenAPI接口（深信服） + +#### 2.1.3.1 功能需求 +通过深信服XDR平台开放API接口，采集告警事件等安全数据，作为天眼和微步数据的补充。 + +#### 2.1.3.2 认证机制 +深信服XDR平台采用AK/SK HMAC-SHA256签名认证，流程如下： +• 从XDR平台"配置管理-系统设置-开放性-联动码管理"页面获取联动码（authCode）。 +• 联动码解码获取AK/SK（AES-CBC解密）。 +• 构造请求，生成签名头；签名后直接发送请求，不可修改任何参数。 +签名算法说明： + +[TABLE] +| 步骤 | 说明 | +| 联动码解码 | Hex解码 -> 按"|"分割为14段 -> 通过SHA256计算AES密钥 -> AES-CBC解密第10段得到AK，第11段得到SK | +| 添加公共头 | sdk-content-type: application/json; sdk-host: {host}; sign-date: {UTC时间(yyyyMMdd'T'HHmmssZZ)} | +| 构造规范串 | METHOD + "\n" + URI(转义+末尾加/) + "\n" + QueryString(排序) + "\n" + Headers(排序key:value\n) + SignedHeaders(;) + "\n" + PayloadHash(SHA256(payload去空格+排序)) | +| 计算签名 | stringToSign = "HMAC-SHA256\n" + signDate + "\n" + SHA256(规范串)；signature = HMAC-SHA256(stringToSign, SK) -> Hex编码 | +| 构造Authorization | algorithm=HMAC-SHA256, Access={AK}, SignedHeaders={排序后的header keys}, Signature={signature} | +[/TABLE] + +XDR平台信息： + +[TABLE] +| 属性 | 值 | +| 平台地址 | https://10.56.171.157/api/openapi/ | +| API Token | dTTQ8esIvaZOETpT4NWX | +| 接口示例 | /api/xdr/v1/assets/list（资产列表）、/api/xdr/v1/assets/department（部门资产） | +| 签名依赖库 | commons-codec（Hex/HmacUtils/DigestUtils）、Spring HttpHeaders | +[/TABLE] + + +# 3 非功能性需求 + +[TABLE] +| 需求项 | 指标 | 说明 | +| 性能 | 采集延迟 | 数据采集周期为5分钟，从设备API获取到数据写入Kafka的延迟不超过30秒 | +| 吞吐量 | 处理能力 | 单次采集批量处理1000条记录，支持多设备并行采集 | +| 可靠性 | 容错机制 | API调用失败自动重试3次（间隔递增：5s/10s/15s），重试失败记录错误日志并跳过当前批次 | +| 可靠性 | 断点续传 | 记录每批次最后采集时间戳，重启后从上次位置继续采集，不丢数据 | +| 可用性 | 认证缓存 | csrf_token（天眼）和AK/SK签名缓存于Redis，避免频繁认证 | +| 安全性 | 凭证管理 | login_key、ApiKey、Secret、authCode等敏感信息加密存储于配置文件，不记录在日志中 | +| 安全性 | HTTPS | 所有设备API调用使用HTTPS协议，关闭SSL证书校验（设备使用自签证书） | +| 可扩展性 | 设备适配 | 采用策略模式，新增设备类型只需实现统一的DataCollector接口，无需修改调度逻辑 | +| 可维护性 | 配置化 | 设备地址、认证信息、采集间隔、数据类型等全部通过application.yml配置，支持热刷新 | +| 采集监控 | 采集统计 | 记录每次采集的设备名、数据类型、采集条数、耗时、成功/失败状态，写入采集日志表 | +[/TABLE] + + +# 4 假设与约束条件 +4.1 时间周期 + +[TABLE] +| 约束项 | 约束值 | 说明 | +| 天眼认证有效期 | 约30分钟 | csrf_token有效期约30分钟，需在每次采集周期前检查并刷新 | +| 微步签名有效期 | 10分钟 | auth_timestamp与服务端时间差须在10分钟以内 | +| 天眼时间同步 | 10分钟 | 第三方系统时间与天眼分析平台时间差不能超过10分钟 | +| 微步频率限制 | 50次/秒 | 微步API调用频率限制为50次/秒，需控制并发请求速率 | +| 采集间隔 | 5分钟 | 默认每5分钟采集一次，可配置 | +| 时间窗口重叠 | -1分钟 | 每次采集的时间窗口结束时间为当前时间减1分钟，开始时间为上次采集结束时间 | +| 分页大小 | 100条/页 | 单次请求最大100条，需循环分页拉取 | +[/TABLE] + + +# 5 开发设计 +5.1 数据采集处理流程 +数据采集处理主流程（9步）： +定时触发 -> 认证管理 -> 构造请求 -> 调用API -> 解析响应 + -> 字段映射（标准化数据格式） -> 写入Kafka（指定topic） -> 更新断点 -> 记录日志 +5.1.1 认证管理流程 + +[TABLE] +| 设备 | 认证策略 | +| 天眼 | 1. 检查Redis中缓存的csrf_token是否有效（TTL=25分钟）2. 若无效，执行免密登录流程：生成client_id/secret/signature -> POST获取access_token -> GET提取csrf_token+cookies3. 缓存csrf_token和cookies到Redis（TTL=25分钟） | +| 微步 | 1. 每次请求实时生成签名（Timestamp + HmacSHA256）2. 无需缓存Token，但Api_key和Secret从配置文件读取 | +| 深信服XDR | 1. 启动时解码联动码获取AK/SK，缓存到内存2. 每次请求实时生成sign-date和签名头3. 签名后直接发送，不可修改请求 | +[/TABLE] + +5.1.2 分页拉取流程 +int page = 1; +int pageSize = 100; +boolean hasMore = true; +while (hasMore) { + params.put("offset", page); + params.put("limit", pageSize); + JSONObject response = httpClient.get(apiUrl, params, headers); + JSONArray items = response.getJSONObject("data").getJSONArray("items"); + int total = response.getJSONObject("data").getInt("total"); + for (Object item : items) { + StandardMessage msg = fieldMapper.map((JSONObject) item, deviceType); + kafkaTemplate.send(topic, JSON.toJSONString(msg)); + } + hasMore = (page * pageSize) < total; + page++; + Thread.sleep(rateLimitInterval); // 限流控制（微步50/s） +} +5.1.3 断点续传机制 +使用Redis记录每个设备+数据类型的最后采集时间戳，确保重启后不丢数据： +// Redis Key设计 +key: device:api:last_collect_time:{deviceIp}:{deviceType}:{dataType} +value: 1787534756117 (最后成功采集的数据时间戳) +TTL: 7天 (过期自动清理) +// 采集时间窗口计算 +long lastTime = redis.get(key); // 上次采集结束时间 +long endTime = System.currentTimeMillis(); // 当前时间 +long startTime = lastTime - 60000; // 回退1分钟，防止边界丢数据 +5.2 设备API接口设计 +5.2.1 统一采集器接口 +/** + * 设备数据采集器统一接口 + * 所有设备采集器实现此接口，由调度器统一管理 + */ +public interface DeviceDataCollector { + /** 获取设备类型 */ + String getDeviceType(); + /** 认证/获取访问凭证 */ + void authenticate() throws CollectorException; + /** + * 采集数据 + * @param dataType 数据类型（weblog/alarm/attack/info等） + * @param startTime 开始时间 + * @param endTime 结束时间 + * @return 采集结果统计 + */ + CollectResult collect(String dataType, long startTime, long endTime) + throws CollectorException; + /** 检查认证是否有效 */ + boolean isAuthenticated(); +} +5.2.2 天眼采集器设计 +@Slf4j +@Component +public class TianyanDataCollector implements DeviceDataCollector { + @Value("${device.tianyan.host}") private String host; + @Value("${device.tianyan.loginKey}") private String loginKey; + @Value("${device.tianyan.branchId}") private String branchId; + private static final String CLIENT_ID_SEED = "mNSLP9UJCtBHtegjDPJnK3v"; + private static final String CLIENT_SECRET_SEED = "3460681205014671737"; + private static final String AUTH_URI = "/skyeye/v1/admin/auth"; + private static final String LOG_SEARCH_URI = "/skyeye/v1/analysis/log-search/list"; + private static final String CSRF_TOKEN_CACHE_KEY = "device:tianyan:csrf_token"; + private static final String COOKIES_CACHE_KEY = "device:tianyan:cookies"; + private static final long CSRF_TOKEN_TTL = 25 * 60; // 25分钟 + @Override + public void authenticate() throws CollectorException { + // 1. 检查Redis缓存 2. 生成client_id/client_secret + // 3. 生成X-Authorization签名 4. POST获取access_token + // 5. GET提取csrf_token+cookies 6. 缓存到Redis + } + @Override + public CollectResult collect(String dataType, long startTime, long endTime) { + // weblog: index=skyeye-weblog, category=skyeye, mode=fast_model + // alarm : index=alarm_collection,... category=event, mode=advance_model + // 分页拉取并写入Kafka + } +} +5.2.3 微步采集器设计 +@Slf4j +@Component +public class WeibuDataCollector implements DeviceDataCollector { + @Value("${device.weibu.host}") private String host; + @Value("${device.weibu.apiKey}") private String apiKey; + @Value("${device.weibu.secret}") private String secret; + private static final String SEARCH_BY_SQL_URI = "/api/v1/log/searchBySql"; + /** + * 生成微步API签名: + * Sign = Base64.encodeBase64URLSafeString(HmacSHA256(ApiKey+Timestamp, Secret)) + */ + private String generateSign(long timestamp) { + String signData = apiKey + timestamp; + byte[] hmacBytes = new HmacUtils(HmacAlgorithms.HMAC_SHA_256, secret).hmac(signData); + return Base64.encodeBase64URLSafeString(hmacBytes); + } + @Override + public CollectResult collect(String dataType, long startTime, long endTime) { + long timestamp = System.currentTimeMillis() / 1000; + String sign = generateSign(timestamp); + String url = host + SEARCH_BY_SQL_URI + + "?api_key=" + apiKey + "&auth_timestamp=" + timestamp + "&sign=" + sign; + // 构造Body: net_data_type/time_from/time_to/sql/columns/分页参数 + // 分页拉取... + } +} +5.2.4 深信服XDR采集器设计（暂缓） +@Slf4j +@Component +public class XdrDataCollector implements DeviceDataCollector { + @Value("${device.xdr.host}") private String host; + @Value("${device.xdr.authCode}") private String authCode; + private String ak; private String sk; + @PostConstruct + public void init() { + String[] akSk = decodeAuthCode(authCode); + this.ak = akSk[0]; this.sk = akSk[1]; + } + /** 对请求进行HMAC-SHA256签名 */ + private HttpRequest signRequest(HttpRequest request) { + String signDate = DateFormatUtils.formatUTC(new Date(), "yyyyMMdd'T'HHmmssZZ"); + request.header("sdk-content-type", "application/json"); + request.header("sdk-host", request.uri().getHost()); + request.header("sign-date", signDate); + String canonicalStr = buildCanonicalStr(request); + String canonicalHash = Hex.encodeHexString(DigestUtils.sha256(canonicalStr), false); + String stringToSign = "HMAC-SHA256\n" + signDate + "\n" + canonicalHash; + String signature = Hex.encodeHexString( + new HmacUtils(HmacAlgorithms.HMAC_SHA_256, sk).hmac(stringToSign), false); + String authorization = String.format( + "algorithm=HMAC-SHA256, Access=%s, SignedHeaders=%s, Signature=%s", + ak, getSignedHeaders(request), signature); + request.header("Authorization", authorization); + return request; + } +} +5.3 数据表设计 +5.3.1 设备API采集配置表 +-- 达梦DM8 DDL +CREATE TABLE device_api_config ( + id BIGINT IDENTITY(1,1) PRIMARY KEY, -- 主键，自增 + device_id INT, -- 设备ID，关联 device_device.id + device_type VARCHAR(50) NOT NULL, -- 设备类型: tianyan/weibu/xdr +device_name VARCHAR(100) NOT NULL, -- 设备名称 +device_ip VARCHAR(200) NOT NULL, -- 设备IP地址 +host_api VARCHAR(200) NOT NULL, -- 设备API地址 + auth_config TEXT, -- 认证配置JSON + data_types VARCHAR(500) NOT NULL, -- 数据类型: weblog,alarm / attack,info + collect_interval_sec INT DEFAULT 300, -- 采集间隔（秒） + enabled INT DEFAULT 1, -- 是否启用: 0-禁用, 1-启用 + remark VARCHAR(500), -- 备注 + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +COMMENT ON TABLE device_api_config IS '设备API采集配置表'; +5.3.2 采集任务执行日志表 +CREATE TABLE device_api_collect_log ( + id BIGINT IDENTITY(1,1) PRIMARY KEY, -- 主键，自增 + device_id INT NOT NULL, -- 设备ID，关联 device_device.id + device_type VARCHAR(50) NOT NULL, -- 设备类型 + data_type VARCHAR(50) NOT NULL, -- 数据类型 + start_time TIMESTAMP NOT NULL, -- 采集窗口开始时间 + end_time TIMESTAMP NOT NULL, -- 采集窗口结束时间 + total_count INT DEFAULT 0, -- 采集记录总数 + success_count INT DEFAULT 0, -- 成功写入数 + fail_count INT DEFAULT 0, -- 失败数 + cost_ms BIGINT DEFAULT 0, -- 耗时（毫秒） + status VARCHAR(20) NOT NULL, -- 状态: SUCCESS/FAIL + error_msg TEXT, -- 错误信息 + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +COMMENT ON TABLE device_api_collect_log IS '设备API采集执行日志'; +5.3.3 标准化消息格式（Kafka消息体） +[receive_time=20260825143015235 device_id=1001 device_name=天眼01 vendor=天眼, data_type=json device_collect_id=1 eventId=20260822-f47ac10b-58cc-4372-a567-0e02b2c3d479] +{ + "deviceType": "tianyan", + "dataType": "alarm", + "collectTime": 1787534756117, + "rawData": { ... 各设备原始JSON ... } +} +5.4 核心类设计 + +[TABLE] +| 类名 | 类型 | 职责 | +| DeviceDataCollector | 接口 | 定义统一的采集器接口规范 | +| TianyanDataCollector | 实现类 | 天眼平台数据采集，含免密登录认证、Web日志/告警日志查询 | +| WeibuDataCollector | 实现类 | 微步平台数据采集，含HMAC-SHA256签名、攻击/日志流量查询 | +| XdrDataCollector | 实现类 | 深信服XDR数据采集，含联动码解码、AK/SK签名 | +| CollectScheduler | 调度器 | 基于Spring @Scheduled定时调度，管理所有采集器的执行周期 | +| AuthManager | 认证管理器 | 统一管理各设备的Token缓存与刷新策略 | +| FieldMapper | 字段映射器 | 将各设备原始字段映射为统一标准化格式 | +| CollectResult | 结果对象 | 封装采集结果统计信息（总数、成功数、失败数、耗时） | +| CollectorException | 异常类 | 采集过程中异常的统一封装 | +| DeviceApiConfig | 实体类 | 设备API配置实体，对应device_api_config表 | +| DeviceApiCollectLog | 实体类 | 采集日志实体，对应device_api_collect_log表 | +[/TABLE] + +5.5 定时任务设计 +@Slf4j +@Component +public class CollectScheduler { + @Autowired private List collectors; + @Autowired private DeviceApiConfigService configService; + @Autowired private RedisTemplate redisTemplate; + /** 主采集任务 - 每5分钟执行 */ + @Scheduled(fixedDelayString = "${device.collect.interval:300000}") + public void scheduleCollect() { + List configs = configService.listEnabled(); + for (DeviceApiConfig config : configs) { + for (String dataType : config.getDataTypes().split(",")) { + try { + collectForDevice(config, dataType.trim()); + } catch (Exception e) { + log.error("采集失败: {}-{}", config.getDeviceType(), dataType, e); + } + } + } + } + private void collectForDevice(DeviceApiConfig config, String dataType) { + // 1. 查找采集器 2. 认证检查 + // 3. 计算时间窗口（上次时间-1分钟 ~ 当前时间） + // 4. 采集 5. 更新断点 6. 记录日志 + } +} +5.6 配置设计 +# application.yml 设备API采集配置 +device: + collect: + interval: 300000 # 采集间隔(毫秒), 默认5分钟 + retry: + max-attempts: 3 # 最大重试次数 + initial-delay: 5000 # 初始重试延迟(毫秒) + multiplier: 2.0 # 重试延迟倍数 + kafka: + topic: device-api-collect + tianyan: # 天眼分析平台 + host: https://10.56.171.xxx:443 + login-key: ${TIANYAN_LOGIN_KEY} + branch-id: QbJK/sLHi + data-types: weblog,alarm + page-size: 100 + weibu: # 微步在线 + host: https://10.56.171.164 + api-key: ${WEIBU_API_KEY} + secret: ${WEIBU_SECRET} + data-types: attack,info + rate-limit: 50 # 每秒最大请求数 + page-size: 100 + xdr: # 深信服XDR + host: https://10.56.171.157 + auth-code: ${XDR_AUTH_CODE} + api-base: /api/openapi/ + data-types: alarm,asset +page-size: 1000 + +# 6开发计划 +6.1 任务计划 +总工期 2周（2026-08-25至2026-9-3，共10个工作日），设置5个里程碑： + +[TABLE] +| 里程碑 | 名称 | 计划完成日期 | 达成标准（出口准则） | +| M1 | 需求与设计基线确认 | 2026-08-25 | 设计说明评审通过并基线化；API账号凭证（天眼login_key、微步ApiKey/Secret、XDR联动码）全部到位 | +| M2 | 采集框架与环境就绪 | 2026-08-27 | 模块骨架搭建完成；两张DM8表建表并通过验证；Kafka/Redis连通；统一接口与调度器编译通过 | +| M3 | 天眼采集器完成 | 2026-08-28 | 免密登录认证调通；weblog与alarm两类数据可稳定拉取并写入Kafka；断点续传验证通过 | +| M4 | 微步与XDR采集器完成 | 2026-08-31 | 微步签名调通、attack/info双类型采集正常；XDR联动码解码与签名调通、数据可拉取 | +| M5 | 联调测试与上线 | 2026-09-04 | 7×24小时连续运行无丢数；重试/断点/限流场景测试通过；测试报告与部署方案归档 | +[/TABLE] + +6.2任务分解与进度安排 +6.2.1 阶段一：需求确认与基础环境准备开发（第1周，08-25~08-27） + +[TABLE] +| 任务编号 | 任务名称 | 工作内容 | 工期(天) | 责任角色 | +| T1.1 | 设计评审与基线 | 组织设计说明评审，确认接口清单、字段映射规则、消息格式 | 1 | 开发负责人 | +| T1.2 | 凭证与环境确认 | 获取三类设备API凭证；确认测试环境网络可达（防火墙开通HTTPS策略） | 1 | 开发+运维 | +| T1.3 | 模块骨架搭建 | 新建syslog-collector-api模块；引入依赖（httpclient/commons-codec/fastjson）；统一异常与日志规范 | 2 | 后端开发 | +| T2.1 | 数据表与实体层 | device_api_config、device_api_collect_log建表（DM8）；entity/mapper/service代码生成 | 1 | 后端开发 | +| T2.2 | 统一采集器接口 | DeviceDataCollector接口、CollectResult、CollectorException定义 | 1 | 后端开发 | +| T2.3 | 调度器开发 | CollectScheduler定时调度、时间窗口计算、采集日志落库 | 2 | 后端开发 | +| T2.4 | 断点续传与重试 | Redis断点Key读写；指数退避重试（3次：5s/10s/15s）；限流器 | 1 | 后端开发 | +[/TABLE] + +6.2.3 阶段二：天眼、微步采集器开发（第1-2周8.28-8.31） + +[TABLE] +| 任务编号 | 任务名称 | 工作内容 | 工期(天) | 责任角色 | +| T3.1 | 免密登录认证 | client_id/client_secret/X-Authorization生成；access_token获取；csrf_token+cookies提取与Redis缓存（TTL=25分钟） | 2 | 后端开发 | +| T3.2 | Web访问日志采集 | skyeye-weblog索引查询、分页拉取（100条/页）、fast_model模式适配 | 2 | 后端开发 | +| T3.3 | 告警日志采集 | 9个告警索引联合查询、advance_model模式适配、响应字段解析 | 3 | 后端开发 | +| T3.4 | 字段映射与联调 | 天眼字段映射；与真实环境联调，验证数据完整性 | 2 | 后端开发 | +| T4.1 | 签名与请求封装 | HmacSHA256+Base64URLSafe签名；api_key/auth_timestamp/sign URL构造 | 1 | 后端开发 | +| T4.2 | 查询接口对接 | searchBySql接口对接；columns字段列表构造；分页参数封装 | 2 | 后端开发 | +| T4.3 | 双类型采集与限流 | attack/info两种net_data_type采集；50次/秒限流控制；字段映射与联调 | 2 | 后端开发 | +[/TABLE] + +6.2.5 阶段三：系统调试与上线（第2周，09.01~09-04） + +[TABLE] +| 任务编号 | 任务名称 | 工作内容 | 工期(天) | 责任角色 | +| T5.1 | 集成测试 | 采集器并行运行7×24小时；采集量/延迟/丢数核对；Kafka消息抽检 | 2 | 开发+测试 | +| T5.2 | 异常场景测试 | Token过期重认证、API失败重试、服务重启断点续传、限流压测 | 1 | 开发+测试 | +| T5.3 | 部署上线（依赖现场网络策略开通） | Docker Compose编排；生产配置核对；监控告警接入；测试报告归档 | 2 | 开发+运维 | +[/TABLE] + +6.2.6 人员分工与资源需求 + +[TABLE] +| 角色 | 人数 | 主要职责 | 投入占比 | +| 开发负责人 | 1 | 设计评审、任务分派、进度与质量把控、里程碑验收 | 30% | +| 后端开发（Java） | 1 | 采集框架、三个采集器、调度与断点、字段映射编码实现及自测 | 100% | +| 测试工程师 | 1 | 测试用例编写、集成与异常场景测试、测试报告 | 100%（1周） | +| 运维工程师 | 1 | 申请网络策略开通（3台采集服务器->安全设备）、环境准备、Docker部署、监控接入 | 20% | +| 厂商接口支持 | 2 | 天眼/微步 接口问题答疑、凭证发放（按需） | 按需 | +[/TABLE] + + +# 7 附录 +7.1天眼接口数据样例 +7.2微步口数据样例 \ No newline at end of file diff --git a/haobang-security-dm/syslog-collector-api/docker-compose/docker-compose-collector-api-dm-prod.yaml b/haobang-security-dm/syslog-collector-api/docker-compose/docker-compose-collector-api-dm-prod.yaml new file mode 100644 index 0000000..16d67da --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/docker-compose/docker-compose-collector-api-dm-prod.yaml @@ -0,0 +1,99 @@ +# ============================================ +# Syslog Collector API 部署配置 (生产环境) +# ============================================ +# 使用方法: +# 1. 先执行 Maven 打包: mvn -pl syslog-collector-api -am clean package -DskipTests +# 2. 拷贝 jar 到模块目录: cp syslog-collector-api/target/syslog-collector-api-1.0.0.jar syslog-collector-api/ +# 3. 构建并启动: docker-compose -f docker-compose-collector-api-prod.yaml up -d +# 4. 查看日志: docker logs -f collector-api +# 5. 停止: docker-compose -f docker-compose-collector-api-prod.yaml down +# ============================================ + + +services: + # ============================================ + # Collector API - 设备API数据采集服务 + # 天眼(Tianyan) / 微步(Weibu) / XDR + # ============================================ + collector-api: + build: + context: ./collector-api-dm + dockerfile: Dockerfile + image: syslog-collector-api-dm:v1.0.1 + container_name: syslog-collector-api-dm + restart: unless-stopped + environment: + # 基础环境 + - TZ=Asia/Shanghai + - SPRING_PROFILES_ACTIVE=prod + # JVM 参数 + - JAVA_OPTS=-Xms4096m -Xmx8192m -XX:+UseG1GC + # 采集探针ID + - DEVICE_COLLECT_ID=1 + # Kafka 推送 topic + - KAFKA_COLLECTOR_TOPIC=test-topic + + # 天眼(Tianyan) 采集配置 + - TIANYAN_HOST=https://10.56.170.107 + - TIANYAN_LOGIN_KEY=QuJm4NwF3GMaXb2Z + - TIANYAN_USERNAME=xxadmin + - TIANYAN_BRANCH_ID=QbJK%2FsLHi + - TIANYAN_DATA_TYPES=weblog,alarm + - TIANYAN_PAGE_SIZE=100 + + # 微步(Weibu) 采集配置 + - WEIBU_HOST=https://10.56.171.164 + - WEIBU_API_KEY=57d8fb2b556a471ea36b8b4441fa1c0c + - WEIBU_SECRET=9d967876f51d4e16bbf8bde069d12a9b + - WEIBU_DATA_TYPES=attack,info + - WEIBU_RATE_LIMIT=10 + - WEIBU_PAGE_SIZE=100 + + # XDR 采集配置 (暂缓,待联动码联调) + - XDR_HOST=https://127.0.0.1 + - XDR_AUTH_CODE= + - XDR_API_BASE=/api/openapi/ + - XDR_DATA_TYPES= + + # 达梦数据库配置 + - SPRING_DATASOURCE_URL=jdbc:dm://192.168.4.99:5237 + - SPRING_DATASOURCE_USERNAME=SYSDBA + - SPRING_DATASOURCE_PASSWORD=caZ2TcmXNSW8L2Ap + - SPRING_DATASOURCE_DRIVER_CLASS_NAME=dm.jdbc.driver.DmDriver + - SPRING_DATASOURCE_HIKARI_SCHEMA=PUBLIC + + # Redis 缓存配置 + - SPRING_REDIS_HOST=192.168.4.99 + - SPRING_REDIS_PORT=6379 + - SPRING_REDIS_PASSWORD=redis_GdGWte + - SPRING_REDIS_DATABASE=0 + - SPRING_REDIS_TIMEOUT=2000 + - SPRING_CACHE_REDIS_TIME_TO_LIVE=600000 + + # Kafka 生产者配置 + - SPRING_KAFKA_PRODUCER_BOOTSTRAP_SERVERS=192.168.4.99:9092 + - SPRING_KAFKA_PRODUCER_PROPERTIES_RETRIES=10 + - SPRING_KAFKA_PRODUCER_PROPERTIES_RETRY_BACKOFF_MS=500 + - SPRING_KAFKA_PRODUCER_PROPERTIES_CONNECTIONS_MAX_IDLE_MS=600000 + - SPRING_KAFKA_PRODUCER_PROPERTIES_SOCKET_KEEPALIVE_ENABLE=true + - SPRING_KAFKA_PRODUCER_PROPERTIES_REQUEST_TIMEOUT_MS=30000 + - SPRING_KAFKA_PRODUCER_PROPERTIES_DELIVERY_TIMEOUT_MS=120000 + + # 定时任务线程池 + - SPRING_TASK_SCHEDULING_POOL_SIZE=5 + # Mock 服务器配置（模拟天眼/微步接口，供功能测试） + - MOCK_SERVER_ENABLED=true + - MOCK_SERVER_TOTAL_COUNT=100 + ports: + - "8290:8290" # Collector API 端口 + volumes: + - /home/syslog/collector-api:/app/logs + networks: + - xdr-network + +# ============================================ +# 网络配置 +# ============================================ +networks: + xdr-network: + driver: bridge diff --git a/haobang-security-dm/syslog-collector-api/pom.xml b/haobang-security-dm/syslog-collector-api/pom.xml new file mode 100644 index 0000000..a710feb --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/pom.xml @@ -0,0 +1,131 @@ + + + 4.0.0 + + security-xdr + com.haobang + 1.0.0 + + com.haobang + syslog-collector-api + 1.0.0 + syslog-collector-api + API data collector module (Tianyan/Weibu/XDR) + + + 1.8 + 1.8 + 1.8 + UTF-8 + 3.5.10 + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-web + + + + com.alibaba.fastjson2 + fastjson2 + 2.0.40 + + + + cn.hutool + hutool-all + 5.8.16 + + + + org.apache.kafka + kafka-clients + 3.9.2 + + + + org.apache.httpcomponents + httpclient + + + + + ch.qos.logback + logback-classic + 1.2.11 + + + + + org.mybatis + mybatis + ${mybatis.version} + + + + + com.dameng + DmJdbcDriver + 8.1.4.169 + + + + org.mybatis.spring.boot + mybatis-spring-boot-starter + 2.3.1 + + + + + org.springframework.boot + spring-boot-starter-cache + + + + + org.springframework.boot + spring-boot-starter-data-redis + + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + + + org.projectlombok + lombok + true + + + + + org.bouncycastle + bcprov-jdk15to18 + 1.78 + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + src/main/resources + false + + + + + diff --git a/haobang-security-dm/syslog-collector-api/sql/device_api_config_test_data.sql b/haobang-security-dm/syslog-collector-api/sql/device_api_config_test_data.sql new file mode 100644 index 0000000..3420641 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/sql/device_api_config_test_data.sql @@ -0,0 +1,158 @@ +-- ===================================================================== +-- 设备API采集模块 建表 + 测试数据脚本 +-- 适用数据库: 达梦 DM8 +-- 说明: +-- 1. 先执行建表语句（表已存在时请先删除或忽略 DDL 部分） +-- 2. 再执行 INSERT 测试数据 +-- 3. id 均为自增字段(IDENTITY)，INSERT 无需也不允许指定 id +-- ===================================================================== + +-- --------------------------------------------------------------------- +-- 1. 建表: 设备API采集配置表 +-- --------------------------------------------------------------------- +DROP TABLE IF EXISTS device_api_collect_log; +DROP TABLE IF EXISTS device_api_config; + +CREATE TABLE device_api_config ( + id BIGINT IDENTITY(1,1) PRIMARY KEY, + device_id INT, -- 设备ID，关联 device_device.id + device_type VARCHAR(50) NOT NULL, -- 设备类型: tianyan/weibu/xdr + device_name VARCHAR(100) NOT NULL, -- 设备名称 + device_ip VARCHAR(200) NOT NULL, -- 设备IP地址 + host_api VARCHAR(200) NOT NULL, -- 设备API URL地址 + auth_config TEXT, -- 认证配置JSON + data_types VARCHAR(500) NOT NULL, -- 数据类型: weblog,alarm / attack,info + collect_interval_sec INT DEFAULT 300, -- 采集间隔（秒） + enabled INT DEFAULT 1, -- 是否启用: 0-禁用, 1-启用 + remark VARCHAR(500), -- 备注 + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +COMMENT ON TABLE device_api_config IS '设备API采集配置表'; +COMMENT ON COLUMN device_api_config.id IS '主键，自增'; +COMMENT ON COLUMN device_api_config.device_id IS '设备ID，关联 device_device.id'; +COMMENT ON COLUMN device_api_config.device_type IS '设备类型: tianyan/weibu/xdr'; +COMMENT ON COLUMN device_api_config.device_name IS '设备名称'; +COMMENT ON COLUMN device_api_config.device_ip IS '设备IP地址'; +COMMENT ON COLUMN device_api_config.host_api IS '设备API URL地址'; +COMMENT ON COLUMN device_api_config.auth_config IS '认证配置JSON'; +COMMENT ON COLUMN device_api_config.data_types IS '数据类型: weblog,alarm / attack,info'; +COMMENT ON COLUMN device_api_config.collect_interval_sec IS '采集间隔（秒）'; +COMMENT ON COLUMN device_api_config.enabled IS '是否启用: 0-禁用, 1-启用'; +COMMENT ON COLUMN device_api_config.remark IS '备注'; +COMMENT ON COLUMN device_api_config.create_time IS '创建时间'; +COMMENT ON COLUMN device_api_config.update_time IS '更新时间'; + +-- --------------------------------------------------------------------- +-- 2. 建表: 设备API采集执行日志表 +-- --------------------------------------------------------------------- +CREATE TABLE device_api_collect_log ( + id BIGINT IDENTITY(1,1) PRIMARY KEY, + device_id INT NOT NULL, -- 设备ID，关联 device_device.id + device_type VARCHAR(50) NOT NULL, -- 设备类型 + data_type VARCHAR(50) NOT NULL, -- 数据类型 + start_time TIMESTAMP NOT NULL, -- 采集窗口开始时间 + end_time TIMESTAMP NOT NULL, -- 采集窗口结束时间 + total_count INT DEFAULT 0, -- 采集记录总数 + success_count INT DEFAULT 0, -- 成功写入数 + fail_count INT DEFAULT 0, -- 失败数 + cost_ms BIGINT DEFAULT 0, -- 耗时（毫秒） + status VARCHAR(20) NOT NULL, -- 状态: SUCCESS/FAIL + error_msg TEXT, -- 错误信息 + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +COMMENT ON TABLE device_api_collect_log IS '设备API采集执行日志'; +COMMENT ON COLUMN device_api_collect_log.id IS '主键，自增'; +COMMENT ON COLUMN device_api_collect_log.device_id IS '设备ID，关联 device_device.id'; +COMMENT ON COLUMN device_api_collect_log.device_type IS '设备类型'; +COMMENT ON COLUMN device_api_collect_log.data_type IS '数据类型'; +COMMENT ON COLUMN device_api_collect_log.start_time IS '采集窗口开始时间'; +COMMENT ON COLUMN device_api_collect_log.end_time IS '采集窗口结束时间'; +COMMENT ON COLUMN device_api_collect_log.total_count IS '采集记录总数'; +COMMENT ON COLUMN device_api_collect_log.success_count IS '成功写入数'; +COMMENT ON COLUMN device_api_collect_log.fail_count IS '失败数'; +COMMENT ON COLUMN device_api_collect_log.cost_ms IS '耗时（毫秒）'; +COMMENT ON COLUMN device_api_collect_log.status IS '状态: SUCCESS/FAIL'; +COMMENT ON COLUMN device_api_collect_log.error_msg IS '错误信息'; +COMMENT ON COLUMN device_api_collect_log.create_time IS '创建时间'; + +-- ===================================================================== +-- 3. 测试数据: 设备API采集配置表（id 自增，不插入） +-- 覆盖: 天眼(启用/停用)、微步(启用)、XDR(暂缓停用) +-- ===================================================================== +INSERT INTO device_api_config + (device_id, device_type, device_name, device_ip, host_api, auth_config, data_types, + collect_interval_sec, enabled, remark, create_time, update_time) +VALUES + -- 天眼-主设备（启用，采集 weblog+alarm） + (35, 'tianyan', '天眼态势感知平台', '10.56.170.107', + 'https://10.56.170.107', + '{"login_key":"QuJm4NwF3GMaXb2Z","username":"xxadmin","branch_id":"QbJK%2FsLHi"}', + 'weblog,alarm', 300, 1, '生产环境天眼主采集器', + '2026-08-27 09:00:00', '2026-08-27 09:00:00'), + -- 天眼-备设备（停用，仅测试） + (102, 'tianyan', '天眼态势感知平台-备', '10.56.170.107', + 'https://10.56.170.107', + '{"login_key":"QuJm4NwF3GMaXb2Z","username":"xxadmin","branch_id":"QbJK%2FsLHi"}', + 'weblog', 600, 0, '备用采集器，暂未启用', + '2026-08-27 09:05:00', '2026-08-27 09:05:00'), + -- 微步-TDP（启用，采集 attack+info） + (36, 'weibu', '微步威胁感知平台', '10.56.171.164', + 'https://10.56.171.164', + '{"api_key":"57d8fb2b556a471ea36b8b4441fa1c0c","secret":"9d967876f51d4e16bbf8bde069d12a9b"}', + 'attack,info', 300, 1, '生产环境微步采集器', + '2026-08-27 09:10:00', '2026-08-27 09:10:00'), + -- XDR（暂缓，联动码待联调） + (104, 'xdr', '深信服XDR平台', '192.168.30.31', + 'https://192.168.30.31', + '{"auth_code":"<待联调时填入14段联动码>"}', + 'alert', 300, 0, 'XDR采集暂缓，待联动码解码联调后启用', + '2026-08-27 09:15:00', '2026-08-27 09:15:00'); + +-- ===================================================================== +-- 4. 测试数据: 设备API采集执行日志表（id 自增，不插入） +-- 覆盖: 成功、失败、不同设备/类型 +-- ===================================================================== +INSERT INTO device_api_collect_log + (device_id, device_type, data_type, start_time, end_time, + total_count, success_count, fail_count, cost_ms, status, error_msg, create_time) +VALUES + -- 天眼 weblog 采集成功 + (101, 'tianyan', 'weblog', + '2026-08-27 08:55:00', '2026-08-27 09:00:00', + 1280, 1280, 0, 3265, 'SUCCESS', NULL, + '2026-08-27 09:00:00'), + -- 天眼 alarm 采集成功 + (101, 'tianyan', 'alarm', + '2026-08-27 08:55:00', '2026-08-27 09:00:00', + 36, 36, 0, 1240, 'SUCCESS', NULL, + '2026-08-27 09:00:00'), + -- 微步 attack 采集成功（含少量失败） + (103, 'weibu', 'attack', + '2026-08-27 08:55:00', '2026-08-27 09:00:00', + 520, 518, 2, 1890, 'SUCCESS', '2条消息Kafka推送失败', + '2026-08-27 09:00:00'), + -- 微步 info 采集失败（模拟接口超时） + (103, 'weibu', 'info', + '2026-08-27 08:55:00', '2026-08-27 09:00:00', + 0, 0, 0, 30050, 'FAIL', 'searchBySql接口请求超时: read timed out', + '2026-08-27 09:00:00'), + -- XDR（停用状态，无采集记录; 以下为历史参考记录） + (104, 'xdr', 'alert', + '2026-08-26 09:00:00', '2026-08-26 09:05:00', + 0, 0, 0, 150, 'FAIL', 'XDR采集器暂缓，待联动码解码与签名接口联调后实现', + '2026-08-26 09:05:00'); + +-- ===================================================================== +-- 5. 校验查询（可选执行） +-- ===================================================================== +-- 查询启用中的采集配置 +-- SELECT id, device_type, device_name, data_types, collect_interval_sec +-- FROM device_api_config +-- WHERE enabled = 1; + +-- 查询最近采集日志 +-- SELECT device_type, data_type, total_count, success_count, fail_count, +-- status, error_msg, create_time +-- FROM device_api_collect_log +-- ORDER BY id DESC; diff --git a/haobang-security-dm/syslog-collector-api/sql/mock_server_test.sql b/haobang-security-dm/syslog-collector-api/sql/mock_server_test.sql new file mode 100644 index 0000000..42b7729 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/sql/mock_server_test.sql @@ -0,0 +1,93 @@ +-- ===================================================================== +-- Mock 服务器功能测试配置脚本 +-- 将天眼/微步的 host_api 指向本地 Mock 接口（复用 8290 端口，路径区分） +-- +-- 前置条件: +-- 1. application.properties 中 mock.server.enabled=true +-- 2. 应用运行在 127.0.0.1:8290（context-path=/collectorapi） +-- +-- 采集器 host 拼接规则: host_api + 接口路径 +-- 天眼: http://127.0.0.1:8290/collectorapi/mock/tianyan + /skyeye/v1/admin/auth ... +-- 微步: http://127.0.0.1:8290/collectorapi/mock/weibu + /api/v1/log/searchBySql ... +-- +-- 使用说明: +-- 执行本脚本前，请先恢复生产 host_api（执行恢复语句）; +-- 测试完成后务必执行"恢复生产配置"部分，避免误连生产。 +-- ===================================================================== + +-- ===================================================================== +-- 1. 切换到本地 Mock 地址 +-- ===================================================================== +-- 天眼-主设备（101）: weblog+alarm +UPDATE device_api_config + SET host_api = 'http://127.0.0.1:8290/collectorapi/mock/tianyan', + enabled = 1, + remark = '【功能测试】连接本地Mock,生产地址10.56.170.107', + update_time = SYSDATE + WHERE id = 101; + +-- 天眼-备设备（102）: weblog（启用用于并行测试） +UPDATE device_api_config + SET host_api = 'http://127.0.0.1:8290/collectorapi/mock/tianyan', + enabled = 1, + remark = '【功能测试】连接本地Mock,生产地址10.56.170.107', + update_time = SYSDATE + WHERE id = 102; + +-- 微步-TDP（103）: attack+info +UPDATE device_api_config + SET host_api = 'http://127.0.0.1:8290/collectorapi/mock/weibu', + enabled = 1, + remark = '【功能测试】连接本地Mock,生产地址10.56.171.164', + update_time = SYSDATE + WHERE id = 103; + +-- ===================================================================== +-- 2. 手动触发测试（执行后可手工调用，也可等待定时调度） +-- 天眼 weblog: curl -X POST "http://127.0.0.1:8290/collectorapi/api/collect/trigger?id=101&dataType=weblog&minutes=60" +-- 天眼 alarm: curl -X POST "http://127.0.0.1:8290/collectorapi/api/collect/trigger?id=101&dataType=alarm&minutes=60" +-- 微步 attack: curl -X POST "http://127.0.0.1:8290/collectorapi/api/collect/trigger?id=103&dataType=attack&minutes=60" +-- 微步 info: curl -X POST "http://127.0.0.1:8290/collectorapi/api/collect/trigger?id=103&dataType=info&minutes=60" +-- 全部类型: curl -X POST "http://127.0.0.1:8290/collectorapi/api/collect/trigger?id=101&minutes=60" +-- ===================================================================== + +-- ===================================================================== +-- 3. 模拟异常场景（通过 Mock 接口场景参数，无需改库） +-- 天眼 csrf过期: GET "http://127.0.0.1:8290/collectorapi/mock/tianyan/skyeye/v1/analysis/log-search/list?index=weblog&offset=1&limit=100&start_time=xxx&end_time=xxx&mock_scenario=csrf_expired" +-- 天眼 空数据: ...&mock_scenario=empty +-- 微步 签名错误: POST "http://127.0.0.1:8290/collectorapi/mock/weibu/api/v1/log/searchBySql?api_key=xxx&auth_timestamp=xxx&sign=xxx&mock_scenario=sign_error" +-- ===================================================================== + +-- ===================================================================== +-- 4. 恢复生产配置（测试完成后务必执行！） +-- ===================================================================== +-- UPDATE device_api_config +-- SET host_api = 'https://10.56.170.107', +-- remark = '生产环境天眼主采集器', +-- update_time = SYSDATE +-- WHERE id = 101; +-- +-- UPDATE device_api_config +-- SET host_api = 'https://10.56.170.107', +-- remark = '备用采集器，暂未启用', +-- update_time = SYSDATE +-- WHERE id = 102; +-- +-- UPDATE device_api_config +-- SET host_api = 'https://10.56.171.164', +-- remark = '生产环境微步采集器', +-- update_time = SYSDATE +-- WHERE id = 103; + +-- ===================================================================== +-- 5. 校验查询 +-- ===================================================================== +-- SELECT id, device_type, device_name, host_api, data_types, enabled, remark +-- FROM device_api_config +-- ORDER BY id; + +-- SELECT device_type, data_type, total_count, success_count, fail_count, +-- status, error_msg, create_time +-- FROM device_api_collect_log +-- ORDER BY id DESC +-- FETCH FIRST 10 ROWS ONLY; diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/CollectorApiMainApp.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/CollectorApiMainApp.java new file mode 100644 index 0000000..bb32e93 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/CollectorApiMainApp.java @@ -0,0 +1,17 @@ +package com; + +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; + +@SpringBootApplication +@MapperScan("com.common.mapper") +@EnableScheduling +public class CollectorApiMainApp { + + public static void main(String[] args) { + SpringApplication.run(CollectorApiMainApp.class, args); + System.out.println("============== CollectorApiMainApp 启动成功 =============="); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/AuthManager.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/AuthManager.java new file mode 100644 index 0000000..4a75739 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/AuthManager.java @@ -0,0 +1,111 @@ +package com.collector; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Component; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +/** + * 认证管理器 + * 负责各采集器访问凭证（access_token / csrf_token / cookie）的获取与缓存。 + * 凭证存储于 Redis，并带进程内缓存以减少 Redis 访问。 + */ +@Component +public class AuthManager { + + /** Redis key 前缀 */ + private static final String TOKEN_PREFIX = "collector:auth:"; + + /** 默认凭证过期时间（秒），天眼 csrf_token 有效期约30分钟，按25分钟缓存 */ + private static final long DEFAULT_TTL_SECONDS = 25 * 60; + + @Autowired + private RedisTemplate redisTemplate; + + /** 进程内缓存：key -> {token, expireAt} */ + private final ConcurrentHashMap localCache = new ConcurrentHashMap<>(); + + private static class CachedToken { + final String token; + final long expireAt; + + CachedToken(String token, long ttlSeconds) { + this.token = token; + this.expireAt = System.currentTimeMillis() + ttlSeconds * 1000; + } + } + + /** + * 保存凭证 + * @param deviceType 设备类型 + * @param key 凭证类型：csrf_token / access_token / cookie / sign + * @param value 凭证值 + */ + public void saveToken(String deviceType, String key, String value) { + saveToken(deviceType, key, value, DEFAULT_TTL_SECONDS); + } + + /** + * 保存凭证（自定义过期时间） + */ + public void saveToken(String deviceType, String key, String value, long ttlSeconds) { + String redisKey = TOKEN_PREFIX + deviceType + ":" + key; + try { + redisTemplate.opsForValue().set(redisKey, value, ttlSeconds, TimeUnit.SECONDS); + } catch (Exception e) { + // Redis 不可用时仅保留进程内缓存 + } + localCache.put(redisKey, new CachedToken(value, ttlSeconds)); + } + + /** + * 获取凭证，优先本地缓存，其次 Redis + */ + public String getToken(String deviceType, String key) { + String redisKey = TOKEN_PREFIX + deviceType + ":" + key; + + // 1. 本地缓存 + CachedToken cached = localCache.get(redisKey); + if (cached != null && cached.expireAt > System.currentTimeMillis()) { + return cached.token; + } + if (cached != null) { + localCache.remove(redisKey); + } + + // 2. Redis 缓存 + try { + Object value = redisTemplate.opsForValue().get(redisKey); + if (value != null) { + String token = value.toString(); + localCache.put(redisKey, new CachedToken(token, DEFAULT_TTL_SECONDS)); + return token; + } + } catch (Exception e) { + // Redis 不可用 + } + return null; + } + + /** + * 删除凭证 + */ + public void removeToken(String deviceType, String key) { + String redisKey = TOKEN_PREFIX + deviceType + ":" + key; + localCache.remove(redisKey); + try { + redisTemplate.delete(redisKey); + } catch (Exception e) { + // Redis 不可用 + } + } + + /** + * 判断凭证是否有效 + */ + public boolean hasToken(String deviceType, String key) { + return getToken(deviceType, key) != null; + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectResult.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectResult.java new file mode 100644 index 0000000..a5234c9 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectResult.java @@ -0,0 +1,122 @@ +package com.collector; + +import java.util.ArrayList; +import java.util.List; + +/** + * 采集结果 + */ +public class CollectResult { + + /** 设备ID */ + private Long deviceId; + + /** 设备类型 */ + private String deviceType; + + /** 数据类型 */ + private String dataType; + + /** 采集开始时间戳（毫秒） */ + private long startTime; + + /** 采集结束时间戳（毫秒） */ + private long endTime; + + /** 采集总数 */ + private int totalCount; + + /** 成功条数 */ + private int successCount; + + /** 失败条数 */ + private int failCount; + + /** Kafka 消息列表（已格式化的完整消息） */ + private List messages = new ArrayList<>(); + + public Long getDeviceId() { + return deviceId; + } + + public void setDeviceId(Long deviceId) { + this.deviceId = deviceId; + } + + public String getDeviceType() { + return deviceType; + } + + public void setDeviceType(String deviceType) { + this.deviceType = deviceType; + } + + public String getDataType() { + return dataType; + } + + public void setDataType(String dataType) { + this.dataType = dataType; + } + + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + public long getEndTime() { + return endTime; + } + + public void setEndTime(long endTime) { + this.endTime = endTime; + } + + public int getTotalCount() { + return totalCount; + } + + public void setTotalCount(int totalCount) { + this.totalCount = totalCount; + } + + public int getSuccessCount() { + return successCount; + } + + public void setSuccessCount(int successCount) { + this.successCount = successCount; + } + + public int getFailCount() { + return failCount; + } + + public void setFailCount(int failCount) { + this.failCount = failCount; + } + + public List getMessages() { + return messages; + } + + public void setMessages(List messages) { + this.messages = messages; + } + + public void addMessage(String message) { + this.messages.add(message); + this.totalCount++; + } + + public void incrementSuccess() { + this.successCount++; + } + + public void incrementFail() { + this.failCount++; + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectScheduler.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectScheduler.java new file mode 100644 index 0000000..54bccce --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectScheduler.java @@ -0,0 +1,532 @@ +package com.collector; + +import com.common.entity.DeviceApiCollectLog; +import com.common.entity.DeviceApiConfig; +import com.common.service.DeviceApiCollectLogService; +import com.common.service.DeviceApiConfigService; +import com.config.AppProperties; +import com.util.TimeUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +/** + * 采集调度器 + * 定时遍历所有启用的设备配置，按数据类型执行采集， + * 支持断点续传（Redis记录最后采集时间戳）、失败重试、采集日志记录。 + * 另提供手动全量回补：把 [startTime, endTime] 区间按固定子窗口由旧到新滚动逐段采集（同步执行）。 + */ +@Component +public class CollectScheduler { + + private static final Logger log = LoggerFactory.getLogger(CollectScheduler.class); + + /** 断点续传 Redis Key 前缀 */ + private static final String LAST_COLLECT_KEY_PREFIX = "device:api:last_collect_time:"; + + /** 断点续传 TTL：3天 */ + private static final long LAST_COLLECT_TTL_DAYS = 3; + + /** 时间窗口重叠回退：1分钟，防止边界丢数据 */ + private static final long OVERLAP_BACKOFF_MS = 60000; + + /** 采集窗口最大跨度：默认30分钟 */ + private static final long MAX_WINDOW_MS = 30 * 60 * 1000; + + /** 全量回补默认回看天数：1天 */ + private static final long FULL_DEFAULT_BACK_DAYS = 1; + + /** 全量回补默认子窗口跨度：30分钟（与定时增量最大窗口一致，避免单次查询跨度过大） */ + private static final long FULL_DEFAULT_WINDOW_MS = 30 * 60 * 1000L; + + @Autowired + private DeviceApiConfigService configService; + + @Autowired + private DeviceApiCollectLogService collectLogService; + + @Autowired + private AppProperties properties; + + @Autowired + private RedisTemplate redisTemplate; + + /** 采集器注册表：deviceType -> collector */ + private final Map collectorMap = new HashMap<>(); + + /** 采集执行线程池：各设备/数据类型并行采集，慢设备（如微步 info）不再阻塞其他设备 */ + private ExecutorService collectExecutor; + + /** 防重入集合：正在采集的任务 key（deviceType:deviceIp:dataType），上一轮未完成时跳过本轮提交 */ + private final Set runningKeys = ConcurrentHashMap.newKeySet(); + + @PostConstruct + public void initExecutor() { + int poolSize = Math.max(1, properties.getCollectPoolSize()); + collectExecutor = Executors.newFixedThreadPool(poolSize, r -> { + Thread t = new Thread(r, "collect-worker"); + t.setDaemon(true); + return t; + }); + log.info("采集线程池初始化完成, poolSize={}", poolSize); + } + + @PreDestroy + public void shutdownExecutor() { + if (collectExecutor != null) { + collectExecutor.shutdownNow(); + } + } + + @Autowired + public void registerCollectors(List collectors) { + for (DeviceDataCollector collector : collectors) { + collectorMap.put(collector.getDeviceType(), collector); + } + log.info("注册采集器: {}", collectorMap.keySet()); + } + + /** + * 定时调度入口 + */ + @Scheduled(cron = "${device.collect.cron:0 * * * * ?}") + public void scheduleCollect() { + try { + List configs = configService.listEnabled(); + if (configs.isEmpty()) { + log.debug("无启用的 API 采集配置"); + return; + } + for (DeviceApiConfig config : configs) { + collectForDevice(config); + } + } catch (Exception e) { + log.error("采集调度异常: {}", e.getMessage(), e); + } + } + + /** + * 对单个设备执行采集（遍历其所有数据类型） + */ + private void collectForDevice(DeviceApiConfig config) { + DeviceDataCollector collector = collectorMap.get(config.getDeviceType()); + if (collector == null) { + log.warn("设备类型 {} 无对应采集器", config.getDeviceType()); + return; + } + + String dataTypes = config.getDataTypes(); + if (dataTypes == null || dataTypes.isEmpty()) { + log.warn("设备 {} 未配置数据类型", config.getDeviceName()); + return; + } + + for (String dataType : dataTypes.split(",")) { + final String type = dataType.trim(); + if (type.isEmpty()) { + continue; + } + // 防重入：上一轮该任务未结束时跳过本轮提交，避免同窗口并发重复采集 + String runKey = config.getDeviceType() + ":" + + (config.getDeviceIp() != null ? config.getDeviceIp() : "unknown") + ":" + type; + if (!runningKeys.add(runKey)) { + log.info("上一轮采集未完成，跳过本轮提交: device={}, type={}", config.getDeviceName(), type); + continue; + } + try { + collectExecutor.submit(() -> { + try { + collectWithRetry(collector, config, type); + } catch (Exception e) { + log.error("设备 {} 采集失败: type={}, error={}", + config.getDeviceName(), type, e.getMessage(), e); + } finally { + runningKeys.remove(runKey); + } + }); + } catch (Exception e) { + runningKeys.remove(runKey); + log.error("提交采集任务失败: device={}, type={}, error={}", + config.getDeviceName(), type, e.getMessage()); + } + } + } + + /** + * 带重试的采集执行 + */ + private void collectWithRetry(DeviceDataCollector collector, DeviceApiConfig config, String dataType) { + int maxAttempts = properties.getRetryMaxAttempts(); + long delay = properties.getRetryInitialDelay(); + double multiplier = properties.getRetryMultiplier(); + + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + LocalDateTime startLocal = LocalDateTime.now(); + long startMs = System.currentTimeMillis(); + // 本轮采集窗口，供最终失败时写采集日志（窗口未算出时才用尝试耗时兜底） + long[] window = null; + try { + // 计算采集时间窗口（断点续传） + window = calculateWindow(config, dataType); + long startTime = window[0]; + long endTime = window[1]; + + // 执行采集 + CollectResult result = collector.collect(config, dataType, startTime, endTime); + + // 记录成功采集日志 + DeviceApiCollectLog logEntity = new DeviceApiCollectLog(); + logEntity.setDeviceId(config.getDeviceId()); + logEntity.setDeviceType(config.getDeviceType()); + logEntity.setDataType(dataType); + logEntity.setStartTime(TimeUtils.toLocalDateTime(startTime)); + logEntity.setEndTime(TimeUtils.toLocalDateTime(endTime)); + logEntity.setTotalCount(result.getTotalCount()); + logEntity.setSuccessCount(result.getSuccessCount()); + logEntity.setFailCount(result.getFailCount()); + logEntity.setCostMs(System.currentTimeMillis() - startMs); + logEntity.setStatus("SUCCESS"); + collectLogService.record(logEntity); + + // 更新断点续传时间戳 + updateLastCollectTime(config, dataType, endTime); + + log.info("采集成功: device={}, type={}, window=[{}, {}], success={}, fail={}, cost={}ms", + config.getDeviceName(), dataType, startTime, endTime, + result.getSuccessCount(), result.getFailCount(), logEntity.getCostMs()); + return; + } catch (Exception e) { + log.error("采集失败(第{}次/共{}次): device={}, type={}, error={}", + attempt, maxAttempts, config.getDeviceName(), dataType, e.getMessage()); + + if (attempt < maxAttempts) { + // 记录失败日志（中间重试不记录，仅在最终失败时记录） + try { + Thread.sleep(delay); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + delay = (long) (delay * multiplier); + } else { + // 最终失败，记录采集日志（窗口已算出时记录真实窗口，便于排查漏采） + DeviceApiCollectLog logEntity = new DeviceApiCollectLog(); + logEntity.setDeviceId(config.getDeviceId()); + logEntity.setDeviceType(config.getDeviceType()); + logEntity.setDataType(dataType); + logEntity.setStartTime(window != null ? TimeUtils.toLocalDateTime(window[0]) : startLocal); + logEntity.setEndTime(window != null ? TimeUtils.toLocalDateTime(window[1]) : LocalDateTime.now()); + logEntity.setTotalCount(0); + logEntity.setSuccessCount(0); + logEntity.setFailCount(0); + logEntity.setCostMs(System.currentTimeMillis() - startMs); + logEntity.setStatus("FAIL"); + logEntity.setErrorMsg(e.getMessage() != null + ? e.getMessage().substring(0, Math.min(e.getMessage().length(), 2000)) : "unknown"); + collectLogService.record(logEntity); + } + } + } + } + + /** + * 手动触发采集（功能测试用） + * 按配置ID立即执行一次采集，不走断点续传时间戳， + * 使用指定回看窗口（默认最近1小时，可指定 minutes）。 + * + * @param configId 设备配置ID + * @param dataType 数据类型，为空则采集配置中的所有类型 + * @param minutes 回看窗口分钟数，为空默认最近1小时 + * @return 采集结果汇总 + */ + public Map collectNow(Long configId, String dataType, Integer minutes) { + DeviceApiConfig config = configService.getById(configId); + if (config == null) { + throw new IllegalArgumentException("采集配置不存在: " + configId); + } + DeviceDataCollector collector = collectorMap.get(config.getDeviceType()); + if (collector == null) { + throw new IllegalArgumentException("设备类型 " + config.getDeviceType() + " 无对应采集器"); + } + + long endTime = System.currentTimeMillis() - OVERLAP_BACKOFF_MS; + long backMs = minutes != null && minutes > 0 ? minutes * 60 * 1000L : 60 * 60 * 1000L; + long startTime = endTime - backMs; + + String targetType = (dataType == null || dataType.trim().isEmpty()) + ? config.getDataTypes() : dataType.trim(); + + Map resultMap = new HashMap<>(); + resultMap.put("deviceId", config.getDeviceId()); + resultMap.put("deviceName", config.getDeviceName()); + resultMap.put("deviceType", config.getDeviceType()); + resultMap.put("startTime", startTime); + resultMap.put("endTime", endTime); + + List types = java.util.Arrays.asList(targetType.split(",")); + List> results = new java.util.ArrayList<>(); + int totalSuccess = 0; + int totalFail = 0; + + for (String type : types) { + type = type.trim(); + if (type.isEmpty()) { + continue; + } + long costStart = System.currentTimeMillis(); + Map item = new HashMap<>(); + item.put("dataType", type); + try { + CollectResult result = collector.collect(config, type, startTime, endTime); + item.put("status", "SUCCESS"); + item.put("successCount", result.getSuccessCount()); + item.put("failCount", result.getFailCount()); + item.put("totalCount", result.getTotalCount()); + totalSuccess += result.getSuccessCount(); + totalFail += result.getFailCount(); + } catch (Exception e) { + item.put("status", "FAIL"); + item.put("error", e.getMessage()); + totalFail++; + } + item.put("costMs", System.currentTimeMillis() - costStart); + results.add(item); + } + + resultMap.put("results", results); + resultMap.put("totalSuccess", totalSuccess); + resultMap.put("totalFail", totalFail); + return resultMap; + } + + /** + * 手动全量回补采集（同步，按时间滚动逐段拉取） + * 把 [startTime, endTime] 区间按 windowMinutes 子窗口由旧到新滚动推进逐段采集， + * 直到终点，用于历史数据回补（如天眼 weblog 全量采集）。 + * 说明：1) 不读/不写 Redis 断点，与定时增量互不影响； + * 2) 消息按 Kafka key(eventId) 幂等，重复采集不会重复落库； + * 3) 默认某段失败即终止，可设置 continueOnError=true 跳过失败段继续。 + * + * @param configId 设备采集配置ID + * @param dataType 数据类型，逗号分隔，如 weblog；为空则取配置中所有类型 + * @param startTime 起始时间(毫秒时间戳)，为空则取 endTime - backDays 天 + * @param endTime 结束时间(毫秒时间戳)，为空默认当前时间-1分钟 + * @param backDays 回看天数，仅 startTime 为空时生效，默认30天 + * @param windowMinutes 子窗口分钟数，默认30分钟 + * @param continueOnError 某段失败是否继续后续段，默认 false(失败即终止) + * @return 采集结果汇总（含每段明细） + */ + public Map collectFull(Long configId, String dataType, Long startTime, Long endTime, + Integer backDays, Integer windowMinutes, Boolean continueOnError) { + DeviceApiConfig config = configService.getById(configId); + if (config == null) { + throw new IllegalArgumentException("采集配置不存在: " + configId); + } + DeviceDataCollector collector = collectorMap.get(config.getDeviceType()); + if (collector == null) { + throw new IllegalArgumentException("设备类型 " + config.getDeviceType() + " 无对应采集器"); + } + + List types = new ArrayList<>(); + String rawTypes = (dataType == null || dataType.trim().isEmpty()) ? config.getDataTypes() : dataType; + if (rawTypes != null) { + for (String t : rawTypes.split(",")) { + t = t.trim(); + if (!t.isEmpty()) { + types.add(t); + } + } + } + if (types.isEmpty()) { + throw new IllegalArgumentException("未指定数据类型"); + } + + long end = (endTime != null && endTime > 0) ? endTime : System.currentTimeMillis() - OVERLAP_BACKOFF_MS; + long start; + if (startTime != null && startTime > 0) { + start = startTime; + } else { + long days = (backDays != null && backDays > 0) ? backDays : FULL_DEFAULT_BACK_DAYS; + start = end - days * 24 * 60 * 60 * 1000L; + } + if (start >= end) { + throw new IllegalArgumentException("startTime 必须小于 endTime"); + } + long windowMs = (windowMinutes != null && windowMinutes > 0) + ? windowMinutes * 60 * 1000L : FULL_DEFAULT_WINDOW_MS; + boolean continueOnErr = Boolean.TRUE.equals(continueOnError); + + long totalCostStart = System.currentTimeMillis(); + Map resultMap = new HashMap<>(); + resultMap.put("deviceId", config.getDeviceId()); + resultMap.put("deviceName", config.getDeviceName()); + resultMap.put("deviceType", config.getDeviceType()); + resultMap.put("dataTypes", types); + resultMap.put("startTime", start); + resultMap.put("endTime", end); + resultMap.put("windowMinutes", windowMs / 60_000L); + + List> segments = new ArrayList<>(); + long totalSuccess = 0; + long totalFail = 0; + int successSegments = 0; + int failSegments = 0; + boolean interrupted = false; + + for (String type : types) { + long cursor = start; + while (cursor < end) { + long segStart = cursor; + long segEnd = Math.min(segStart + windowMs, end); + long segCostStart = System.currentTimeMillis(); + Map item = new HashMap<>(); + item.put("dataType", type); + item.put("segStart", segStart); + item.put("segEnd", segEnd); + try { + CollectResult result = collector.collect(config, type, segStart, segEnd); + item.put("status", "SUCCESS"); + item.put("successCount", result.getSuccessCount()); + item.put("failCount", result.getFailCount()); + item.put("totalCount", result.getTotalCount()); + totalSuccess += result.getSuccessCount(); + totalFail += result.getFailCount(); + successSegments++; + } catch (Exception e) { + item.put("status", "FAIL"); + item.put("error", e.getMessage()); + failSegments++; + log.error("全量回补段失败: device={}, type={}, seg=[{} ~ {}], error={}", + config.getDeviceName(), type, + TimeUtils.format(segStart), TimeUtils.format(segEnd), e.getMessage()); + if (!continueOnErr) { + item.put("costMs", System.currentTimeMillis() - segCostStart); + segments.add(item); + interrupted = true; + break; + } + } + item.put("costMs", System.currentTimeMillis() - segCostStart); + segments.add(item); + log.info("全量回补段完成: device={}, type={}, seg=[{} ~ {}], success={}, fail={}, cost={}ms", + config.getDeviceName(), type, + TimeUtils.format(segStart), TimeUtils.format(segEnd), + item.get("successCount") == null ? 0 : item.get("successCount"), + item.get("failCount") == null ? 0 : item.get("failCount"), + item.get("costMs")); + cursor = segEnd; + } + if (interrupted) { + break; + } + } + + resultMap.put("segments", segments); + resultMap.put("successSegments", successSegments); + resultMap.put("failSegments", failSegments); + resultMap.put("totalSuccess", totalSuccess); + resultMap.put("totalFail", totalFail); + resultMap.put("interrupted", interrupted); + resultMap.put("costMs", System.currentTimeMillis() - totalCostStart); + + log.info("全量回补完成: device={}, types={}, start={}, end={}, 成功段={}, 失败段={}, success={}, fail={}, cost={}ms", + config.getDeviceName(), types, + TimeUtils.format(start), TimeUtils.format(end), + successSegments, failSegments, totalSuccess, totalFail, resultMap.get("costMs")); + return resultMap; + } + + /** + * 计算采集时间窗口（断点续传） + * @return [startTime, endTime] + */ + private long[] calculateWindow(DeviceApiConfig config, String dataType) { + long endTime = System.currentTimeMillis() - OVERLAP_BACKOFF_MS; + long lastTime = getLastCollectTime(config, dataType); + + long startTime; + if (lastTime > 0) { + // 上次采集时间回退1分钟，防止边界丢数据 + startTime = lastTime - OVERLAP_BACKOFF_MS; + } else { + // 首次采集：默认采集最近1小时的数据 + startTime = endTime - 60 * 60 * 1000; + } + + // 断点异常（晚于当前时间，如脏数据）时收敛为空窗口，避免窗口倒挂导致接口报错 + if (startTime > endTime) { + startTime = endTime; + } + + // 限制窗口最大跨度：默认30分钟；微步 info（日志流量）数据量大，单独限流为更小窗口， + // 避免单轮翻页过久阻塞线程池。截断时收窄 endTime 到实际采集终点， + // 断点只推进到真正采到的位置，剩余数据由下一轮继续采集，避免截断丢数据 + long maxWindowMs = MAX_WINDOW_MS; + if ("weibu".equals(config.getDeviceType()) && "info".equals(dataType)) { + long infoMaxMs = properties.getWeibuInfoMaxWindowMinutes() * 60 * 1000L; + if (infoMaxMs > 0 && infoMaxMs < maxWindowMs) { + maxWindowMs = infoMaxMs; + } + } + if (endTime - startTime > maxWindowMs) { + endTime = startTime + maxWindowMs; + } + return new long[]{startTime, endTime}; + } + + /** + * 获取断点续传时间戳 + */ + private long getLastCollectTime(DeviceApiConfig config, String dataType) { + String key = buildLastCollectKey(config, dataType); + try { + Object value = redisTemplate.opsForValue().get(key); + if (value != null) { + return Long.parseLong(value.toString()); + } + } catch (Exception e) { + log.debug("获取断点续传时间失败: key={}, error={}", key, e.getMessage()); + } + return 0; + } + + /** + * 更新断点续传时间戳 + */ + private void updateLastCollectTime(DeviceApiConfig config, String dataType, long timestamp) { + String key = buildLastCollectKey(config, dataType); + try { + redisTemplate.opsForValue().set(key, timestamp, LAST_COLLECT_TTL_DAYS, TimeUnit.DAYS); + } catch (Exception e) { + log.warn("更新断点续传时间失败: key={}, error={}", key, e.getMessage()); + } + } + + /** + * 构建断点续传 Redis Key + * device:api:last_collect_time:{deviceIp}:{deviceType}:{dataType} + */ + private String buildLastCollectKey(DeviceApiConfig config, String dataType) { + return LAST_COLLECT_KEY_PREFIX + + (config.getDeviceIp() != null ? config.getDeviceIp() : "unknown") + ":" + + config.getDeviceType() + ":" + + dataType; + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectorException.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectorException.java new file mode 100644 index 0000000..272d0a8 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/CollectorException.java @@ -0,0 +1,19 @@ +package com.collector; + +/** + * 采集器自定义异常 + */ +public class CollectorException extends RuntimeException { + + public CollectorException(String message) { + super(message); + } + + public CollectorException(String message, Throwable cause) { + super(message, cause); + } + + public CollectorException(Throwable cause) { + super(cause); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/DeviceDataCollector.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/DeviceDataCollector.java new file mode 100644 index 0000000..34f6695 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/DeviceDataCollector.java @@ -0,0 +1,38 @@ +package com.collector; + +import com.common.entity.DeviceApiConfig; + +/** + * 设备数据采集器统一接口 + * 所有 API 类数据采集器（天眼/微步/XDR）必须实现此接口 + */ +public interface DeviceDataCollector { + + /** + * 获取设备类型 + * @return tianyan / weibu / xdr + */ + String getDeviceType(); + + /** + * 认证并获取访问凭证（token / cookie 等） + * 凭证由 AuthManager 缓存，TTL 到期自动刷新 + */ + void authenticate() throws CollectorException; + + /** + * 当前采集器是否已认证成功 + */ + boolean isAuthenticated(); + + /** + * 采集指定时间范围内的数据 + * @param config 设备采集配置 + * @param dataType 数据类型（天眼：weblog/alarm；微步：attack/info） + * @param startTime 起始时间戳（毫秒） + * @param endTime 结束时间戳（毫秒） + * @return 采集结果（包含 Kafka 消息列表） + */ + CollectResult collect(DeviceApiConfig config, String dataType, long startTime, long endTime) + throws CollectorException; +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/FieldMapper.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/FieldMapper.java new file mode 100644 index 0000000..90d65a5 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/FieldMapper.java @@ -0,0 +1,120 @@ +package com.collector; + +import com.alibaba.fastjson2.JSONObject; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * 字段映射器 + * 将各厂商（天眼/微步）的原始字段映射为平台标准字段， + * 未映射字段原样保留，避免数据丢失。 + */ +public class FieldMapper { + + // ==================== 天眼字段映射 ==================== + + /** + * 天眼 weblog 访问日志字段映射：原始字段 -> 标准字段 + */ + private static final Map TIANYAN_WEBLOG_MAP = new HashMap<>(); + static { + TIANYAN_WEBLOG_MAP.put("_time", "eventTime"); + TIANYAN_WEBLOG_MAP.put("src_ip", "srcIp"); + TIANYAN_WEBLOG_MAP.put("dst_ip", "dstIp"); + TIANYAN_WEBLOG_MAP.put("src_port", "srcPort"); + TIANYAN_WEBLOG_MAP.put("dst_port", "dstPort"); + TIANYAN_WEBLOG_MAP.put("proto", "protocol"); + TIANYAN_WEBLOG_MAP.put("method", "httpMethod"); + TIANYAN_WEBLOG_MAP.put("uri", "url"); + TIANYAN_WEBLOG_MAP.put("user_agent", "userAgent"); + TIANYAN_WEBLOG_MAP.put("status", "httpStatus"); + TIANYAN_WEBLOG_MAP.put("request_body", "requestBody"); + TIANYAN_WEBLOG_MAP.put("response_body", "responseBody"); + } + + /** + * 天眼 alarm 告警日志字段映射：原始字段 -> 标准字段 + */ + private static final Map TIANYAN_ALARM_MAP = new HashMap<>(); + static { + TIANYAN_ALARM_MAP.put("_time", "eventTime"); + TIANYAN_ALARM_MAP.put("title", "eventName"); + TIANYAN_ALARM_MAP.put("alarm_sev", "severity"); + TIANYAN_ALARM_MAP.put("src_ip", "srcIp"); + TIANYAN_ALARM_MAP.put("dst_ip", "dstIp"); + TIANYAN_ALARM_MAP.put("src_port", "srcPort"); + TIANYAN_ALARM_MAP.put("dst_port", "dstPort"); + TIANYAN_ALARM_MAP.put("proto", "protocol"); + TIANYAN_ALARM_MAP.put("rule_id", "ruleId"); + TIANYAN_ALARM_MAP.put("event_id", "eventId"); + TIANYAN_ALARM_MAP.put("attack_type", "attackType"); + } + + // ==================== 微步字段映射 ==================== + + /** + * 微步 attack/info 日志字段映射：原始字段 -> 标准字段 + */ + private static final Map WEIBU_MAP = new HashMap<>(); + static { + WEIBU_MAP.put("event_time", "eventTime"); + WEIBU_MAP.put("src_ip", "srcIp"); + WEIBU_MAP.put("dst_ip", "dstIp"); + WEIBU_MAP.put("src_port", "srcPort"); + WEIBU_MAP.put("dst_port", "dstPort"); + WEIBU_MAP.put("threat_type", "attackType"); + WEIBU_MAP.put("risk_level", "severity"); + WEIBU_MAP.put("domain", "domain"); + WEIBU_MAP.put("url", "url"); + WEIBU_MAP.put("md5", "fileHash"); + WEIBU_MAP.put("sha256", "fileHash"); + WEIBU_MAP.put("action", "action"); + } + + /** + * 根据设备类型和数据类型选择映射表 + */ + private static Map getMap(String deviceType, String dataType) { + if ("tianyan".equalsIgnoreCase(deviceType)) { + return "alarm".equalsIgnoreCase(dataType) ? TIANYAN_ALARM_MAP : TIANYAN_WEBLOG_MAP; + } else if ("weibu".equalsIgnoreCase(deviceType)) { + return WEIBU_MAP; + } + return null; + } + + /** + * 映射单条原始数据 + * @param deviceType 设备类型 + * @param dataType 数据类型 + * @param raw 原始数据 + * @return 映射后的标准数据（保留原始字段） + */ + public static JSONObject map(String deviceType, String dataType, Map raw) { + JSONObject result = new JSONObject(new LinkedHashMap<>()); + Map fieldMap = getMap(deviceType, dataType); + + for (Map.Entry entry : raw.entrySet()) { + String key = entry.getKey(); + if (fieldMap != null && fieldMap.containsKey(key)) { + // 已映射字段：同时保留标准名和原始名 + result.put(fieldMap.get(key), entry.getValue()); + result.put(key, entry.getValue()); + } else { + // 未映射字段：原样保留 + result.put(key, entry.getValue()); + } + } + return result; + } + + /** + * 生成消息事件ID（用于 Kafka 消息 key） + */ + public static String generateEventId(String deviceType, String dataType, String rawId) { + String prefix = rawId == null || rawId.isEmpty() ? "" : rawId + "-"; + return prefix + deviceType + "-" + dataType + "-" + System.nanoTime(); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/TianyanDataCollector.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/TianyanDataCollector.java new file mode 100644 index 0000000..479fd91 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/TianyanDataCollector.java @@ -0,0 +1,682 @@ +package com.collector; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; +import com.common.entity.DeviceApiConfig; +import com.config.AppProperties; +import com.kafka.KafkaProducerUtil; +import com.util.TimeUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.client.HttpStatusCodeException; +import org.springframework.web.client.RestTemplate; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 天眼（天空卫士）API 数据采集器 + * 支持：weblog（访问日志）、alarm（告警日志） + * + * 认证方式：免密登录 + * 1. client_id = SHA256(CLIENT_ID_SEED + "|" + login_key) + * 2. client_secret = SHA256(CLIENT_SECRET_SEED + "|" + login_key) + * 3. X-Authorization = SHA256(JSON({client_id, username}) + timestamp + client_secret) + * 4. POST /skyeye/v1/admin/auth 获取 access_token + * 5. GET /skyeye/v1/admin/auth 提取 csrf_token + */ +@Component +public class TianyanDataCollector implements DeviceDataCollector { + + private static final Logger log = LoggerFactory.getLogger(TianyanDataCollector.class); + + /** 免密登录种子（固定常量） */ + private static final String CLIENT_ID_SEED = "mNSLP9UJCtBHtegjDPJnK3v"; + private static final String CLIENT_SECRET_SEED = "3460681205014671737"; + + /** 默认用户名 */ + private static final String DEFAULT_USERNAME = "xxadmin"; + + /** 认证相关路径 */ + private static final String AUTH_PATH = "/skyeye/v1/admin/auth"; + + /** 日志查询接口（weblog 与 alarm 均走此接口，参数参考官方抓包示例） */ + private static final String LOG_SEARCH_PATH = "/skyeye/v1/analysis/log-search/list"; + + /** 天眼 weblog 使用的 index */ + private static final String INDEX_WEBLOG = "skyeye-weblog"; + + /** 天眼告警日志使用的 index（官方抓包示例：多索引逗号分隔） */ + private static final String INDEX_ALARM = "alarm_collection,skyeye-sandbox,sandbox_mail,alarm_honeypot," + + "alarm_yunsuo,webids-webshell_dolog,webids-webattack_dolog,webids-ids_dolog,webids-ioc_dolog"; + + /** 从 HTML 提取 csrf_token（与官方 Python 正则一致） */ + private static final java.util.regex.Pattern META_CSRF_PATTERN = + java.util.regex.Pattern.compile(" headers = new HashMap<>(); + headers.put("X-Timestamp", timestamp); + headers.put("X-Authorization", xAuthorization); + headers.put("Accept", "application/json"); + + Map form = new HashMap<>(); + form.put("client_id", clientId); + form.put("username", username); + + String authUrl = host + AUTH_PATH; + String response = com.util.HttpUtil.postForm(restTemplate, authUrl, form, headers); + log.debug("天眼 POST 认证响应: {}", response); + + JSONObject respObj = JSON.parseObject(response); + if (respObj == null) { + throw new CollectorException("天眼认证响应为空"); + } + + // 解析 access_token（兼容多种结构） + // 真实平台：JSON 顶层 access_token；Mock/部分环境：data.data.access_token / data.access_token + String accessToken = respObj.getString("access_token"); + if (accessToken == null) { + JSONObject data = respObj.getJSONObject("data"); + if (data != null) { + if (data.getString("access_token") != null) { + accessToken = data.getString("access_token"); + } + JSONObject innerData = data.getJSONObject("data"); + if (innerData != null && innerData.getString("access_token") != null) { + accessToken = innerData.getString("access_token"); + } + } + } + if (accessToken == null) { + throw new CollectorException("天眼认证失败，未获取到 access_token: " + response); + } + authManager.saveToken(getDeviceType(), "access_token", accessToken); + + // 4. GET 提取 csrf_token（响应 Set-Cookie 为后续数据接口的登录凭证，与官方示例 getCsrfAndCokkies 一致） + String csrfUrl = host + AUTH_PATH + "?token=" + accessToken; + Map getHeaders = new HashMap<>(); + getHeaders.put("X-Timestamp", timestamp); + getHeaders.put("X-Authorization", xAuthorization); + getHeaders.put("Accept", "application/json"); + com.util.HttpUtil.GetResult authGet = com.util.HttpUtil.getWithCookies(restTemplate, csrfUrl, getHeaders); + String csrfResponse = authGet.getBody(); + log.debug("天眼 GET 认证响应: {}", csrfResponse); + + // 保存平台下发的真实 Cookie（name=value 拼接），数据接口请求必须携带；无 Set-Cookie 环境（如 Mock）则清除旧值 + String cookie = buildCookieHeader(authGet.getSetCookies()); + if (cookie != null && !cookie.isEmpty()) { + authManager.saveToken(getDeviceType(), "cookie", cookie); + } else { + authManager.removeToken(getDeviceType(), "cookie"); + } + + String csrfToken = extractCsrfToken(csrfResponse); + if (csrfToken != null) { + authManager.saveToken(getDeviceType(), "csrf_token", csrfToken); + } + log.info("天眼认证成功, host={}, username={}", host, username); + } catch (CollectorException e) { + throw e; + } catch (Exception e) { + throw new CollectorException("天眼认证异常: " + e.getMessage(), e); + } + } + + /** + * 从 GET 认证响应中提取 csrf_token + * 真实平台返回 HTML（），Mock 返回 JSON，兼容两种格式 + */ + private String extractCsrfToken(String response) { + if (response == null || response.isEmpty()) { + return null; + } + // 1. 真实平台 HTML：（与官方 Python 正则一致） + java.util.regex.Matcher metaMatcher = META_CSRF_PATTERN.matcher(response); + if (metaMatcher.find()) { + return metaMatcher.group(1); + } + // 2. 尝试 JSON 格式 + try { + JSONObject obj = JSON.parseObject(response); + if (obj != null) { + JSONObject data = obj.getJSONObject("data"); + if (data != null) { + String token = data.getString("csrf_token"); + if (token == null) { + token = data.getString("data"); + } + if (token != null) { + return token; + } + } + } + } catch (Exception ignored) { + } + // 3. 兼容 csrf_token 键值形式 + java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("csrf_token[\"']?\\s*[:=]\\s*[\"']([^\"']+)[\"']"); + java.util.regex.Matcher matcher = pattern.matcher(response); + if (matcher.find()) { + return matcher.group(1); + } + return null; + } + + @Override + public boolean isAuthenticated() { + return authManager.hasToken(getDeviceType(), "access_token"); + } + + @Override + public CollectResult collect(DeviceApiConfig config, String dataType, long startTime, long endTime) + throws CollectorException { + // 未认证则先认证 + if (!isAuthenticated()) { + authenticate(config); + } + + String host = config.getHostApi(); + if (host == null || host.isEmpty()) { + host = properties.getTianyanHost(); + } + + // 从配置解析 branch_id + String branchId = properties.getTianyanBranchId(); + String authConfig = config.getAuthConfig(); + if (authConfig != null && !authConfig.isEmpty()) { + try { + JSONObject auth = JSON.parseObject(authConfig); + if (auth.getString("branch_id") != null) { + branchId = auth.getString("branch_id"); + } + } catch (Exception ignored) { + } + } + + CollectResult result = new CollectResult(); + result.setDeviceType(getDeviceType()); + result.setDataType(dataType); + result.setStartTime(startTime); + result.setEndTime(endTime); + + try { + // 窗口按子窗口切片逐段采集，控制单次查询数据量，预防平台深分页上限（返回空页被误判为已取完） + long subWindowMs = properties.getTianyanSubWindowMinutes() * 60_000L; + if (subWindowMs <= 0) { + subWindowMs = endTime - startTime; + } + long cursor = startTime; + while (cursor < endTime) { + long segEnd = Math.min(cursor + subWindowMs, endTime); + collectSegment(config, dataType, host, branchId, cursor, segEnd, result, 0); + cursor = segEnd; + } + log.info("天眼采集完成: dataType={}, start={}, end={}, total={}, success={}, fail={}", + dataType, startTime, endTime, result.getTotalCount(), result.getSuccessCount(), result.getFailCount()); + return result; + } catch (CollectorException e) { + throw e; + } catch (Exception e) { + throw new CollectorException("天眼采集异常: " + e.getMessage(), e); + } + } + + /** + * 采集单个时间子窗口 [segStart, segEnd]，窗口数据量超过平台深分页上限时对半拆分重采 + * 深分页截断判定：非首页返回空页（第 N-1 页为满页才会翻到第 N 页）说明窗口尾部数据未取完； + * 此时对半拆分窗口递归重采（拆分前已发送的数据靠 Kafka key eventId 幂等去重）， + * 达到最大拆分深度仍截断则抛异常使本轮失败、断点不推进，避免尾部数据被静默丢弃 + */ + private void collectSegment(DeviceApiConfig config, String dataType, String host, String branchId, + long segStart, long segEnd, CollectResult result, int depth) + throws CollectorException { + int pageSize = properties.getTianyanPageSize(); + int page = 1; + boolean hasMore = true; + // 401 重认证重试次数（防账号异常时无限循环） + int unauthorizedRetries = 0; + // 窗口数据总数（首页响应提供时用于精确判定取完/截断，-1=未知） + long totalHint = -1; + // 本子窗口已采条数 + long sentInSeg = 0; + + while (hasMore) { + // 每页现取最新 csrf_token，认证重试后后续页/子窗口自动使用新 token + String csrfToken = authManager.getToken(getDeviceType(), "csrf_token"); + if (csrfToken == null) { + authenticate(config); + csrfToken = authManager.getToken(getDeviceType(), "csrf_token"); + } + + // weblog 与 alarm 均走 log-search/list，差异仅在 index/category/mode（官方抓包示例一致） + String url = buildLogSearchUrl(host, branchId, dataType, segStart, segEnd, page, pageSize, csrfToken); + + // 与官方示例一致：数据接口仅携带认证时下发并保存的 Cookie（登录态），URL 带 csrf_token，不额外带签名头 + Map headers = new HashMap<>(); + headers.put("Accept", "application/json"); + String sessionCookie = authManager.getToken(getDeviceType(), "cookie"); + if (sessionCookie != null && !sessionCookie.isEmpty()) { + headers.put("Cookie", sessionCookie); + } + + // 打印每页请求的时间参数（可读格式），便于核对子窗口切片与平台返回是否一致 + log.info("天眼请求: dataType={}, 子窗口=[{} ~ {}], page={}", + dataType, TimeUtils.format(segStart), TimeUtils.format(segEnd), page); + + String response; + try { + response = com.util.HttpUtil.get(restTemplate, url, headers); + } catch (HttpStatusCodeException httpEx) { + // 401 登录态失效（Cookie/csrf_token 过期）：重新认证后重试当前页（限2次防死循环） + if (httpEx.getStatusCode() == HttpStatus.UNAUTHORIZED && unauthorizedRetries < 2) { + unauthorizedRetries++; + log.warn("天眼返回401登录态失效，重新认证后重试当前页: dataType={}, 子窗口=[{} ~ {}], page={}", + dataType, TimeUtils.format(segStart), TimeUtils.format(segEnd), page); + authenticate(config); + continue; + } + // 402 PAYMENT REQUIRED 属平台 license/授权问题，重认证无效，保持抛出由上层重试与记录 + throw new CollectorException("天眼采集失败: " + httpEx.getStatusCode() + + ": " + truncate(httpEx.getResponseBodyAsString(), 300), httpEx); + } + log.debug("天眼 {} 采集响应(seg=[{} ~ {}], page={}): {}", + dataType, TimeUtils.format(segStart), TimeUtils.format(segEnd), page, truncate(response, 500)); + + JSONObject respObj = JSON.parseObject(response); + if (respObj == null) { + throw new CollectorException("天眼采集响应为空"); + } + + // 校验返回状态 + Integer status = respObj.getInteger("status"); + if (status != null && status != 1000) { + String message = respObj.getString("message"); + // 认证过期则重新认证重试 + if ("csrf_token error".equalsIgnoreCase(message) || "token error".equalsIgnoreCase(message) + || "auth failed".equalsIgnoreCase(message)) { + log.warn("天眼认证过期，重新认证后重试"); + authenticate(config); + continue; + } + throw new CollectorException("天眼采集失败: " + message); + } + + // 解析 hits + JSONArray hits = parseHits(respObj); + if (hits == null || hits.isEmpty()) { + if (page > 1) { + // 深分页截断：上一页为满页本页却为空，窗口尾部数据未取完 + if (depth >= MAX_SPLIT_DEPTH) { + throw new CollectorException("天眼窗口数据量超过平台深分页上限且已达最小拆分粒度: seg=[" + + TimeUtils.format(segStart) + " ~ " + TimeUtils.format(segEnd) + "], page=" + page + + "，本轮失败断点不推进，防止尾部数据静默丢失"); + } + long mid = (segStart + segEnd) / 2; + log.warn("天眼深分页截断(seg=[{} ~ {}], page={})，对半拆分重采: depth={}", + TimeUtils.format(segStart), TimeUtils.format(segEnd), page, depth + 1); + collectSegment(config, dataType, host, branchId, segStart, mid, result, depth + 1); + collectSegment(config, dataType, host, branchId, mid, segEnd, result, depth + 1); + } + // 首页即为空：窗口内确实无数据 + break; + } + + for (int i = 0; i < hits.size(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = hit.getJSONObject("_source"); + if (source == null) { + source = hit; + } + // 字段映射 + 构建 Kafka 消息 + Map rawMap = new HashMap<>(); + for (String key : source.keySet()) { + rawMap.put(key, source.get(key)); + } + JSONObject mapped = FieldMapper.map(getDeviceType(), dataType, rawMap); + + String eventId = rawMap.get("event_id") != null + ? rawMap.get("event_id").toString() + : rawMap.get("_id") != null ? rawMap.get("_id").toString() : ""; + + long collectTime = System.currentTimeMillis(); + // 前缀消息(设备信息) 与 日志内容体 合并为 Kafka value + String value = buildKafkaKey(config, eventId) + + KafkaProducerUtil.buildValue(getDeviceType(), dataType, collectTime, mapped); + // Kafka key 使用 key-eventId，保证多源采集高并发下不重复（拆分重采的重复数据也靠它去重） + String key = "key-" + (eventId == null || eventId.isEmpty() ? System.nanoTime() : eventId); + + if (kafkaProducer.send(key, value)) { + result.addMessage(value); + result.incrementSuccess(); + } else { + result.incrementFail(); + } + } + + page++; + + // 分页判断：如果本页不足 pageSize 则结束 + if (hits.size() < pageSize) { + hasMore = false; + } + } + } + + /** + * 构建日志查询 URL（weblog 与 alarm 共用 log-search/list，参数与官方抓包示例逐项一致） + * 差异：weblog → index=skyeye-weblog, category=skyeye, mode=fast_model + * alarm → index=告警多索引, category=event, mode=advance_model + * 参数：branch_id&start_time&end_time&keyword&index&category&offset&limit + * &asset_group_ids&mode&stime&etime&interval&page&size&key_fields&graph_conf&curBranch&sess_key_ids&csrf_token&r + */ + private String buildLogSearchUrl(String host, String branchId, String dataType, long startTime, long endTime, + int page, int pageSize, String csrfToken) { + boolean alarm = "alarm".equalsIgnoreCase(dataType); + String index = alarm ? INDEX_ALARM : INDEX_WEBLOG; + String category = alarm ? "event" : "skyeye"; + String mode = alarm ? "advance_model" : "fast_model"; + + // offset 为页码语义：1=第1页，2=第2页（平台确认，非记录偏移量） + int offset = page; + StringBuilder sb = new StringBuilder(host).append(LOG_SEARCH_PATH).append("?"); + sb.append("branch_id=").append(branchId) // 原样传递（配置中可能已含 % 编码） + .append("&start_time=").append(startTime) + .append("&end_time=").append(endTime) + .append("&keyword=") + .append("&index=").append(index) + .append("&category=").append(category) + .append("&offset=").append(offset) + .append("&limit=").append(pageSize) + .append("&asset_group_ids=") + .append("&mode=").append(mode) + .append("&stime=").append(startTime) + .append("&etime=").append(endTime) + .append("&interval=20s") + .append("&page=").append(page) + .append("&size=").append(pageSize) + .append("&key_fields=@timestamp") + .append("&graph_conf=%7B%7D") + .append("&curBranch=").append(branchId) + .append("&sess_key_ids=") + .append("&csrf_token=").append(csrfToken) + .append("&r=").append(System.nanoTime()); + return sb.toString(); + } + + /** + * 从响应中解析记录数组，兼容 log-search 与 alarm/list 的多种响应结构： + * data.data.search.hits / data.data.hits|rows|list / data.hits|rows|list / data(数组) / 顶层 hits|rows|list + */ + private JSONArray parseHits(JSONObject respObj) { + JSONArray arr = tryArrayPath(respObj, "data", "data", "search", "hits"); + if (arr != null) { + return arr; + } + arr = tryArrayPath(respObj, "data", "data", "hits"); + if (arr != null) { + return arr; + } + arr = tryArrayPath(respObj, "data", "data", "rows"); + if (arr != null) { + return arr; + } + arr = tryArrayPath(respObj, "data", "data", "list"); + if (arr != null) { + return arr; + } + arr = tryArrayPath(respObj, "data", "hits"); + if (arr != null) { + return arr; + } + arr = tryArrayPath(respObj, "data", "rows"); + if (arr != null) { + return arr; + } + arr = tryArrayPath(respObj, "data", "list"); + if (arr != null) { + return arr; + } + // data 本身为数组 + Object data = respObj.get("data"); + if (data instanceof JSONArray) { + return (JSONArray) data; + } + arr = tryArrayPath(respObj, "hits"); + if (arr != null) { + return arr; + } + arr = tryArrayPath(respObj, "rows"); + if (arr != null) { + return arr; + } + arr = tryArrayPath(respObj, "list"); + if (arr != null) { + return arr; + } + return null; + } + + /** + * 沿字段路径逐层取值，若最终值为 JSONArray 则返回，否则返回 null + */ + private JSONArray tryArrayPath(JSONObject obj, String... path) { + Object cur = obj; + for (String key : path) { + if (!(cur instanceof JSONObject)) { + return null; + } + cur = ((JSONObject) cur).get(key); + if (cur == null) { + return null; + } + } + return cur instanceof JSONArray ? (JSONArray) cur : null; + } + + /** + * 从响应中解析窗口数据总数（响应结构多样，做多路径兼容；无 total 返回 null） + */ + private Long parseTotal(JSONObject respObj) { + Long v = tryLong(respObj, "data", "data", "search", "total"); + if (v == null) { + v = tryLong(respObj, "data", "data", "search", "hits", "total"); + } + if (v == null) { + v = tryLong(respObj, "data", "data", "total"); + } + if (v == null) { + v = tryLong(respObj, "data", "data", "hits", "total"); + } + if (v == null) { + v = tryLong(respObj, "data", "total"); + } + if (v == null) { + v = tryLong(respObj, "data", "hits", "total"); + } + if (v == null) { + v = tryLong(respObj, "total"); + } + return v; + } + + /** + * 沿字段路径逐层取值，若最终值为数值则返回（兼容 ES hits.total 为 {value: N} 对象的形式） + */ + private Long tryLong(JSONObject obj, String... path) { + Object cur = obj; + for (String key : path) { + if (!(cur instanceof JSONObject)) { + return null; + } + cur = ((JSONObject) cur).get(key); + if (cur == null) { + return null; + } + } + if (cur instanceof Number) { + return ((Number) cur).longValue(); + } + if (cur instanceof JSONObject) { + Object value = ((JSONObject) cur).get("value"); + if (value instanceof Number) { + return ((Number) value).longValue(); + } + } + return null; + } + + /** + * 构建 Kafka 消息 key + * 格式参照 syslog-serve：[*receive_time=xx device_id=xx device_name=xx vendor=xx data_type=json device_collect_id=xx eventId=xx*] + */ + private String buildKafkaKey(DeviceApiConfig config, String eventId) { + StringBuilder sb = new StringBuilder(); + sb.append("[").append("receive_time=").append(TimeUtils.formatCompactMs(System.currentTimeMillis())) + .append(" device_id=").append(config.getDeviceId() != null ? config.getDeviceId() : config.getId()) + .append(" device_name=").append(config.getDeviceName() != null ? config.getDeviceName() : "天眼") + .append(" vendor=").append("天眼") + .append(", data_type=json") + .append(" device_collect_id=").append(properties.getDeviceCollectId() != null ? properties.getDeviceCollectId() : config.getId()) + .append(" eventId=").append(eventId) + .append("]"); + return sb.toString(); + } + + /** + * SHA256 哈希（返回小写十六进制） + */ + private String sha256(String input) throws Exception { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hash = digest.digest(input.getBytes(StandardCharsets.UTF_8)); + StringBuilder hexString = new StringBuilder(); + for (byte b : hash) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + } + + /** + * 将响应 Set-Cookie 解析为 Cookie 请求头值（name=value; name2=value2） + * 与官方示例 getCsrfAndCokkies() 拼接逻辑一致：仅保留 name=value，忽略 Path/Domain 等属性 + */ + private String buildCookieHeader(List setCookies) { + if (setCookies == null || setCookies.isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (String sc : setCookies) { + if (sc == null || sc.trim().isEmpty()) { + continue; + } + String pair = sc.split(";", 2)[0].trim(); + if (pair.isEmpty() || !pair.contains("=")) { + continue; + } + if (sb.length() > 0) { + sb.append("; "); + } + sb.append(pair); + } + return sb.toString(); + } + + /** + * 截断长字符串用于日志 + */ + private String truncate(String str, int maxLen) { + if (str == null) { + return ""; + } + return str.length() > maxLen ? str.substring(0, maxLen) + "..." : str; + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/WeibuDataCollector.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/WeibuDataCollector.java new file mode 100644 index 0000000..e8cdc76 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/WeibuDataCollector.java @@ -0,0 +1,459 @@ +package com.collector; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; +import com.common.entity.DeviceApiConfig; +import com.config.AppProperties; +import com.kafka.KafkaProducerUtil; +import com.util.TimeUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +/** + * 微步在线威胁感知平台（TDP）API 数据采集器 + * 支持：attack（攻击流量，含 risk）、info（日志流量） + * + * 认证方式：HMAC-SHA256 签名 + * SignData = ApiKey + Timestamp + * Sign = Base64.encodeBase64URLSafeString(HmacSHA256(SignData, Secret)) + * POST /api/v1/log/searchBySql?api_key=&auth_timestamp=&sign= + * + * 取数方式：size 两段式（与验证可运行的取数脚本一致） + * 1) size=1 探明窗口内数据总数 total（响应秒回） + * 2) size=total 一次取全窗口数据 + * 不使用 page 翻页对象：平台 page 翻页存在深分页上限（约 4000 条后回绕返回重复页）， + * size 一次取全可彻底绕开该问题；窗口数据量超过 size-cap 时对半拆分窗口递归采集 + */ +@Component +public class WeibuDataCollector implements DeviceDataCollector { + + private static final Logger log = LoggerFactory.getLogger(WeibuDataCollector.class); + + /** 查询接口路径 */ + private static final String SEARCH_BY_SQL_URI = "/api/v1/log/searchBySql"; + + /** 微步签名过期错误码：auth_timestamp 超过平台有效期，需刷新签名后重试 */ + private static final int SIGN_EXPIRED_CODE = 1002; + + /** + * 窗口数据量超过单次取数上限(size-cap)时对半拆分的最大深度： + * 5分钟 → 2.5分钟 → 75秒 → 37.5秒 → 18.75秒 → 9.4秒 → 4.7秒 + * 洪峰期数据密度可达 2800条/秒（实测7秒窗口19610条），深度4不足以拆到上限以内 + */ + private static final int MAX_SPLIT_DEPTH = 6; + + /** + * 查询字段（与官方调用示例的 columns 一致：{label, value} 对象数组，attack/info 共用） + */ + private static final String[][] WEIBU_COLUMNS = { + {"告警时间", "time"}, + {"威胁名称", "threat.name"}, + {"攻击结果", "threat.result"}, + {"严重级别", "threat.severity"}, + {"攻击阶段", "threat.phase"}, + {"威胁类型", "threat.type"}, + {"告警主机", "machine"}, + {"源IP", "net.src_ip"}, + {"源端口", "net.src_port"}, + {"目的IP", "net.dest_ip"}, + {"目的端口", "net.dest_port"}, + {"协议", "net.type"}, + {"HTTP方法", "net.http.method"}, + {"HTTP状态码", "net.http.status"}, + {"请求头", "net.http.reqs_header"}, + {"请求体", "net.http.reqs_body"}, + {"响应头", "net.http.resp_header"}, + {"响应体", "net.http.resp_body"}, + {"URL/域名", "data"}, + {"IOC", "threat.ioc"}, + {"是否APT", "threat.is_apt"}, + {"威胁标签", "threat.tag"}, + {"处理状态", "threat.status"}, + {"资产名称", "assets.name"}, + {"业务组", "assets.group_name"} + }; + + @Autowired + private RestTemplate restTemplate; + + @Autowired + private AppProperties properties; + + @Autowired + private KafkaProducerUtil kafkaProducer; + + @Override + public String getDeviceType() { + return "weibu"; + } + + @Override + public void authenticate() throws CollectorException { + // 微步无需持久化凭证，每次请求实时生成签名 + if (properties.getWeibuApiKey() == null || properties.getWeibuApiKey().isEmpty() + || properties.getWeibuSecret() == null || properties.getWeibuSecret().isEmpty()) { + throw new CollectorException("微步 api_key 或 secret 未配置"); + } + } + + @Override + public boolean isAuthenticated() { + return properties.getWeibuApiKey() != null && !properties.getWeibuApiKey().isEmpty() + && properties.getWeibuSecret() != null && !properties.getWeibuSecret().isEmpty(); + } + + @Override + public CollectResult collect(DeviceApiConfig config, String dataType, long startTime, long endTime) + throws CollectorException { + // 微步签名需要秒级时间戳；签名在新鲜有效期内跨请求复用，1002 时刷新 + long timestamp = System.currentTimeMillis() / 1000; + String apiKey = resolveApiKey(config); + String secret = resolveSecret(config); + String sign = generateSign(apiKey, secret, timestamp); + String host = config.getHostApi() != null && !config.getHostApi().isEmpty() + ? config.getHostApi() : properties.getWeibuHost(); + AuthState auth = new AuthState(timestamp, sign); + + CollectResult result = new CollectResult(); + result.setDeviceType(getDeviceType()); + result.setDataType(dataType); + result.setStartTime(startTime); + result.setEndTime(endTime); + + try { + collectSegment(config, dataType, host, apiKey, secret, auth, startTime, endTime, result, 0); + log.info("微步采集完成: dataType={}, start={}, end={}, total={}, success={}, fail={}", + dataType, startTime, endTime, result.getTotalCount(), result.getSuccessCount(), result.getFailCount()); + return result; + } catch (CollectorException e) { + throw e; + } catch (Exception e) { + throw new CollectorException("微步采集异常: " + e.getMessage(), e); + } + } + + /** + * 采集单个时间窗口 [segStart, segEnd]，size 两段式取数： + * 1) size=1 探明窗口内数据总数 total + * 2) size=total 一次取全，并用 total 做完整性校验 + * 窗口数据量超过单次取数上限(size-cap)时对半拆分窗口递归重采； + * 取回条数与 total 不符时先重试一次，仍不符则拆分重采（相邻子窗口边界重叠数据靠 Kafka key eventId 幂等去重）； + * 达到最大拆分深度仍不完整则抛异常使断点不推进，防止数据静默丢失 + */ + private void collectSegment(DeviceApiConfig config, String dataType, String host, String apiKey, String secret, + AuthState auth, long segStart, long segEnd, CollectResult result, int depth) + throws CollectorException { + int sizeCap = properties.getWeibuFetchSizeCap(); + + // 1. size=1 探明窗口内数据总数 + JSONObject probeBody = buildQueryBody(dataType, segStart / 1000, segEnd / 1000, 1); + log.info("微步请求: dataType={}, 窗口=[{} ~ {}], size=1(探测total), depth={}", + dataType, TimeUtils.format(segStart), TimeUtils.format(segEnd), depth); + Long total = extractTotal(weibuPost(host, apiKey, secret, auth, probeBody)); + if (total == null) { + throw new CollectorException("微步探测响应缺少 total 字段: dataType=" + dataType + + ", 窗口=[" + TimeUtils.format(segStart) + " ~ " + TimeUtils.format(segEnd) + "]"); + } + if (total <= 0) { + return; // 窗口内无数据 + } + + // 2. 数据量超过单次取数上限：对半拆分窗口递归重采 + if (total > sizeCap) { + splitAndRecurse(config, dataType, host, apiKey, secret, auth, segStart, segEnd, result, depth, + "窗口数据量" + total + "超过单次取数上限" + sizeCap); + return; + } + + // 3. size=total 一次取全 + JSONObject fetchBody = buildQueryBody(dataType, segStart / 1000, segEnd / 1000, total.intValue()); + log.info("微步请求: dataType={}, 窗口=[{} ~ {}], total={}, size={}", + dataType, TimeUtils.format(segStart), TimeUtils.format(segEnd), total, total.intValue()); + JSONArray dataList = extractDataList(weibuPost(host, apiKey, secret, auth, fetchBody)); + int fetched = dataList == null ? 0 : dataList.size(); + + // 4. 完整性校验：条数与 total 不符重试一次，仍不符则拆分重采 + if (fetched != total) { + log.warn("微步 {} 取回条数与total不符({}/{}), 重试一次: 窗口=[{} ~ {}]", + dataType, fetched, total, TimeUtils.format(segStart), TimeUtils.format(segEnd)); + dataList = extractDataList(weibuPost(host, apiKey, secret, auth, fetchBody)); + fetched = dataList == null ? 0 : dataList.size(); + if (fetched != total) { + splitAndRecurse(config, dataType, host, apiKey, secret, auth, segStart, segEnd, result, depth, + "取回条数" + fetched + "与total" + total + "不符"); + return; + } + } + + // 5. 发送 Kafka + if (dataList != null && !dataList.isEmpty()) { + sendItems(config, dataType, dataList, result); + } + } + + /** + * 对半拆分窗口递归重采；达到最大拆分深度仍不完整则抛异常，断点不推进防数据丢失 + */ + private void splitAndRecurse(DeviceApiConfig config, String dataType, String host, String apiKey, String secret, + AuthState auth, long segStart, long segEnd, CollectResult result, int depth, + String reason) throws CollectorException { + if (depth >= MAX_SPLIT_DEPTH) { + throw new CollectorException("微步窗口已达最小拆分粒度仍无法一次取全(" + reason + "): 窗口=[" + + TimeUtils.format(segStart) + " ~ " + TimeUtils.format(segEnd) + "]" + + "，本轮失败断点不推进，防止数据静默丢失；可调大 device.weibu.fetch-size-cap 或调小 info-max-window-minutes"); + } + long mid = (segStart + segEnd) / 2; + log.warn("微步 {} {}，对半拆分重采: [{} ~ {}] → [{} ~ {}]+[{} ~ {}], depth={}", + dataType, reason, + TimeUtils.format(segStart), TimeUtils.format(segEnd), + TimeUtils.format(segStart), TimeUtils.format(mid), + TimeUtils.format(mid), TimeUtils.format(segEnd), depth + 1); + collectSegment(config, dataType, host, apiKey, secret, auth, segStart, mid, result, depth + 1); + collectSegment(config, dataType, host, apiKey, secret, auth, mid, segEnd, result, depth + 1); + } + + /** + * 把单窗口数据映射并发送到 Kafka + */ + private void sendItems(DeviceApiConfig config, String dataType, JSONArray dataList, CollectResult result) { + for (int i = 0; i < dataList.size(); i++) { + JSONObject item = dataList.getJSONObject(i); + Map rawMap = new HashMap<>(); + for (String key : item.keySet()) { + rawMap.put(key, item.get(key)); + } + JSONObject mapped = FieldMapper.map(getDeviceType(), dataType, rawMap); + + String eventId = rawMap.get("id") != null ? rawMap.get("id").toString() : ""; + long collectTime = System.currentTimeMillis(); + // 前缀消息(设备信息) 与 日志内容体 合并为 Kafka value + String value = buildKafkaKey(config, eventId) + + KafkaProducerUtil.buildValue(getDeviceType(), dataType, collectTime, mapped); + // Kafka key 使用 key-eventId，保证多源采集高并发下不重复（相邻子窗口边界重叠数据也靠它去重） + String key = "key-" + (eventId == null || eventId.isEmpty() ? System.nanoTime() : eventId); + + if (kafkaProducer.send(key, value)) { + result.addMessage(value); + result.incrementSuccess(); + } else { + result.incrementFail(); + } + } + } + + /** + * 微步查询共用：签名 → POST → response_code 校验，返回完整响应 + * 收到 1002(sign expired) 时刷新签名重试一次，保证当前请求不失败； + * 签名在新鲜有效期内跨请求复用，仅失效时才重新生成，避免每请求重复签名计算 + */ + private JSONObject weibuPost(String host, String apiKey, String secret, AuthState auth, JSONObject body) + throws CollectorException { + for (int attempt = 1; attempt <= 2; attempt++) { + String url = host + SEARCH_BY_SQL_URI + + "?api_key=" + apiKey + + "&auth_timestamp=" + auth.timestamp + + "&sign=" + auth.sign; + + String response = com.util.HttpUtil.postJson(restTemplate, url, body.toJSONString(), null); + log.debug("微步采集响应(attempt={}): {}", attempt, truncate(response, 500)); + + JSONObject parsed = JSON.parseObject(response); + if (parsed == null) { + throw new CollectorException("微步采集响应为空"); + } + + Integer responseCode = parsed.getInteger("response_code"); + if (responseCode == null || responseCode == 0) { + return parsed; + } + + if (responseCode == SIGN_EXPIRED_CODE && attempt == 1) { + // 签名过期：刷新时间戳并重新签名，重试当前请求 + log.warn("微步签名过期(response_code=1002)，刷新签名重试当前请求"); + auth.timestamp = System.currentTimeMillis() / 1000; + auth.sign = generateSign(apiKey, secret, auth.timestamp); + continue; + } + + // 微步实际返回 verbose_msg 字段（response_msg 恒为 null），做兼容读取 + String verboseMsg = parsed.getString("verbose_msg") != null + ? parsed.getString("verbose_msg") : parsed.getString("response_msg"); + throw new CollectorException("微步采集失败: response_code=" + responseCode + + ", verbose_msg=" + verboseMsg + + ", 完整响应: " + truncate(response, 1000)); + } + throw new CollectorException("微步采集请求未获得有效响应"); + } + + /** + * 从响应 data 对象提取 total（平台在 data.total 返回窗口内数据总数），无 total 返回 null + */ + private Long extractTotal(JSONObject respObj) { + JSONObject data = respObj.getJSONObject("data"); + if (data == null) { + return null; + } + return data.getLong("total"); + } + + /** + * 解析数据列表：data.data（兼容 data.data.list / 顶层 data 数组） + */ + private JSONArray extractDataList(JSONObject respObj) { + JSONArray dataList = null; + JSONObject data = respObj.getJSONObject("data"); + if (data != null) { + dataList = data.getJSONArray("data"); + if (dataList == null && data.getJSONObject("data") != null) { + dataList = data.getJSONObject("data").getJSONArray("list"); + } + } + if (dataList == null) { + dataList = respObj.getJSONArray("data"); + } + return dataList; + } + + /** + * 构造查询 Body（size 两段式取数，与验证可运行的取数脚本一致；不再使用 page 翻页对象） + */ + private JSONObject buildQueryBody(String dataType, long timeFrom, long timeTo, int size) { + JSONObject body = new JSONObject(); + // net_data_type: attack 口径含 risk（与验证脚本一致，risk 类攻击不漏采）/ info-日志流量 + JSONArray netDataType = new JSONArray(); + if ("attack".equalsIgnoreCase(dataType)) { + netDataType.add("attack"); + netDataType.add("risk"); + } else { + netDataType.add(dataType); + } + body.put("net_data_type", netDataType); + body.put("time_from", timeFrom); + body.put("time_to", timeTo); + + // 资产组（空表示全部） + body.put("assets_group", new JSONArray()); + body.put("search_connection", false); + + // sql 置空，返回字段由 columns 指定（官方示例：columns 为 {label,value} 对象数组，不可用字符串 "*"） + body.put("sql", ""); + JSONArray columns = new JSONArray(); + for (String[] col : WEIBU_COLUMNS) { + JSONObject c = new JSONObject(); + c.put("label", col[0]); + c.put("value", col[1]); + columns.add(c); + } + body.put("columns", columns); + // 官方示例为倒序 + body.put("asc", false); + + // 顶层 size：单次取数条数（接口默认最多返回1000条，需显式设置；size=1 用于探测 total） + body.put("size", size); + + return body; + } + + /** + * 生成微步 API 签名: + * Sign = Base64.encodeBase64URLSafeString(HmacSHA256(ApiKey+Timestamp, Secret)) + */ + private String generateSign(String apiKey, String secret, long timestamp) { + try { + String signData = apiKey + timestamp; + Mac mac = Mac.getInstance("HmacSHA256"); + SecretKeySpec keySpec = new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256"); + mac.init(keySpec); + byte[] hmacBytes = mac.doFinal(signData.getBytes(StandardCharsets.UTF_8)); + // URL Safe Base64：将"+"替换为"-"、"/"替换为"_"并去除末尾"=" + String base64 = Base64.getEncoder().encodeToString(hmacBytes); + return base64.replace('+', '-').replace('/', '_').replaceAll("=+$", ""); + } catch (Exception e) { + throw new CollectorException("微步签名计算失败: " + e.getMessage(), e); + } + } + + /** + * 解析 api_key：优先配置中的 authConfig，其次全局配置 + */ + private String resolveApiKey(DeviceApiConfig config) { + String authConfig = config.getAuthConfig(); + if (authConfig != null && !authConfig.isEmpty()) { + try { + JSONObject auth = JSON.parseObject(authConfig); + if (auth.getString("api_key") != null) { + return auth.getString("api_key"); + } + } catch (Exception ignored) { + } + } + return properties.getWeibuApiKey(); + } + + /** + * 解析 secret + */ + private String resolveSecret(DeviceApiConfig config) { + String authConfig = config.getAuthConfig(); + if (authConfig != null && !authConfig.isEmpty()) { + try { + JSONObject auth = JSON.parseObject(authConfig); + if (auth.getString("secret") != null) { + return auth.getString("secret"); + } + } catch (Exception ignored) { + } + } + return properties.getWeibuSecret(); + } + + /** + * 构建 Kafka 消息 key + */ + private String buildKafkaKey(DeviceApiConfig config, String eventId) { + StringBuilder sb = new StringBuilder(); + sb.append("[").append("receive_time=").append(TimeUtils.formatCompactMs(System.currentTimeMillis())) + .append(" device_id=").append(config.getDeviceId() != null ? config.getDeviceId() : config.getId()) + .append(" device_name=").append(config.getDeviceName() != null ? config.getDeviceName() : "微步") + .append(" vendor=").append("微步") + .append(", data_type=json") + .append(" device_collect_id=").append(properties.getDeviceCollectId() != null ? properties.getDeviceCollectId() : config.getId()) + .append(" eventId=").append(eventId) + .append("]"); + return sb.toString(); + } + + /** + * 截断长字符串用于日志 + */ + private String truncate(String str, int maxLen) { + if (str == null) { + return ""; + } + return str.length() > maxLen ? str.substring(0, maxLen) + "..." : str; + } + + /** + * 签名状态（timestamp + sign），1002 时刷新后跨请求复用 + */ + private static class AuthState { + private long timestamp; + private String sign; + + AuthState(long timestamp, String sign) { + this.timestamp = timestamp; + this.sign = sign; + } + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/XdrDataCollector.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/XdrDataCollector.java new file mode 100644 index 0000000..6da0cb6 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/collector/XdrDataCollector.java @@ -0,0 +1,100 @@ +package com.collector; + +import com.common.entity.DeviceApiConfig; +import com.config.AppProperties; +import com.kafka.KafkaProducerUtil; +import com.util.Sm4Util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; + +/** + * 深信服 XDR API 数据采集器（暂缓） + * + * 认证方式：AK/SK HMAC-SHA256 签名 + * 1. 联动码解码：Hex解码 -> 按"|"分割为14段 -> 通过SHA256计算AES密钥 -> AES-CBC解密第10段得到AK，第11段得到SK + * 2. 添加公共头：sdk-content-type / sdk-host / sign-date + * 3. 构造规范串、计算签名、构造 Authorization + */ +@Component +public class XdrDataCollector implements DeviceDataCollector { + + private static final Logger log = LoggerFactory.getLogger(XdrDataCollector.class); + + @Autowired + private RestTemplate restTemplate; + + @Autowired + private AppProperties properties; + + @Autowired + private KafkaProducerUtil kafkaProducer; + + /** AK / SK（联动码解码获得） */ + private String ak; + private String sk; + + @Override + public String getDeviceType() { + return "xdr"; + } + + /** + * 联动码解码获取 AK/SK + * Hex解码 -> 按"|"分割为14段 -> SHA256计算AES密钥 -> AES-CBC解密第10段(AK)、第11段(SK) + */ + public void decodeAuthCode(String authCode) throws CollectorException { + try { + byte[] hexBytes = cn.hutool.core.util.HexUtil.decodeHex(authCode); + String decoded = new String(hexBytes, StandardCharsets.UTF_8); + String[] segments = decoded.split("\\|"); + if (segments.length < 14) { + throw new CollectorException("XDR 联动码格式错误，段数不足14"); + } + + // 计算 AES 密钥：SHA256(前几段拼接) + StringBuilder seed = new StringBuilder(); + for (int i = 0; i < 9; i++) { + seed.append(segments[i]).append("|"); + } + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] aesKey = digest.digest(seed.toString().getBytes(StandardCharsets.UTF_8)); + + // 第10段、第11段解密（AES-CBC，iv 固定为前16字节或配置） + // 说明：XDR 联动码的 AES 密钥与 IV 算法细节需在联调阶段确认，此处为骨架实现 + this.ak = segments[9]; + this.sk = segments[10]; + log.info("XDR 联动码解码成功, ak={}", ak); + } catch (CollectorException e) { + throw e; + } catch (Exception e) { + throw new CollectorException("XDR 联动码解码失败: " + e.getMessage(), e); + } + } + + @Override + public void authenticate() throws CollectorException { + String authCode = properties.getXdrAuthCode(); + if (authCode == null || authCode.isEmpty()) { + throw new CollectorException("XDR auth-code 未配置，XDR 采集暂缓"); + } + decodeAuthCode(authCode); + } + + @Override + public boolean isAuthenticated() { + return ak != null && sk != null; + } + + @Override + public CollectResult collect(DeviceApiConfig config, String dataType, long startTime, long endTime) + throws CollectorException { + // 暂缓实现：XDR 接口联调完成后实现 + throw new CollectorException("XDR 采集器暂缓，待联动码解码与签名接口联调后实现"); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/common/entity/DeviceApiCollectLog.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/entity/DeviceApiCollectLog.java new file mode 100644 index 0000000..4b2346b --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/entity/DeviceApiCollectLog.java @@ -0,0 +1,170 @@ +package com.common.entity; + +import java.time.LocalDateTime; + +/** + * API 采集执行日志表 + * 对应数据表 device_api_collect_log + */ +public class DeviceApiCollectLog { + + /** 主键 */ + private Long id; + + /** 设备ID，关联 device_device.id，非空 */ + private Integer deviceId; + + /** 设备类型：tianyan / weibu / xdr */ + private String deviceType; + + /** 采集数据类型：weblog / alarm / attack / info */ + private String dataType; + + /** 采集开始时间 */ + private LocalDateTime startTime; + + /** 采集结束时间 */ + private LocalDateTime endTime; + + /** 采集总数 */ + private Integer totalCount; + + /** 成功条数 */ + private Integer successCount; + + /** 失败条数 */ + private Integer failCount; + + /** 耗时（毫秒） */ + private Long costMs; + + /** 采集状态：SUCCESS-成功 FAIL-失败 */ + private String status; + + /** 错误信息 */ + private String errorMsg; + + /** 创建时间 */ + private LocalDateTime createTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Integer getDeviceId() { + return deviceId; + } + + public void setDeviceId(Integer deviceId) { + this.deviceId = deviceId; + } + + public String getDeviceType() { + return deviceType; + } + + public void setDeviceType(String deviceType) { + this.deviceType = deviceType; + } + + public String getDataType() { + return dataType; + } + + public void setDataType(String dataType) { + this.dataType = dataType; + } + + public LocalDateTime getStartTime() { + return startTime; + } + + public void setStartTime(LocalDateTime startTime) { + this.startTime = startTime; + } + + public LocalDateTime getEndTime() { + return endTime; + } + + public void setEndTime(LocalDateTime endTime) { + this.endTime = endTime; + } + + public Integer getTotalCount() { + return totalCount; + } + + public void setTotalCount(Integer totalCount) { + this.totalCount = totalCount; + } + + public Integer getSuccessCount() { + return successCount; + } + + public void setSuccessCount(Integer successCount) { + this.successCount = successCount; + } + + public Integer getFailCount() { + return failCount; + } + + public void setFailCount(Integer failCount) { + this.failCount = failCount; + } + + public Long getCostMs() { + return costMs; + } + + public void setCostMs(Long costMs) { + this.costMs = costMs; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getErrorMsg() { + return errorMsg; + } + + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } + + public LocalDateTime getCreateTime() { + return createTime; + } + + public void setCreateTime(LocalDateTime createTime) { + this.createTime = createTime; + } + + @Override + public String toString() { + return "DeviceApiCollectLog{" + + "id=" + id + + ", deviceId=" + deviceId + + ", deviceType='" + deviceType + '\'' + + ", dataType='" + dataType + '\'' + + ", startTime=" + startTime + + ", endTime=" + endTime + + ", totalCount=" + totalCount + + ", successCount=" + successCount + + ", failCount=" + failCount + + ", costMs=" + costMs + + ", status=" + status + + '}'; + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/common/entity/DeviceApiConfig.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/entity/DeviceApiConfig.java new file mode 100644 index 0000000..c566d54 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/entity/DeviceApiConfig.java @@ -0,0 +1,167 @@ +package com.common.entity; + +import java.time.LocalDateTime; + +/** + * API 采集设备配置表 + * 对应数据表 device_api_config + */ +public class DeviceApiConfig { + + /** 主键 */ + private Long id; + + /** 设备ID，关联 device_device.id */ + private Integer deviceId; + + /** 设备类型：tianyan-天眼 / weibu-微步 / xdr-XDR */ + private String deviceType; + + /** 设备名称 */ + private String deviceName; + + /** 设备IP */ + private String deviceIp; + + /** API 基础地址 */ + private String hostApi; + + /** 认证配置(JSON)：login_key / api_key / secret / username / branch_id 等 */ + private String authConfig; + + /** 采集数据类型，逗号分隔：weblog,alarm / attack,info */ + private String dataTypes; + + /** 采集间隔（秒） */ + private Integer collectIntervalSec; + + /** 是否启用：1-启用 0-停用 */ + private Integer enabled; + + /** 备注 */ + private String remark; + + /** 创建时间 */ + private LocalDateTime createTime; + + /** 更新时间 */ + private LocalDateTime updateTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Integer getDeviceId() { + return deviceId; + } + + public void setDeviceId(Integer deviceId) { + this.deviceId = deviceId; + } + + public String getDeviceType() { + return deviceType; + } + + public void setDeviceType(String deviceType) { + this.deviceType = deviceType; + } + + public String getDeviceName() { + return deviceName; + } + + public void setDeviceName(String deviceName) { + this.deviceName = deviceName; + } + + public String getDeviceIp() { + return deviceIp; + } + + public void setDeviceIp(String deviceIp) { + this.deviceIp = deviceIp; + } + + public String getHostApi() { + return hostApi; + } + + public void setHostApi(String hostApi) { + this.hostApi = hostApi; + } + + public String getAuthConfig() { + return authConfig; + } + + public void setAuthConfig(String authConfig) { + this.authConfig = authConfig; + } + + public String getDataTypes() { + return dataTypes; + } + + public void setDataTypes(String dataTypes) { + this.dataTypes = dataTypes; + } + + public Integer getCollectIntervalSec() { + return collectIntervalSec; + } + + public void setCollectIntervalSec(Integer collectIntervalSec) { + this.collectIntervalSec = collectIntervalSec; + } + + public Integer getEnabled() { + return enabled; + } + + public void setEnabled(Integer enabled) { + this.enabled = enabled; + } + + public String getRemark() { + return remark; + } + + public void setRemark(String remark) { + this.remark = remark; + } + + public LocalDateTime getCreateTime() { + return createTime; + } + + public void setCreateTime(LocalDateTime createTime) { + this.createTime = createTime; + } + + public LocalDateTime getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(LocalDateTime updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + return "DeviceApiConfig{" + + "id=" + id + + ", deviceId=" + deviceId + + ", deviceType='" + deviceType + '\'' + + ", deviceName='" + deviceName + '\'' + + ", deviceIp='" + deviceIp + '\'' + + ", hostApi='" + hostApi + '\'' + + ", dataTypes='" + dataTypes + '\'' + + ", enabled=" + enabled + + '}'; + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/common/mapper/DeviceApiCollectLogMapper.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/mapper/DeviceApiCollectLogMapper.java new file mode 100644 index 0000000..095410d --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/mapper/DeviceApiCollectLogMapper.java @@ -0,0 +1,43 @@ +package com.common.mapper; + +import com.common.entity.DeviceApiCollectLog; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * API 采集执行日志 Mapper + */ +@Mapper +public interface DeviceApiCollectLogMapper { + + /** + * 插入采集日志 + */ + int insert(DeviceApiCollectLog log); + + /** + * 根据ID查询 + */ + DeviceApiCollectLog selectById(@Param("id") Long id); + + /** + * 根据设备类型查询 + */ + List selectByDeviceType(@Param("deviceType") String deviceType); + + /** + * 按时间范围查询 + */ + List selectByTimeRange( + @Param("startTime") java.time.LocalDateTime startTime, + @Param("endTime") java.time.LocalDateTime endTime); + + /** + * 删除旧数据 + */ + int deleteByTimeRange( + @Param("startTime") java.time.LocalDateTime startTime, + @Param("endTime") java.time.LocalDateTime endTime); +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/common/mapper/DeviceApiConfigMapper.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/mapper/DeviceApiConfigMapper.java new file mode 100644 index 0000000..916c9c3 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/mapper/DeviceApiConfigMapper.java @@ -0,0 +1,54 @@ +package com.common.mapper; + +import com.common.entity.DeviceApiConfig; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * API 采集设备配置 Mapper + */ +@Mapper +public interface DeviceApiConfigMapper { + + /** + * 查询所有启用的采集配置 + */ + List selectEnabled(); + + /** + * 根据ID查询 + */ + DeviceApiConfig selectById(@Param("id") Long id); + + /** + * 根据设备类型查询 + */ + List selectByDeviceType(@Param("deviceType") String deviceType); + + /** + * 查询所有配置 + */ + List selectAll(); + + /** + * 新增配置 + */ + int insert(DeviceApiConfig config); + + /** + * 更新配置 + */ + int update(DeviceApiConfig config); + + /** + * 更新启用状态 + */ + int updateEnabled(@Param("id") Long id, @Param("enabled") Integer enabled); + + /** + * 删除配置 + */ + int deleteById(@Param("id") Long id); +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/common/service/DeviceApiCollectLogService.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/service/DeviceApiCollectLogService.java new file mode 100644 index 0000000..fc80f34 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/service/DeviceApiCollectLogService.java @@ -0,0 +1,43 @@ +package com.common.service; + +import com.common.entity.DeviceApiCollectLog; +import com.common.mapper.DeviceApiCollectLogMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.util.List; + +/** + * API 采集执行日志 Service + */ +@Service +public class DeviceApiCollectLogService { + + @Autowired + private DeviceApiCollectLogMapper logMapper; + + /** + * 记录采集日志 + */ + public void record(DeviceApiCollectLog log) { + if (log.getCreateTime() == null) { + log.setCreateTime(LocalDateTime.now()); + } + logMapper.insert(log); + } + + /** + * 按设备类型查询 + */ + public List listByDeviceType(String deviceType) { + return logMapper.selectByDeviceType(deviceType); + } + + /** + * 清理指定时间之前的旧日志 + */ + public int cleanBefore(LocalDateTime startTime) { + return logMapper.deleteByTimeRange(startTime, LocalDateTime.now()); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/common/service/DeviceApiConfigService.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/service/DeviceApiConfigService.java new file mode 100644 index 0000000..501090d --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/common/service/DeviceApiConfigService.java @@ -0,0 +1,78 @@ +package com.common.service; + +import com.common.entity.DeviceApiConfig; +import com.common.mapper.DeviceApiConfigMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Collections; +import java.util.List; + +/** + * API 采集设备配置 Service + */ +@Service +public class DeviceApiConfigService { + + @Autowired + private DeviceApiConfigMapper configMapper; + + /** + * 查询所有启用的采集配置 + */ + public List listEnabled() { + List list = configMapper.selectEnabled(); + return list == null ? Collections.emptyList() : list; + } + + /** + * 根据ID查询 + */ + public DeviceApiConfig getById(Long id) { + return configMapper.selectById(id); + } + + /** + * 根据设备类型查询 + */ + public List listByDeviceType(String deviceType) { + List list = configMapper.selectByDeviceType(deviceType); + return list == null ? Collections.emptyList() : list; + } + + /** + * 查询所有配置 + */ + public List listAll() { + List list = configMapper.selectAll(); + return list == null ? Collections.emptyList() : list; + } + + /** + * 新增配置 + */ + public int add(DeviceApiConfig config) { + return configMapper.insert(config); + } + + /** + * 更新配置 + */ + public int update(DeviceApiConfig config) { + return configMapper.update(config); + } + + /** + * 更新启用状态 + */ + public int updateEnabled(Long id, Integer enabled) { + return configMapper.updateEnabled(id, enabled); + } + + /** + * 删除配置 + */ + public int delete(Long id) { + return configMapper.deleteById(id); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/config/AppProperties.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/config/AppProperties.java new file mode 100644 index 0000000..27dc8d6 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/config/AppProperties.java @@ -0,0 +1,103 @@ +package com.config; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +/** + * API 采集器应用配置属性 + */ +@Data +@Component +public class AppProperties { + + // ========== 采集调度配置 ========== + /** 定时采集间隔（毫秒），默认5分钟 */ + @Value("${device.collect.interval:300000}") + private long collectInterval; + + /** 最大重试次数 */ + @Value("${device.collect.retry.max-attempts:3}") + private int retryMaxAttempts; + + /** 重试初始延迟（毫秒） */ + @Value("${device.collect.retry.initial-delay:5000}") + private long retryInitialDelay; + + /** 重试延迟倍数 */ + @Value("${device.collect.retry.multiplier:2.0}") + private double retryMultiplier; + + /** Kafka 推送 topic */ + @Value("${device.collect.kafka.topic:device-api-collect}") + private String kafkaTopic; + + /** 采集探针ID */ + @Value("${app.service.device_collect_id:1}") + private Long deviceCollectId; + + // ========== 天眼配置 ========== + @Value("${device.tianyan.host:https://127.0.0.1:443}") + private String tianyanHost; + + @Value("${device.tianyan.login-key:}") + private String tianyanLoginKey; + + @Value("${device.tianyan.username:xxadmin}") + private String tianyanUsername; + + @Value("${device.tianyan.branch-id:}") + private String tianyanBranchId; + + @Value("${device.tianyan.data-types:weblog,alarm}") + private String tianyanDataTypes; + + @Value("${device.tianyan.page-size:100}") + private int tianyanPageSize; + + /** 天眼单次查询子窗口跨度（分钟）：采集窗口切片预防平台深分页上限，<=0 表示不切片 */ + @Value("${device.tianyan.sub-window-minutes:5}") + private int tianyanSubWindowMinutes; + + /** 采集执行线程池大小：各设备/数据类型并行采集 */ + @Value("${device.collect.pool-size:15}") + private int collectPoolSize; + + /** 微步 info（日志流量）单轮采集窗口上限（分钟）：数据量大，限流防止单轮过久，积压时逐轮追赶 */ + @Value("${device.weibu.info-max-window-minutes:5}") + private int weibuInfoMaxWindowMinutes; + + // ========== 微步配置 ========== + @Value("${device.weibu.host:https://127.0.0.1:443}") + private String weibuHost; + + @Value("${device.weibu.api-key:}") + private String weibuApiKey; + + @Value("${device.weibu.secret:}") + private String weibuSecret; + + @Value("${device.weibu.data-types:attack,info}") + private String weibuDataTypes; + + /** 微步限流：每秒最大请求数 */ + @Value("${device.weibu.rate-limit:10}") + private int weibuRateLimit; + + /** 微步单次取数上限（size 两段式取数一次取全的条数上限，超出对半拆分窗口重采） */ + @Value("${device.weibu.fetch-size-cap:10000}") + private int weibuFetchSizeCap; + + // ========== XDR 配置（暂缓） ========== + @Value("${device.xdr.host:https://127.0.0.1}") + private String xdrHost; + + @Value("${device.xdr.auth-code:}") + private String xdrAuthCode; + + @Value("${device.xdr.api-base:/api/openapi/}") + private String xdrApiBase; + + @Value("${device.xdr.data-types:}") + private String xdrDataTypes; +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/config/CacheConfig.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/config/CacheConfig.java new file mode 100644 index 0000000..d949257 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/config/CacheConfig.java @@ -0,0 +1,38 @@ +package com.config; + +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.cache.RedisCacheConfiguration; +import org.springframework.data.redis.cache.RedisCacheManager; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer; +import org.springframework.data.redis.serializer.RedisSerializationContext; +import org.springframework.data.redis.serializer.StringRedisSerializer; + +import java.time.Duration; + +@Configuration +@EnableCaching +public class CacheConfig { + + @Bean + public RedisCacheConfiguration redisCacheConfiguration() { + return RedisCacheConfiguration.defaultCacheConfig() + .entryTtl(Duration.ofMinutes(30)) + .disableCachingNullValues() + .serializeKeysWith(RedisSerializationContext.SerializationPair + .fromSerializer(new StringRedisSerializer())) + .serializeValuesWith(RedisSerializationContext.SerializationPair + .fromSerializer(new GenericJackson2JsonRedisSerializer())); + } + + @Bean + public CacheManager cacheManager(RedisConnectionFactory factory) { + return RedisCacheManager.builder(factory) + .cacheDefaults(redisCacheConfiguration()) + .transactionAware() + .build(); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/config/RedisConfig.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/config/RedisConfig.java new file mode 100644 index 0000000..05e1b03 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/config/RedisConfig.java @@ -0,0 +1,41 @@ +package com.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; +import org.springframework.data.redis.serializer.StringRedisSerializer; + +/** + * Redis 序列化配置 + */ +@Configuration +public class RedisConfig { + + @Bean + public RedisTemplate redisTemplate(RedisConnectionFactory factory) { + RedisTemplate template = new RedisTemplate<>(); + template.setConnectionFactory(factory); + + Jackson2JsonRedisSerializer serializer = new Jackson2JsonRedisSerializer<>(Object.class); + + ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new JavaTimeModule()); + mapper.activateDefaultTyping( + mapper.getPolymorphicTypeValidator(), + ObjectMapper.DefaultTyping.NON_FINAL + ); + serializer.setObjectMapper(mapper); + + template.setKeySerializer(new StringRedisSerializer()); + template.setValueSerializer(serializer); + template.setHashKeySerializer(new StringRedisSerializer()); + template.setHashValueSerializer(serializer); + template.afterPropertiesSet(); + + return template; + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/config/RestTemplateConfig.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/config/RestTemplateConfig.java new file mode 100644 index 0000000..4f7842c --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/config/RestTemplateConfig.java @@ -0,0 +1,40 @@ +package com.config; + +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.ssl.SSLContextBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; + +import javax.net.ssl.SSLContext; + +/** + * RestTemplate 配置（忽略 SSL 证书验证，适用于内网/测试环境） + */ +@Configuration +public class RestTemplateConfig { + + @Bean + public RestTemplate restTemplate() throws Exception { + // 忽略 SSL 证书验证（仅用于内网/测试环境，生产环境建议使用真实证书） + SSLContext sslContext = SSLContextBuilder.create() + .loadTrustMaterial((chain, authType) -> true) + .build(); + + SSLConnectionSocketFactory socketFactory = new SSLConnectionSocketFactory( + sslContext, NoopHostnameVerifier.INSTANCE); + + CloseableHttpClient httpClient = HttpClients.custom() + .setSSLSocketFactory(socketFactory) + .build(); + + HttpComponentsClientHttpRequestFactory factory = new HttpComponentsClientHttpRequestFactory(httpClient); + factory.setConnectTimeout(10000); + factory.setReadTimeout(60000); + return new RestTemplate(factory); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/controller/DeviceApiCollectController.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/controller/DeviceApiCollectController.java new file mode 100644 index 0000000..e69ee52 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/controller/DeviceApiCollectController.java @@ -0,0 +1,183 @@ +package com.controller; + +import com.collector.CollectScheduler; +import com.common.entity.DeviceApiCollectLog; +import com.common.entity.DeviceApiConfig; +import com.common.service.DeviceApiCollectLogService; +import com.common.service.DeviceApiConfigService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.*; + +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * API 采集配置管理接口 + */ +@RestController +@RequestMapping("/api/collect") +public class DeviceApiCollectController { + + @Autowired + private DeviceApiConfigService configService; + + @Autowired + private DeviceApiCollectLogService collectLogService; + + @Autowired + private CollectScheduler collectScheduler; + + /** + * 全量回补功能开关，默认禁用 + */ + @Value("${device.collect.full.enabled:false}") + private boolean fullCollectEnabled; + + /** + * 手动触发采集（功能测试用） + * 按配置ID立即执行一次采集，不走定时调度。 + * + * @param id 设备配置ID + * @param dataType 数据类型（可选，为空则采集配置中所有类型） + * @param minutes 回看窗口分钟数（可选，默认最近1小时） + */ + @PostMapping("/trigger") + public Map trigger(@RequestParam Long id, + @RequestParam(required = false) String dataType, + @RequestParam(required = false) Integer minutes) { + Map result = collectScheduler.collectNow(id, dataType, minutes); + return success(result); + } + + /** + * 手动全量回补采集（同步执行，默认禁用） + * 把 [startTime, endTime] 区间内历史数据（如天眼 weblog）按子窗口由旧到新滚动逐段完整采集， + * 用于首次部署全量导入 / 漏采补数。区间较大时请求会持续较久，请按需控制 backDays/startTime。 + * 该功能由配置项 device.collect.full.enabled 控制，默认 false（禁用）。 + * + * @param id 设备配置ID + * @param dataType 数据类型，逗号分隔，如 weblog；为空则取配置中所有类型 + * @param startTime 起始时间（毫秒时间戳，可选） + * @param endTime 结束时间（毫秒时间戳，可选，默认当前时间-1分钟） + * @param backDays 回看天数（startTime 为空时生效，默认1天） + * @param windowMinutes 子窗口分钟数（可选，默认30分钟） + * @param continueOnError 单个子窗口失败是否继续（可选，默认false，失败即终止） + */ + @PostMapping("/full") + public Map submitFull(@RequestParam Long id, + @RequestParam(required = false) String dataType, + @RequestParam(required = false) Long startTime, + @RequestParam(required = false) Long endTime, + @RequestParam(required = false) Integer backDays, + @RequestParam(required = false) Integer windowMinutes, + @RequestParam(required = false) Boolean continueOnError) { + if (!fullCollectEnabled) { + Map result = new HashMap<>(); + result.put("code", 1); + result.put("message", "全量回补功能已禁用，请先设置 device.collect.full.enabled=true 后重启服务"); + return result; + } + return success(collectScheduler.collectFull(id, dataType, startTime, endTime, + backDays, windowMinutes, continueOnError)); + } + + /** + * 查询所有采集配置 + */ + @GetMapping("/config/list") + public Map listConfig() { + List list = configService.listAll(); + return success(list); + } + + /** + * 查询启用的采集配置 + */ + @GetMapping("/config/enabled") + public Map listEnabled() { + List list = configService.listEnabled(); + return success(list); + } + + /** + * 新增采集配置 + */ + @PostMapping("/config/add") + public Map addConfig(@RequestBody DeviceApiConfig config) { + if (config.getEnabled() == null) { + config.setEnabled(1); + } + configService.add(config); + return success("新增成功"); + } + + /** + * 更新采集配置 + */ + @PostMapping("/config/update") + public Map updateConfig(@RequestBody DeviceApiConfig config) { + configService.update(config); + return success("更新成功"); + } + + /** + * 启用/停用采集配置 + */ + @PostMapping("/config/enabled") + public Map updateEnabled(@RequestParam Long id, @RequestParam Integer enabled) { + configService.updateEnabled(id, enabled); + return success("操作成功"); + } + + /** + * 删除采集配置 + */ + @PostMapping("/config/delete") + public Map deleteConfig(@RequestParam Long id) { + configService.delete(id); + return success("删除成功"); + } + + /** + * 查询采集日志（按设备类型） + */ + @GetMapping("/log/device-type") + public Map listLogByType(@RequestParam String deviceType) { + List list = collectLogService.listByDeviceType(deviceType); + return success(list); + } + + /** + * 清理指定日期前的采集日志 + */ + @PostMapping("/log/clean") + public Map cleanLog(@RequestParam String beforeDate) { + LocalDateTime before = LocalDateTime.parse(beforeDate); + int count = collectLogService.cleanBefore(before); + Map data = new HashMap<>(); + data.put("deleted", count); + return success(data); + } + + /** + * 采集器健康检查 + */ + @GetMapping("/health") + public Map health() { + Map data = new HashMap<>(); + data.put("status", "UP"); + data.put("time", LocalDateTime.now().toString()); + return success(data); + } + + private Map success(Object data) { + Map result = new HashMap<>(); + result.put("code", 0); + result.put("message", "success"); + result.put("data", data); + return result; + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/kafka/KafkaProducerUtil.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/kafka/KafkaProducerUtil.java new file mode 100644 index 0000000..94aa3d2 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/kafka/KafkaProducerUtil.java @@ -0,0 +1,129 @@ +package com.kafka; + +import com.alibaba.fastjson2.JSONObject; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import com.util.Sm4Util; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Properties; + +/** + * Kafka 生产者工具 + * 将采集到的数据推送至 Kafka，供下游消费处理 + */ +@Component +public class KafkaProducerUtil { + + private static final Logger log = LoggerFactory.getLogger(KafkaProducerUtil.class); + + @Value("${spring.kafka.producer.bootstrap-servers:192.168.4.99:9092}") + private String bootstrapServers; + + @Value("${device.collect.kafka.topic:device-api-collect}") + private String topic; + + @Value("${syslog.sm4.generateKey:}") + private String sm4Key; + + /** producer 单条消息上限（字节）：需与 broker 的 message.max.bytes、topic 的 max.message.bytes 配合调整 */ + @Value("${device.kafka.max-request-size:10485760}") + private int maxRequestSize; + + private KafkaProducer producer; + + @PostConstruct + public void init() { + Properties props = new Properties(); + props.put("bootstrap.servers", bootstrapServers); + props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + props.put("acks", "1"); + props.put("retries", 10); + props.put("retry.backoff.ms", 500); + props.put("connections.max.idle.ms", 600000); + props.put("socket.keepalive.enable", true); + props.put("request.timeout.ms", 30000); + props.put("delivery.timeout.ms", 120000); + // 单条消息大小上限：超限时 send() 同步抛 RecordTooLargeException（fail 计数准确） + props.put("max.request.size", maxRequestSize); + producer = new KafkaProducer<>(props); + log.info("KafkaProducerUtil 初始化成功, brokers={}, topic={}", bootstrapServers, topic); + } + + @PreDestroy + public void destroy() { + if (producer != null) { + producer.close(); + } + } + + /** + * 对消息内容进行 SM4 加密（CBC 模式，格式 Base64(IV):Base64(密文)） + * 未配置密钥(syslog.sm4.generateKey)时返回明文，保持兼容 + */ + private String encryptValue(String value) { + String key = sm4Key == null ? "" : sm4Key.trim(); + if (key.isEmpty()) { + log.warn("未配置 syslog.sm4.generateKey，Kafka 消息以明文发送"); + return value; + } + String encrypted = Sm4Util.encryptCbc(value, key); + log.debug("Kafka 消息已 SM4 加密: 明文长度={}, 密文长度={}", value.length(), encrypted.length()); + return encrypted; + } + + /** + * 发送单条消息（异步提交，不阻塞采集线程） + * 发送结果通过回调日志观察；producer 配置 retries=10、delivery.timeout.ms=120s 保证最终投递 + * @param key 消息 key + * @param value 消息内容 + * @return 是否成功提交到 producer 发送缓冲 + */ + public boolean send(String key, String value) { + try { + ProducerRecord record = new ProducerRecord<>(topic, key, encryptValue(value)); + producer.send(record, (metadata, exception) -> { + if (exception != null) { + log.error("Kafka 异步发送失败: key={}, error={}", key, exception.getMessage()); + } else { + log.debug("Kafka 发送成功: key={}, partition={}, offset={}", + key, metadata.partition(), metadata.offset()); + } + }); + return true; + } catch (Exception e) { + log.error("Kafka 提交失败: key={}, error={}", key, e.getMessage()); + return false; + } + } + + /** + * 获取 topic + */ + public String getTopic() { + return topic; + } + + /** + * 构建 Kafka 消息 value（统一数据模型） + * @param deviceType 设备类型：tianyan/weibu/xdr + * @param dataType 数据类型：weblog/alarm/attack/info + * @param collectTime 采集时间（毫秒） + * @param rawData 原始数据 + */ + public static String buildValue(String deviceType, String dataType, long collectTime, Object rawData) { + JSONObject value = new JSONObject(); + value.put("deviceType", deviceType); + value.put("dataType", dataType); + value.put("collectTime", collectTime); + value.put("rawData", rawData); + return value.toJSONString(); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/mock/MockDataGenerator.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/mock/MockDataGenerator.java new file mode 100644 index 0000000..387aea8 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/mock/MockDataGenerator.java @@ -0,0 +1,166 @@ +package com.mock; + +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +/** + * Mock 数据生成器 + * 模拟天眼/微步的原始数据，字段与真实 API 返回保持一致， + * 用于功能测试跑通：采集 -> 字段映射 -> Kafka -> 日志落库 全流程。 + * + * 说明：同一时间窗口内，按序号(seq)确定性生成数据， + * 保证 mock 接口分页连续、重复请求数据一致。 + */ +public class MockDataGenerator { + + private MockDataGenerator() { + } + + /** 内网源 IP 段 */ + private static final String[] SRC_IP_PREFIX = {"192.168.1", "192.168.2", "192.168.10", "172.16.1", "172.16.5", "10.10.1"}; + + /** 目的 IP（模拟攻击目标/被访问服务器） */ + private static final String[] DST_IPS = {"10.56.170.107", "10.56.171.164", "10.56.100.88", "10.56.100.120"}; + + /** 天眼 weblog URI */ + private static final String[] WEBLOG_URIS = { + "/login", "/index.html", "/api/user/info", "/admin", "/download/file?id=1", + "/report/list", "/upload", "/cgi-bin/test.cgi", "/webroot/decision/login.jsp", "/console" + }; + + /** 天眼 alarm 攻击类型 */ + private static final String[] ALARM_TITLES = { + "SQL注入攻击", "XSS跨站脚本攻击", "命令注入攻击", "Webshell上传", "暴力破解", + "目录遍历攻击", "文件包含攻击", "SSRF服务端请求伪造", "CSRF跨站请求伪造", "恶意扫描" + }; + + /** 微步 attack 威胁类型 */ + private static final String[] WEIBU_THREAT = { + "木马远控", "蠕虫病毒", "挖矿木马", "僵尸网络", "暴力破解", + "勒索病毒", "恶意下载", "DDoS攻击", "端口扫描", "数据泄露" + }; + + /** 微步风险等级 */ + private static final String[] RISK_LEVELS = {"high", "medium", "low"}; + + /** 域名后缀 */ + private static final String[] DOMAIN_SUFFIX = {".com", ".net", ".org", ".cn", ".xyz", ".top"}; + + /** HTTP 方法 */ + private static final String[] HTTP_METHODS = {"GET", "POST", "PUT", "DELETE"}; + + /** 协议 */ + private static final String[] PROTOS = {"TCP", "UDP", "HTTP", "HTTPS"}; + + /** 微步 action */ + private static final String[] WEIBU_ACTIONS = {"block", "allow", "detect"}; + + /** + * 生成天眼单条 hit（含 _source 包装，结构与真实 API 一致） + * + * @param dataType weblog / alarm + * @param startTime 窗口开始（毫秒） + * @param endTime 窗口结束（毫秒） + * @param seq 数据序号（从1开始），用于确定性分页 + */ + public static Map buildTianyanHit(String dataType, long startTime, long endTime, int seq) { + Random random = new Random(startTime + dataType.hashCode() + seq * 31L); + Map hit = new HashMap<>(); + hit.put("_index", "alarm".equalsIgnoreCase(dataType) ? "alarm_collection" : "skyeye-weblog"); + hit.put("_source", "alarm".equalsIgnoreCase(dataType) + ? buildTianyanAlarm(random, startTime, endTime, seq) + : buildTianyanWeblog(random, startTime, endTime, seq)); + return hit; + } + + /** + * 生成微步单条数据 + * + * @param dataType attack / info + * @param startTime 窗口开始（毫秒） + * @param endTime 窗口结束（毫秒） + * @param seq 数据序号（从1开始），用于确定性分页 + */ + public static Map buildWeibuItem(String dataType, long startTime, long endTime, int seq) { + Random random = new Random(startTime + dataType.hashCode() + seq * 31L); + long eventTime = startTime + (long) (random.nextDouble() * (endTime - startTime)); + Map item = new HashMap<>(); + item.put("event_time", eventTime); + item.put("src_ip", randomIp(random)); + item.put("dst_ip", DST_IPS[random.nextInt(DST_IPS.length)]); + item.put("src_port", 1024 + random.nextInt(60000)); + item.put("dst_port", random.nextInt(2) == 0 ? 80 : 443); + item.put("threat_type", WEIBU_THREAT[random.nextInt(WEIBU_THREAT.length)]); + item.put("risk_level", RISK_LEVELS[random.nextInt(RISK_LEVELS.length)]); + item.put("domain", "evil" + (random.nextInt(999) + 1) + DOMAIN_SUFFIX[random.nextInt(DOMAIN_SUFFIX.length)]); + item.put("url", "http://" + item.get("domain") + "/payload?cmd=" + (random.nextInt(9999) + 1)); + item.put("md5", randomHex(random, 32)); + item.put("sha256", randomHex(random, 64)); + item.put("action", WEIBU_ACTIONS[random.nextInt(WEIBU_ACTIONS.length)]); + item.put("id", "wb_" + eventTime + "_" + seq + "_" + random.nextInt(99999)); + return item; + } + + /** + * 构建天眼 weblog 单条数据 + */ + private static Map buildTianyanWeblog(Random random, long startTime, long endTime, int seq) { + long eventTime = startTime + (long) (random.nextDouble() * (endTime - startTime)); + Map source = new HashMap<>(); + source.put("_time", eventTime); + source.put("src_ip", randomIp(random)); + source.put("dst_ip", DST_IPS[random.nextInt(DST_IPS.length)]); + source.put("src_port", 1024 + random.nextInt(60000)); + source.put("dst_port", random.nextInt(2) == 0 ? 80 : 443); + source.put("proto", PROTOS[random.nextInt(PROTOS.length)]); + source.put("method", HTTP_METHODS[random.nextInt(HTTP_METHODS.length)]); + source.put("uri", WEBLOG_URIS[random.nextInt(WEBLOG_URIS.length)]); + source.put("user_agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"); + source.put("status", random.nextInt(2) == 0 ? 200 : 404); + source.put("request_body", random.nextInt(3) == 0 ? "{\"username\":\"admin\",\"password\":\"******\"}" : ""); + source.put("response_body", "mock weblog page"); + source.put("_id", "ty_wb_" + eventTime + "_" + seq); + return source; + } + + /** + * 构建天眼 alarm 单条数据 + */ + private static Map buildTianyanAlarm(Random random, long startTime, long endTime, int seq) { + long eventTime = startTime + (long) (random.nextDouble() * (endTime - startTime)); + Map source = new HashMap<>(); + source.put("_time", eventTime); + source.put("title", ALARM_TITLES[random.nextInt(ALARM_TITLES.length)]); + source.put("alarm_sev", 1 + random.nextInt(5)); + source.put("src_ip", randomIp(random)); + source.put("dst_ip", DST_IPS[random.nextInt(DST_IPS.length)]); + source.put("src_port", 1024 + random.nextInt(60000)); + source.put("dst_port", 80 + random.nextInt(1000)); + source.put("proto", PROTOS[random.nextInt(PROTOS.length)]); + source.put("rule_id", "TY-RULE-" + (1000 + random.nextInt(9000))); + source.put("event_id", "ty_al_" + eventTime + "_" + seq + "_" + random.nextInt(99999)); + source.put("attack_type", ALARM_TITLES[random.nextInt(ALARM_TITLES.length)]); + return source; + } + + /** + * 随机内网 IP + */ + private static String randomIp(Random random) { + String prefix = SRC_IP_PREFIX[random.nextInt(SRC_IP_PREFIX.length)]; + return prefix + "." + (random.nextInt(254) + 1); + } + + /** + * 随机十六进制字符串（md5/sha256） + */ + private static String randomHex(Random random, int length) { + StringBuilder sb = new StringBuilder(); + String chars = "0123456789abcdef"; + for (int i = 0; i < length; i++) { + sb.append(chars.charAt(random.nextInt(chars.length()))); + } + return sb.toString(); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/mock/TianyanMockController.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/mock/TianyanMockController.java new file mode 100644 index 0000000..9ab387b --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/mock/TianyanMockController.java @@ -0,0 +1,216 @@ +package com.mock; + +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +/** + * 天眼(Tianyan) API Mock 接口 + *

+ * 模拟真实天眼的认证与日志查询接口，供功能测试使用。 + * 采集器将 device_api_config.host_api 配置为 + * http://127.0.0.1:8290/collectorapi/mock/tianyan 即可无缝对接。 + *

+ * 模拟接口： + * 1. POST /mock/tianyan/skyeye/v1/admin/auth 免密登录，返回 access_token + * 2. GET /mock/tianyan/skyeye/v1/admin/auth 提取 csrf_token + * 3. GET /mock/tianyan/skyeye/v1/analysis/log-search/list 日志查询（weblog） + * 4. GET /mock/tianyan/skyeye/v1/alarm/alarm/list 告警查询（alarm） + * + * 开关配置：mock.server.enabled=true 时启用，默认关闭（生产不暴露）。 + */ +@RestController +@RequestMapping("/mock/tianyan") +@ConditionalOnProperty(name = "mock.server.enabled", havingValue = "true") +public class TianyanMockController { + + private static final Logger log = LoggerFactory.getLogger(TianyanMockController.class); + + /** 模拟数据总量（每类），分页测试用，默认250条 */ + @Value("${mock.server.total-count:250}") + private int totalCount; + + /** + * 免密登录，返回 access_token + * 请求：POST /skyeye/v1/admin/auth + * Headers: X-Authorization=SHA256(JSON(jsonBody)+timestamp+clientSecret) + * Body: {"client_id":"xxx","username":"xxx"} + */ + @PostMapping("/skyeye/v1/admin/auth") + public String auth(@RequestBody(required = false) String body, + @RequestHeader(value = "X-Authorization", required = false) String xAuthorization) { + log.info("[Mock] 天眼认证请求, X-Authorization={}, body={}", maskAuth(xAuthorization), body); + JSONObject resp = new JSONObject(); + resp.put("status", 1000); + resp.put("message", "success"); + JSONObject inner = new JSONObject(); + inner.put("access_token", "mock_tianyan_token_" + System.currentTimeMillis()); + JSONObject data = new JSONObject(); + data.put("data", inner); + resp.put("data", data); + return resp.toJSONString(); + } + + /** + * 提取 csrf_token + * 请求：GET /skyeye/v1/admin/auth?token=access_token + */ + @GetMapping("/skyeye/v1/admin/auth") + public String getCsrf(@RequestParam(value = "token", required = false) String token) { + log.info("[Mock] 天眼获取 csrf_token, token={}", maskToken(token)); + JSONObject resp = new JSONObject(); + resp.put("status", 1000); + resp.put("message", "success"); + JSONObject data = new JSONObject(); + data.put("csrf_token", "mock_tianyan_csrf_" + System.currentTimeMillis()); + resp.put("data", data); + return resp.toJSONString(); + } + + /** + * 日志查询（weblog / alarm） + * 请求：GET /skyeye/v1/analysis/log-search/list + * 参数：branch_id, start_time(ms), end_time(ms), index, category, offset, limit, mode, csrf_token, r + * 响应：{"status":1000,"data":{"data":{"search":{"hits":[{"_source":{...}}]}}}} + * + * 可选场景参数 mock_scenario： + * - empty 返回空数据 + * - csrf_expired 返回 csrf_token error（触发采集器重新认证） + * - delay 延迟 5 秒返回（模拟超时） + */ + @GetMapping("/skyeye/v1/analysis/log-search/list") + public String searchLog(@RequestParam(value = "index", required = false) String index, + @RequestParam(value = "offset", defaultValue = "1") int offset, + @RequestParam(value = "limit", defaultValue = "100") int limit, + @RequestParam(value = "start_time", required = false) Long startTime, + @RequestParam(value = "end_time", required = false) Long endTime, + @RequestParam(value = "mock_scenario", required = false) String scenario) throws InterruptedException { + boolean isAlarm = index != null && index.toLowerCase().contains("alarm"); + String dataType = isAlarm ? "alarm" : "weblog"; + log.info("[Mock] 天眼日志查询: index={}, dataType={}, offset={}, limit={}, start={}, end={}, scenario={}", + index, dataType, offset, limit, startTime, endTime, scenario); + + JSONObject resp = new JSONObject(); + + // 模拟场景：csrf_token 过期 + if ("csrf_expired".equalsIgnoreCase(scenario)) { + resp.put("status", 2000); + resp.put("message", "csrf_token error"); + return resp.toJSONString(); + } + + // 模拟场景：响应延迟 + if ("delay".equalsIgnoreCase(scenario)) { + Thread.sleep(5000); + } + + resp.put("status", 1000); + resp.put("message", "success"); + + // 模拟场景：空数据 + long now = System.currentTimeMillis(); + long start = startTime != null && startTime > 0 ? startTime : now - 60 * 60 * 1000L; + long end = endTime != null && endTime > 0 ? endTime : now; + int size = "empty".equalsIgnoreCase(scenario) ? 0 + : Math.max(0, Math.min(limit, totalCount - (offset - 1))); + + JSONArray hits = new JSONArray(); + for (int i = 0; i < size; i++) { + Map hit = MockDataGenerator.buildTianyanHit(dataType, start, end, offset + i); + hits.add(new JSONObject(hit)); + } + + JSONObject search = new JSONObject(); + search.put("hits", hits); + JSONObject inner = new JSONObject(); + inner.put("search", search); + JSONObject data = new JSONObject(); + data.put("data", inner); + resp.put("data", data); + return resp.toJSONString(); + } + + /** + * 告警查询（alarm） + * 请求：GET /skyeye/v1/alarm/alarm/list + * 参数：start_time(ms), end_time(ms), hazard_level, status, host_state, limit, offset, csrf_token, r + * 响应：{"status":1000,"data":{"data":{"rows":[{"_source":{...}}]}}} + * + * 可选场景参数 mock_scenario： + * - empty 返回空数据 + * - csrf_expired 返回 csrf_token error（触发采集器重新认证） + * - delay 延迟 5 秒返回（模拟超时） + */ + @GetMapping("/skyeye/v1/alarm/alarm/list") + public String alarmList(@RequestParam(value = "offset", defaultValue = "1") int offset, + @RequestParam(value = "limit", defaultValue = "100") int limit, + @RequestParam(value = "start_time", required = false) Long startTime, + @RequestParam(value = "end_time", required = false) Long endTime, + @RequestParam(value = "mock_scenario", required = false) String scenario) throws InterruptedException { + log.info("[Mock] 天眼告警查询: offset={}, limit={}, start={}, end={}, scenario={}", + offset, limit, startTime, endTime, scenario); + + JSONObject resp = new JSONObject(); + + // 模拟场景：csrf_token 过期 + if ("csrf_expired".equalsIgnoreCase(scenario)) { + resp.put("status", 2000); + resp.put("message", "csrf_token error"); + return resp.toJSONString(); + } + + // 模拟场景：响应延迟 + if ("delay".equalsIgnoreCase(scenario)) { + Thread.sleep(5000); + } + + resp.put("status", 1000); + resp.put("message", "success"); + + // 模拟场景：空数据 + long now = System.currentTimeMillis(); + long start = startTime != null && startTime > 0 ? startTime : now - 60 * 60 * 1000L; + long end = endTime != null && endTime > 0 ? endTime : now; + int size = "empty".equalsIgnoreCase(scenario) ? 0 + : Math.max(0, Math.min(limit, totalCount - (offset - 1))); + + JSONArray rows = new JSONArray(); + for (int i = 0; i < size; i++) { + Map row = MockDataGenerator.buildTianyanHit("alarm", start, end, offset + i); + rows.add(new JSONObject(row)); + } + + JSONObject inner = new JSONObject(); + inner.put("rows", rows); + JSONObject data = new JSONObject(); + data.put("data", inner); + resp.put("data", data); + return resp.toJSONString(); + } + + private String maskAuth(String v) { + if (v == null || v.length() < 8) { + return v; + } + return v.substring(0, 8) + "***"; + } + + private String maskToken(String v) { + if (v == null || v.length() < 8) { + return v; + } + return v.substring(0, 8) + "***"; + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/mock/WeibuMockController.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/mock/WeibuMockController.java new file mode 100644 index 0000000..4659f72 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/mock/WeibuMockController.java @@ -0,0 +1,137 @@ +package com.mock; + +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +/** + * 微步(Weibu) API Mock 接口 + *

+ * 模拟真实微步的数据查询接口，供功能测试使用。 + * 采集器将 device_api_config.host_api 配置为 + * http://127.0.0.1:8290/collectorapi/mock/weibu 即可无缝对接。 + *

+ * 模拟接口： + * 1. POST /mock/weibu/api/v1/log/searchBySql 数据查询（attack/info） + * + * 开关配置：mock.server.enabled=true 时启用，默认关闭（生产不暴露）。 + */ +@RestController +@RequestMapping("/mock/weibu") +@ConditionalOnProperty(name = "mock.server.enabled", havingValue = "true") +public class WeibuMockController { + + private static final Logger log = LoggerFactory.getLogger(WeibuMockController.class); + + /** 模拟数据总量（每类），分页测试用，默认250条 */ + @Value("${mock.server.total-count:250}") + private int totalCount; + + /** + * 数据查询 + * 请求：POST /api/v1/log/searchBySql?api_key=xxx&auth_timestamp=xxx&sign=xxx + * Body: {"net_data_type":["attack"],"time_from":秒,"time_to":秒, + * "assets_group":[],"search_connection":false,"sql":"","columns":["*"], + * "asc":true,"page":{"cur_page":1,"page_size":100}} + * 响应：{"response_code":0,"response_msg":"OK","data":{"data":[...]}} + * + * 可选场景参数 mock_scenario： + * - empty 返回空数据 + * - sign_error 返回签名错误（response_code 非0，触发采集失败分支） + * - delay 延迟 5 秒返回（模拟超时） + */ + @PostMapping("/api/v1/log/searchBySql") + public String searchBySql(@RequestParam(value = "api_key", required = false) String apiKey, + @RequestParam(value = "auth_timestamp", required = false) String authTimestamp, + @RequestParam(value = "sign", required = false) String sign, + @RequestParam(value = "mock_scenario", required = false) String scenario, + @RequestBody(required = false) String body) throws InterruptedException { + JSONObject req = body != null ? JSONObject.parseObject(body) : new JSONObject(); + log.info("[Mock] 微步查询请求: api_key={}, ts={}, sign={}, body={}, scenario={}", + maskAuth(apiKey), authTimestamp, maskAuth(sign), req, scenario); + + JSONObject resp = new JSONObject(); + + // 模拟场景：签名错误 + if ("sign_error".equalsIgnoreCase(scenario)) { + resp.put("response_code", 401); + resp.put("response_msg", "signature verification failed"); + return resp.toJSONString(); + } + + // 模拟场景：响应延迟 + if ("delay".equalsIgnoreCase(scenario)) { + Thread.sleep(5000); + } + + // 解析 net_data_type（attack/info） + String dataType = "attack"; + JSONArray netDataType = req.getJSONArray("net_data_type"); + if (netDataType != null && !netDataType.isEmpty()) { + String first = netDataType.getString(0); + if (first != null) { + dataType = first.toLowerCase(); + } + } + + // 解析时间窗口（秒级 -> 毫秒），缺失时默认最近1小时 + long nowSec = System.currentTimeMillis() / 1000; + long timeFromSec = req.getLongValue("time_from", 0L); + long timeToSec = req.getLongValue("time_to", 0L); + long start; + long end; + if (timeFromSec > 0 && timeToSec > 0 && timeToSec >= timeFromSec) { + start = timeFromSec * 1000; + end = timeToSec * 1000; + } else { + start = (nowSec - 60 * 60) * 1000; + end = nowSec * 1000; + } + + // 解析分页 + int curPage = 1; + int pageSize = 100; + JSONObject page = req.getJSONObject("page"); + if (page != null) { + curPage = page.getIntValue("cur_page", 1); + pageSize = page.getIntValue("page_size", 100); + } + + log.info("[Mock] 微步查询: dataType={}, start={}, end={}, curPage={}, pageSize={}", + dataType, start, end, curPage, pageSize); + + // 模拟场景：空数据 + int size = "empty".equalsIgnoreCase(scenario) ? 0 + : Math.max(0, Math.min(pageSize, totalCount - (curPage - 1) * pageSize)); + + JSONArray dataList = new JSONArray(); + for (int i = 0; i < size; i++) { + Map item = MockDataGenerator.buildWeibuItem(dataType, start, end, (curPage - 1) * pageSize + i + 1); + dataList.add(new JSONObject(item)); + } + + resp.put("response_code", 0); + resp.put("response_msg", "OK"); + JSONObject data = new JSONObject(); + data.put("data", dataList); + resp.put("data", data); + return resp.toJSONString(); + } + + private String maskAuth(String v) { + if (v == null || v.length() < 8) { + return v; + } + return v.substring(0, 8) + "***"; + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/util/HttpUtil.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/util/HttpUtil.java new file mode 100644 index 0000000..3969e1d --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/util/HttpUtil.java @@ -0,0 +1,130 @@ +package com.util; + +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.RestTemplate; + +import java.net.URI; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * HTTP 请求工具（基于 RestTemplate） + */ +public class HttpUtil { + + private HttpUtil() { + } + + /** + * 将已构建好的 URL 字符串转为 URI 对象 + * URL 中的参数已按平台要求预编码（如 branch_id 中的 %2F）， + * 必须以 URI 类型传给 RestTemplate，否则默认 UriTemplateHandler 会对 % 再次编码（%2F → %252F） + */ + private static URI toUri(String url) { + return URI.create(url); + } + + /** + * 发送 GET 请求 + * @param restTemplate RestTemplate + * @param url URL + * @param headers 请求头（可为 null） + * @return 响应体字符串 + */ + public static String get(RestTemplate restTemplate, String url, Map headers) { + HttpHeaders httpHeaders = new HttpHeaders(); + if (headers != null) { + headers.forEach(httpHeaders::set); + } + HttpEntity entity = new HttpEntity<>(httpHeaders); + ResponseEntity response = restTemplate.exchange(toUri(url), HttpMethod.GET, entity, String.class); + return response.getBody(); + } + + /** + * 发送 GET 请求，同时返回响应体与 Set-Cookie（供需要登录 Cookie 的接口使用，如天眼免密登录） + * @param restTemplate RestTemplate + * @param url URL + * @param headers 请求头（可为 null） + * @return 响应结果（body + Set-Cookie 原始列表） + */ + public static GetResult getWithCookies(RestTemplate restTemplate, String url, Map headers) { + HttpHeaders httpHeaders = new HttpHeaders(); + if (headers != null) { + headers.forEach(httpHeaders::set); + } + HttpEntity entity = new HttpEntity<>(httpHeaders); + ResponseEntity response = restTemplate.exchange(toUri(url), HttpMethod.GET, entity, String.class); + List setCookies = response.getHeaders() != null + ? response.getHeaders().get(HttpHeaders.SET_COOKIE) : null; + return new GetResult(response.getBody(), setCookies); + } + + /** + * GET 请求响应结果封装 + */ + public static class GetResult { + private final String body; + private final List setCookies; + + GetResult(String body, List setCookies) { + this.body = body; + this.setCookies = setCookies; + } + + public String getBody() { + return body; + } + + /** Set-Cookie 原始串列表（可能为 null/空） */ + public List getSetCookies() { + return setCookies == null ? Collections.emptyList() : setCookies; + } + } + + /** + * 发送 POST 请求（JSON 体） + * @param restTemplate RestTemplate + * @param url URL + * @param jsonBody JSON 请求体 + * @param headers 请求头（可为 null） + * @return 响应体字符串 + */ + public static String postJson(RestTemplate restTemplate, String url, String jsonBody, Map headers) { + HttpHeaders httpHeaders = new HttpHeaders(); + if (headers != null) { + headers.forEach(httpHeaders::set); + } + httpHeaders.set(HttpHeaders.CONTENT_TYPE, "application/json"); + HttpEntity entity = new HttpEntity<>(jsonBody, httpHeaders); + ResponseEntity response = restTemplate.exchange(toUri(url), HttpMethod.POST, entity, String.class); + return response.getBody(); + } + + /** + * 发送 POST 请求（表单体） + * @param restTemplate RestTemplate + * @param url URL + * @param formParams 表单参数 + * @param headers 请求头（可为 null） + * @return 响应体字符串 + */ + public static String postForm(RestTemplate restTemplate, String url, Map formParams, Map headers) { + HttpHeaders httpHeaders = new HttpHeaders(); + if (headers != null) { + headers.forEach(httpHeaders::set); + } + httpHeaders.set(HttpHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded"); + org.springframework.util.LinkedMultiValueMap body = new org.springframework.util.LinkedMultiValueMap<>(); + if (formParams != null) { + formParams.forEach(body::add); + } + HttpEntity> entity = new HttpEntity<>(body, httpHeaders); + ResponseEntity response = restTemplate.exchange(toUri(url), HttpMethod.POST, entity, String.class); + return response.getBody(); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/util/Sm4Util.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/util/Sm4Util.java new file mode 100644 index 0000000..5d03f88 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/util/Sm4Util.java @@ -0,0 +1,114 @@ +package com.util; + +import cn.hutool.core.util.HexUtil; +import cn.hutool.core.util.RandomUtil; +import cn.hutool.crypto.symmetric.SM4; +import cn.hutool.crypto.symmetric.SymmetricCrypto; +import org.bouncycastle.jce.provider.BouncyCastleProvider; + +import javax.crypto.SecretKey; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.security.Security; +import java.util.Base64; + +/** + * 国密 SM4 加解密工具类 + * 基于 Hutool + Bouncy Castle 实现，与 syslog-serve / syslog-consumer 保持一致 + * 支持 ECB 和 CBC 模式；推荐 CBC 模式，输出格式 "Base64(IV):Base64(密文)" + */ +public class Sm4Util { + + static { + // 统一注册 Provider，确保全局生效 + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + + /** + * 生成 SM4 密钥（16字节，128位） + * + * @return 十六进制字符串格式的密钥 + */ + public static String generateKey() { + byte[] key = cn.hutool.crypto.SmUtil.sm4().getSecretKey().getEncoded(); + return HexUtil.encodeHexStr(key); + } + + // ==================== ECB 模式（电子密码本模式） ==================== + // 注意：ECB 模式不需要 IV，但安全性较低，不建议用于加密大量数据 + + /** + * ECB 模式加密 + * + * @param plainText 明文 + * @param hexKey 十六进制字符串格式的密钥 + * @return Base64 编码的密文 + */ + public static String encryptEcb(String plainText, String hexKey) { + SM4 sm4 = new SM4(HexUtil.decodeHex(hexKey)); + return sm4.encryptBase64(plainText); + } + + /** + * ECB 模式解密 + * + * @param cipherText Base64 编码的密文 + * @param hexKey 十六进制字符串格式的密钥 + * @return 明文 + */ + public static String decryptEcb(String cipherText, String hexKey) { + SM4 sm4 = new SM4(HexUtil.decodeHex(hexKey)); + return sm4.decryptStr(cipherText); + } + + // ==================== CBC 模式（密码分组链接模式，推荐） ==================== + // 需要随机 IV（初始化向量），安全性更高；密文携带 IV，格式 "Base64(IV):Base64(密文)" + + /** + * CBC 模式加密（推荐） + * + * @param plainText 明文 + * @param hexKey 十六进制字符串格式的密钥 + * @return 格式为 "Base64(IV):Base64(密文)" 的字符串 + */ + public static String encryptCbc(String plainText, String hexKey) { + // 1. 随机生成16字节 IV + byte[] iv = RandomUtil.randomBytes(16); + IvParameterSpec ivSpec = new IvParameterSpec(iv); + // 2. 将十六进制密钥转为字节数组并包装为 SecretKey + byte[] keyBytes = HexUtil.decodeHex(hexKey); + SecretKey secretKey = new SecretKeySpec(keyBytes, "SM4"); + // 3. 创建加解密器 + SymmetricCrypto sm4 = new SymmetricCrypto("SM4/CBC/PKCS5Padding", secretKey, ivSpec); + // 4. 执行加密 + String encryptedBase64 = sm4.encryptBase64(plainText); + String ivBase64 = Base64.getEncoder().encodeToString(iv); + return ivBase64 + ":" + encryptedBase64; + } + + /** + * CBC 模式解密 + * + * @param cipherTextWithIv 由 {@link #encryptCbc(String, String)} 生成的字符串 + * @param hexKey 十六进制字符串格式的密钥 + * @return 明文 + */ + public static String decryptCbc(String cipherTextWithIv, String hexKey) { + String[] parts = cipherTextWithIv.split(":"); + if (parts.length != 2) { + throw new IllegalArgumentException("无效的密文格式，应为 'Base64(IV):Base64(密文)'"); + } + // 1. 解析 IV + byte[] iv = Base64.getDecoder().decode(parts[0]); + IvParameterSpec ivSpec = new IvParameterSpec(iv); + // 2. 解析密钥 + byte[] keyBytes = HexUtil.decodeHex(hexKey); + SecretKey secretKey = new SecretKeySpec(keyBytes, "SM4"); + // 3. 创建解密器（算法参数必须与加密时完全一致） + SymmetricCrypto sm4 = new SymmetricCrypto("SM4/CBC/PKCS5Padding", secretKey, ivSpec); + // 4. 解密 + return sm4.decryptStr(parts[1]); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/util/SpringContextUtil.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/util/SpringContextUtil.java new file mode 100644 index 0000000..395d2f0 --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/util/SpringContextUtil.java @@ -0,0 +1,40 @@ +package com.util; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +/** + * Spring 上下文工具类 + */ +@Component +public class SpringContextUtil implements ApplicationContextAware { + + private static ApplicationContext applicationContext; + + @Override + public void setApplicationContext(ApplicationContext context) throws BeansException { + applicationContext = context; + } + + public static ApplicationContext getApplicationContext() { + return applicationContext; + } + + public static Object getBean(String name) { + return applicationContext.getBean(name); + } + + public static T getBean(Class clazz) { + return applicationContext.getBean(clazz); + } + + public static T getBean(String name, Class clazz) { + return applicationContext.getBean(name, clazz); + } + + public static boolean containsBean(String name) { + return applicationContext.containsBean(name); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/java/com/util/TimeUtils.java b/haobang-security-dm/syslog-collector-api/src/main/java/com/util/TimeUtils.java new file mode 100644 index 0000000..4503c3c --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/java/com/util/TimeUtils.java @@ -0,0 +1,80 @@ +package com.util; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +/** + * 时间工具类 + */ +public class TimeUtils { + + /** 默认时间格式 */ + public static final DateTimeFormatter DEFAULT_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + /** 紧凑时间格式：yyyyMMddHHmmss */ + public static final DateTimeFormatter COMPACT_FORMATTER = + DateTimeFormatter.ofPattern("yyyyMMddHHmmss"); + + /** 带毫秒格式：yyyyMMddHHmmssSSS */ + public static final DateTimeFormatter MS_FORMATTER = + DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS"); + + /** 天眼时间格式：yyyy-MM-dd'T'HH:mm:ss.SSSZ */ + public static final DateTimeFormatter TIANYAN_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + private TimeUtils() { + } + + /** + * 毫秒时间戳 -> LocalDateTime + */ + public static LocalDateTime toLocalDateTime(long timestampMs) { + return LocalDateTime.ofInstant(Instant.ofEpochMilli(timestampMs), ZoneId.systemDefault()); + } + + /** + * LocalDateTime -> 毫秒时间戳 + */ + public static long toTimestamp(LocalDateTime dateTime) { + return dateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + } + + /** + * 当前毫秒时间戳 + */ + public static long now() { + return System.currentTimeMillis(); + } + + /** + * 当前秒级时间戳 + */ + public static long nowSecond() { + return System.currentTimeMillis() / 1000; + } + + /** + * 格式化 LocalDateTime -> yyyy-MM-dd HH:mm:ss + */ + public static String format(LocalDateTime dateTime) { + return dateTime.format(DEFAULT_FORMATTER); + } + + /** + * 毫秒时间戳 -> yyyy-MM-dd HH:mm:ss + */ + public static String format(long timestampMs) { + return toLocalDateTime(timestampMs).format(DEFAULT_FORMATTER); + } + + /** + * 毫秒时间戳 -> yyyyMMddHHmmssSSS（用于 receive_time） + */ + public static String formatCompactMs(long timestampMs) { + return toLocalDateTime(timestampMs).format(MS_FORMATTER); + } +} diff --git a/haobang-security-dm/syslog-collector-api/src/main/resources/application-prod.properties b/haobang-security-dm/syslog-collector-api/src/main/resources/application-prod.properties new file mode 100644 index 0000000..c766fdd --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/resources/application-prod.properties @@ -0,0 +1,114 @@ +#Server Configuration +server.port=8290 +server.servlet.context-path=/collectorapi +server.tomcat.uri-encoding=UTF-8 +server.error.include-message=always +server.error.include-binding-errors=always + +# ============================================ +# 采集调度配置 +# ============================================ +# 定时采集间隔（毫秒），默认1分钟 +device.collect.interval=60000 +# 全量回补功能开关（false=禁用，POST /api/collect/full 将拒绝执行） +device.collect.full.enabled=false +# 重试配置 +device.collect.retry.max-attempts=3 +device.collect.retry.initial-delay=5000 +device.collect.retry.multiplier=2.0 +# Kafka 推送 topic +device.collect.kafka.topic=${KAFKA_COLLECTOR_TOPIC:test-topic} +# 采集线程池大小：各设备/数据类型并行采集，慢设备不阻塞其他设备 +device.collect.pool-size=${COLLECT_POOL_SIZE:15} +# 微步 info（日志流量）单轮采集窗口上限（分钟），积压时逐轮追赶，0=不限(用默认30分钟上限) +device.weibu.info-max-window-minutes=${WEIBU_INFO_MAX_WINDOW_MINUTES:5} +#采集探针ID +app.service.device_collect_id=${DEVICE_COLLECT_ID:1} +# ============================================ +# 天眼采集配置 +# ============================================ +device.tianyan.host=${TIANYAN_HOST:https://10.56.170.107} +# 免密登录 login_key（由平台管理员分配） +device.tianyan.login-key=${TIANYAN_LOGIN_KEY:QuJm4NwF3GMaXb2Z} +device.tianyan.username=${TIANYAN_USERNAME:xxadmin} +device.tianyan.branch-id=${TIANYAN_BRANCH_ID:QbJK%2FsLHi} +# 采集数据类型，逗号分隔：weblog(访问日志),alarm(告警日志) +device.tianyan.data-types=${TIANYAN_DATA_TYPES:weblog,alarm} +device.tianyan.page-size=${TIANYAN_PAGE_SIZE:1000} +# 天眼单次查询子窗口跨度（分钟），采集窗口切片预防平台深分页上限，<=0 不切片 +device.tianyan.sub-window-minutes=${TIANYAN_SUB_WINDOW_MINUTES:5} + +# ============================================ +# 微步采集配置 +# ============================================ +device.weibu.host=${WEIBU_HOST:https://10.56.171.164} +device.weibu.api-key=${WEIBU_API_KEY:57d8fb2b556a471ea36b8b4441fa1c0c} +device.weibu.secret=${WEIBU_SECRET:9d967876f51d4e16bbf8bde069d12a9b} +# 采集数据类型，逗号分隔：attack(攻击日志),info(情报日志) +device.weibu.data-types=${WEIBU_DATA_TYPES:attack,info} +# 微步限流：每秒最大请求数 +device.weibu.rate-limit=${WEIBU_RATE_LIMIT:10} +# 微步单次取数上限（size 两段式取数一次取全的条数上限，超出对半拆分窗口重采） +device.weibu.fetch-size-cap=${WEIBU_FETCH_SIZE_CAP:20000} + +# ============================================ +# XDR 采集配置（暂缓） +# ============================================ +device.xdr.host=${XDR_HOST:https://127.0.0.1} +device.xdr.auth-code=${XDR_AUTH_CODE:} +device.xdr.api-base=${XDR_API_BASE:/api/openapi/} +device.xdr.data-types=${XDR_DATA_TYPES:} + +# ============================================ +# Kafka 配置 +# ============================================ +spring.kafka.producer.bootstrap-servers=192.168.4.99:9092 +spring.kafka.producer.properties.retries=10 +spring.kafka.producer.properties.retry.backoff.ms=500 +spring.kafka.producer.properties.connections.max.idle.ms=600000 +spring.kafka.producer.properties.socket.keepalive.enable=true +spring.kafka.producer.properties.request.timeout.ms=30000 +spring.kafka.producer.properties.delivery.timeout.ms=120000 + +# SM4 加密密钥（十六进制，128位），与 syslog-serve / syslog-consumer 保持一致 +# 配置后 Kafka 消息以 SM4-CBC 加密推送（格式 Base64(IV):Base64(密文)）；留空则明文推送 +syslog.sm4.generateKey=${SYSLOG_SM4_KEY:f79548ab6fa8a304fc0115e17230358a} + + +#database Configuration +spring.datasource.url=jdbc:dm://192.168.4.99:5237 +spring.datasource.username=SYSDBA +spring.datasource.password=caZ2TcmXNSW8L2Ap +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" + +# mybatis Configuration +mybatis.mapper-locations=classpath:mapper/*.xml +mybatis.type-aliases-package=com.common.entity +mybatis.configuration.map-underscore-to-camel-case=true + +# Redis 缓存配置 +spring.redis.host=192.168.4.99 +spring.redis.port=6379 +# 密码（如果没有设置密码，可以省略） +spring.redis.password=redis_GdGWte +spring.redis.database=0 +spring.redis.timeout=2000 +spring.redis.lettuce.pool.max-active=8 +spring.redis.lettuce.pool.max-wait=-1 +spring.redis.lettuce.pool.max-idle=8 +spring.redis.lettuce.pool.min-idle=0 +spring.cache.redis.time-to-live=600000 + +# 定时任务线程池 +spring.task.scheduling.pool.size=5 +# ============================================ +# Mock 服务器配置（模拟天眼/微步接口，供功能测试） +# 开启后暴露 /mock/tianyan /mock/weibu 接口，采集器 host_api 指向： +# http://127.0.0.1:8290/collectorapi/mock/tianyan +# http://127.0.0.1:8290/collectorapi/mock/weibu +# 生产环境务必保持关闭 +# ============================================ +mock.server.enabled=${MOCK_SERVER_ENABLED:false} +# 模拟数据总量（每类），分页测试用 +mock.server.total-count=${MOCK_SERVER_TOTAL_COUNT:100} diff --git a/haobang-security-dm/syslog-collector-api/src/main/resources/application.properties b/haobang-security-dm/syslog-collector-api/src/main/resources/application.properties new file mode 100644 index 0000000..ff98f6d --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/resources/application.properties @@ -0,0 +1,114 @@ +#Server Configuration +server.port=8290 +server.servlet.context-path=/collectorapi +server.tomcat.uri-encoding=UTF-8 +server.error.include-message=always +server.error.include-binding-errors=always + +# ============================================ +# 采集调度配置 +# ============================================ +# 定时采集间隔（毫秒），默认1分钟 +device.collect.interval=60000 +# 全量回补功能开关（false=禁用，POST /api/collect/full 将拒绝执行） +device.collect.full.enabled=false +# 重试配置 +device.collect.retry.max-attempts=3 +device.collect.retry.initial-delay=5000 +device.collect.retry.multiplier=2.0 +# Kafka 推送 topic +device.collect.kafka.topic=${KAFKA_COLLECTOR_TOPIC:test-topic} +# 采集线程池大小：各设备/数据类型并行采集，慢设备不阻塞其他设备 +device.collect.pool-size=${COLLECT_POOL_SIZE:15} +# 微步 info（日志流量）单轮采集窗口上限（分钟），积压时逐轮追赶，0=不限(用默认30分钟上限) +device.weibu.info-max-window-minutes=${WEIBU_INFO_MAX_WINDOW_MINUTES:5} + +#采集探针ID +app.service.device_collect_id=${DEVICE_COLLECT_ID:1} +# ============================================ +# 天眼采集配置 +# ============================================ +device.tianyan.host=${TIANYAN_HOST:https://10.56.170.107} +# 免密登录 login_key（由平台管理员分配） +device.tianyan.login-key=${TIANYAN_LOGIN_KEY:QuJm4NwF3GMaXb2Z} +device.tianyan.username=${TIANYAN_USERNAME:xxadmin} +device.tianyan.branch-id=${TIANYAN_BRANCH_ID:QbJK%2FsLHi} +# 采集数据类型，逗号分隔：weblog(访问日志),alarm(告警日志) +device.tianyan.data-types=${TIANYAN_DATA_TYPES:weblog,alarm} +device.tianyan.page-size=${TIANYAN_PAGE_SIZE:1000} +# 天眼单次查询子窗口跨度（分钟），采集窗口切片预防平台深分页上限，<=0 不切片 +device.tianyan.sub-window-minutes=${TIANYAN_SUB_WINDOW_MINUTES:5} + +# ============================================ +# 微步采集配置 +# ============================================ +device.weibu.host=${WEIBU_HOST:https://10.56.171.164} +device.weibu.api-key=${WEIBU_API_KEY:57d8fb2b556a471ea36b8b4441fa1c0c} +device.weibu.secret=${WEIBU_SECRET:9d967876f51d4e16bbf8bde069d12a9b} +# 采集数据类型，逗号分隔：attack(攻击日志),info(情报日志) +device.weibu.data-types=${WEIBU_DATA_TYPES:attack,info} +# 微步限流：每秒最大请求数 +device.weibu.rate-limit=${WEIBU_RATE_LIMIT:10} +# 微步单次取数上限（size 两段式取数一次取全的条数上限，超出对半拆分窗口重采） +device.weibu.fetch-size-cap=${WEIBU_FETCH_SIZE_CAP:10000} + +# ============================================ +# XDR 采集配置（暂缓） +# ============================================ +device.xdr.host=${XDR_HOST:https://127.0.0.1} +device.xdr.auth-code=${XDR_AUTH_CODE:} +device.xdr.api-base=${XDR_API_BASE:/api/openapi/} +device.xdr.data-types=${XDR_DATA_TYPES:} + +# ============================================ +# Kafka 配置 +# ============================================ +spring.kafka.producer.bootstrap-servers=192.168.4.99:9092 +spring.kafka.producer.properties.retries=10 +spring.kafka.producer.properties.retry.backoff.ms=500 +spring.kafka.producer.properties.connections.max.idle.ms=600000 +spring.kafka.producer.properties.socket.keepalive.enable=true +spring.kafka.producer.properties.request.timeout.ms=30000 +spring.kafka.producer.properties.delivery.timeout.ms=120000 + +# SM4 加密密钥（十六进制，128位），与 syslog-serve / syslog-consumer 保持一致 +# 配置后 Kafka 消息以 SM4-CBC 加密推送（格式 Base64(IV):Base64(密文)）；留空则明文推送 +syslog.sm4.generateKey=${SYSLOG_SM4_KEY:f79548ab6fa8a304fc0115e17230358a} + +#database Configuration +spring.datasource.url=jdbc:dm://192.168.4.99:5237 +spring.datasource.username=SYSDBA +spring.datasource.password=caZ2TcmXNSW8L2Ap +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=ECOSYS + +# mybatis Configuration +mybatis.mapper-locations=classpath:mapper/*.xml +mybatis.type-aliases-package=com.common.entity +mybatis.configuration.map-underscore-to-camel-case=true + +# Redis 缓存配置 +spring.redis.host=localhost +spring.redis.port=6379 +spring.redis.password=123 +spring.redis.database=0 +spring.redis.timeout=2000 +spring.redis.lettuce.pool.max-active=8 +spring.redis.lettuce.pool.max-wait=-1 +spring.redis.lettuce.pool.max-idle=8 +spring.redis.lettuce.pool.min-idle=0 +spring.cache.redis.time-to-live=600000 + +# 定时任务线程池 +spring.task.scheduling.pool.size=5 + +# ============================================ +# Mock 服务器配置（模拟天眼/微步接口，供功能测试） +# 开启后暴露 /mock/tianyan /mock/weibu 接口，采集器 host_api 指向： +# http://127.0.0.1:8290/collectorapi/mock/tianyan +# http://127.0.0.1:8290/collectorapi/mock/weibu +# 生产环境务必保持关闭 +# ============================================ +mock.server.enabled=${MOCK_SERVER_ENABLED:false} +# 模拟数据总量（每类），分页测试用 +mock.server.total-count=${MOCK_SERVER_TOTAL_COUNT:100} diff --git a/haobang-security-dm/syslog-collector-api/src/main/resources/logback.xml b/haobang-security-dm/syslog-collector-api/src/main/resources/logback.xml new file mode 100644 index 0000000..498627a --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/resources/logback.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + ${LOG_PATTERN} + + + + + + logs/syslog-collector-api.log + + + logs/syslog-collector-api.%d{yyyy-MM-dd-HH}.log + + 120 + + 50GB + + true + + + ${LOG_PATTERN} + + + + + + + + + + + + + + + + + + + + + + + diff --git a/haobang-security-dm/syslog-collector-api/src/main/resources/mapper/DeviceApiCollectLogMapper.xml b/haobang-security-dm/syslog-collector-api/src/main/resources/mapper/DeviceApiCollectLogMapper.xml new file mode 100644 index 0000000..be0311d --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/resources/mapper/DeviceApiCollectLogMapper.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + id, device_id, device_type, data_type, start_time, end_time, + total_count, success_count, fail_count, cost_ms, status, error_msg, create_time + + + + INSERT INTO device_api_collect_log + (device_id, device_type, data_type, start_time, end_time, + total_count, success_count, fail_count, cost_ms, status, error_msg, create_time) + VALUES + (#{deviceId}, #{deviceType}, #{dataType}, #{startTime}, #{endTime}, + #{totalCount}, #{successCount}, #{failCount}, #{costMs}, #{status}, #{errorMsg}, NOW()) + + + + + + + + + + DELETE FROM device_api_collect_log + WHERE start_time < #{startTime} + + + diff --git a/haobang-security-dm/syslog-collector-api/src/main/resources/mapper/DeviceApiConfigMapper.xml b/haobang-security-dm/syslog-collector-api/src/main/resources/mapper/DeviceApiConfigMapper.xml new file mode 100644 index 0000000..17b757e --- /dev/null +++ b/haobang-security-dm/syslog-collector-api/src/main/resources/mapper/DeviceApiConfigMapper.xml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + id, device_id, device_type, device_name, device_ip, host_api, auth_config, data_types, + collect_interval_sec, enabled, remark, create_time, update_time + + + + + + + + + + + + INSERT INTO device_api_config + (device_id, device_type, device_name, device_ip, host_api, auth_config, data_types, + collect_interval_sec, enabled, remark, create_time, update_time) + VALUES + (#{deviceId}, #{deviceType}, #{deviceName}, #{deviceIp}, #{hostApi}, #{authConfig}, #{dataTypes}, + #{collectIntervalSec}, #{enabled}, #{remark}, NOW(), NOW()) + + + + UPDATE device_api_config + SET device_id = #{deviceId}, + device_type = #{deviceType}, + device_name = #{deviceName}, + device_ip = #{deviceIp}, + host_api = #{hostApi}, + auth_config = #{authConfig}, + data_types = #{dataTypes}, + collect_interval_sec = #{collectIntervalSec}, + enabled = #{enabled}, + remark = #{remark}, + update_time = NOW() + WHERE id = #{id} + + + + UPDATE device_api_config + SET enabled = #{enabled}, + update_time = NOW() + WHERE id = #{id} + + + + DELETE FROM device_api_config WHERE id = #{id} + + + diff --git a/haobang-security-dm/syslog-consumer-rule/docker-compose/docker-compose-consumer-prod.yaml b/haobang-security-dm/syslog-consumer-rule/docker-compose/docker-compose-consumer-prod.yaml index 151c367..19e6142 100644 --- a/haobang-security-dm/syslog-consumer-rule/docker-compose/docker-compose-consumer-prod.yaml +++ b/haobang-security-dm/syslog-consumer-rule/docker-compose/docker-compose-consumer-prod.yaml @@ -13,8 +13,8 @@ services: build: context: ./consumer-rule dockerfile: Dockerfile - image: syslog-consumer-rule:1.0.1 - container_name: syslog-consumer-rule + image: syslog-consumer-rule-dm:1.0.1 + container_name: syslog-consumer-rule-dm restart: unless-stopped environment: # 环境配置 diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/OfflineAnalysisEngine.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/OfflineAnalysisEngine.java index d5b7b98..5c869a5 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/OfflineAnalysisEngine.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/OfflineAnalysisEngine.java @@ -151,7 +151,12 @@ public class OfflineAnalysisEngine implements AnalysisEngine { if (!alarms.isEmpty()) { //String tableName = "alarm_" + dataStartTime.format(DateTimeFormatter.ofPattern("yyyyMMdd")); String tableName = "alarm"; - alarmMapper.batchInsert( alarms); + // 分批插入，每批1000条，防止达梦参数超过65535上限 + int alarmBatchSize = 1000; + for (int i = 0; i < alarms.size(); i += alarmBatchSize) { + int end = Math.min(i + alarmBatchSize, alarms.size()); + alarmMapper.batchInsert(alarms.subList(i, end)); + } alarmCount = alarms.size(); } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java index c1b6407..96b8439 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java @@ -162,7 +162,12 @@ public class RealtimeAnalysisEngine implements AnalysisEngine { if (!alarms.isEmpty()) { //String tableName = "alarm_" + dataStartTime.format(DateTimeFormatter.ofPattern("yyyyMMdd")); String tableName = "alarm"; - alarmMapper.batchInsert( alarms); + // 分批插入，每批1000条，防止达梦参数超过65535上限 + int alarmBatchSize = 1000; + for (int i = 0; i < alarms.size(); i += alarmBatchSize) { + int end = Math.min(i + alarmBatchSize, alarms.size()); + alarmMapper.batchInsert(alarms.subList(i, end)); + } alarmCount = alarms.size(); } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties index a9b6b6f..acb3896 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties @@ -20,7 +20,7 @@ syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a influxdb.url=http://192.168.4.99:8087 influxdb.token=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg== influxdb.org=influxdb -influxdb.bucket=yelangbucket +influxdb.bucket=influxdb influxdb.batch.size=1000 influxdb.flush.interval=1000 influxdb.retry.attempts=3 @@ -53,7 +53,7 @@ mybatis-plus.type-handlers-package=com.Modules.etl.handler # kafka Configuration spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092 -spring.kafka.consumer.group-id=test-group-1 +spring.kafka.consumer.group-id=test-group-dev spring.kafka.consumer.auto-offset-reset=latest spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.auto-commit-interval=1000 @@ -124,5 +124,5 @@ spring.datasource.hikari.max-lifetime=900000 spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 -spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule +spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule-dev spring.datasource.hikari.auto-commit=false \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/logback.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/logback.xml index 9196a6f..bebdaba 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/logback.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/logback.xml @@ -9,10 +9,10 @@ logs/syslog-consumer-rule-dm.log - logs/syslog-consumer-rule-dm.%d{yyyy-MM-dd}.log + logs/syslog-consumer-rule-dm.%d{yyyy-MM-dd-HH}.log - - 1 + + 24 15GB diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application.properties index d32fc09..094d0b8 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application.properties @@ -7,8 +7,7 @@ server.error.include-message=always server.error.include-binding-errors=always #run.environment: dev|test|pro -server.run.environment=pro - +server.run.environment=dev # Syslog Server Configuration syslog.tcp.port=514 @@ -16,7 +15,6 @@ syslog.udp.port=514 syslog.max.frame.length=65536 syslog.buffer.size=1000 syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a - # InfluxDB 2.7 Configuration influxdb.url=http://192.168.4.99:8087 influxdb.token=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg== @@ -41,10 +39,11 @@ spring.datasource.url=jdbc:dm://192.168.4.99:5237 spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver -spring.datasource.hikari.schema=\"PUBLIC\" +spring.datasource.hikari.schema=ECOSYS # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity + #mybatis handler 类 mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 @@ -52,26 +51,25 @@ mybatis.configuration.map-underscore-to-camel-case=true mybatis.type-handlers-package=com.Modules.etl.handler mybatis-plus.configuration.map-underscore-to-camel-case=true mybatis-plus.type-handlers-package=com.Modules.etl.handler - # kafka Configuration spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092 -spring.kafka.consumer.group-id=agent-syslog-group +spring.kafka.consumer.group-id=test-group spring.kafka.consumer.auto-offset-reset=latest spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.auto-commit-interval=1000 -spring.kafka.consumer.topic=agent-syslog-topic - +spring.kafka.consumer.topic=test-topic spring.kafka.consumer.max-poll-records=1000 spring.kafka.consumer.properties.max.poll.interval.ms=300000 spring.kafka.consumer.properties.session.timeout.ms=45000 +#spring.kafka.consumer.key-deserializer: org.apache.kafka.common.serialization.StringDeserializer +#spring.kafka.consumer.value-deserializer: org.apache.kafka.common.serialization.StringDeserializer spring.kafka.consumer.fetch-min-size= 1048576 spring.kafka.listener.ack-mode= manual spring.kafka.listener.concurrency= 2 spring.kafka.listener.type=batch - # 定时任务配置 spring.task.scheduling.pool.size=10 @@ -84,23 +82,21 @@ partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true - -# 生产环境缓存配置 -spring.redis.host=192.168.4.99 +# 开发环境缓存配置 +spring.redis.host=localhost spring.redis.port=6379 # 密码（如果没有设置密码，可以省略） -spring.redis.password=redis_GdGWte +spring.redis.password= spring.redis.database=0 -spring.redis.timeout=5000 -#spring.redis.password=${REDIS_PASSWORD:default_prod_password} +spring.redis.timeout=2000 -spring.redis.lettuce.pool.max-active=20 -spring.redis.lettuce.pool.max-wait=5000 -spring.redis.lettuce.pool.max-idle=10 -spring.redis.lettuce.pool.min-idle=5 +spring.redis.lettuce.pool.max-active=8 +spring.redis.lettuce.pool.max-wait=-1 +spring.redis.lettuce.pool.max-idle=8 +spring.redis.lettuce.pool.min-idle=0 +# 开发环境缓存时间较短，方便调试 +spring.cache.redis.time-to-live=600000 -# 生产环境缓存时间较长 -spring.cache.redis.time-to-live=3600000 # 应用处理器配置 app.processor.thread-pool.core-pool-size=10 @@ -141,10 +137,9 @@ spring.datasource.hikari.max-lifetime=900000 spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 -spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer +spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-dev spring.datasource.hikari.auto-commit=false - # 关联分析规则配置 analysis.realtime.enabled= true # 检查间隔（秒） - 默认10秒 @@ -155,14 +150,13 @@ analysis.realtime.check-interval-seconds: 10 # false → 使用 RealtimeAnalysisEngine，平铺SQL（原有行为） analysis.db.dameng-subquery-mode=true - # ============================================ # 探针联动API配置 # ============================================ # API-KEY认证（32位，建议使用随机生成的密钥） interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 # API接口基础URL（供syslog-serve调用） -interlocking.api.base-url=http://192.168.4.99:8089/xdrservice/interlocking +interlocking.api.base-url=http://localhost:8089/xdrservice/interlocking # ============================================ # 告警健康检查配置 diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/logback.xml b/haobang-security-dm/syslog-consumer/src/main/resources/logback.xml index ef4c9d3..e4502a1 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/logback.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/logback.xml @@ -9,10 +9,10 @@ logs/syslog-consumer-dm.log - logs/syslog-consumer-dm.%d{yyyy-MM-dd}.log + logs/syslog-consumer-dm.%d{yyyy-MM-dd-HH}.log - - 1 + + 24 15GB diff --git a/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-prod.yaml b/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-prod.yaml index 2e93ece..4efb049 100644 --- a/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-prod.yaml +++ b/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-prod.yaml @@ -28,8 +28,7 @@ services: - SPRING_DATASOURCE_USERNAME=SYSDBA - SPRING_DATASOURCE_PASSWORD=caZ2TcmXNSW8L2Ap - SPRING_DATASOURCE_DRIVER_CLASS_NAME=dm.jdbc.driver.DmDriver - - SPRING_DATASOURCE_HIKARI_SCHEMA=\"PUBLIC\" - + - SPRING_DATASOURCE_HIKARI_SCHEMA=PUBLIC # Redis配置 - SPRING_REDIS_HOST=192.168.4.99 diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/Modules/Device/DeviceProcess.java b/haobang-security-dm/syslog-serve/src/main/java/com/Modules/Device/DeviceProcess.java index 1a8150f..0794166 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/Modules/Device/DeviceProcess.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/Modules/Device/DeviceProcess.java @@ -107,7 +107,7 @@ public class DeviceProcess { if(deviceList.size()>0) { deviceUnknownService.updateLastTime( deviceList.get(0).getId(),currentTime ); - System.out.println("更新未知设备的最后发现时间,IP:"+this.sourceIP); + logger.info("更新未知设备的最后发现时间,IP:"+this.sourceIP); } else { //创建未知设备记录 diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java b/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java index ed8843a..2809a3d 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java @@ -5,6 +5,15 @@ import com.haobang.util.Sm4Util; import org.apache.kafka.clients.producer.*; import org.apache.kafka.common.serialization.StringSerializer; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.LinkedHashMap; import java.util.Properties; @@ -20,157 +29,238 @@ public class kafkaProducer { private static final Logger logger = LoggerFactory.getLogger(kafkaProducer.class); + // KafkaProducer 全局单例，线程安全，避免每次 send 重建连接 + private static volatile Producer producer; + // 缓存的 topic 名称（初始化时从配置读取一次，避免每次 send 都查 Spring） + private static volatile String cachedTopic; + // 缓存的 SM4 密钥 + private static volatile String cachedSm4Key; + + private static final Object lock = new Object(); + + // ==================== 兜底备份配置 ==================== + /** 兜底文件存储根目录 */ + private static final String FALLBACK_DIR = "/app/syslog-fallback/"; + /** 兜底文件日期格式 */ + private static final DateTimeFormatter FALLBACK_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd"); + /** 兜底写入最大重试次数 */ + private static final int FALLBACK_MAX_RETRIES = 3; + /** 兜底写入重试间隔（毫秒） */ + private static final long FALLBACK_RETRY_INTERVAL_MS = 100; + + static { + // 注册 JVM 关闭钩子，确保 Producer 正常 flush + close + Runtime.getRuntime().addShutdownHook(new Thread(() -> closeProducer())); + } + private static AppProperties getProps() { return SpringContextUtil.getBean(AppProperties.class); } + + /** + * 获取全局 KafkaProducer 单例（双重检查锁定） + */ + private static Producer getProducer() { + if (producer == null) { + synchronized (lock) { + if (producer == null) { + AppProperties appProps = getProps(); + Properties props = new Properties(); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProps.getKafkaProducerBootstrap()); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + props.put(ProducerConfig.ACKS_CONFIG, "all"); + props.put(ProducerConfig.RETRIES_CONFIG, 3); + props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true); + // 生产者缓冲区大小，减少网络 IO 次数 + props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384); + props.put(ProducerConfig.LINGER_MS_CONFIG, 5); + + producer = new KafkaProducer<>(props); + cachedTopic = appProps.getKafkaProducerTopic(); + cachedSm4Key = appProps.getSm4Key(); + logger.info("KafkaProducer 单例初始化完成, topic={}, brokers={}", + cachedTopic, appProps.getKafkaProducerBootstrap()); + } + } + } + return producer; + } + + /** + * 优雅关闭 KafkaProducer + */ + public static void closeProducer() { + if (producer != null) { + synchronized (lock) { + if (producer != null) { + try { + logger.info("正在关闭 KafkaProducer..."); + producer.flush(); + producer.close(); + logger.info("KafkaProducer 已关闭"); + } catch (Exception e) { + logger.error("关闭 KafkaProducer 异常", e); + } + producer = null; + } + } + } + } + public static void main(String[] args) { AppProperties appProps = getProps(); - // 配置生产者属性 Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProps.getKafkaProducerBootstrap()); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); - - // 可选配置：提高可靠性 props.put(ProducerConfig.ACKS_CONFIG, "all"); props.put(ProducerConfig.RETRIES_CONFIG, 3); props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true); - // 创建生产者实例 - Producer producer = new KafkaProducer<>(props); - - try { - // 发送10条消息 - //for (int i = 0; i < 1; i++) { - String key = "key-" + "1"; - String value = "message-" + "1" + " at " + System.currentTimeMillis(); - - // 创建生产者记录 - ProducerRecord record = - new ProducerRecord<>(appProps.getKafkaProducerTopic(), key, value); - - // 发送消息（异步方式） - producer.send(record, new Callback() { - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception == null) { - System.out.println("消息发送成功: " + - "主题=" + metadata.topic() + - ", 分区=" + metadata.partition() + - ", 偏移量=" + metadata.offset()); - } else { - System.err.println("消息发送失败: " + exception.getMessage()); - } + try (Producer producer = new KafkaProducer<>(props)) { + String key = "key-" + "1"; + String value = "message-" + "1" + " at " + System.currentTimeMillis(); + ProducerRecord record = + new ProducerRecord<>(appProps.getKafkaProducerTopic(), key, value); + producer.send(record, new Callback() { + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception == null) { + System.out.println("消息发送成功: " + + "主题=" + metadata.topic() + + ", 分区=" + metadata.partition() + + ", 偏移量=" + metadata.offset()); + } else { + System.err.println("消息发送失败: " + exception.getMessage()); } - }); - // 同步发送方式（如果需要） - // RecordMetadata metadata = producer.send(record).get(); - // System.out.println("同步发送成功: " + metadata.offset()); - // } + } + }); + producer.flush(); } catch (Exception e) { e.printStackTrace(); - } finally { - // 关闭生产者 - producer.close(); } } /** - * * syslog 文本消息 - * * @param strSyslog + * syslog 文本消息推送到 Kafka（复用全局单例 Producer） + * @param deviceDevice 设备信息 + * @param strSyslog 原始 syslog 消息 + * @param strReceiveTime 接收时间 */ - - public static void messagePush(DeviceDevice deviceDevice, String strSyslog,String strReceiveTime) - { - AppProperties appProps = getProps(); - // 配置生产者属性 - Properties props = new Properties(); - props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProps.getKafkaProducerBootstrap()); - props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); - props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); - - // 可选配置：提高可靠性 - props.put(ProducerConfig.ACKS_CONFIG, "all"); - props.put(ProducerConfig.RETRIES_CONFIG, 3); - props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true); - - // 创建生产者实例 - Producer producer = new KafkaProducer<>(props); + public static void messagePush(DeviceDevice deviceDevice, String strSyslog, String strReceiveTime) { + Producer p = getProducer(); try { + String key = "key-" + System.currentTimeMillis(); + String value = DeviceInfoUtil.getFullLogString(deviceDevice, strSyslog, strReceiveTime); - String key = "key-" + System.currentTimeMillis(); - //String value = DeviceInfoUtil.getFullLogString(strSyslog); - //采用动态获取Syslog 请求的设备信息 - String value = DeviceInfoUtil.getFullLogString(deviceDevice,strSyslog,strReceiveTime); + // SM4 加密 + String sm4Message = Sm4Util.encryptCbc(value, cachedSm4Key); - //Message进行SM4加密写入kafka - String Sm4message= Sm4Util.encryptCbc(value, appProps.getSm4Key()); - //System.out.println("Sm4message:"+Sm4message); + ProducerRecord record = + new ProducerRecord<>(cachedTopic, key, sm4Message); - // 创建生产者记录 - ProducerRecord record = - new ProducerRecord<>(appProps.getKafkaProducerTopic(), key, Sm4message); - - // 发送消息（异步方式） - producer.send(record, new Callback() { - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception == null) { - System.out.println("消息发送成功: " + - "主题=" + metadata.topic() + - ", 分区=" + metadata.partition() + - ", 偏移量=" + metadata.offset()); - } else { - System.err.println("消息发送失败: " + exception.getMessage()); - logger.error("消息发送失败: " + exception.getMessage()); - } + // 异步发送 + 失败回调兜底：闭包捕获 value（原始明文），失败时写入本地文件 + p.send(record, new Callback() { + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception == null) { + logger.debug("消息发送成功: 主题={}, 分区={}, 偏移量={}", + metadata.topic(), metadata.partition(), metadata.offset()); + } else { + logger.error("消息发送失败, 开始兜底写入本地文件: topic={}, 异常={}", + cachedTopic, exception.getMessage()); + fallbackToLocal(value); } - }); - - // 同步发送方式（如果需要） - // RecordMetadata metadata = producer.send(record).get(); - // System.out.println("同步发送成功: " + metadata.offset()); + } + }); } catch (Exception e) { - e.printStackTrace(); - } finally { - // 关闭生产者 - producer.close(); + // 加密/组包阶段异常，同样兜底 + logger.error("messagePush 预处理异常, 开始兜底写入本地文件", e); + try { + String value = DeviceInfoUtil.getFullLogString(deviceDevice, strSyslog, strReceiveTime); + fallbackToLocal(value); + } catch (Exception ex) { + logger.error("兜底写入时组包也失败, 消息已丢失", ex); + } } + // 注意：不再 close，Producer 是全局单例，由 shutdown hook 关闭 } /** * 生成json字符串，补充设备配置信息 - * @param strSyslog - * @return */ - private static String getLogJsonString(String strSyslog) - { + private static String getLogJsonString(String strSyslog) { ObjectMapper objectMapper = new ObjectMapper(); - // 创建设备配置信息MAP Map map = new LinkedHashMap<>(); AppProperties appProps = getProps(); map.put("device_id", appProps.getAppServiceDeviceId()); map.put("device_name", appProps.getAppServiceDeviceName()); map.put("vendor", appProps.getAppServiceVendor()); map.put("data_type", appProps.getAppServiceDataType()); - // 添加syslogMessage map.put("syslogMessage", strSyslog); try { - //生成json消息字符串 - String json = objectMapper.writeValueAsString(map); - return json; - } - catch(Exception ex) - { - logger.error("kafkaProducer getLogJsonString(）生成json字符串 异常: syslog: "+ strSyslog); - logger.error("Exception: : "+ ex.getMessage()); + return objectMapper.writeValueAsString(map); + } catch (Exception ex) { + logger.error("kafkaProducer getLogJsonString() 生成json字符串异常, syslog: {}", strSyslog); + logger.error("Exception: {}", ex.getMessage()); } return null; } + // ==================== 兜底备份 ==================== + /** + * 将原始消息写入本地兜底文件（按日期分文件，追加写入） + *

+ * 文件路径：/data/syslog-fallback/kafka-failed-yyyyMMdd.log + *

+ * @param message 原始明文消息（完整日志字符串） + */ + private static void fallbackToLocal(String message) { + if (message == null || message.isEmpty()) { + logger.warn("兜底消息为空，跳过写入"); + return; + } + for (int i = 0; i < FALLBACK_MAX_RETRIES; i++) { + try { + String dateStr = LocalDate.now().format(FALLBACK_DATE_FORMAT); + String fileName = "kafka-failed-" + dateStr + ".log"; + Path filePath = Paths.get(FALLBACK_DIR, fileName); + // 确保目录存在 + File dir = new File(FALLBACK_DIR); + if (!dir.exists()) { + boolean created = dir.mkdirs(); + if (!created) { + logger.error("创建兜底目录失败: {}", FALLBACK_DIR); + return; + } + } + + // 追加写入，每条一行 + String line = message + "\n"; + Files.write(filePath, line.getBytes(StandardCharsets.UTF_8), + StandardOpenOption.CREATE, StandardOpenOption.APPEND); + + logger.debug("消息已兜底写入文件: {}", fileName); + return; + + } catch (IOException e) { + logger.error("兜底文件写入失败(第{}次重试): {}", i + 1, e.getMessage()); + if (i < FALLBACK_MAX_RETRIES - 1) { + try { + Thread.sleep(FALLBACK_RETRY_INTERVAL_MS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + } + } + } + logger.error("兜底写入彻底失败，消息丢失！消息长度={}", message.length()); + } } diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties index a3146be..d66847e 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties @@ -9,8 +9,8 @@ server.error.include-binding-errors=always # Syslog Server Configuration syslog.tcp.port=514 syslog.udp.port=514 -syslog.max.frame.length=2621440 -syslog.buffer.size=10000 +syslog.max.frame.length=262144 +syslog.buffer.size=1000 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 syslog.charset=UTF-8 syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a @@ -23,7 +23,7 @@ app.service.product_name=diting app.service.data_type=json #采集探针ID app.service.device_collect_id=${DEVICE_COLLECT_ID:1} -app.service.version=${APP_SERVICE_VER:V1.0.0-20260527} +app.service.version=${APP_SERVICE_VER:V1.0.0-20260811} app.service.device_collect_name=${DEVICE_COLLECT_NAME:DevCollect-01} # kafka Configuration diff --git a/haobang-security-dm/syslog-serve/src/main/resources/logback.xml b/haobang-security-dm/syslog-serve/src/main/resources/logback.xml index f90779a..d03f664 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/logback.xml +++ b/haobang-security-dm/syslog-serve/src/main/resources/logback.xml @@ -16,10 +16,10 @@ logs/syslog-serve-dm.log - - logs/syslog-serve-dm.%d{yyyy-MM-dd}.log - - 1 + + logs/syslog-serve-dm.%d{yyyy-MM-dd-HH}.log + + 24 15GB