nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增syslog-collector-api 采集模块，通过api 方式采集天眼、微步的设备日志。

Browse Source
This commit is contained in:
nanChen
2026-09-07 15:34:17 +08:00
parent a3852fc0f9
commit 037cca32dd
53 changed files with 6358 additions and 488 deletions
Download Patch File Download Diff File Expand all files Collapse all files
haobang-security-dm/syslog-consumer-rule/docker-compose/docker-compose-consumer-prod.yaml
+2 -2
View File
@@ -13,8 +13,8 @@ services:
build:
context: ./consumer-rule
dockerfile: Dockerfile
image: syslog-consumer-rule:1.0.1
container_name: syslog-consumer-rule
image: syslog-consumer-rule-dm:1.0.1
container_name: syslog-consumer-rule-dm
restart: unless-stopped
environment:
# 环境配置
haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/OfflineAnalysisEngine.java
+6 -1
View File
@@ -151,7 +151,12 @@ public class OfflineAnalysisEngine implements AnalysisEngine {
if (!alarms.isEmpty()) {
//String tableName = "alarm_" + dataStartTime.format(DateTimeFormatter.ofPattern("yyyyMMdd"));
String tableName = "alarm";
alarmMapper.batchInsert( alarms);
// 分批插入，每批1000条，防止达梦参数超过65535上限
int alarmBatchSize = 1000;
for (int i = 0; i < alarms.size(); i += alarmBatchSize) {
int end = Math.min(i + alarmBatchSize, alarms.size());
alarmMapper.batchInsert(alarms.subList(i, end));
}
alarmCount = alarms.size();
}
}
haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java
+6 -1
View File
@@ -162,7 +162,12 @@ public class RealtimeAnalysisEngine implements AnalysisEngine {
if (!alarms.isEmpty()) {
//String tableName = "alarm_" + dataStartTime.format(DateTimeFormatter.ofPattern("yyyyMMdd"));
String tableName = "alarm";
alarmMapper.batchInsert( alarms);
// 分批插入，每批1000条，防止达梦参数超过65535上限
int alarmBatchSize = 1000;
for (int i = 0; i < alarms.size(); i += alarmBatchSize) {
int end = Math.min(i + alarmBatchSize, alarms.size());
alarmMapper.batchInsert(alarms.subList(i, end));
}
alarmCount = alarms.size();
}
}
haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties
+3 -3
View File
@@ -20,7 +20,7 @@ syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a
influxdb.url=http://192.168.4.99:8087
influxdb.token=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg==
influxdb.org=influxdb
influxdb.bucket=yelangbucket
influxdb.bucket=influxdb
influxdb.batch.size=1000
influxdb.flush.interval=1000
influxdb.retry.attempts=3
@@ -53,7 +53,7 @@ mybatis-plus.type-handlers-package=com.Modules.etl.handler
# kafka Configuration
spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092
spring.kafka.consumer.group-id=test-group-1
spring.kafka.consumer.group-id=test-group-dev
spring.kafka.consumer.auto-offset-reset=latest
spring.kafka.consumer.enable-auto-commit=false
spring.kafka.consumer.auto-commit-interval=1000
@@ -124,5 +124,5 @@ spring.datasource.hikari.max-lifetime=900000
spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL
spring.datasource.hikari.validation-timeout=5000
spring.datasource.hikari.leak-detection-threshold=30000
spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule
spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule-dev
spring.datasource.hikari.auto-commit=false
haobang-security-dm/syslog-consumer-rule/src/main/resources/logback.xml
+3 -3
View File
@@ -9,10 +9,10 @@
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>logs/syslog-consumer-rule-dm.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>logs/syslog-consumer-rule-dm.%d{yyyy-MM-dd}.log</fileNamePattern>
<fileNamePattern>logs/syslog-consumer-rule-dm.%d{yyyy-MM-dd-HH}.log</fileNamePattern>
<!-- 保留的日志文件的最大天数 -->
<maxHistory>1</maxHistory>
<!-- 保留的日志文件的最大小时数 -->
<maxHistory>24</maxHistory>
<!-- 所有归档日志文件的总大小上限 -->
<totalSizeCap>15GB</totalSizeCap>
<!-- 启动时是否清理过期日志 -->