nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1、适配部署环境变量自动注入

Browse Source 
2、适配南网达梦数据库安全版SQL语句Group by 不支持自定义函数
3、达梦数据库驱动改为达梦提供的指定版本
4、修改syslog-consumer模块yaml 文件kafka 变量名称
5、修改访问日志告警表新增4个字段，完善异常行为详情页面显示字段
This commit is contained in:
nanChen
2026-07-22 18:50:44 +08:00
parent 5fe0e124bb
commit 16b73dffee
89 changed files with 4507 additions and 2186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
haobang-security-dm/syslog-consumer-rule/src/main/java/com/kafka/kafkalogconsumer.java
+16 -24
View File
@@ -18,7 +18,9 @@ import com.common.util.JsonParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.common.util.JsonParser;
import com.config.AppConfig;
import com.common.util.SpringContextUtil;
import com.config.AppProperties;
import com.config.KafkaConsumerProperties;
import com.Modules.NormalData.LogNormalProcessor;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
@@ -38,34 +40,27 @@ public class kafkalogconsumer {
}
public static void Run()
{
// 配置消费者属性
//Properties props = new Properties();
//props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.222.130:9092");
// props.put(ConsumerConfig.GROUP_ID_CONFIG, "test-group-app");
KafkaConsumerProperties kafkaProps = SpringContextUtil.getBean(KafkaConsumerProperties.class);
LogNormalProcessor logNormalProcessor = SpringContextUtil.getBean(LogNormalProcessor.class);
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfig.getBootstrapServers());
props.put(ConsumerConfig.GROUP_ID_CONFIG, AppConfig.getGroupId());
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProps.getBootstrapServers());
props.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaProps.getGroupId());
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
// 可选配置
//props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none"); // 从最早的消息开始消费
//props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 从最早的消息开始消费
//props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); // 自动提交偏移量
//props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000"); // 自动提交间隔
//props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1000); // 设置单次拉取最大消息数[citation:6]
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, AppConfig.getAutoOffsetReset()); // 从last开始消费
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, AppConfig.getEnableAutoCommit()); // 自动提交偏移量
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,AppConfig.getAutoCommitIntervalMS()); // 自动提交间隔
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaProps.getAutoOffsetReset());
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, kafkaProps.isEnableAutoCommit());
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, kafkaProps.getAutoCommitInterval());
// 创建消费者实例
Consumer<String, String> consumer = new KafkaConsumer<>(props);
try {
// 订阅主题
consumer.subscribe(Collections.singletonList(AppConfig.getTopic()));
consumer.subscribe(Collections.singletonList(kafkaProps.getTopic()));
System.out.println("开始消费消息...");
com.influx.InfluxDBClient influxClient = new InfluxDBClient();
com.influx.InfluxDBClient influxClient = SpringContextUtil.getBean(com.influx.InfluxDBClient.class);
// 持续消费消息
while (true) {
// 拉取消息（等待最多100毫秒）
@@ -88,7 +83,7 @@ public class kafkalogconsumer {
Point point = Point.measurement("syslog_security")
.addTag("deviceid", mapdev.get("device_id")) // 添加标签
.addTag("uuid", sysLogUUID) //syslog uuid
.addTag("topic", AppConfig.getTopic()) //kafka topic
.addTag("topic", kafkaProps.getTopic()) //kafka topic
.addField("message", record.value()) // 添加字段
.time(System.currentTimeMillis(), WritePrecision.MS) ;// 毫秒级时间戳
influxClient.writePointBlocking(point);
@@ -97,12 +92,9 @@ public class kafkalogconsumer {
//insertSingleRecord( record.value());
System.out.println("insert postgres syslog ,value:"+ record.key());
//String syslogMessage= AppConfig.geRunEnvironment‌().equals("test")? record.value().substring(34) : record.value();
String syslogMessage= record.value();
//剔除测试环境本机syslog新增的头部信息
LogNormalProcessor logNormalProcessor = new LogNormalProcessor(syslogMessage,sysLogUUID,AppConfig.getTopic());
//LogNormalProcessor logNormalProcessor =new LogNormalProcessor(record.value());
logNormalProcessor.init();
//使用注入的 Spring Bean 进行标准化处理
logNormalProcessor.process(syslogMessage, sysLogUUID, null);
}
// 手动提交偏移量（如果禁用自动提交）
consumer.commitSync();