nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1、新增功能探针联动处置、心跳在线检测

Browse Source 
2、syslog-consumer模块拆分 syslog-consumer-rule模块实现日志数据消费、解析、泛化入库。
This commit is contained in:
nanChen
2026-05-28 14:30:06 +08:00
parent 19c563b3f3
commit a360895292
1479 changed files with 116572 additions and 4549 deletions
Download Patch File Download Diff File Expand all files Collapse all files
haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessage.java
+75
View File
@@ -0,0 +1,75 @@
package com.netty;
import java.util.Date;
/**
* Syslog 消息实体类
*/
public class SyslogMessage {
private Integer facility;
private Integer severity;
private Date timestamp;
private String hostname;
private String message;
// 构造函数
public SyslogMessage() {}
// Getter 和 Setter 方法
public Integer getFacility() {
return facility;
}
public void setFacility(Integer facility) {
this.facility = facility;
}
public Integer getSeverity() {
return severity;
}
public void setSeverity(Integer severity) {
this.severity = severity;
}
public Date getTimestamp() {
return timestamp;
}
public void setTimestamp(String timestamp) {
// 这里可以添加时间戳解析逻辑
// 简化实现，直接使用当前时间
this.timestamp = new Date();
}
public void setTimestamp(Date timestamp) {
this.timestamp = timestamp;
}
public String getHostname() {
return hostname;
}
public void setHostname(String hostname) {
this.hostname = hostname;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
@Override
public String toString() {
return "SyslogMessage{" +
"facility=" + facility +
", severity=" + severity +
", timestamp=" + timestamp +
", hostname='" + hostname + '\'' +
", message='" + message + '\'' +
'}';
}
}
haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java
+151
View File
@@ -0,0 +1,151 @@
package com.netty;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.channel.socket.DatagramPacket;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.InetSocketAddress;
import com.kafka.kafkaProducer;
import com.Modules.Device.DeviceProcess;
import com.haobang.util.Sm4Util;
import com.haobang.config.AppConfig;
/**
* Syslog 消息处理器
*/
public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
private static final Logger logger = LoggerFactory.getLogger(SyslogMessageHandler.class);
@Override
protected void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception {
String message;
String source;
String source_ip;
String networkProtocol;
int source_port;
if (msg instanceof DatagramPacket) {
// UDP 消息处理
DatagramPacket packet = (DatagramPacket) msg;
message = packet.content().toString(io.netty.util.CharsetUtil.UTF_8);
source_ip=packet.sender().getAddress().getHostAddress();
source_port=packet.sender().getPort();
source = packet.sender().getAddress().getHostAddress() + ":" + packet.sender().getPort();
networkProtocol="UDP";
logger.info("Received syslog from {}: {}", source, message);
} else if (msg instanceof String) {
// TCP 消息处理
message = (String) msg;
InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress();
source = remoteAddress.getAddress().getHostAddress() + ":" + remoteAddress.getPort();
source_ip=remoteAddress.getAddress().getHostAddress();
source_port=remoteAddress.getPort();
networkProtocol="TCP";
logger.info("Received syslog from {}: {}", source, message);
} else {
logger.warn("Received unknown message type: {}", msg.getClass().getName());
return;
}
DeviceProcess deviceProcess = new DeviceProcess(source_ip);
try {
//日志推送的设备信息进行鉴权处理
int deviceId = deviceProcess.getDeviceID(source_ip);
if (deviceId < 0) {
logger.info("syslog message 的请求设备IP:{}非系统注册，暂不做处理！",source_ip);
//创建未知设备
deviceProcess.saveDeviceUnknow(networkProtocol);
return;
}
if (!deviceProcess.IsBelongDeviceCollectTask()) {
logger.info("syslog message 的请求设备IP:{}不归属当前采集探针，暂不做处理！",source_ip);
return;
}
//插入PG库接收记录
deviceProcess.saveDeviceReceiveLog(message,true);
kafkaProducer.messagePush(deviceProcess.getDeviceDevice(), message, deviceProcess.getDeviceReceiveLog().getReceiveTimeStr());
} catch (Exception e) {
//记录日志推送失败
deviceProcess.saveDeviceReceiveLog(message,false);
logger.warn("Failed to Push syslog message", e);
}
// 处理 Syslog 消息
//processSyslogMessage(message.trim(), source);
}
private void processSyslogMessage(String message, String source) {
// 这里可以添加 Syslog 消息解析逻辑
// 根据 RFC 3164 或 RFC 5424 解析消息
System.out.println("Received syslog from "+ source +", msg:"+ message);
logger.info("Received syslog from {}: {}", source, message);
try {
SyslogMessage syslogMsg = parseSyslogMessage(message);
logger.info("Parsed syslog - Facility: {}, Severity: {}, Timestamp: {}, Host: {}, Message: {}",
syslogMsg.getFacility(),
syslogMsg.getSeverity(),
syslogMsg.getTimestamp(),
syslogMsg.getHostname(),
syslogMsg.getMessage());
// 这里可以添加消息存储或其他处理逻辑
// storeToDatabase(syslogMsg);
} catch (Exception e) {
logger.warn("Failed to parse syslog message: {}", message, e);
}
}
/**
* 简单的 Syslog 消息解析（RFC 3164 格式）
*/
private SyslogMessage parseSyslogMessage(String message) {
SyslogMessage syslogMsg = new SyslogMessage();
// 尝试解析 PRI 部分
if (message.startsWith("<")) {
int priEnd = message.indexOf(">");
if (priEnd > 0) {
String priStr = message.substring(1, priEnd);
try {
int pri = Integer.parseInt(priStr);
int facility = pri >> 3;
int severity = pri & 0x07;
syslogMsg.setFacility(facility);
syslogMsg.setSeverity(severity);
message = message.substring(priEnd + 1).trim();
} catch (NumberFormatException e) {
// 忽略 PRI 解析错误
}
}
}
// 尝试解析时间戳和主机名
// 这是一个简化的解析，实际实现可能需要更复杂的逻辑
String[] parts = message.split(" ", 5);
if (parts.length >= 4) {
// 假设前三个部分是时间戳，第四个部分是主机名
String timestamp = parts[0] + " " + parts[1] + " " + parts[2];
syslogMsg.setTimestamp(timestamp);
syslogMsg.setHostname(parts[3]);
if (parts.length == 5) {
syslogMsg.setMessage(parts[4]);
}
} else {
syslogMsg.setMessage(message);
}
return syslogMsg;
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
logger.error("Exception in channel handler", cause);
ctx.close();
}
}
haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogServer.java
+84
View File
@@ -0,0 +1,84 @@
package com.netty;
import io.netty.bootstrap.Bootstrap;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioDatagramChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Syslog 服务器主类，支持 TCP 和 UDP 协议
*/
public class SyslogServer {
private static final Logger logger = LoggerFactory.getLogger(SyslogServer.class);
private final int tcpPort;
private final int udpPort;
public SyslogServer(int tcpPort, int udpPort) {
this.tcpPort = tcpPort;
this.udpPort = udpPort;
}
public void start() throws InterruptedException {
// 启动 TCP 服务器
startTcpServer();
// 启动 UDP 服务器
startUdpServer();
}
private void startTcpServer() throws InterruptedException {
EventLoopGroup bossGroup = new NioEventLoopGroup(1);
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
ServerBootstrap b = new ServerBootstrap();
b.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.childHandler(new SyslogTcpChannelInitializer())
.option(ChannelOption.SO_BACKLOG, 128)
.childOption(ChannelOption.SO_KEEPALIVE, true);
ChannelFuture f = b.bind(tcpPort).sync();
logger.info("TCP Syslog server started on port {}", tcpPort);
f.channel().closeFuture().sync();
} finally {
workerGroup.shutdownGracefully();
bossGroup.shutdownGracefully();
}
}
private void startUdpServer() throws InterruptedException {
EventLoopGroup group = new NioEventLoopGroup();
try {
Bootstrap b = new Bootstrap();
b.group(group)
.channel(NioDatagramChannel.class)
.handler(new SyslogUdpChannelInitializer())
.option(ChannelOption.SO_BROADCAST, true);
ChannelFuture f = b.bind(udpPort).sync();
logger.info("UDP Syslog server started on port {}", udpPort);
f.channel().closeFuture().sync();
} finally {
group.shutdownGracefully();
}
}
public static void main(String[] args) throws Exception {
int tcpPort = args.length > 0 ? Integer.parseInt(args[0]) : 514;
int udpPort = args.length > 1 ? Integer.parseInt(args[1]) : 514;
SyslogServer server = new SyslogServer(tcpPort, udpPort);
server.start();
}
}
haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogServerBoth.java
+202
View File
@@ -0,0 +1,202 @@
package com.netty;
import io.netty.bootstrap.Bootstrap;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioDatagramChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* Syslog 服务器主类，支持 TCP 和 UDP 协议
*/
public class SyslogServerBoth {
private static final Logger logger = LoggerFactory.getLogger(SyslogServer.class);
private final int tcpPort;
private final int udpPort;
// 共享的EventLoopGroup以优化资源使用
private EventLoopGroup bossGroup;
private EventLoopGroup workerGroup;
private EventLoopGroup udpGroup;
private ChannelFuture tcpChannelFuture;
private ChannelFuture udpChannelFuture;
private final ExecutorService serverExecutor = Executors.newFixedThreadPool(2);
public SyslogServerBoth(int tcpPort, int udpPort) {
this.tcpPort = tcpPort;
this.udpPort = udpPort;
}
public void start() throws InterruptedException {
logger.info("Starting Syslog server with TCP port {} and UDP port {}", tcpPort, udpPort);
// 创建共享的EventLoopGroup
bossGroup = new NioEventLoopGroup(1);
workerGroup = new NioEventLoopGroup();
udpGroup = new NioEventLoopGroup();
final CountDownLatch latch = new CountDownLatch(2);
try {
// 启动UDP服务器（使用独立的线程）
serverExecutor.submit(() -> {
try {
startUdpServer();
latch.countDown();
} catch (Exception e) {
logger.error("UDP server failed to start", e);
latch.countDown();
throw new RuntimeException(e);
}
});
// 启动TCP服务器（使用独立的线程）
serverExecutor.submit(() -> {
try {
startTcpServer();
latch.countDown();
} catch (Exception e) {
logger.error("TCP server failed to start", e);
latch.countDown();
throw new RuntimeException(e);
}
});
// 等待两个服务器都启动完成
latch.await();
logger.info("Both TCP and UDP Syslog servers are running");
// 注册关闭钩子
Runtime.getRuntime().addShutdownHook(new Thread(this::shutdown));
// 保持主线程运行
Thread.currentThread().join();
} catch (Exception e) {
logger.error("Failed to start servers", e);
shutdown();
}
}
private void startTcpServer() throws InterruptedException {
ServerBootstrap b = new ServerBootstrap();
b.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.childHandler(new SyslogTcpChannelInitializer())
.option(ChannelOption.SO_BACKLOG, 128)
.childOption(ChannelOption.SO_KEEPALIVE, true);
tcpChannelFuture = b.bind(tcpPort).sync();
logger.info("TCP Syslog server started on port {}", tcpPort);
// 添加关闭监听器
tcpChannelFuture.channel().closeFuture().addListener(future -> {
if (future.isSuccess()) {
logger.info("TCP server channel closed");
}
});
}
private void startUdpServer() throws InterruptedException {
Bootstrap b = new Bootstrap();
b.group(udpGroup)
.channel(NioDatagramChannel.class)
.handler(new SyslogUdpChannelInitializer())
.option(ChannelOption.SO_BROADCAST, true);
udpChannelFuture = b.bind(udpPort).sync();
logger.info("UDP Syslog server started on port {}", udpPort);
// 添加关闭监听器
udpChannelFuture.channel().closeFuture().addListener(future -> {
if (future.isSuccess()) {
logger.info("UDP server channel closed");
}
});
}
/**
* 优雅关闭服务器
*/
public void shutdown() {
logger.info("Shutting down Syslog servers...");
try {
// 关闭TCP服务器
if (tcpChannelFuture != null) {
tcpChannelFuture.channel().close().sync();
}
// 关闭UDP服务器
if (udpChannelFuture != null) {
udpChannelFuture.channel().close().sync();
}
// 关闭EventLoopGroup
if (bossGroup != null) {
bossGroup.shutdownGracefully().sync();
}
if (workerGroup != null) {
workerGroup.shutdownGracefully().sync();
}
if (udpGroup != null) {
udpGroup.shutdownGracefully().sync();
}
// 关闭线程池
serverExecutor.shutdown();
logger.info("Syslog servers shutdown complete");
} catch (Exception e) {
logger.error("Error during shutdown", e);
}
}
/**
* 检查服务器是否在运行
*/
public boolean isRunning() {
return (tcpChannelFuture != null && tcpChannelFuture.channel().isActive()) ||
(udpChannelFuture != null && udpChannelFuture.channel().isActive());
}
public static void main(String[] args) throws Exception {
int tcpPort = 514;
int udpPort = 514;
// 解析命令行参数
if (args.length >= 1) {
tcpPort = Integer.parseInt(args[0]);
}
if (args.length >= 2) {
udpPort = Integer.parseInt(args[1]);
}
// 如果两个端口相同，UDP端口自动+1避免冲突
if (tcpPort == udpPort) {
udpPort = tcpPort + 1;
logger.warn("TCP and UDP ports cannot be the same. UDP port changed to {}", udpPort);
}
SyslogServerBoth server = new SyslogServerBoth(tcpPort, udpPort);
// 注册全局异常处理器
Thread.setDefaultUncaughtExceptionHandler((t, e) -> {
logger.error("Uncaught exception in thread {}", t.getName(), e);
});
server.start();
}
}
haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogTcpChannelInitializer.java
+36
View File
@@ -0,0 +1,36 @@
package com.netty;
import com.haobang.config.AppConfig;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.socket.SocketChannel;
import io.netty.handler.codec.DelimiterBasedFrameDecoder;
import io.netty.handler.codec.Delimiters;
import io.netty.handler.codec.string.StringDecoder;
import io.netty.handler.codec.string.StringEncoder;
import io.netty.util.CharsetUtil;
/**
* TCP 通道初始化器
*/
public class SyslogTcpChannelInitializer extends ChannelInitializer<SocketChannel> {
private static final int MAX_FRAME_LENGTH = AppConfig.getSyslogMaxFrameLength();
@Override
protected void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
// 添加基于分隔符的帧解码器，处理以换行符结尾的 Syslog 消息
pipeline.addLast("framer", new DelimiterBasedFrameDecoder(
MAX_FRAME_LENGTH,
Delimiters.lineDelimiter()));
// 添加字符串解码器和编码器
pipeline.addLast("decoder", new StringDecoder(CharsetUtil.UTF_8));
pipeline.addLast("encoder", new StringEncoder(CharsetUtil.UTF_8));
// 添加业务处理器
pipeline.addLast("handler", new SyslogMessageHandler());
}
}
haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogUdpChannelInitializer.java
+25
View File
@@ -0,0 +1,25 @@
package com.netty;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.socket.DatagramChannel;
import io.netty.handler.codec.string.StringDecoder;
import io.netty.handler.codec.string.StringEncoder;
import io.netty.util.CharsetUtil;
/**
* UDP 通道初始化器
*/
public class SyslogUdpChannelInitializer extends ChannelInitializer<DatagramChannel> {
@Override
protected void initChannel(DatagramChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
// 添加字符串解码器和编码器
pipeline.addLast("decoder", new StringDecoder(CharsetUtil.UTF_8));
pipeline.addLast("encoder", new StringEncoder(CharsetUtil.UTF_8));
// 添加业务处理器
pipeline.addLast("handler", new SyslogMessageHandler());
}
}