1、适配部署环境变量自动注入

2、适配南网达梦数据库安全版SQL语句Group by 不支持自定义函数
3、达梦数据库驱动改为达梦提供的指定版本
4、修改syslog-consumer模块yaml 文件kafka 变量名称
5、修改访问日志告警表新增4个字段,完善异常行为详情页面显示字段
This commit is contained in:
2026-07-22 18:50:44 +08:00
parent 5fe0e124bb
commit 16b73dffee
89 changed files with 4507 additions and 2186 deletions
@@ -24,38 +24,38 @@ services:
# 探针标识
- DEVICE_COLLECT_ID=1
# 数据库配置
- spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys
- spring.datasource.username=postgres
- spring.datasource.password=TnLanWaidYSwTSG5
- spring.datasource.driver-class-name=org.postgresql.Driver
- SPRING_DATASOURCE_URL=jdbc:postgresql://117.72.68.72:54329/ecosys
- SPRING_DATASOURCE_USERNAME=postgres
- SPRING_DATASOURCE_PASSWORD=TnLanWaidYSwTSG5
- SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
# Redis配置
- spring.redis.host=192.168.222.131
- spring.redis.port=6379
- spring.redis.password=
- spring.redis.database=0
- spring.redis.timeout=2000
- spring.cache.redis.time-to-live=600000
- SPRING_REDIS_HOST=192.168.222.131
- SPRING_REDIS_PORT=6379
- SPRING_REDIS_PASSWORD=
- SPRING_REDIS_DATABASE=0
- SPRING_REDIS_TIMEOUT=2000
- SPRING_CACHE_REDIS_TIME_TO_LIVE=600000
# Kafka配置
- spring.kafka.producer.bootstrap-servers=192.168.222.130:9092
- spring.kafka.producer.topic=test-topic
- SPRING_KAFKA_PRODUCER_BOOTSTRAP_SERVERS=192.168.222.130:9092
- SPRING_KAFKA_PRODUCER_TOPIC=test-topic
# 防火墙配置
- firewall.enabled=true
- firewall.api-key=your-secure-api-key
- FIREWALL_ENABLED=true
- FIREWALL_API_KEY=your-secure-api-key
# 联动配置 (syslog-consumer 地址)
- interlocking.enabled=true
- interlocking.api.base-url=http://192.168.222.131:8089/xdrservice/interlocking
- interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
- INTERLOCKING_ENABLED=true
- INTERLOCKING_API_BASE_URL=http://192.168.222.131:8089/xdrservice/interlocking
- INTERLOCKING_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
# 探针心跳配置
- probe.heartbeat.enabled=true
- probe.heartbeat.interval-seconds=60
- probe.heartbeat.initial-delay-ms=5000
- probe.collect-id=probe-001
- probe.collect-name=日志采集探针
- probe.app-version=1.0.0
- probe.platform.api-url=http://192.168.222.131:8089/xdrservice/interlocking/probe/heartbeat
- probe.platform.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
- PROBE_HEARTBEAT_ENABLED=true
- PROBE_HEARTBEAT_INTERVAL_SECONDS=60
- PROBE_HEARTBEAT_INITIAL_DELAY_MS=5000
- PROBE_COLLECT_ID=probe-001
- PROBE_COLLECT_NAME=日志采集探针
- PROBE_APP_VERSION=1.0.0
- PROBE_PLATFORM_API_URL=http://192.168.222.131:8089/xdrservice/interlocking/probe/heartbeat
- PROBE_PLATFORM_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
# 定时任务配置
- spring.task.scheduling.pool.size=5
- SPRING_TASK_SCHEDULING_POOL_SIZE=5
# JVM配置
- JAVA_OPTS=-Xms256m -Xmx1024m -XX:+UseG1GC
ports:
@@ -12,58 +12,57 @@ services:
# ============================================
syslog-serve:
build:
context: ./syslog-serve
context: ./serve-dm
dockerfile: Dockerfile
image: syslog-serve:1.0.1
container_name: syslog-serve
image: syslog-serve-dm:v1.2.1
container_name: syslog-serve-dm
restart: unless-stopped
environment:
# 环境配置
- SPRING_PROFILES_ACTIVE=prod
- TZ=Asia/Shanghai
# 探针标识
- DEVICE_COLLECT_ID=probe-001
# 探针标识ID
- APP_SERVICE_DEVICE_COLLECT_ID=1
# 数据库配置
- spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys
- spring.datasource.username=postgres
- spring.datasource.password=TnLanWaidYSwTSG5
- spring.datasource.driver-class-name=org.postgresql.Driver
- SPRING_DATASOURCE_URL=jdbc:dm://192.168.4.99:5237
- SPRING_DATASOURCE_USERNAME=SYSDBA
- SPRING_DATASOURCE_PASSWORD=caZ2TcmXNSW8L2Ap
- SPRING_DATASOURCE_DRIVER_CLASS_NAME=dm.jdbc.driver.DmDriver
- SPRING_DATASOURCE_HIKARI_SCHEMA=\"PUBLIC\"
# Redis配置
- spring.redis.host=127.0.0.1
- spring.redis.port=6379
- spring.redis.password=
- spring.redis.database=0
- spring.redis.timeout=2000
- spring.cache.redis.time-to-live=600000
- SPRING_REDIS_HOST=192.168.4.99
- SPRING_REDIS_PORT=6379
- SPRING_REDIS_PASSWORD=redis_GdGWte
- SPRING_REDIS_DATABASE=0
- SPRING_REDIS_TIMEOUT=2000
- SPRING_CACHE_REDIS_TIME_TO_LIVE=600000
# Kafka配置
- spring.kafka.producer.bootstrap-servers=192.168.222.130:9092
- spring.kafka.producer.topic=test-topic
# 防火墙配置
- firewall.enabled=true
- firewall.api-key=your-secure-api-key
- SPRING_KAFKA_PRODUCER_BOOTSTRAP_SERVERS=192.168.4.99:9092
- SPRING_KAFKA_PRODUCER_TOPIC=agent-01-syslog-topic
# 防火墙配置
- FIREWALL_ENABLED=true
- FIREWALL_API_KEY=your-secure-api-key
# 联动配置 (syslog-consumer 地址)
- interlocking.enabled=true
- interlocking.api.base-url=http://117.72.68.72:8089/xdrservice/interlocking
- interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
- INTERLOCKING_ENABLED=true
- INTERLOCKING_API_BASE_URL=http://192.168.4.99:8089/xdrservice/interlocking
- INTERLOCKING_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
# 探针心跳配置
- probe.heartbeat.enabled=true
- probe.heartbeat.interval-seconds=60
- probe.heartbeat.initial-delay-ms=5000
- probe.collect-id=probe-001
- probe.collect-name=日志采集探针
- probe.app-version=1.0.0
- probe.platform.api-url=http://117.72.68.72:8089/xdrservice/interlocking/probe/heartbeat
- probe.platform.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
- PROBE_HEARTBEAT_ENABLED=true
- PROBE_HEARTBEAT_INTERVAL_SECONDS=60
- PROBE_HEARTBEAT_INITIAL_DELAY_MS=5000
- PROBE_PLATFORM_API_URL=http://192.168.4.99:8089/xdrservice/interlocking/probe/heartbeat
# 定时任务配置
- spring.task.scheduling.pool.size=5
- SPRING_TASK_SCHEDULING_POOL_SIZE=5
# JVM配置
- JAVA_OPTS=-Xms256m -Xmx512m -XX:+UseG1GC
- JAVA_OPTS=-Xms2048m -Xmx4096m -XX:+UseG1GC
ports:
- "8189:8189" # API端口
- "514:514/udp" # Syslog UDP
- "514:514/tcp" # Syslog TCP
volumes:
- syslog-serve-logs:/app/logs
- /home/syslog/logs:/app/logs
networks:
- xdr-network
cap_add:
@@ -24,38 +24,38 @@ services:
# 探针标识
- DEVICE_COLLECT_ID=1
# 数据库配置
- spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys
- spring.datasource.username=postgres
- spring.datasource.password=TnLanWaidYSwTSG5
- spring.datasource.driver-class-name=org.postgresql.Driver
- SPRING_DATASOURCE_URL=jdbc:postgresql://117.72.68.72:54329/ecosys
- SPRING_DATASOURCE_USERNAME=postgres
- SPRING_DATASOURCE_PASSWORD=TnLanWaidYSwTSG5
- SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
# Redis配置
- spring.redis.host=192.168.222.131
- spring.redis.port=6379
- spring.redis.password=
- spring.redis.database=0
- spring.redis.timeout=2000
- spring.cache.redis.time-to-live=600000
- SPRING_REDIS_HOST=192.168.222.131
- SPRING_REDIS_PORT=6379
- SPRING_REDIS_PASSWORD=
- SPRING_REDIS_DATABASE=0
- SPRING_REDIS_TIMEOUT=2000
- SPRING_CACHE_REDIS_TIME_TO_LIVE=600000
# Kafka配置
- spring.kafka.producer.bootstrap-servers=192.168.222.130:9092
- spring.kafka.producer.topic=test-topic
- SPRING_KAFKA_PRODUCER_BOOTSTRAP_SERVERS=192.168.222.130:9092
- SPRING_KAFKA_PRODUCER_TOPIC=test-topic
# 防火墙配置
- firewall.enabled=true
- firewall.api-key=your-secure-api-key
- FIREWALL_ENABLED=true
- FIREWALL_API_KEY=your-secure-api-key
# 联动配置 (syslog-consumer 地址)
- interlocking.enabled=true
- interlocking.api.base-url=http://192.168.222.131:8089/xdrservice/interlocking
- interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
- INTERLOCKING_ENABLED=true
- INTERLOCKING_API_BASE_URL=http://192.168.222.131:8089/xdrservice/interlocking
- INTERLOCKING_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
# 探针心跳配置
- probe.heartbeat.enabled=true
- probe.heartbeat.interval-seconds=60
- probe.heartbeat.initial-delay-ms=5000
- probe.collect-id=probe-001
- probe.collect-name=日志采集探针
- probe.app-version=1.0.0
- probe.platform.api-url=http://192.168.222.131:8089/xdrservice/interlocking/probe/heartbeat
- probe.platform.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
- PROBE_HEARTBEAT_ENABLED=true
- PROBE_HEARTBEAT_INTERVAL_SECONDS=60
- PROBE_HEARTBEAT_INITIAL_DELAY_MS=5000
- PROBE_COLLECT_ID=probe-001
- PROBE_COLLECT_NAME=日志采集探针
- PROBE_APP_VERSION=1.0.0
- PROBE_PLATFORM_API_URL=http://192.168.222.131:8089/xdrservice/interlocking/probe/heartbeat
- PROBE_PLATFORM_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
# 定时任务配置
- spring.task.scheduling.pool.size=5
- SPRING_TASK_SCHEDULING_POOL_SIZE=5
# JVM配置
- JAVA_OPTS=-Xms256m -Xmx1024m -XX:+UseG1GC
ports:
+3 -3
View File
@@ -56,7 +56,7 @@
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>3.4.0</version>
<version>3.9.2</version>
</dependency>
@@ -134,8 +134,8 @@
<!-- 达梦数据库驱动 JDK1.8 -->
<dependency>
<groupId>com.dameng</groupId>
<artifactId>DmJdbcDriver18</artifactId>
<version>8.1.2.141</version>
<artifactId>DmJdbcDriver</artifactId>
<version>8.1.4.169</version>
</dependency>
<dependency>
@@ -2,7 +2,7 @@ package com.Modules.Device;
import com.common.entity.DeviceDevice;
import com.haobang.config.AppConfig;
import com.haobang.config.AppProperties;
import com.kafka.kafkaProducer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -85,7 +85,8 @@ public class DeviceProcess {
//当前判断是否归属
if(deviceDevice!=null )
{
int deviceCollectId =AppConfig.getDeviceCollectId();
AppProperties props = SpringContextUtil.getBean(AppProperties.class);
int deviceCollectId = props.getAppServiceDeviceCollectId();
if(deviceDevice.getDeviceCollectId()!=deviceCollectId)
{
logger.error("设备请求的Host IP不属于当前采集探针任务,请联系管理员核实!");
@@ -110,8 +111,10 @@ public class DeviceProcess {
}
else {
//创建未知设备记录
DeviceCollectTask deviceCollectTask= deviceCollectTaskService.getById(AppConfig.getDeviceCollectId());
deviceUnknown.setDeviceCollectId( AppConfig.getDeviceCollectId());
AppProperties props = SpringContextUtil.getBean(AppProperties.class);
int deviceCollectId = props.getAppServiceDeviceCollectId();
DeviceCollectTask deviceCollectTask= deviceCollectTaskService.getById(deviceCollectId);
deviceUnknown.setDeviceCollectId(deviceCollectId);
deviceUnknown.setDeviceIp( this.sourceIP);
deviceUnknown.setFirstTime(currentTime);
if(deviceCollectTask!=null) deviceUnknown.setDeviceCollectName(deviceCollectTask.getTaskName());
@@ -141,9 +144,10 @@ public class DeviceProcess {
try {
// 创建设备接收日志对象
AppProperties props = SpringContextUtil.getBean(AppProperties.class);
devicelog= DeviceReceiveLog.builder()
.deviceId(deviceDevice.getId())
.deviceCollectId(AppConfig.getDeviceCollectId())
.deviceCollectId(props.getAppServiceDeviceCollectId())
.deviceIp(deviceDevice.getIp())
.receiveTime(now)
.receiveTimeStr(timeString)
@@ -1,4 +1,6 @@
package com;
import com.haobang.config.AppProperties;
import com.haobang.config.SyslogConfig;
import com.kafka.kafkaProducer;
import org.mybatis.spring.annotation.MapperScan;
@@ -7,8 +9,9 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.core.env.Environment;
import com.netty.SyslogServer;
import com.haobang.config.AppConfig;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableScheduling;
@@ -23,12 +26,13 @@ public class SyslogServeMainApp {
private static final Logger logger = LoggerFactory.getLogger(SyslogServeMainApp.class);
public static void main(String[] args) {
SpringApplication.run(SyslogServeMainApp.class, args);
ConfigurableApplicationContext context = SpringApplication.run(SyslogServeMainApp.class, args);
Environment env = context.getEnvironment();
try {
//System.out.println("syslogConfig TcpPort: "+ AppConfig.getSyslogTcpPort());
int tcpPort = args.length > 0 ? Integer.parseInt(args[0]) : AppConfig.getSyslogTcpPort() ;
int udpPort = args.length > 1 ? Integer.parseInt(args[1]) : AppConfig.getSyslogUdpPort();
//SyslogServer server = new SyslogServer(tcpPort, udpPort);
int tcpPort = args.length > 0 ? Integer.parseInt(args[0])
: Integer.parseInt(env.getProperty("syslog.tcp.port", "514"));
int udpPort = args.length > 1 ? Integer.parseInt(args[1])
: Integer.parseInt(env.getProperty("syslog.udp.port", "515"));
SyslogServerBoth server = new SyslogServerBoth(tcpPort, udpPort);
logger.info("Application SyslogServer start !");
server.start();
@@ -21,6 +21,10 @@ public class DeviceCollectTask {
private Integer taskCount;
private OffsetDateTime recentDiscoverTime;
private Integer epmUpperLimit;
private OffsetDateTime expireTime;
private String remark;
private String deviceIp;
private Long organizationId;
// Getter and Setter 方法
public Integer getId() { return id; }
@@ -74,6 +78,18 @@ public class DeviceCollectTask {
public Integer getEpmUpperLimit() { return epmUpperLimit; }
public void setEpmUpperLimit(Integer epmUpperLimit) { this.epmUpperLimit = epmUpperLimit; }
public OffsetDateTime getExpireTime() { return expireTime; }
public void setExpireTime(OffsetDateTime expireTime) { this.expireTime = expireTime; }
public String getRemark() { return remark; }
public void setRemark(String remark) { this.remark = remark; }
public String getDeviceIp() { return deviceIp; }
public void setDeviceIp(String deviceIp) { this.deviceIp = deviceIp; }
public Long getOrganizationId() { return organizationId; }
public void setOrganizationId(Long organizationId) { this.organizationId = organizationId; }
@Override
public String toString() {
return "DeviceCollectTask{" +
@@ -94,6 +110,10 @@ public class DeviceCollectTask {
", taskCount=" + taskCount +
", recentDiscoverTime=" + recentDiscoverTime +
", epmUpperLimit=" + epmUpperLimit +
", expireTime=" + expireTime +
", remark='" + remark + '\'' +
", deviceIp='" + deviceIp + '\'' +
", organizationId=" + organizationId +
'}';
}
}
@@ -60,10 +60,12 @@ public interface DeviceCollectTaskMapper {
*/
@Insert("INSERT INTO device_collect_task (created_at, updated_at, device_id, method, task_name, " +
"first_time, last_success_time, last_failed_time, detail_id, epm, epm_peak, " +
"process_architecture, task_count, recent_discover_time, epm_upper_limit) " +
"process_architecture, task_count, recent_discover_time, epm_upper_limit, " +
"expire_time, remark, device_ip, organization_id) " +
"VALUES (NOW(), NOW(), #{deviceId}, #{method}, #{taskName}, #{firstTime}, " +
"#{lastSuccessTime}, #{lastFailedTime}, #{detailId}, #{epm}, #{epmPeak}, " +
"#{processArchitecture}, #{taskCount}, #{recentDiscoverTime}, #{epmUpperLimit})")
"#{processArchitecture}, #{taskCount}, #{recentDiscoverTime}, #{epmUpperLimit}, " +
"#{expireTime}, #{remark}, #{deviceIp}, #{organizationId})")
@Options(useGeneratedKeys = true, keyProperty = "id")
int insert(DeviceCollectTask task);
@@ -84,7 +86,11 @@ public interface DeviceCollectTaskMapper {
"process_architecture = #{processArchitecture}, " +
"task_count = #{taskCount}, " +
"recent_discover_time = #{recentDiscoverTime}, " +
"epm_upper_limit = #{epmUpperLimit} " +
"epm_upper_limit = #{epmUpperLimit}, " +
"expire_time = #{expireTime}, " +
"remark = #{remark}, " +
"device_ip = #{deviceIp}, " +
"organization_id = #{organizationId} " +
"WHERE id = #{id}")
int update(DeviceCollectTask task);
@@ -0,0 +1,60 @@
package com.haobang.config;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
/**
* 应用配置属性 - 替代静态 AppConfigsyslog-serve 模块)
* 使用 Spring Boot 标准 @Value 机制,支持 docker-compose 环境变量覆盖
*/
@Data
@Component
public class AppProperties {
// ========== Syslog 配置 ==========
@Value("${syslog.tcp.port:514}")
private int syslogTcpPort;
@Value("${syslog.udp.port:515}")
private int syslogUdpPort;
@Value("${syslog.max.frame.length:65536}")
private int syslogMaxFrameLength;
@Value("${syslog.buffer.size:1024}")
private int syslogBufferSize;
@Value("${syslog.charset:GBK}")
private String syslogCharset;
// ========== App Service 配置(探针设备信息)==========
@Value("${app.service.device_id:}")
private String appServiceDeviceId;
@Value("${app.service.device_name:}")
private String appServiceDeviceName;
@Value("${app.service.vendor:}")
private String appServiceVendor;
@Value("${app.service.product_name:}")
private String appServiceProductName;
@Value("${app.service.data_type:json}")
private String appServiceDataType;
@Value("${app.service.device_collect_id:1}")
private int appServiceDeviceCollectId;
// ========== Kafka Producer 配置 ==========
@Value("${spring.kafka.producer.bootstrap-servers:localhost:9092}")
private String kafkaProducerBootstrap;
@Value("${spring.kafka.producer.topic:agent-syslog-topic}")
private String kafkaProducerTopic;
// ========== SM4 加密配置 ==========
@Value("${syslog.sm4.generateKey:}")
private String sm4Key;
}
@@ -3,7 +3,8 @@ package com.haobang.util;
import com.common.entity.DeviceDevice;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.haobang.config.AppConfig;
import com.haobang.config.AppProperties;
import com.haobang.util.SpringContextUtil;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -11,6 +12,10 @@ import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
public class DeviceInfoUtil {
private static AppProperties getProps() {
return SpringContextUtil.getBean(AppProperties.class);
}
/**
* 补充设备配置信息 key value值
* @param strSyslog
@@ -18,6 +23,7 @@ public class DeviceInfoUtil {
*/
public static String getFullLogString(String strSyslog)
{
AppProperties appProps = getProps();
ObjectMapper objectMapper = new ObjectMapper();
// 创建设备配置信息MAP
// 获取当前时间(精度到毫秒)
@@ -29,11 +35,11 @@ public class DeviceInfoUtil {
Map<String, String> map = new LinkedHashMap<>();
map.put("receive_time",timeString);
map.put("device_id", AppConfig.getAppServieDeviceId());
map.put("device_name", AppConfig.getAppServieDeviceName());
map.put("vendor", AppConfig.getAppServieVendor());
map.put("data_type", AppConfig.getAppServieDataType());
map.put("device_collect_id", Long.toString(AppConfig.getDeviceCollectId()));
map.put("device_id", appProps.getAppServiceDeviceId());
map.put("device_name", appProps.getAppServiceDeviceName());
map.put("vendor", appProps.getAppServiceVendor());
map.put("data_type", appProps.getAppServiceDataType());
map.put("device_collect_id", Long.toString(appProps.getAppServiceDeviceCollectId()));
String formattedString = formatDeviceInfo(map);
//return formattedString + "" + strSyslog.substring(34);
@@ -48,6 +54,7 @@ public class DeviceInfoUtil {
*/
public static String getFullLogString(DeviceDevice deviceDevice,String strSyslog)
{
AppProperties appProps = getProps();
ObjectMapper objectMapper = new ObjectMapper();
// 创建设备配置信息MAP
// 获取当前时间(精度到毫秒)
@@ -61,8 +68,8 @@ public class DeviceInfoUtil {
map.put("device_id",deviceDevice.getId().toString());
map.put("device_name", deviceDevice.getName());
map.put("vendor", deviceDevice.getVendor());
map.put("data_type", AppConfig.getAppServieDataType());
map.put("device_collect_id", Long.toString(AppConfig.getDeviceCollectId()));
map.put("data_type", appProps.getAppServiceDataType());
map.put("device_collect_id", Long.toString(appProps.getAppServiceDeviceCollectId()));
String formattedString = formatDeviceInfo(map);
//return formattedString + "" + strSyslog.substring(34);
//测试环境截取34位之后字符串
@@ -71,6 +78,7 @@ public class DeviceInfoUtil {
public static String getFullLogString(DeviceDevice deviceDevice,String strSyslog,String receive_time)
{
AppProperties appProps = getProps();
ObjectMapper objectMapper = new ObjectMapper();
// 创建设备配置信息MAP
// 获取当前时间(精度到毫秒)
@@ -84,8 +92,8 @@ public class DeviceInfoUtil {
map.put("device_id",deviceDevice.getId().toString());
map.put("device_name", deviceDevice.getName());
map.put("vendor", deviceDevice.getVendor());
map.put("data_type", AppConfig.getAppServieDataType());
map.put("device_collect_id", Long.toString(AppConfig.getDeviceCollectId()));
map.put("data_type", appProps.getAppServiceDataType());
map.put("device_collect_id", Long.toString(appProps.getAppServiceDeviceCollectId()));
String formattedString = formatDeviceInfo(map);
//return formattedString + "" + strSyslog.substring(34);
//测试环境截取34位之后字符串
@@ -1,5 +1,6 @@
package com.kafka;
import com.common.entity.DeviceDevice;
import com.haobang.config.AppProperties;
import com.haobang.util.Sm4Util;
import org.apache.kafka.clients.producer.*;
import org.apache.kafka.common.serialization.StringSerializer;
@@ -13,21 +14,20 @@ import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.haobang.util.DeviceInfoUtil;
import com.haobang.util.SpringContextUtil;
public class kafkaProducer {
private static final Logger logger = LoggerFactory.getLogger(kafkaProducer.class);
private static String strhexKey=AppConfig.getSM4Key();
private static AppProperties getProps() {
return SpringContextUtil.getBean(AppProperties.class);
}
public static void main(String[] args) {
// System.out.println(getFullLogString("syslogmessage"));
AppProperties appProps = getProps();
// 配置生产者属性
Properties props = new Properties();
//props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.222.130:9092");
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfig.getKafkaProducerBootstrap());
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProps.getKafkaProducerBootstrap());
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
@@ -47,7 +47,7 @@ public class kafkaProducer {
// 创建生产者记录
ProducerRecord<String, String> record =
new ProducerRecord<>(AppConfig.getKafkaProducerTopic(), key, value);
new ProducerRecord<>(appProps.getKafkaProducerTopic(), key, value);
// 发送消息(异步方式)
producer.send(record, new Callback() {
@@ -82,9 +82,10 @@ public class kafkaProducer {
public static void messagePush(DeviceDevice deviceDevice, String strSyslog,String strReceiveTime)
{
AppProperties appProps = getProps();
// 配置生产者属性
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfig.getKafkaProducerBootstrap());
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProps.getKafkaProducerBootstrap());
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
@@ -103,12 +104,12 @@ public class kafkaProducer {
String value = DeviceInfoUtil.getFullLogString(deviceDevice,strSyslog,strReceiveTime);
//Message进行SM4加密写入kafka
String Sm4message= Sm4Util.encryptCbc(value, strhexKey);
System.out.println("Sm4message:"+Sm4message);
String Sm4message= Sm4Util.encryptCbc(value, appProps.getSm4Key());
//System.out.println("Sm4message:"+Sm4message);
// 创建生产者记录
ProducerRecord<String, String> record =
new ProducerRecord<>(AppConfig.getKafkaProducerTopic(), key, Sm4message);
new ProducerRecord<>(appProps.getKafkaProducerTopic(), key, Sm4message);
// 发送消息(异步方式)
producer.send(record, new Callback() {
@@ -148,10 +149,11 @@ public class kafkaProducer {
ObjectMapper objectMapper = new ObjectMapper();
// 创建设备配置信息MAP
Map<String, Object> map = new LinkedHashMap<>();
map.put("device_id",AppConfig.getAppServieDeviceId() );
map.put("device_name", AppConfig.getAppServieDeviceName());
map.put("vendor", AppConfig.getAppServieVendor());
map.put("data_type", AppConfig.getAppServieDataType());
AppProperties appProps = getProps();
map.put("device_id", appProps.getAppServiceDeviceId());
map.put("device_name", appProps.getAppServiceDeviceName());
map.put("vendor", appProps.getAppServiceVendor());
map.put("data_type", appProps.getAppServiceDataType());
// 添加syslogMessage
map.put("syslogMessage", strSyslog);
try {
@@ -21,7 +21,7 @@ app.service.device_name=honeypot
app.service.vendor=changting
app.service.product_name=diting
#采集探针ID
app.service.device_collect_id=${DEVICE_COLLECT_ID:1}
app.service.device_collect_id=${DEVICE_COLLECT_ID:7}
app.service.version=${APP_SERVICE_VER:V1.0.0-20260527}
app.service.device_collect_name=${DEVICE_COLLECT_NAME:DevCollect-01}
# syslog message data_type
@@ -29,7 +29,7 @@ app.service.data_type=json
# kafka Configuration
spring.kafka.producer.bootstrap-servers=192.168.4.99:9092
spring.kafka.producer.topic =${KAFKA_PRODUCER_TOPIC:agent-syslog-topic}
spring.kafka.producer.topic =${KAFKA_PRODUCER_TOPIC:test-topic}
# kafka Configuration 新增优化配置
spring.kafka.producer.properties.retries=10
spring.kafka.producer.properties.retry.backoff.ms=500
@@ -43,7 +43,7 @@ spring.datasource.url=jdbc:dm://192.168.4.99:5237
spring.datasource.username=SYSDBA
spring.datasource.password=caZ2TcmXNSW8L2Ap
spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver
spring.datasource.hikari.schema=\"PUBLIC\"
spring.datasource.hikari.schema=ECOSYS
# mybatis Configuration
mybatis.mapper-locations=classpath:mapper/*.xml
@@ -30,7 +30,7 @@ spring.kafka.producer.bootstrap-servers=10.11.2.142:9092
spring.kafka.producer.topic =agent-syslog-topic
#database Configuration
spring.datasource.url=jdbc:dm://192.163.4.99:5237/DM_ecosys
spring.datasource.url=jdbc:dm://192.163.4.99:5237
spring.datasource.username=SYSDBA
spring.datasource.password=caZ2TcmXNSW8L2Ap
spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver
@@ -12,7 +12,7 @@ syslog.udp.port=514
syslog.max.frame.length=262144
syslog.buffer.size=1000
# syslog 消息字符编码,国内安全设备普遍使用 GBK,如需 UTF-8 改为 syslog.charset=UTF-8
syslog.charset=GBK
syslog.charset=UTF-8
syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a
# APP Service Configuration
app.service.device_id=1
@@ -37,7 +37,7 @@ spring.kafka.producer.properties.socket.keepalive.enable=true
spring.kafka.producer.properties.request.timeout.ms=30000
spring.kafka.producer.properties.delivery.timeout.ms=120000
#database Configuration
spring.datasource.url=jdbc:dm://192.163.4.99:5237
spring.datasource.url=jdbc:dm://192.168.4.99:5237
spring.datasource.username=SYSDBA
spring.datasource.password=caZ2TcmXNSW8L2Ap
spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver
@@ -52,7 +52,7 @@ mybatis.configuration.map-underscore-to-camel-case=true
spring.redis.host=192.168.4.99
spring.redis.port=6379
# 密码(如果没有设置密码,可以省略)
spring.redis.password=123456
spring.redis.password=redis_GdGWte
spring.redis.database=0
spring.redis.timeout=5000
#spring.redis.password=${REDIS_PASSWORD:default_prod_password}
@@ -1,4 +1,4 @@
#Server Configuration [192.168.4.26]
#Server Configuration
server.port=8189
server.servlet.context-path=/syslogserve
#server.address=0.0.0.0
@@ -14,21 +14,22 @@ syslog.buffer.size=1000
# syslog 消息字符编码,国内安全设备普遍使用 GBK,如需 UTF-8 改为 syslog.charset=UTF-8
syslog.charset=GBK
syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a
# APP Service Configuration
app.service.device_id=1
app.service.device_name=honeypot
app.service.vendor=changting
app.service.product_name=diting
# syslog message data_type
app.service.data_type=json
#采集探针ID
app.service.device_collect_id=${DEVICE_COLLECT_ID:1}
app.service.device_collect_id=${DEVICE_COLLECT_ID:7}
app.service.version=${APP_SERVICE_VER:V1.0.0-20260527}
app.service.device_collect_name=${DEVICE_COLLECT_NAME:DevCollect-01}
# syslog message data_type
app.service.data_type=json
# kafka Configuration
spring.kafka.producer.bootstrap-servers=192.168.4.99:9092
spring.kafka.producer.topic =agent-01-syslog-topic
spring.kafka.producer.topic =${KAFKA_PRODUCER_TOPIC:test-topic}
# kafka Configuration 新增优化配置
spring.kafka.producer.properties.retries=10
spring.kafka.producer.properties.retry.backoff.ms=500
@@ -36,50 +37,48 @@ spring.kafka.producer.properties.connections.max.idle.ms=600000
spring.kafka.producer.properties.socket.keepalive.enable=true
spring.kafka.producer.properties.request.timeout.ms=30000
spring.kafka.producer.properties.delivery.timeout.ms=120000
#database Configuration
spring.datasource.url=jdbc:dm://192.163.4.99:5237
spring.datasource.url=jdbc:dm://192.168.4.99:5237
spring.datasource.username=SYSDBA
spring.datasource.password=caZ2TcmXNSW8L2Ap
spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver
spring.datasource.hikari.schema=\"PUBLIC\"
spring.datasource.hikari.schema=ECOSYS
# mybatis Configuration
mybatis.mapper-locations=classpath:mapper/*.xml
mybatis.type-aliases-package=com.common.entity
mybatis.configuration.map-underscore-to-camel-case=true
# 生产环境缓存配置
spring.redis.host=192.168.4.99
# 开发环境缓存配置
spring.redis.host=localhost
spring.redis.port=6379
# 密码(如果没有设置密码,可以省略)
spring.redis.password=123456
spring.redis.password=
spring.redis.database=0
spring.redis.timeout=5000
#spring.redis.password=${REDIS_PASSWORD:default_prod_password}
spring.redis.timeout=2000
spring.redis.lettuce.pool.max-active=20
spring.redis.lettuce.pool.max-wait=5000
spring.redis.lettuce.pool.max-idle=10
spring.redis.lettuce.pool.min-idle=5
# 生产环境缓存时间较长
spring.cache.redis.time-to-live=3600000
spring.redis.lettuce.pool.max-active=8
spring.redis.lettuce.pool.max-wait=-1
spring.redis.lettuce.pool.max-idle=8
spring.redis.lettuce.pool.min-idle=0
# 开发环境缓存时间较短,方便调试
spring.cache.redis.time-to-live=600000
# 黑名单API配置
blacklist.api.url= https://103.43.84.11/api/v3/Objects/Blacklist
blacklist.api.username=apt-admin103
blacklist.api.password=C9W2xYgfc%SN1
# 白名单API配置
whitelist.api.url=https://103.43.84.11/api/v3/Policies/GlobalWhitelist
whitelist.api.url=https://103.43.84.11/api/v3/Policies/GlobalWhitelist
# ============================================
# 探针联动配置
# ============================================
# 是否启用联动功能
interlocking.enabled=true
# syslog-consumer API基础URL(安全平台外网映射地址)
interlocking.api.base-url=http://192.168.4.99:8089/xdrservice/interlocking
interlocking.api.base-url=http://localhost:8089/xdrservice/interlocking
# API-KEY认证(32位,需与syslog-consumer配置一致)
interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
# 定时任务执行间隔(毫秒),默认30秒
@@ -94,8 +93,11 @@ probe.heartbeat.enabled=true
probe.heartbeat.interval-seconds=60
# 心跳发送初始延迟(毫秒)
probe.heartbeat.initial-delay-ms=5000
# 平台端心跳接收接口URL
probe.platform.api-url=http://192.168.4.99:8089/xdrservice/interlocking/probe/heartbeat
probe.platform.api-url=http://localhost:8089/xdrservice/interlocking/probe/heartbeat
# 平台API Key
probe.platform.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
# ============================================
# 定时任务配置
@@ -22,12 +22,17 @@
<result column="task_count" property="taskCount" />
<result column="recent_discover_time" property="recentDiscoverTime" />
<result column="epm_upper_limit" property="epmUpperLimit" />
<result column="expire_time" property="expireTime" />
<result column="remark" property="remark" />
<result column="device_ip" property="deviceIp" />
<result column="organization_id" property="organizationId" />
</resultMap>
<sql id="Base_Column_List">
id, created_at, updated_at, deleted_at, device_id, method, task_name,
first_time, last_success_time, last_failed_time, detail_id, epm, epm_peak,
process_architecture, task_count, recent_discover_time, epm_upper_limit
process_architecture, task_count, recent_discover_time, epm_upper_limit,
expire_time, remark, device_ip, organization_id
</sql>
<!-- 多条件组合查询 -->
@@ -60,6 +65,16 @@
<if test="lastSuccessTime != null">
AND last_success_time >= #{lastSuccessTime}
</if>
<!-- 新增字段查询 -->
<if test="deviceIp != null and deviceIp != ''">
AND device_ip = #{deviceIp}
</if>
<if test="organizationId != null">
AND organization_id = #{organizationId}
</if>
<if test="remark != null and remark != ''">
AND remark LIKE CONCAT('%', #{remark}, '%')
</if>
</where>
ORDER BY updated_at DESC
</select>
@@ -0,0 +1,174 @@
package com.haobang.syslog;
import com.common.entity.DeviceDevice;
import com.common.mapper.DeviceDeviceMapper;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import java.util.List;
/**
* 达梦数据库连接测试 — 查询 device_device 表
*/
@SpringBootTest
@ActiveProfiles("dev")
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
class DeviceDeviceTest {
@Autowired
private DeviceDeviceMapper deviceDeviceMapper;
// ==================== 基础查询测试 ====================
@Test
@Order(1)
@DisplayName("查询设备总数")
void testCount() {
System.out.println("\n========== 查询设备总数 ==========");
Long count = deviceDeviceMapper.count();
System.out.println("device_device 表数据总量: " + count);
assert count != null && count >= 0;
}
@Test
@Order(2)
@DisplayName("查询所有设备")
void testSelectAll() {
System.out.println("\n========== 查询所有设备 ==========");
List<DeviceDevice> list = deviceDeviceMapper.selectAll();
System.out.println("查询到 " + list.size() + " 条记录:");
for (int i = 0; i < Math.min(list.size(), 10); i++) {
DeviceDevice d = list.get(i);
System.out.printf(" [%d] id=%-6d name=%-20s ip=%-18s status=%-4s vendor=%-15s delFlag=%s%n",
i + 1, d.getId(), d.getName(), d.getIp(), d.getStatus(), d.getVendor(), d.getDelFlag());
}
if (list.size() > 10) {
System.out.println(" ... 仅展示前10条,共 " + list.size() + "");
}
assert list != null;
}
@Test
@Order(3)
@DisplayName("查询未删除的有效设备 (del_flag='0')")
void testSelectActiveDevices() {
System.out.println("\n========== 查询有效设备 (del_flag='0') ==========");
List<DeviceDevice> list = deviceDeviceMapper.selectActiveDevices();
System.out.println("查询到 " + list.size() + " 条有效记录:");
for (int i = 0; i < Math.min(list.size(), 10); i++) {
DeviceDevice d = list.get(i);
System.out.printf(" [%d] id=%-6d name=%-20s ip=%-18s status=%-4s vendor=%-15s%n",
i + 1, d.getId(), d.getName(), d.getIp(), d.getStatus(), d.getVendor());
}
if (list.size() > 10) {
System.out.println(" ... 仅展示前10条,共 " + list.size() + "");
}
assert list != null;
}
@Test
@Order(4)
@DisplayName("分页查询 — 前5条")
void testSelectByPage() {
System.out.println("\n========== 分页查询 (第1页, 每页5条) ==========");
List<DeviceDevice> list = deviceDeviceMapper.selectByPage(0, 5);
System.out.println("查询到 " + list.size() + " 条记录:");
for (DeviceDevice d : list) {
System.out.println(" " + d);
}
assert list != null && list.size() <= 5;
}
// ==================== 精确查询测试 ====================
@Test
@Order(5)
@DisplayName("根据ID查询设备")
void testSelectById() {
System.out.println("\n========== 根据ID查询 ==========");
// 先查所有取第一条的ID
List<DeviceDevice> all = deviceDeviceMapper.selectAll();
if (all != null && !all.isEmpty()) {
Integer firstId = all.get(0).getId();
DeviceDevice d = deviceDeviceMapper.selectById(firstId);
System.out.println("ID=" + firstId + " 查询结果:");
System.out.println(" " + d);
assert d != null && d.getId().equals(firstId);
} else {
System.out.println("device_device 表无数据,跳过按ID查询");
}
}
// ==================== 分页查询测试 ====================
@Test
@Order(6)
@DisplayName("分页查询 — 第2页")
void testSelectByPage2() {
System.out.println("\n========== 分页查询 (第2页, 每页5条) ==========");
List<DeviceDevice> list = deviceDeviceMapper.selectByPage(5, 5);
System.out.println("查询到 " + list.size() + " 条记录:");
for (DeviceDevice d : list) {
System.out.printf(" id=%-6d name=%-20s ip=%-18s%n", d.getId(), d.getName(), d.getIp());
}
assert list != null;
}
// ==================== 完整字段展示 ====================
@Test
@Order(7)
@DisplayName("完整字段展示 — 查询1条设备的所有字段")
void testShowFullFields() {
System.out.println("\n========== 完整字段展示 ==========");
List<DeviceDevice> all = deviceDeviceMapper.selectAll();
if (all != null && !all.isEmpty()) {
DeviceDevice d = all.get(0);
System.out.println("┌─────────────────────────────────────────────┐");
System.out.printf("│ %-12s: %-30s │%n", "id", d.getId());
System.out.printf("│ %-12s: %-30s │%n", "name", d.getName());
System.out.printf("│ %-12s: %-30s │%n", "ip", d.getIp());
System.out.printf("│ %-12s: %-30s │%n", "vendor", d.getVendor());
System.out.printf("│ %-12s: %-30s │%n", "productName", d.getProductName());
System.out.printf("│ %-12s: %-30s │%n", "deviceGroup", d.getDeviceGroup());
System.out.printf("│ %-12s: %-30s │%n", "deviceType", d.getDeviceType());
System.out.printf("│ %-12s: %-30s │%n", "status", d.getStatus());
System.out.printf("│ %-12s: %-30s │%n", "isMonitoring", d.getIsMonitoring());
System.out.printf("│ %-12s: %-30s │%n", "organizationId", d.getOrganizationId());
System.out.printf("│ %-12s: %-30s │%n", "agentId", d.getAgentId());
System.out.printf("│ %-12s: %-30s │%n", "tenantId", d.getTenantId());
System.out.printf("│ %-12s: %-30s │%n", "delFlag", d.getDelFlag());
System.out.printf("│ %-12s: %-30s │%n", "managerName", d.getManagerName());
System.out.printf("│ %-12s: %-30s │%n", "createdAt", d.getCreatedAt());
System.out.printf("│ %-12s: %-30s │%n", "updatedAt", d.getUpdatedAt());
System.out.printf("│ %-12s: %-30s │%n", "lastReceiveTime", d.getLastReceiveTime());
System.out.printf("│ %-12s: %-30s │%n", "createTime", d.getCreateTime());
System.out.printf("│ %-12s: %-30s │%n", "updateTime", d.getUpdateTime());
System.out.printf("│ %-12s: %-30s │%n", "licenseStartTime", d.getLicenseStartTime());
System.out.printf("│ %-12s: %-30s │%n", "licenseEndTime", d.getLicenseEndTime());
System.out.printf("│ %-12s: %-30s │%n", "decodeType", d.getDecodeType());
System.out.printf("│ %-12s: %-30s │%n", "missPolicy", d.getMissPolicy());
System.out.printf("│ %-12s: %-30s │%n", "securityScopeId", d.getSecurityScopeId());
System.out.printf("│ %-12s: %-30s │%n", "ownerId", d.getOwnerId());
System.out.printf("│ %-12s: %-30s │%n", "sshConfigId", d.getSshConfigId());
System.out.printf("│ %-12s: %-30s │%n", "createdById", d.getCreatedById());
System.out.printf("│ %-12s: %-30s │%n", "createBy", d.getCreateBy());
System.out.printf("│ %-12s: %-30s │%n", "updateBy", d.getUpdateBy());
System.out.printf("│ %-12s: %-30s │%n", "createDept", d.getCreateDept());
System.out.printf("│ %-12s: %-30s │%n", "deviceCollectId", d.getDeviceCollectId());
System.out.printf("│ %-12s: %-30s │%n", "todayParseCount", d.getTodayParseCount());
System.out.printf("│ %-12s: %-30s │%n", "todayNonLogCount", d.getTodayNonLogCount());
System.out.printf("│ %-12s: %-30s │%n", "detailId", d.getDetailId());
System.out.printf("│ %-12s: %-30s │%n", "controlAgentId", d.getControlAgentId());
System.out.println("└─────────────────────────────────────────────┘");
} else {
System.out.println("device_device 表无数据");
}
}
}