diff --git a/haobang-security-dm/.idea/workspace.xml b/haobang-security-dm/.idea/workspace.xml index 358da24..7035a1c 100644 --- a/haobang-security-dm/.idea/workspace.xml +++ b/haobang-security-dm/.idea/workspace.xml @@ -1,265 +1,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -360,24 +102,56 @@ - + - - - + + + + + + + + + + - - + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + @@ -388,91 +162,87 @@ - + + + + + + + + + + + + + + + + + + + - + - + + + + + + - - + + - + - - + + - + - - - - - - - - - - - + + - + - - + + - + - - + + - + - - - - - - - - - - - - - - - - - - - - + + @@ -488,20 +258,6 @@ - bigint - device_id - event_level - event_category - dataMap.event_category - dataMap.attack_result - dataMap.event_type - dataMap.fall - dataMap.collect_method - dataMap.device_type - dataMap.srcip_id - dataMap.origin_total_bytes - dataMap.target_port - print_pages srcAddress src_ip window @@ -518,6 +274,20 @@ syslog-consumer @Scheduled 配置 Elasticsearch + dameng + spring.jackson.serialization.fail-on-self-references=false + 处理初始化异常 + getCropperParams + ARRAY_AGG + JSONB_AGG + AS window_time + tumble_start + hop_start + session + toString + 补充原始记录日志字段异 + 补充原始记录日志字段异常 + processAlgorithm com.common. @@ -532,57 +302,57 @@ @@ -598,11 +368,6 @@ - @@ -611,6 +376,24 @@ + + + + + + + + + + + + + + + + + + @@ -629,18 +412,13 @@ - - - - - - - @@ -784,10 +583,11 @@ - - + + + @@ -853,6 +653,20 @@ + @@ -1007,7 +823,15 @@ - + + + + + + + + + 1769412046230 @@ -1033,8 +857,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - @@ -1048,7 +904,7 @@ - + @@ -1058,24 +914,24 @@ - - + + - + - + - + @@ -1159,28 +1015,16 @@ - + - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1534,9 +1054,326 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/haobang-security-dm/syslog-consumer-rule/docker_run.txt b/haobang-security-dm/syslog-consumer-rule/docker_run.txt index 6ddbaf2..e8248e7 100644 --- a/haobang-security-dm/syslog-consumer-rule/docker_run.txt +++ b/haobang-security-dm/syslog-consumer-rule/docker_run.txt @@ -8,15 +8,19 @@ docker ps -a --2.���docker image�ļ� (Dockerfile ��ǰĿ¼�� docker build -t syslog-consumer-rule:v1.X.X . - +--dm +docker build -t syslog-consumer-rule-dm:v1.2.X . --3.ֹͣ���� ��ɾ�� docker stop syslog-consumer-rule && docker rm syslog-consumer-rule --4.����docker �ļ� -docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name ct-syslog-consumer-rule -p 8289:8289 -v /home/syslog/logs:/app/logs --privileged=true syslog-consumer-rule:v1.X.X +--dm +docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name syslog-consumer-rule-dm -v /home/syslog/logs:/app/logs --privileged=true syslog-consumer-rule-dm:v1.2.X +--pg +docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name syslog-consumer-rule -v /home/syslog/logs:/app/logs --privileged=true syslog-consumer-rule:v1.2.X �ڳ�CMD�� -docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name ct-syslog-consumer-rule -p 8289:8289 -v /data/syslog/logs:/app/logs --privileged=true syslog-consumer-rule:v1.X.X +docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name syslog-consumer-rule-dm -p 8289:8289 -v /data/syslog/logs:/app/logs --privileged=true syslog-consumer-rule-dm:v1.X.X --������������ docker run -d --name ct-syslog-consumer-rule -p 8089:8089 --privileged=true syslog-consumer-rule:v1.X.X diff --git a/haobang-security-dm/syslog-consumer-rule/pom.xml b/haobang-security-dm/syslog-consumer-rule/pom.xml index 11b588e..cfb977b 100644 --- a/haobang-security-dm/syslog-consumer-rule/pom.xml +++ b/haobang-security-dm/syslog-consumer-rule/pom.xml @@ -92,12 +92,21 @@ ${mybatis.version} - + + + + + + com.dameng + DmJdbcDriver18 + 8.1.2.141 + com.fasterxml.jackson.core @@ -193,11 +202,7 @@ - - com.dameng - DmJdbcDriver18 - 8.1.2.141 - + org.bouncycastle diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayByteTypeHandler.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayByteTypeHandler.java index 484cb34..ae919b7 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayByteTypeHandler.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayByteTypeHandler.java @@ -1,47 +1,86 @@ package com.Modules.etl.handler; - - +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.ibatis.type.BaseTypeHandler; import org.apache.ibatis.type.JdbcType; import org.apache.ibatis.type.MappedJdbcTypes; import org.apache.ibatis.type.MappedTypes; -import java.sql.*; +import java.sql.CallableStatement; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Base64; +import java.util.List; + +/** + * 字节数组类型处理器 - 达梦数据库兼容版本（JSON格式） + * + * 将 Java byte[][] 与数据库 VARCHAR 列进行互转。 + * 存储格式: JSON 数组，每个元素为 Base64 编码字符串，如 ["YWJj","ZGVm"] + * 空数组: 存储为 "[]" + * null值: 存储为 NULL + * + * 原 PostgreSQL 版本使用 createArrayOf("bytea", ...) 创建原生 bytea 数组， + * 达梦数据库不兼容此 API，改为 VARCHAR + JSON + Base64 存储。 + */ @MappedTypes(byte[][].class) -@MappedJdbcTypes(JdbcType.ARRAY) +@MappedJdbcTypes(JdbcType.VARCHAR) public class ArrayByteTypeHandler extends BaseTypeHandler { + private static final ObjectMapper MAPPER = new ObjectMapper(); + @Override public void setNonNullParameter(PreparedStatement ps, int i, byte[][] parameter, JdbcType jdbcType) throws SQLException { - Array array = ps.getConnection().createArrayOf("bytea", parameter); - ps.setArray(i, array); + try { + String[] encoded = new String[parameter.length]; + for (int j = 0; j < parameter.length; j++) { + encoded[j] = Base64.getEncoder().encodeToString(parameter[j]); + } + ps.setString(i, MAPPER.writeValueAsString(encoded)); + } catch (JsonProcessingException e) { + throw new SQLException("Failed to serialize byte[][] to JSON", e); + } } @Override public byte[][] getNullableResult(ResultSet rs, String columnName) throws SQLException { - return getArray(rs.getArray(columnName)); + return parseArray(rs.getString(columnName)); } @Override public byte[][] getNullableResult(ResultSet rs, int columnIndex) throws SQLException { - return getArray(rs.getArray(columnIndex)); + return parseArray(rs.getString(columnIndex)); } @Override public byte[][] getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { - return getArray(cs.getArray(columnIndex)); + return parseArray(cs.getString(columnIndex)); } - private byte[][] getArray(Array array) throws SQLException { - if (array != null) { - Object[] objArray = (Object[]) array.getArray(); - byte[][] result = new byte[objArray.length][]; - for (int i = 0; i < objArray.length; i++) { - result[i] = (byte[]) objArray[i]; + private byte[][] parseArray(String value) { + if (value == null || value.isEmpty()) { + return new byte[0][]; + } + try { + // JSON 格式: ["abc","def"] + List list = MAPPER.readValue(value, MAPPER.getTypeFactory().constructCollectionType(List.class, String.class)); + byte[][] result = new byte[list.size()][]; + for (int i = 0; i < list.size(); i++) { + String s = list.get(i); + result[i] = (s == null || s.isEmpty()) ? new byte[0] : Base64.getDecoder().decode(s); + } + return result; + } catch (JsonProcessingException e) { + // 兼容旧的逗号分隔格式 + String[] parts = value.split(",", -1); + byte[][] result = new byte[parts.length][]; + for (int i = 0; i < parts.length; i++) { + String part = parts[i].trim(); + result[i] = part.isEmpty() ? new byte[0] : Base64.getDecoder().decode(part); } return result; } - return null; } -} \ No newline at end of file +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayIntegerTypeHandler.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayIntegerTypeHandler.java index 0a40ce1..bb70315 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayIntegerTypeHandler.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayIntegerTypeHandler.java @@ -1,43 +1,73 @@ package com.Modules.etl.handler; - +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.ibatis.type.BaseTypeHandler; import org.apache.ibatis.type.JdbcType; import org.apache.ibatis.type.MappedJdbcTypes; import org.apache.ibatis.type.MappedTypes; -import java.sql.*; -import java.util.Arrays; +import java.sql.CallableStatement; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * 整型数组类型处理器 - 达梦数据库兼容版本（JSON格式） + * + * 将 Java Integer[] 与数据库 VARCHAR 列进行互转。 + * 存储格式: JSON 数组，如 [1,2,3] + * 空数组: 存储为 "[]" + * null值: 存储为 NULL + * + * 原 PostgreSQL 版本使用 createArrayOf("integer", ...) 创建原生数组， + * 达梦数据库不兼容此 API，改为 VARCHAR + JSON 存储。 + */ @MappedTypes(Integer[].class) -@MappedJdbcTypes(JdbcType.ARRAY) +@MappedJdbcTypes(JdbcType.VARCHAR) public class ArrayIntegerTypeHandler extends BaseTypeHandler { + private static final ObjectMapper MAPPER = new ObjectMapper(); + @Override public void setNonNullParameter(PreparedStatement ps, int i, Integer[] parameter, JdbcType jdbcType) throws SQLException { - Array array = ps.getConnection().createArrayOf("integer", parameter); - ps.setArray(i, array); + try { + ps.setString(i, MAPPER.writeValueAsString(parameter)); + } catch (JsonProcessingException e) { + throw new SQLException("Failed to serialize Integer[] to JSON", e); + } } @Override public Integer[] getNullableResult(ResultSet rs, String columnName) throws SQLException { - return getArray(rs.getArray(columnName)); + return parseArray(rs.getString(columnName)); } @Override public Integer[] getNullableResult(ResultSet rs, int columnIndex) throws SQLException { - return getArray(rs.getArray(columnIndex)); + return parseArray(rs.getString(columnIndex)); } @Override public Integer[] getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { - return getArray(cs.getArray(columnIndex)); + return parseArray(cs.getString(columnIndex)); } - private Integer[] getArray(Array array) throws SQLException { - if (array != null) { - Object[] objArray = (Object[]) array.getArray(); - return Arrays.copyOf(objArray, objArray.length, Integer[].class); + private Integer[] parseArray(String value) { + if (value == null || value.isEmpty()) { + return new Integer[0]; + } + try { + return MAPPER.readValue(value, Integer[].class); + } catch (JsonProcessingException e) { + // 兼容旧的逗号分隔格式 + String[] parts = value.split(",", -1); + Integer[] result = new Integer[parts.length]; + for (int i = 0; i < parts.length; i++) { + String part = parts[i].trim(); + result[i] = part.isEmpty() ? null : Integer.parseInt(part); + } + return result; } - return null; } -} \ No newline at end of file +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayStringTypeHandler.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayStringTypeHandler.java index 174364f..cac69e9 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayStringTypeHandler.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/etl/handler/ArrayStringTypeHandler.java @@ -1,43 +1,67 @@ package com.Modules.etl.handler; - +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.ibatis.type.BaseTypeHandler; import org.apache.ibatis.type.JdbcType; import org.apache.ibatis.type.MappedJdbcTypes; import org.apache.ibatis.type.MappedTypes; -import java.sql.*; -import java.util.Arrays; +import java.sql.CallableStatement; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * 字符串数组类型处理器 - 达梦数据库兼容版本（JSON格式） + * + * 将 Java String[] 与数据库 VARCHAR 列进行互转。 + * 存储格式: JSON 数组，如 ["value1","value2","value3"] + * 空数组: 存储为 "[]" + * null值: 存储为 NULL + * + * 原 PostgreSQL 版本使用 createArrayOf("text", ...) 创建原生数组， + * 达梦数据库不兼容此 API，改为 VARCHAR + JSON 存储。 + */ @MappedTypes(String[].class) -@MappedJdbcTypes(JdbcType.ARRAY) +@MappedJdbcTypes(JdbcType.VARCHAR) public class ArrayStringTypeHandler extends BaseTypeHandler { + private static final ObjectMapper MAPPER = new ObjectMapper(); + @Override public void setNonNullParameter(PreparedStatement ps, int i, String[] parameter, JdbcType jdbcType) throws SQLException { - Array array = ps.getConnection().createArrayOf("text", parameter); - ps.setArray(i, array); + try { + ps.setString(i, MAPPER.writeValueAsString(parameter)); + } catch (JsonProcessingException e) { + throw new SQLException("Failed to serialize String[] to JSON", e); + } } @Override public String[] getNullableResult(ResultSet rs, String columnName) throws SQLException { - return getArray(rs.getArray(columnName)); + return parseArray(rs.getString(columnName)); } @Override public String[] getNullableResult(ResultSet rs, int columnIndex) throws SQLException { - return getArray(rs.getArray(columnIndex)); + return parseArray(rs.getString(columnIndex)); } @Override public String[] getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { - return getArray(cs.getArray(columnIndex)); + return parseArray(cs.getString(columnIndex)); } - private String[] getArray(Array array) throws SQLException { - if (array != null) { - Object[] objArray = (Object[]) array.getArray(); - return Arrays.copyOf(objArray, objArray.length, String[].class); + private String[] parseArray(String value) { + if (value == null || value.isEmpty()) { + return new String[0]; + } + try { + return MAPPER.readValue(value, String[].class); + } catch (JsonProcessingException e) { + // 兼容旧的逗号分隔格式 + return value.split(",", -1); } - return null; } -} \ No newline at end of file +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlocking.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlocking.java index 6ce07ee..1c16f59 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlocking.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlocking.java @@ -1,6 +1,6 @@ package com.common.entity; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; /** * 联动设备表实体类（防火墙设备信息） @@ -17,9 +17,9 @@ public class DeviceInterlocking { private String tenantId; private Long createDept; private Long createBy; - private OffsetDateTime createTime; + private LocalDateTime createTime; private Long updateBy; - private OffsetDateTime updateTime; + private LocalDateTime updateTime; private String remark; private String authUsername; // 用户名 private String authPassword; // 密码 @@ -55,14 +55,14 @@ public class DeviceInterlocking { public Long getCreateBy() { return createBy; } public void setCreateBy(Long createBy) { this.createBy = createBy; } - public OffsetDateTime getCreateTime() { return createTime; } - public void setCreateTime(OffsetDateTime createTime) { this.createTime = createTime; } + public LocalDateTime getCreateTime() { return createTime; } + public void setCreateTime(LocalDateTime createTime) { this.createTime = createTime; } public Long getUpdateBy() { return updateBy; } public void setUpdateBy(Long updateBy) { this.updateBy = updateBy; } - public OffsetDateTime getUpdateTime() { return updateTime; } - public void setUpdateTime(OffsetDateTime updateTime) { this.updateTime = updateTime; } + public LocalDateTime getUpdateTime() { return updateTime; } + public void setUpdateTime(LocalDateTime updateTime) { this.updateTime = updateTime; } public String getRemark() { return remark; } public void setRemark(String remark) { this.remark = remark; } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlockingCmd.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlockingCmd.java index 29343a0..c01ab29 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlockingCmd.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlockingCmd.java @@ -4,9 +4,8 @@ import com.Modules.etl.handler.ArrayIntegerTypeHandler; import com.Modules.etl.handler.ArrayStringTypeHandler; import org.apache.ibatis.type.JdbcType; import org.apache.ibatis.type.TypeHandler; -import org.apache.ibatis.type.ArrayTypeHandler; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; import java.util.Arrays; import java.util.List; @@ -25,8 +24,8 @@ public class DeviceInterlockingCmd { private String banType; // 封禁类型（1:白名单、0:黑名单） private String cmdStatus; // 指令状态（0:未执行、1:已完成、2:执行中） private Integer banDuration; // 封禁时长（秒，-1表示永久） - private OffsetDateTime createTime; - private OffsetDateTime updateTime; + private LocalDateTime createTime; + private LocalDateTime updateTime; private String tenantId; private Long createDept; private Long createBy; @@ -65,11 +64,11 @@ public class DeviceInterlockingCmd { public Integer getBanDuration() { return banDuration; } public void setBanDuration(Integer banDuration) { this.banDuration = banDuration; } - public OffsetDateTime getCreateTime() { return createTime; } - public void setCreateTime(OffsetDateTime createTime) { this.createTime = createTime; } + public LocalDateTime getCreateTime() { return createTime; } + public void setCreateTime(LocalDateTime createTime) { this.createTime = createTime; } - public OffsetDateTime getUpdateTime() { return updateTime; } - public void setUpdateTime(OffsetDateTime updateTime) { this.updateTime = updateTime; } + public LocalDateTime getUpdateTime() { return updateTime; } + public void setUpdateTime(LocalDateTime updateTime) { this.updateTime = updateTime; } public String getTenantId() { return tenantId; } public void setTenantId(String tenantId) { this.tenantId = tenantId; } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlockingLog.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlockingLog.java index 516c829..c4147ee 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlockingLog.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/entity/DeviceInterlockingLog.java @@ -1,6 +1,6 @@ package com.common.entity; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; /** * 封禁记录表实体类 @@ -12,15 +12,15 @@ public class DeviceInterlockingLog { private Long deviceInterlockingId; // 封禁设备ID private String banIp; // 封禁IP地址 private String deviceName; // 封禁设备名称 - private OffsetDateTime banTime; // 封禁时间 + private LocalDateTime banTime; // 封禁时间 private String banMethod; // 封禁方式（0.人工、1.自动化封禁） private Integer banResult; // 联动结果（成功：1、失败：0） private String tenantId; private Long createDept; private Long createBy; - private OffsetDateTime createTime; + private LocalDateTime createTime; private Long updateBy; - private OffsetDateTime updateTime; + private LocalDateTime updateTime; private String remark; private String respBody; // 响应body private String reqBody; // 请求body @@ -41,8 +41,8 @@ public class DeviceInterlockingLog { public String getDeviceName() { return deviceName; } public void setDeviceName(String deviceName) { this.deviceName = deviceName; } - public OffsetDateTime getBanTime() { return banTime; } - public void setBanTime(OffsetDateTime banTime) { this.banTime = banTime; } + public LocalDateTime getBanTime() { return banTime; } + public void setBanTime(LocalDateTime banTime) { this.banTime = banTime; } public String getBanMethod() { return banMethod; } public void setBanMethod(String banMethod) { this.banMethod = banMethod; } @@ -59,14 +59,14 @@ public class DeviceInterlockingLog { public Long getCreateBy() { return createBy; } public void setCreateBy(Long createBy) { this.createBy = createBy; } - public OffsetDateTime getCreateTime() { return createTime; } - public void setCreateTime(OffsetDateTime createTime) { this.createTime = createTime; } + public LocalDateTime getCreateTime() { return createTime; } + public void setCreateTime(LocalDateTime createTime) { this.createTime = createTime; } public Long getUpdateBy() { return updateBy; } public void setUpdateBy(Long updateBy) { this.updateBy = updateBy; } - public OffsetDateTime getUpdateTime() { return updateTime; } - public void setUpdateTime(OffsetDateTime updateTime) { this.updateTime = updateTime; } + public LocalDateTime getUpdateTime() { return updateTime; } + public void setUpdateTime(LocalDateTime updateTime) { this.updateTime = updateTime; } public String getRemark() { return remark; } public void setRemark(String remark) { this.remark = remark; } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/AlarmMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/AlarmMapper.java index b8a5ffc..f06eefb 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/AlarmMapper.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/AlarmMapper.java @@ -15,7 +15,7 @@ public interface AlarmMapper { "INSERT INTO alarm (", "id, created_at, alarm_name, alarm_level, alarm_type, ", "alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, ", - "device_id, comment,origin_log_ids,log_start_at, log_end_at, window_time, http_status, ", + "device_id, \"comment\",origin_log_ids,log_start_at, log_end_at, window_time, http_status, ", "attack_port, victim_port, attack_method, etl_time, log_count, ", "attack_chain_phase, disposition_advice, attack_direction, ", "judged_state, disposed_state, attack_result, fall, payload, dns_info, engine_type, " , @@ -52,7 +52,7 @@ public interface AlarmMapper { @Insert("INSERT INTO alarm (" + "id, created_at, alarm_name, alarm_level, alarm_type, " + "alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, " + - "device_id, comment,origin_log_ids, log_start_at, log_end_at, window_time, http_status, " + + "device_id, \"comment\",origin_log_ids, log_start_at, log_end_at, window_time, http_status, " + "attack_port, victim_port, attack_method, etl_time, log_count, " + "attack_chain_phase, disposition_advice, attack_direction, " + "judged_state, disposed_state, attack_result, fall, payload, dns_info, engine_type, " + diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/AlarmVisitMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/AlarmVisitMapper.java index 00b1948..9a20855 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/AlarmVisitMapper.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/AlarmVisitMapper.java @@ -19,7 +19,7 @@ public interface AlarmVisitMapper { "INSERT INTO alarm_visit (", "id, created_at, alarm_name, alarm_level, alarm_type, ", "alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, ", - "device_id, comment,origin_log_ids,log_start_at, log_end_at,window_time, http_status, ", + "device_id, \"comment\",origin_log_ids,log_start_at, log_end_at,window_time, http_status, ", "attack_port, victim_port, attack_method, etl_time, log_count, ", "attack_chain_phase, disposition_advice, attack_direction, ", "judged_state, disposed_state, attack_result, fall, payload, dns_info, engine_type, " , @@ -56,7 +56,7 @@ public interface AlarmVisitMapper { @Insert("INSERT INTO alarm_visit (" + "id, created_at, alarm_name, alarm_level, alarm_type, " + "alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, " + - "device_id, comment,origin_log_ids, log_start_at, log_end_at, window_time,http_status, " + + "device_id, \"comment\",origin_log_ids, log_start_at, log_end_at, window_time,http_status, " + "attack_port, victim_port, attack_method, etl_time, log_count, " + "attack_chain_phase, disposition_advice, attack_direction, " + "judged_state, disposed_state, attack_result, fall, payload, dns_info,engine_type, " + diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceCollectHeartbeatMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceCollectHeartbeatMapper.java index b74af1b..a169f07 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceCollectHeartbeatMapper.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceCollectHeartbeatMapper.java @@ -37,22 +37,37 @@ public interface DeviceCollectHeartbeatMapper { /** * 插入或更新（根据collect_id） + * 达梦数据库使用 MERGE INTO 实现 upsert */ - @Insert("INSERT INTO device_collect_heartbeat (" + + @Update("MERGE INTO device_collect_heartbeat t " + + "USING (SELECT " + + "#{collectId} AS collect_id, " + + "#{collectName} AS collect_name, " + + "#{deviceIp} AS device_ip, " + + "#{appVersion} AS app_version, " + + "#{lastHeartbeat} AS last_heartbeat, " + + "#{heartbeatCount} AS heartbeat_count, " + + "#{status} AS status, " + + "#{failCount} AS fail_count, " + + "#{updateTime} AS update_time " + + "FROM DUAL) s " + + "ON (t.collect_id = s.collect_id) " + + "WHEN MATCHED THEN UPDATE SET " + + "t.collect_name = s.collect_name, " + + "t.device_ip = s.device_ip, " + + "t.app_version = s.app_version, " + + "t.last_heartbeat = s.last_heartbeat, " + + "t.heartbeat_count = s.heartbeat_count, " + + "t.status = s.status, " + + "t.fail_count = s.fail_count, " + + "t.update_time = s.update_time " + + "WHEN NOT MATCHED THEN INSERT (" + "collect_id, collect_name, device_ip, app_version, last_heartbeat, " + - "heartbeat_count, status, fail_count, update_time " + + "heartbeat_count, status, fail_count, update_time" + ") VALUES (" + - "#{collectId}, #{collectName}, #{deviceIp}, #{appVersion}, #{lastHeartbeat}, " + - "#{heartbeatCount}, #{status}, #{failCount}, #{updateTime} " + - ") ON CONFLICT (collect_id) DO UPDATE SET " + - "collect_name = EXCLUDED.collect_name, " + - "device_ip = EXCLUDED.device_ip, " + - "app_version = EXCLUDED.app_version, " + - "last_heartbeat = EXCLUDED.last_heartbeat, " + - "heartbeat_count = EXCLUDED.heartbeat_count, " + - "status = EXCLUDED.status, " + - "fail_count = EXCLUDED.fail_count, " + - "update_time = EXCLUDED.update_time") + "s.collect_id, s.collect_name, s.device_ip, s.app_version, s.last_heartbeat, " + + "s.heartbeat_count, s.status, s.fail_count, s.update_time" + + ")") int upsert(DeviceCollectHeartbeat heartbeat); /** diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java index 3a05666..221223e 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java @@ -155,12 +155,12 @@ public interface DeviceCollectTaskMapper extends BaseMapper{ "" + "UPDATE device_collect_task SET " + " first_time = CASE " + - " WHEN first_time IS NULL AND #{task.firstTime}::TIMESTAMP IS NOT NULL THEN #{task.firstTime}::TIMESTAMP " + + " WHEN first_time IS NULL AND #{task.firstTime} IS NOT NULL THEN #{task.firstTime} " + " ELSE first_time " + " END, " + - " last_success_time = #{task.lastSuccessTime}::TIMESTAMP, " + - " last_failed_time = #{task.lastFailedTime}::TIMESTAMP, " + - " updated_at = #{task.updatedAt}::TIMESTAMP " + + " last_success_time = #{task.lastSuccessTime}, " + + " last_failed_time = #{task.lastFailedTime}, " + + " updated_at = #{task.updatedAt} " + "WHERE id = #{task.id}" + "" + "") @@ -171,12 +171,12 @@ public interface DeviceCollectTaskMapper extends BaseMapper{ */ @Update("UPDATE device_collect_task " + "SET first_time = CASE " + - " WHEN first_time IS NULL AND #{firstTime}::TIMESTAMP IS NOT NULL THEN #{firstTime}::TIMESTAMP " + + " WHEN first_time IS NULL AND #{firstTime} IS NOT NULL THEN #{firstTime} " + " ELSE first_time " + " END, " + - " last_success_time = #{lastSuccessTime}::TIMESTAMP, " + - " last_failed_time = #{lastFailTime}::TIMESTAMP, " + - " updated_at = #{updateTime}::TIMESTAMP " + + " last_success_time = #{lastSuccessTime}, " + + " last_failed_time = #{lastFailTime}, " + + " updated_at = #{updateTime} " + "WHERE id = #{deviceCollectId}") int updateTaskTime(@Param("deviceCollectId") String deviceCollectId, @Param("firstTime") LocalDateTime firstTime, diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceInterlockingCmdMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceInterlockingCmdMapper.java index 57bfe7e..a51f63c 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceInterlockingCmdMapper.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceInterlockingCmdMapper.java @@ -61,8 +61,11 @@ public interface DeviceInterlockingCmdMapper { @Insert("INSERT INTO device_interlocking_cmd (probe_id, probe_ip, device_interlocking_id, device_interlocking_ip, " + "ban_ips, ban_method, ban_type, cmd_status, ban_duration, create_time, update_time, " + "tenant_id, create_dept, create_by, remark, ban_operation_type) " + - "VALUES (#{probeId}, #{probeIp}, ARRAY[:ids], ARRAY[:ips], ARRAY[:banIps], " + - "#{banMethod}, #{banType}, #{cmdStatus}, #{banDuration}, NOW(), NOW(), " + + "VALUES (#{probeId}, #{probeIp}, " + + "#{deviceInterlockingId, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, " + + "#{deviceInterlockingIp, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " + + "#{banIps, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " + + "#{banMethod}, #{banType}, #{cmdStatus}, #{banDuration}, SYSDATE, SYSDATE, " + "#{tenantId}, #{createDept}, #{createBy}, #{remark}, #{banOperationType})") @Options(useGeneratedKeys = true, keyProperty = "id") int insert(DeviceInterlockingCmd cmd); @@ -73,25 +76,25 @@ public interface DeviceInterlockingCmdMapper { * @param cmdStatus 新状态 * @return 影响行数 */ - @Update("UPDATE device_interlocking_cmd SET cmd_status = #{cmdStatus}, update_time = NOW() WHERE id = #{id}") + @Update("UPDATE device_interlocking_cmd SET cmd_status = #{cmdStatus}, update_time = SYSDATE WHERE id = #{id}") int updateStatus(@Param("id") Long id, @Param("cmdStatus") String cmdStatus); /** * 更新指令状态为执行中 */ - @Update("UPDATE device_interlocking_cmd SET cmd_status = '2', update_time = NOW() WHERE id = #{id}") + @Update("UPDATE device_interlocking_cmd SET cmd_status = '2', update_time = SYSDATE WHERE id = #{id}") int updateStatusToExecuting(@Param("id") Long id); /** * 更新指令状态为执行完成 */ - @Update("UPDATE device_interlocking_cmd SET cmd_status = '1', update_time = NOW() WHERE id = #{id}") + @Update("UPDATE device_interlocking_cmd SET cmd_status = '1', update_time = SYSDATE WHERE id = #{id}") int updateStatusToCompleted(@Param("id") Long id); /** * 更新指令状态为执行失败 */ - @Update("UPDATE device_interlocking_cmd SET cmd_status = '3', update_time = NOW() WHERE id = #{id}") + @Update("UPDATE device_interlocking_cmd SET cmd_status = '3', update_time = SYSDATE WHERE id = #{id}") int updateStatusToFailed(@Param("id") Long id); /** diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java index cf38883..6124c49 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java @@ -84,7 +84,7 @@ public interface DeviceReceiveLogMapper { */ @Select("SELECT device_collect_id, MAX(created_at) AS last_success_time " + "FROM device_receive_log " + - "WHERE push_success = true " + + "WHERE push_success = 1 " + "AND created_at >= CURRENT_DATE " + "GROUP BY device_collect_id") List selectDailySuccessTimes(); @@ -94,7 +94,7 @@ public interface DeviceReceiveLogMapper { */ @Select("SELECT device_collect_id, MAX(created_at) AS last_fail_time " + "FROM device_receive_log " + - "WHERE push_success = false " + + "WHERE push_success = 0 " + "AND created_at >= CURRENT_DATE " + "GROUP BY device_collect_id") List selectDailyFailTimes(); @@ -104,7 +104,7 @@ public interface DeviceReceiveLogMapper { */ @Select("SELECT device_collect_id, MIN(created_at) AS first_success_time " + "FROM device_receive_log " + - "WHERE push_success = true " + + "WHERE push_success = 1 " + "GROUP BY device_collect_id") List selectFirstSuccessTimes(); @@ -114,7 +114,7 @@ public interface DeviceReceiveLogMapper { @Select("SELECT device_collect_id, MIN(created_at) AS first_time, " + "MAX(created_at) AS last_success_time " + "FROM device_receive_log " + - "WHERE push_success = true " + + "WHERE push_success = 1 " + "AND created_at >= #{startTime} " + "AND created_at < #{endTime} " + "GROUP BY device_collect_id") diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java index e11ffaa..c9addc0 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java @@ -49,30 +49,29 @@ public interface SyslogNormalAlarmMapper { */ @Select("SELECT " + "to_char(log_time, 'YYYYMMDD') as log_date, " + - "ARRAY_AGG(DISTINCT host(src_ip)::text) as attack_ips, " + + "WM_CONCAT(DISTINCT src_ip) as attack_ips, " + "origin_event_name, " + "MAX(attack_result) as attack_result, " + "MIN(log_time) as min_log_time, " + "MAX(log_time) as max_log_time, " + "COUNT(1) as log_count, " + - "ARRAY_AGG(DISTINCT host(dest_ip)::text) as victim_ips, " + - "ARRAY_AGG(DISTINCT http_url) as victim_web_urls, " + - "ARRAY_AGG(DISTINCT device_id) as device_ids, " + - "ARRAY_AGG(DISTINCT id) as origin_log_ids, " + + "WM_CONCAT(DISTINCT dest_ip) as victim_ips, " + + "WM_CONCAT(DISTINCT http_url) as victim_web_urls, " + + "WM_CONCAT(DISTINCT device_id) as device_ids, " + + "WM_CONCAT(DISTINCT id) as origin_log_ids, " + "MAX(event_level) as max_event_level, " + "MIN(origin_event_type) AS first_event_type, " + "MAX(origin_event_type) as event_type, " + "MIN(event_type) as min_event_type, " + - "ARRAY_AGG(DISTINCT src_port::int4) as attack_ports, " + - "ARRAY_AGG(DISTINCT dest_port::int4) as victim_ports, " + - "ARRAY_AGG(DISTINCT http_resp_codes::text) as http_status_codes, " + - "ARRAY_AGG(DISTINCT payload::BYTEA) as payload_samples, " + - "ARRAY_AGG(DISTINCT http_req_header) as httpReqHeaders, " + - "ARRAY_AGG(DISTINCT http_req_body) as httpReqBodys, " + - "ARRAY_AGG(DISTINCT http_resp_header) as httpRespHeaders, " + - "ARRAY_AGG(DISTINCT http_resp_body) as httpRespBodys, " + - "MODE() WITHIN GROUP (ORDER BY dest_domain) as dns_info, " + - "STRING_AGG(DISTINCT COALESCE(host(dest_ip)::text, ''), ',') as victim_ips_str " + + "WM_CONCAT(DISTINCT src_port) as attack_ports, " + + "WM_CONCAT(DISTINCT dest_port) as victim_ports, " + + "WM_CONCAT(DISTINCT http_resp_codes) as http_status_codes, " + + "WM_CONCAT(DISTINCT payload) as payload_samples, " + + "WM_CONCAT(DISTINCT http_req_header) as httpReqHeaders, " + + "WM_CONCAT(DISTINCT http_req_body) as httpReqBodys, " + + "WM_CONCAT(DISTINCT http_resp_header) as httpRespHeaders, " + + "WM_CONCAT(DISTINCT http_resp_body) as httpRespBodys, " + + "LISTAGG(DISTINCT COALESCE(dest_ip, ''), ',') as victim_ips_str " + "FROM syslog_normal_alarm " + "WHERE log_time >= #{startTime} AND log_time < #{endTime} " + "AND event_level >= 1 AND src_ip NOT IN ('127.0.0.1', '127.0.0.2') " + diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java index 358218c..7ef7737 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java @@ -46,30 +46,29 @@ public interface SyslogNormalDataMapper { */ @Select("SELECT " + "to_char(log_time, 'YYYYMMDD') as log_date, " + - "ARRAY_AGG(DISTINCT host(src_ip)::text) as attack_ips, " + + "WM_CONCAT(DISTINCT src_ip) as attack_ips, " + "origin_event_name, " + "MAX(attack_result) as attack_result, " + "MIN(log_time) as min_log_time, " + "MAX(log_time) as max_log_time, " + "COUNT(1) as log_count, " + - "ARRAY_AGG(DISTINCT host(dest_ip)::text) as victim_ips, " + - "ARRAY_AGG(DISTINCT http_url) as victim_web_urls, " + - "ARRAY_AGG(DISTINCT device_id) as device_ids, " + - "ARRAY_AGG(DISTINCT id) as origin_log_ids, " + + "WM_CONCAT(DISTINCT dest_ip) as victim_ips, " + + "WM_CONCAT(DISTINCT http_url) as victim_web_urls, " + + "WM_CONCAT(DISTINCT device_id) as device_ids, " + + "WM_CONCAT(DISTINCT id) as origin_log_ids, " + "MAX(event_level) as max_event_level, " + "MIN(origin_event_type) AS first_event_type, " + "MAX(origin_event_type) as event_type, " + "MIN(event_type) as min_event_type, " + - "ARRAY_AGG(DISTINCT src_port::int4) as attack_ports, " + - "ARRAY_AGG(DISTINCT dest_port::int4) as victim_ports, " + - "ARRAY_AGG(DISTINCT http_resp_codes::text) as http_status_codes, " + - "ARRAY_AGG(DISTINCT payload::BYTEA) as payload_samples, " + - "ARRAY_AGG(DISTINCT http_req_header) as httpReqHeaders, " + - "ARRAY_AGG(DISTINCT http_req_body) as httpReqBodys, " + - "ARRAY_AGG(DISTINCT http_resp_header) as httpRespHeaders, " + - "ARRAY_AGG(DISTINCT http_resp_body) as httpRespBodys, " + - "MODE() WITHIN GROUP (ORDER BY dest_domain) as dns_info, " + - "STRING_AGG(DISTINCT COALESCE(host(dest_ip)::text, ''), ',') as victim_ips_str " + + "WM_CONCAT(DISTINCT src_port) as attack_ports, " + + "WM_CONCAT(DISTINCT dest_port) as victim_ports, " + + "WM_CONCAT(DISTINCT http_resp_codes) as http_status_codes, " + + "WM_CONCAT(DISTINCT payload) as payload_samples, " + + "WM_CONCAT(DISTINCT http_req_header) as httpReqHeaders, " + + "WM_CONCAT(DISTINCT http_req_body) as httpReqBodys, " + + "WM_CONCAT(DISTINCT http_resp_header) as httpRespHeaders, " + + "WM_CONCAT(DISTINCT http_resp_body) as httpRespBodys, " + + "LISTAGG(DISTINCT COALESCE(dest_ip, ''), ',') as victim_ips_str " + "FROM syslog_normal_data " + "WHERE log_time >= #{startTime} AND log_time < #{endTime} " + "AND http_resp_codes =200 and origin_event_type <> '' and origin_event_name='访问日志' AND src_ip NOT IN ('127.0.0.1', '127.0.0.2') " + diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/WecomNotificationMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/WecomNotificationMapper.java index abd6f61..24d81eb 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/WecomNotificationMapper.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/WecomNotificationMapper.java @@ -18,12 +18,12 @@ public interface WecomNotificationMapper { "wecom_notification_time, tenant_id, create_dept, create_by, create_time, " + "update_by, update_time, remark, wecom_notification_status" + ") VALUES (" + - "nextval('seq_wecom_notification'), #{userId}, #{wecomNotificationName}, #{wecomNotificationIp}, " + + "seq_wecom_notification.NEXTVAL, #{userId}, #{wecomNotificationName}, #{wecomNotificationIp}, " + "#{wecomNotificationType}, #{wecomNotificationLevel}, #{wecomNotificationContent}, " + "#{wecomNotificationTime}, #{tenantId}, #{createDept}, #{createBy}, #{createTime}, " + "#{updateBy}, #{updateTime}, #{remark}, #{wecomNotificationStatus}" + ")") - @SelectKey(statement = "SELECT currval('seq_wecom_notification')", keyProperty = "wecomNotificationId", resultType = Long.class, before = false) + @SelectKey(statement = "SELECT seq_wecom_notification.currval", keyProperty = "wecomNotificationId", resultType = Long.class, before = false) int insert(WecomNotification notification); /** @@ -41,7 +41,7 @@ public interface WecomNotificationMapper { /** * 更新通知状态 */ - @Update("UPDATE wecom_notification SET wecom_notification_status = #{status}, update_time = NOW() " + + @Update("UPDATE wecom_notification SET wecom_notification_status = #{status}, update_time = SYSDATE " + "WHERE wecom_notification_id = #{wecomNotificationId}") int updateStatus(@Param("wecomNotificationId") Long wecomNotificationId, @Param("status") String status); } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DeviceInterlockingLogService.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DeviceInterlockingLogService.java index b251092..af885b8 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DeviceInterlockingLogService.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DeviceInterlockingLogService.java @@ -4,7 +4,7 @@ import com.common.entity.DeviceInterlockingLog; import com.common.mapper.DeviceInterlockingLogMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; import java.util.List; @Service @@ -32,7 +32,7 @@ public class DeviceInterlockingLogService { */ public int insert(DeviceInterlockingLog log) { if (log.getBanTime() == null) { - log.setBanTime(OffsetDateTime.now()); + log.setBanTime(LocalDateTime.now()); } return logMapper.insert(log); } @@ -44,7 +44,7 @@ public class DeviceInterlockingLogService { if (logs != null && !logs.isEmpty()) { for (DeviceInterlockingLog log : logs) { if (log.getBanTime() == null) { - log.setBanTime(OffsetDateTime.now()); + log.setBanTime(LocalDateTime.now()); } } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmNormalizeRuleService.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmNormalizeRuleService.java index 58e7090..2bf5387 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmNormalizeRuleService.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmNormalizeRuleService.java @@ -77,7 +77,7 @@ public class DmNormalizeRuleService { List> ruleMap=dmNormalizeRuleMapper.selectByDeviceId(id); sqlSession.commit(); - return ruleMap; + return convertClobToString(ruleMap); } catch (Exception e) { logger.error("DmNormalizeRuleService MyBatisUtil getSqlSession 异常", e); @@ -95,7 +95,41 @@ public class DmNormalizeRuleService { { System.out.println("调用selectByDeviceIdAuto 方法,id:"+id); List> ruleMap=dmNormalizeRuleMapper.selectByDeviceId(id); - return ruleMap; + return convertClobToString(ruleMap); + } + + /** + * 将达梦 JDBC CLOB/NCLOB 对象转换为 String，避免缓存序列化报错 + * 达梦驱动返回的 TEXT/CLOB 列可能是 dm.jdbc.driver.DmdbNClob 等内部类型， + * toString() 只返回对象引用（如 DmdbNClob@xxx），必须通过 Clob 接口获取实际文本 + */ + private List> convertClobToString(List> list) { + if (list == null) return null; + for (Map map : list) { + if (map == null) continue; + for (Map.Entry entry : map.entrySet()) { + Object value = entry.getValue(); + if (value != null && value.getClass().getName().startsWith("dm.jdbc.")) { + try { + if (value instanceof java.sql.Clob) { + java.sql.Clob clob = (java.sql.Clob) value; + long length = clob.length(); + if (length > 0) { + entry.setValue(clob.getSubString(1, (int) length)); + } else { + entry.setValue(""); + } + } else { + entry.setValue(value.toString()); + } + } catch (Exception e) { + logger.warn("CLOB/NCLOB 转换 String 失败: " + e.getMessage()); + entry.setValue(null); + } + } + } + } + return list; } } \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java index 03e9a81..c1b6407 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java @@ -734,7 +734,25 @@ public class RealtimeAnalysisEngine implements AnalysisEngine { return result; } // PostgreSQL数组以字符串形式返回，如 "{ip1,ip2,ip3}" + // 达梦 JSONB_AGG 返回 JSON 数组格式，如 "[41614, 8080]" String str = value.toString(); + if (str.startsWith("[") && str.endsWith("]")) { + str = str.substring(1, str.length() - 1).trim(); + if (str.isEmpty()) { + return new String[0]; + } + // 拆分 JSON 数组元素（兼容带引号和纯数字） + String[] parts = str.split(","); + String[] result = new String[parts.length]; + for (int i = 0; i < parts.length; i++) { + String part = parts[i].trim(); + if (part.startsWith("\"") && part.endsWith("\"")) { + part = part.substring(1, part.length() - 1); + } + result[i] = part; + } + return result; + } if (str.startsWith("{") && str.endsWith("}")) { str = str.substring(1, str.length() - 1); return str.split(","); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java index eab92de..c7ce093 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java @@ -747,19 +747,19 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { return "AVG(" + columnName + ")"; case "DUPLICATESANDSPLICE": if (StringUtils.isNotBlank(argsStr)) { - return "STRING_AGG(DISTINCT " + columnName + ", '" + argsStr + "')"; + return "LISTAGG(DISTINCT " + columnName + ", '" + argsStr + "')"; } - return "STRING_AGG(DISTINCT " + columnName + ", ',')"; + return "LISTAGG(DISTINCT " + columnName + ", ',')"; case "CONCAT_AGG": if (StringUtils.isNotBlank(argsStr)) { - return "STRING_AGG(" + columnName + ", '" + argsStr + "')"; + return "LISTAGG(" + columnName + ", '" + argsStr + "')"; } - return "STRING_AGG(" + columnName + ", ',')"; + return "LISTAGG(" + columnName + ", ',')"; case "CONCAT_AGG_ID": if (StringUtils.isNotBlank(argsStr)) { - return "STRING_AGG(" + columnName + ", '" + argsStr + "')"; + return "LISTAGG(" + columnName + ", '" + argsStr + "')"; } - return "STRING_AGG(" + columnName + ", ',')"; + return "LISTAGG(" + columnName + ", ',')"; case "SPLIT_DISTINCT_CONCAT": if (StringUtils.isNotBlank(argsStr)) { String[] splitArgs = argsStr.split(","); @@ -767,23 +767,23 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { String separator = splitArgs[0].trim(); String delimiter = splitArgs[1].trim(); String limit = splitArgs[2].trim(); - return "STRING_AGG(DISTINCT REGEXP_SPLIT(" + columnName + ", '" + delimiter + "'), '" + separator + "') LIMIT " + limit; + return "LISTAGG(DISTINCT REGEXP_SPLIT(" + columnName + ", '" + delimiter + "'), '" + separator + "') LIMIT " + limit; } } return columnName; //自定添加方法 case "MODE_WITH_GROUP": - return "MODE() WITHIN GROUP (ORDER BY " + columnName + ")"; + return columnName; // 聚合函数（兼容旧代码） case "ARRAY_AGG": - return "ARRAY_AGG(DISTINCT " + columnName + ")"; + return "WM_CONCAT(DISTINCT " + columnName + ")"; case "STRING_AGG": if (StringUtils.isNotBlank(argsStr)) { - return "STRING_AGG(" + columnSafeWrap(columnName) + ", " + argsStr + ")"; + return "LISTAGG(" + columnSafeWrap(columnName) + ", " + argsStr + ")"; } - return "STRING_AGG(DISTINCT " + columnName + ", ',')"; + return "LISTAGG(DISTINCT " + columnName + ", ',')"; // 时间函数 case "YEAR": @@ -847,7 +847,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { case "TO_CHAR": return "TO_CHAR(" + columnName + ", 'YYYYMMDD')"; case "HOST": - return "HOST(" + columnName + ")::text"; + return columnName; default: return functionName + "(" + columnName + ")"; } @@ -938,12 +938,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { * 列名安全包装（处理类型转换） */ private String columnSafeWrap(String columnName) { - if (columnName.toLowerCase().contains("ip")) { - return "host(" + columnName + ")::text"; - } - if (columnName.toLowerCase().contains("port")) { - return columnName + "::int4"; - } + // DM不需要类型转换，直接返回列名 return columnName; } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/JsonbUtil.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/JsonbUtil.java index a3d0ca5..3b05f88 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/JsonbUtil.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/JsonbUtil.java @@ -22,6 +22,18 @@ public class JsonbUtil { return null; } + // 达梦数据库：TEXT/CLOB 列返回 dm.jdbc.driver.DmdbNClob 对象， + // toString() 只返回对象引用而非实际内容，必须通过 Clob 接口读取 + if (value instanceof java.sql.Clob) { + try { + java.sql.Clob clob = (java.sql.Clob) value; + long length = clob.length(); + value = length > 0 ? clob.getSubString(1, (int) length) : ""; + } catch (Exception e) { + value = value.toString(); + } + } + // 如果已经是字符串，直接返回 if (value instanceof String) { String strValue = (String) value; diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/CacheConfig.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/CacheConfig.java index 4daedcf..6ac6023 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/CacheConfig.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/CacheConfig.java @@ -1,8 +1,16 @@ package com.config; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.BeanDescription; +import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.jsontype.BasicPolymorphicTypeValidator; import com.fasterxml.jackson.databind.jsontype.PolymorphicTypeValidator; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.BeanSerializerModifier; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.EnableCaching; @@ -15,6 +23,7 @@ import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSeriali import org.springframework.data.redis.serializer.RedisSerializationContext; import org.springframework.data.redis.serializer.StringRedisSerializer; import com.fasterxml.jackson.databind.DeserializationFeature; +import java.io.IOException; import java.time.Duration; import java.util.Collections; import org.springframework.context.annotation.Primary; @@ -35,6 +44,33 @@ public class CacheConfig { // 禁用将日期序列化为时间戳 mapper.disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + // 注册达梦 JDBC 安全序列化器 — 拦截 dm.jdbc.* 类，直接返回 null 防止循环引用 StackOverflow + SimpleModule dmSafeModule = new SimpleModule("dm-safe"); + dmSafeModule.setSerializerModifier(new BeanSerializerModifier() { + @Override + public JsonSerializer modifySerializer(SerializationConfig config, + BeanDescription beanDesc, + JsonSerializer serializer) { + if (beanDesc.getBeanClass().getName().startsWith("dm.jdbc.")) { + return new JsonSerializer() { + @Override + public void serialize(Object value, JsonGenerator gen, + SerializerProvider serializers) throws IOException { + gen.writeNull(); + } + + @Override + public void serializeWithType(Object value, JsonGenerator gen, + SerializerProvider serializers, TypeSerializer typeSer) throws IOException { + gen.writeNull(); + } + }; + } + return serializer; + } + }); + mapper.registerModule(dmSafeModule); + // 启用类型信息，解决 LinkedHashMap 转换问题 PolymorphicTypeValidator ptv = BasicPolymorphicTypeValidator.builder() .allowIfSubType("com.common.entity.") // 允许你的实体类包 diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/RedisConfig.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/RedisConfig.java index 3bd7321..134af0f 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/RedisConfig.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/RedisConfig.java @@ -1,6 +1,14 @@ package com.config; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.BeanDescription; +import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.BeanSerializerModifier; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -9,6 +17,8 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; +import java.io.IOException; + @Configuration public class RedisConfig { @@ -23,6 +33,32 @@ public class RedisConfig { ObjectMapper mapper = new ObjectMapper(); mapper.registerModule(new JavaTimeModule()); + // 注册达梦 JDBC 安全序列化器 — 拦截 dm.jdbc.* 类，直接返回 null 防止循环引用 StackOverflow + SimpleModule dmSafeModule = new SimpleModule("dm-safe"); + dmSafeModule.setSerializerModifier(new BeanSerializerModifier() { + @Override + public JsonSerializer modifySerializer(SerializationConfig config, + BeanDescription beanDesc, + JsonSerializer serializer) { + if (beanDesc.getBeanClass().getName().startsWith("dm.jdbc.")) { + return new JsonSerializer() { + @Override + public void serialize(Object value, JsonGenerator gen, + SerializerProvider serializers) throws IOException { + gen.writeNull(); + } + + @Override + public void serializeWithType(Object value, JsonGenerator gen, + SerializerProvider serializers, TypeSerializer typeSer) throws IOException { + gen.writeNull(); + } + }; + } + return serializer; + } + }); + mapper.registerModule(dmSafeModule); mapper.activateDefaultTyping( mapper.getPolymorphicTypeValidator(), ObjectMapper.DefaultTyping.NON_FINAL diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/WebConfig.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/WebConfig.java index 5f52013..7330f38 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/WebConfig.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/WebConfig.java @@ -1,8 +1,16 @@ package com.config; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.BeanDescription; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.BeanSerializerModifier; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -10,6 +18,7 @@ import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import java.io.IOException; import java.util.List; @Configuration @@ -28,6 +37,33 @@ public class WebConfig implements WebMvcConfigurer { // 禁用将日期序列化为时间戳 mapper.disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + // 注册达梦 JDBC 安全序列化器 — 拦截 dm.jdbc.* 类，直接返回 null 防止循环引用 StackOverflow + SimpleModule dmSafeModule = new SimpleModule("dm-safe"); + dmSafeModule.setSerializerModifier(new BeanSerializerModifier() { + @Override + public JsonSerializer modifySerializer(SerializationConfig config, + BeanDescription beanDesc, + JsonSerializer serializer) { + if (beanDesc.getBeanClass().getName().startsWith("dm.jdbc.")) { + return new JsonSerializer() { + @Override + public void serialize(Object value, JsonGenerator gen, + SerializerProvider serializers) throws IOException { + gen.writeNull(); + } + + @Override + public void serializeWithType(Object value, JsonGenerator gen, + SerializerProvider serializers, TypeSerializer typeSer) throws IOException { + gen.writeNull(); + } + }; + } + return serializer; + } + }); + mapper.registerModule(dmSafeModule); + // 忽略未知属性 mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-dev.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-dev.properties index 1968b74..b499174 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-dev.properties +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-dev.properties @@ -24,7 +24,7 @@ influxdb.batch.size=1000 influxdb.flush.interval=1000 influxdb.retry.attempts=3 influxdb.retry.delay=1000 -# InfluxDB 2.7 ӳʱ +# InfluxDB 2.7 连接超时配置 influxdb.connection.timeout=30s influxdb.connection.read-timeout=30s influxdb.connection.write-timeout=60s @@ -35,16 +35,16 @@ app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys -spring.datasource.username=postgres -spring.datasource.password=TnLanWaidYSwTSG5 -spring.datasource.driver-class-name=org.postgresql.Driver - +spring.datasource.url=jdbc:dm://192.163.4.99:5237/DM_ecosys +spring.datasource.username=SYSDBA +spring.datasource.password=caZ2TcmXNSW8L2Ap +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity -#mybatis handler +#mybatis handler 类 mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 mybatis.configuration.map-underscore-to-camel-case=true @@ -71,22 +71,22 @@ spring.kafka.listener.concurrency= 2 spring.kafka.listener.type=batch -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# +# 开发环境缓存配置 spring.redis.host=localhost spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password= spring.redis.database=0 spring.redis.timeout=2000 @@ -95,11 +95,11 @@ spring.redis.lettuce.pool.max-active=8 spring.redis.lettuce.pool.max-wait=-1 spring.redis.lettuce.pool.max-idle=8 spring.redis.lettuce.pool.min-idle=0 -# ʱ϶̣ +# 开发环境缓存时间较短，方便调试 spring.cache.redis.time-to-live=600000 -# Ӧô +# 应用处理器配置 app.processor.thread-pool.core-pool-size=10 app.processor.thread-pool.max-pool-size=20 app.processor.thread-pool.queue-capacity=2000 @@ -107,7 +107,7 @@ app.processor.thread-pool.keep-alive-seconds=60 app.processor.batch-size=100 app.processor.process-timeout-ms=30000 -# ETL +# ETL配置 etl.batch.page-size=1000 etl.batch.insert-batch-size=500 etl.schedule.cron=0 0 2 * * ? @@ -120,10 +120,9 @@ spring.datasource.hikari.minimum-idle=5 spring.datasource.hikari.connection-timeout=30000 spring.datasource.hikari.idle-timeout=600000 spring.datasource.hikari.max-lifetime=900000 -spring.datasource.hikari.connection-test-query=SELECT 1 +spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule spring.datasource.hikari.auto-commit=false -spring.datasource.hikari.schema=public diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-pre-nw.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-pre-nw.properties index 21f8b04..d7eeab2 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-pre-nw.properties +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-pre-nw.properties @@ -26,7 +26,7 @@ influxdb.batch.size=1000 influxdb.flush.interval=1000 influxdb.retry.attempts=3 influxdb.retry.delay=1000 -# InfluxDB 2.7 ӳʱ +# InfluxDB 2.7 连接超时配置 influxdb.connection.timeout=30s influxdb.connection.read-timeout=30s influxdb.connection.write-timeout=60s @@ -37,15 +37,15 @@ app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://10.150 81.209:5432/ecosys -spring.datasource.username=postgres +spring.datasource.url=jdbc:dm://192.163.4.99:5237/DM_ecosys +spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap -spring.datasource.driver-class-name=org.postgresql.Driver +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity -#mybatis handler +#mybatis handler 类 mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 mybatis.configuration.map-underscore-to-camel-case=true @@ -72,23 +72,23 @@ spring.kafka.listener.concurrency= 2 spring.kafka.listener.type=batch -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# +# 生产环境缓存配置 spring.redis.host=192.168.4.26 spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password=123456 spring.redis.database=0 spring.redis.timeout=5000 @@ -99,10 +99,10 @@ spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 -# Ӧô +# 应用处理器配置 app.processor.thread-pool.core-pool-size=10 app.processor.thread-pool.max-pool-size=20 app.processor.thread-pool.queue-capacity=2000 @@ -111,20 +111,20 @@ app.processor.batch-size=100 app.processor.process-timeout-ms=30000 -# Elasticsearch -# Elasticsearchӵַ +# 配置 Elasticsearch +# Elasticsearch连接地址 spring.elasticsearch.uris=http://192.168.1.174:9200 -# Elasticsearch û +# 配置 Elasticsearch 用户名 spring.elasticsearch.username=CONTAINER_NAME -# Elasticsearch +# 配置 Elasticsearch 密码 spring.elasticsearch.password=t2NZCiajmdazxBrF -# ӳʱʱ +# 连接超时时间 spring.elasticsearch.connection-timeout=10s -# Socket ʱʱ +# Socket 超时时间 spring.elasticsearch.socket-timeout=30s -# ETL +# ETL配置 etl.batch.page-size=1000 etl.batch.insert-batch-size=500 etl.schedule.cron=0 0 2 * * ? @@ -138,51 +138,50 @@ spring.datasource.hikari.minimum-idle=5 spring.datasource.hikari.connection-timeout=30000 spring.datasource.hikari.idle-timeout=600000 spring.datasource.hikari.max-lifetime=900000 -spring.datasource.hikari.connection-test-query=SELECT 1 +spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 -spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer +spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule spring.datasource.hikari.auto-commit=false -spring.datasource.hikari.schema=public -# +# 关联分析规则配置 analysis.realtime.enabled= true -# 룩 - Ĭ10 +# 检查间隔（秒） - 默认10秒 analysis.realtime.check-interval-seconds: 10 # ============================================ -# ̽API +# 探针联动API配置 # ============================================ -# API-KEY֤32λʹɵԿ +# API-KEY认证（32位，建议使用随机生成的密钥） interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 -# APIӿڻURLsyslog-serveã +# API接口基础URL（供syslog-serve调用） interlocking.api.base-url=http://10.150 81.210:8089/xdrservice/interlocking # ============================================ -# 澯 +# 告警健康检查配置 # ============================================ -# 澯ֵСʱ +# 告警表无数据阈值（小时） alarm.health-check.alarm-hours=2 -# 澯־ֵСʱ +# 告警日志表无数据阈值（小时） alarm.health-check.alarm-visit-hours=4 -# ǷöʱѲ +# 是否启用定时巡检 alarm.health-check.enabled=true # ============================================ -# ̽ +# 探针心跳检测配置 # ============================================ -# Ƿ +# 是否启用心跳检测 probe.heartbeat.enabled=true -# ֵ̽ӣʱδյжΪ +# 探针离线阈值（分钟），超过此时间未收到心跳则判定为离线 probe.heartbeat.offline-threshold-minutes=10 -# ״̬CronʽĬÿ10ӣ +# 状态检查Cron表达式（默认每10分钟） probe.status.check.cron=0 */10 * * * ? -# ̽⻧ID +# 探针租户ID probe.heartbeat.tenant-id=000000 -# ʷ +# 心跳历史保留天数 probe.heartbeat.history.keep-days=10 -# Ƿʷ +# 是否启用历史清理 probe.heartbeat.history.cleanup-enabled=true -# ʷCronʽĬÿ賿1㣩 +# 历史清理Cron表达式（默认每天凌晨1点） probe.history.cleanup.cron=0 0 1 * * ? \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod-zc.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod-zc.properties index 597fdfa..f4e3637 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod-zc.properties +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod-zc.properties @@ -25,7 +25,7 @@ influxdb.batch.size=1000 influxdb.flush.interval=1000 influxdb.retry.attempts=3 influxdb.retry.delay=1000 -# InfluxDB 2.7 ӳʱ +# InfluxDB 2.7 连接超时配置 influxdb.connection.timeout=30s influxdb.connection.read-timeout=30s influxdb.connection.write-timeout=60s @@ -36,16 +36,16 @@ app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://10.11.2.141:5432/ecosys -spring.datasource.username=ecosys -spring.datasource.password=wsYDPjrpNZPrkPrR -spring.datasource.driver-class-name=org.postgresql.Driver - +spring.datasource.url=jdbc:dm://192.163.4.99:5237/DM_ecosys +spring.datasource.username=SYSDBA +spring.datasource.password=caZ2TcmXNSW8L2Ap +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity -#mybatis handler +#mybatis handler 类 mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 mybatis.configuration.map-underscore-to-camel-case=true @@ -73,23 +73,23 @@ spring.kafka.listener.ack-mode= manual spring.kafka.listener.concurrency= 2 spring.kafka.listener.type=batch -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# +# 生产环境缓存配置 spring.redis.host=10.11.2.142 spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password=redis_edP6N6 spring.redis.database=0 spring.redis.timeout=5000 @@ -99,10 +99,10 @@ spring.redis.lettuce.pool.max-active=20 spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 -# Ӧô +# 应用处理器配置 app.processor.thread-pool.core-pool-size=10 app.processor.thread-pool.max-pool-size=20 app.processor.thread-pool.queue-capacity=2000 @@ -110,20 +110,20 @@ app.processor.thread-pool.keep-alive-seconds=60 app.processor.batch-size=100 app.processor.process-timeout-ms=30000 -# Elasticsearch -# Elasticsearchӵַ +# 配置 Elasticsearch +# Elasticsearch连接地址 spring.elasticsearch.uris=http://192.168.1.174:9200 -# Elasticsearch û +# 配置 Elasticsearch 用户名 spring.elasticsearch.username=CONTAINER_NAME -# Elasticsearch +# 配置 Elasticsearch 密码 spring.elasticsearch.password=t2NZCiajmdazxBrF -# ӳʱʱ +# 连接超时时间 spring.elasticsearch.connection-timeout=10s -# Socket ʱʱ +# Socket 超时时间 spring.elasticsearch.socket-timeout=30s -# ETL +# ETL配置 etl.batch.page-size=1000 etl.batch.insert-batch-size=500 etl.schedule.cron=0 0 2 * * ? @@ -137,15 +137,14 @@ spring.datasource.hikari.minimum-idle=5 spring.datasource.hikari.connection-timeout=30000 spring.datasource.hikari.idle-timeout=600000 spring.datasource.hikari.max-lifetime=900000 -spring.datasource.hikari.connection-test-query=SELECT 1 +spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer spring.datasource.hikari.auto-commit=false -spring.datasource.hikari.schema=public -# +# 关联分析规则配置 analysis.realtime.enabled= true -# 룩 - Ĭ10 +# 检查间隔（秒） - 默认10秒 analysis.realtime.check-interval-seconds: 10 \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties index 43ccc54..fbc2bc1 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-prod.properties @@ -17,15 +17,15 @@ syslog.buffer.size=1000 syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # InfluxDB 2.7 Configuration -influxdb.url=http://192.168.4.26:8087 -influxdb.token=LFjXZyRxTf1V84oN-wwjhSjS4qIK-ZMoHzQJB67ir3qHNSBVJbMcTkPuNmM0cNxvzFEDWLYNzrz1VJKMitY5hw== +influxdb.url=http://192.168.4.99:8087 +influxdb.token=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg== influxdb.org=influxdb influxdb.bucket=yelangbucket influxdb.batch.size=1000 influxdb.flush.interval=1000 influxdb.retry.attempts=3 influxdb.retry.delay=1000 -# InfluxDB 2.7 ӳʱ +# InfluxDB 2.7 连接超时配置 influxdb.connection.timeout=30s influxdb.connection.read-timeout=30s influxdb.connection.write-timeout=60s @@ -35,15 +35,15 @@ app.max.queue.size=10000 app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://192.168.4.26:5432/ecosys -spring.datasource.username=postgres +spring.datasource.url=jdbc:dm://192.163.4.99:5237 +spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap -spring.datasource.driver-class-name=org.postgresql.Driver - +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity -#mybatis handler +#mybatis handler 类 mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 mybatis.configuration.map-underscore-to-camel-case=true @@ -53,7 +53,7 @@ mybatis-plus.type-handlers-package=com.Modules.etl.handler # kafka Configuration spring.kafka.consumer.bootstrap-servers=192.168.4.26:9092 -spring.kafka.consumer.group-id=agent-01-syslog-group +spring.kafka.consumer.group-id=agent-01-syslog-group-dm spring.kafka.consumer.auto-offset-reset=latest spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.auto-commit-interval=1000 @@ -70,24 +70,24 @@ spring.kafka.listener.concurrency= 2 spring.kafka.listener.type=batch -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# -spring.redis.host=192.168.4.26 +# 生产环境缓存配置 +spring.redis.host=192.168.4.99 spring.redis.port=6379 -# 루û룬ʡԣ -spring.redis.password=123456 +# 密码（如果没有设置密码，可以省略） +spring.redis.password=redis_GdGWte spring.redis.database=0 spring.redis.timeout=5000 #spring.redis.password=${REDIS_PASSWORD:default_prod_password} @@ -97,10 +97,10 @@ spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 -# Ӧô +# 应用处理器配置 app.processor.thread-pool.core-pool-size=10 app.processor.thread-pool.max-pool-size=20 app.processor.thread-pool.queue-capacity=2000 @@ -108,7 +108,7 @@ app.processor.thread-pool.keep-alive-seconds=60 app.processor.batch-size=100 app.processor.process-timeout-ms=30000 -# ETL +# ETL配置 etl.batch.page-size=1000 etl.batch.insert-batch-size=500 etl.schedule.cron=0 0 2 * * ? @@ -121,9 +121,8 @@ spring.datasource.hikari.minimum-idle=5 spring.datasource.hikari.connection-timeout=30000 spring.datasource.hikari.idle-timeout=600000 spring.datasource.hikari.max-lifetime=900000 -spring.datasource.hikari.connection-test-query=SELECT 1 +spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 -spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer -spring.datasource.hikari.auto-commit=false -spring.datasource.hikari.schema=public \ No newline at end of file +spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule +spring.datasource.hikari.auto-commit=false \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-test.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-test.properties index 4cfb03b..0274086 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-test.properties +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application-test.properties @@ -33,10 +33,11 @@ app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://192.168.4.32:5432/ecosys -spring.datasource.username=user_eSER8N -spring.datasource.password=password_QCYKj6 -spring.datasource.driver-class-name=org.postgresql.Driver +spring.datasource.url=jdbc:dm://192.163.4.99:5237/DM_ecosys +spring.datasource.username=SYSDBA +spring.datasource.password=caZ2TcmXNSW8L2Ap +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml @@ -54,23 +55,23 @@ spring.kafka.consumer.topic=agent-syslog-topic -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# +# 生产环境缓存配置 spring.redis.host=192.168.4.32 spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password=redis_edP6N6 spring.redis.database=0 spring.redis.timeout=5000 @@ -81,5 +82,5 @@ spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties index 43ccc54..faf832d 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties @@ -17,15 +17,15 @@ syslog.buffer.size=1000 syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # InfluxDB 2.7 Configuration -influxdb.url=http://192.168.4.26:8087 -influxdb.token=LFjXZyRxTf1V84oN-wwjhSjS4qIK-ZMoHzQJB67ir3qHNSBVJbMcTkPuNmM0cNxvzFEDWLYNzrz1VJKMitY5hw== +influxdb.url=http://192.168.4.99:8087 +influxdb.token=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg== influxdb.org=influxdb influxdb.bucket=yelangbucket influxdb.batch.size=1000 influxdb.flush.interval=1000 influxdb.retry.attempts=3 influxdb.retry.delay=1000 -# InfluxDB 2.7 ӳʱ +# InfluxDB 2.7 连接超时配置 influxdb.connection.timeout=30s influxdb.connection.read-timeout=30s influxdb.connection.write-timeout=60s @@ -35,15 +35,15 @@ app.max.queue.size=10000 app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://192.168.4.26:5432/ecosys -spring.datasource.username=postgres +spring.datasource.url=jdbc:dm://192.168.4.99:5237 +spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap -spring.datasource.driver-class-name=org.postgresql.Driver - +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity -#mybatis handler +#mybatis handler 类 mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 mybatis.configuration.map-underscore-to-camel-case=true @@ -53,7 +53,7 @@ mybatis-plus.type-handlers-package=com.Modules.etl.handler # kafka Configuration spring.kafka.consumer.bootstrap-servers=192.168.4.26:9092 -spring.kafka.consumer.group-id=agent-01-syslog-group +spring.kafka.consumer.group-id=agent-01-syslog-group-dm spring.kafka.consumer.auto-offset-reset=latest spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.auto-commit-interval=1000 @@ -70,24 +70,24 @@ spring.kafka.listener.concurrency= 2 spring.kafka.listener.type=batch -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# -spring.redis.host=192.168.4.26 +# 生产环境缓存配置 +spring.redis.host=192.168.4.99 spring.redis.port=6379 -# 루û룬ʡԣ -spring.redis.password=123456 +# 密码（如果没有设置密码，可以省略） +spring.redis.password=redis_GdGWte spring.redis.database=0 spring.redis.timeout=5000 #spring.redis.password=${REDIS_PASSWORD:default_prod_password} @@ -97,10 +97,10 @@ spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 -# Ӧô +# 应用处理器配置 app.processor.thread-pool.core-pool-size=10 app.processor.thread-pool.max-pool-size=20 app.processor.thread-pool.queue-capacity=2000 @@ -108,7 +108,7 @@ app.processor.thread-pool.keep-alive-seconds=60 app.processor.batch-size=100 app.processor.process-timeout-ms=30000 -# ETL +# ETL配置 etl.batch.page-size=1000 etl.batch.insert-batch-size=500 etl.schedule.cron=0 0 2 * * ? @@ -121,9 +121,8 @@ spring.datasource.hikari.minimum-idle=5 spring.datasource.hikari.connection-timeout=30000 spring.datasource.hikari.idle-timeout=600000 spring.datasource.hikari.max-lifetime=900000 -spring.datasource.hikari.connection-test-query=SELECT 1 +spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 -spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer -spring.datasource.hikari.auto-commit=false -spring.datasource.hikari.schema=public \ No newline at end of file +spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule +spring.datasource.hikari.auto-commit=false \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/logback.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/logback.xml index 966b2eb..9196a6f 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/logback.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/logback.xml @@ -7,9 +7,9 @@ - logs/syslog-consumer-rule.log + logs/syslog-consumer-rule-dm.log - logs/syslog-consumer-rule.%d{yyyy-MM-dd}.log + logs/syslog-consumer-rule-dm.%d{yyyy-MM-dd}.log 1 diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml index 9402d7e..c09d908 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml @@ -1,4 +1,4 @@ - + @@ -53,7 +53,7 @@ SELECT FROM analysis_analysis_rule - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND del_flag = '0' @@ -62,7 +62,7 @@ SELECT FROM analysis_analysis_rule - WHERE rule_id = #{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id = #{ruleId, jdbcType=OTHER} AND del_flag = '0' @@ -70,11 +70,11 @@ UPDATE analysis_analysis_rule SET task_status = #{taskStatus}, - update_time = NOW() + update_time = SYSDATE ,update_by = #{updateBy} - WHERE rule_id = #{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id = #{ruleId, jdbcType=OTHER} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisFieldMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisFieldMapper.xml index 9416648..45df145 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisFieldMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisFieldMapper.xml @@ -1,4 +1,4 @@ - + @@ -42,7 +42,7 @@ SELECT FROM analysis_field - WHERE rule_id = #{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id = #{ruleId, jdbcType=OTHER} AND del_flag = '0' ORDER BY id ASC @@ -52,7 +52,7 @@ SELECT FROM analysis_field - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND del_flag = '0' AND type IN ('measure', 'calc') ORDER BY id ASC @@ -63,7 +63,7 @@ SELECT FROM analysis_field - WHERE rule_id = #{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id = #{ruleId, jdbcType=OTHER} AND del_flag = '0' AND type = 'dimension' ORDER BY id ASC diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisFilterMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisFilterMapper.xml index b8f9dd6..c98190c 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisFilterMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisFilterMapper.xml @@ -1,4 +1,4 @@ - + @@ -13,9 +13,9 @@ - + - + @@ -42,7 +42,7 @@ SELECT FROM analysis_filter - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND del_flag = '0' diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByColumnMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByColumnMapper.xml index 64048ae..a5de79f 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByColumnMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByColumnMapper.xml @@ -1,4 +1,4 @@ - + @@ -44,7 +44,7 @@ create_time, update_time, create_by, update_by, remark, tenant_id, rule_id, group_id, field_id, sort FROM analysis_group_by_column - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND del_flag = '0' ORDER BY sort ASC diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByHavingMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByHavingMapper.xml index ca3cdb1..8cd682a 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByHavingMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByHavingMapper.xml @@ -1,4 +1,4 @@ - + @@ -13,9 +13,9 @@ - + - + @@ -53,7 +53,7 @@ h.update_time, h.create_by, h.update_by, h.remark, h.tenant_id FROM analysis_group_by_having h INNER JOIN analysis_group_by g ON h.group_by_id = g.id - WHERE g.rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE g.rule_id =#{ruleId, jdbcType=OTHER} AND h.del_flag = '0' AND g.del_flag = '0' ORDER BY h.id ASC diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByMapper.xml index 1e426e8..f9d0bba 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisGroupByMapper.xml @@ -1,10 +1,10 @@ - + - + @@ -33,7 +33,7 @@ SELECT FROM analysis_group_by - WHERE rule_id = #{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id = #{ruleId, jdbcType=OTHER} AND del_flag = '0' diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisTaskHistoryMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisTaskHistoryMapper.xml index 10eb841..99dccbe 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisTaskHistoryMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisTaskHistoryMapper.xml @@ -1,4 +1,4 @@ - + @@ -33,7 +33,7 @@ INSERT INTO analysis_task_history ( ) VALUES ( - #{id}, #{ruleId}::uuid, #{startTime}, #{endTime}, #{durationTime}, #{progressPercent}, + #{id}, #{ruleId}, #{startTime}, #{endTime}, #{durationTime}, #{progressPercent}, #{inputCount}, #{outputCount}, #{status}, #{createDept}, #{delFlag}, #{createTime}, #{updateTime}, #{createBy}, #{updateBy}, #{remark}, #{tenantId} ) @@ -48,7 +48,7 @@ input_count = #{inputCount}, output_count = #{outputCount}, status = #{status}, - update_time = NOW() + update_time = SYSDATE ,remark = #{remark} @@ -60,7 +60,7 @@ SELECT FROM analysis_task_history - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND del_flag = '0' ORDER BY create_time DESC @@ -73,7 +73,7 @@ SELECT FROM analysis_task_history - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND status = #{status} AND del_flag = '0' ORDER BY create_time DESC diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisWhereConditionMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisWhereConditionMapper.xml index 541bebb..8b65090 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisWhereConditionMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/AnalysisWhereConditionMapper.xml @@ -1,4 +1,4 @@ - + @@ -29,7 +29,7 @@ SELECT FROM analysis_where_condition - WHERE rule_id = #{ruleId}::uuid + WHERE rule_id = #{ruleId} AND del_flag = '0' ORDER BY seq_num ASC @@ -39,7 +39,7 @@ SELECT FROM analysis_where_condition - WHERE rule_id = #{ruleId}::uuid + WHERE rule_id = #{ruleId} AND (parent_cond_id IS NULL OR parent_cond_id = 0) AND del_flag = '0' ORDER BY seq_num ASC diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DeviceDeviceMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DeviceDeviceMapper.xml index e6f8c47..be096be 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DeviceDeviceMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DeviceDeviceMapper.xml @@ -1,4 +1,4 @@ - + @@ -46,11 +46,11 @@ - id, created_at::timestamp , updated_at::timestamp, deleted_at::timestamp, name, ip, device_group, device_type, - vendor, product_name, organization_id, last_receive_time::timestamp, agent_id, detail_id, - control_agent_id, license_start_time::timestamp, license_end_time::timestamp, is_monitoring, + id, created_at , updated_at, deleted_at, name, ip, device_group, device_type, + vendor, product_name, organization_id, last_receive_time, agent_id, detail_id, + control_agent_id, license_start_time, license_end_time, is_monitoring, security_scope_id, owner_id, ssh_config_id, status, created_by_id, decode_type, - miss_policy, tenant_id, create_time::timestamp, update_time::timestamp, create_by, update_by, del_flag, + miss_policy, tenant_id, create_time, update_time, create_by, update_by, del_flag, manager_name, today_parse_count, today_non_log_count, create_dept, device_collect_id diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DeviceReceiveLogMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DeviceReceiveLogMapper.xml index e07532e..1b1af5e 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DeviceReceiveLogMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DeviceReceiveLogMapper.xml @@ -1,4 +1,4 @@ - + @@ -26,10 +26,10 @@ receive_time_str, syslog_message ) VALUES ( - COALESCE(#{createdAt}, NOW() AT TIME ZONE 'utc'), + COALESCE(#{createdAt}, SYSDATE), #{deviceCollectId}, #{deviceId}, - #{deviceIp}::inet, + #{deviceIp}, #{receiveTime}, #{receiveTimeStr}, #{syslogMessage} @@ -49,10 +49,10 @@ ) VALUES ( - COALESCE(#{item.createdAt}, NOW() AT TIME ZONE 'utc'), + COALESCE(#{item.createdAt}, SYSDATE), #{item.deviceCollectId}, #{item.deviceId}, - #{item.deviceIp}::inet, + #{item.deviceIp}, #{item.receiveTime}, #{item.receiveTimeStr}, #{item.syslogMessage} @@ -80,10 +80,10 @@ ORDER BY receive_time DESC - + @@ -105,7 +105,7 @@ AND device_collect_id = #{deviceCollectId} - AND device_ip >>= #{deviceIp}::inet + AND device_ip = #{deviceIp} AND receive_time >= #{receiveTime} @@ -128,7 +128,7 @@ AND device_collect_id = #{deviceCollectId} - AND device_ip >>= #{deviceIp}::inet + AND device_ip = #{deviceIp} AND receive_time >= #{receiveTime} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DmColumnMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DmColumnMapper.xml index c72ee92..2eadc56 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DmColumnMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/DmColumnMapper.xml @@ -1,4 +1,4 @@ - + @@ -65,13 +65,13 @@ @@ -80,13 +80,13 @@ @@ -137,7 +137,7 @@ UPDATE syslog_non_normal_message SET del_flag = '1', - update_time = NOW() + update_time = SYSDATE WHERE id IN #{item.id} diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml index 004a2b3..486c430 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml @@ -1,4 +1,4 @@ - + @@ -406,9 +406,9 @@ #{dataMap.container_name}, #{dataMap.container_id}, #{dataMap.http_resp_server}, - #{dataMap.srcip_id}::int8, - #{dataMap.cdnip}::inet, - #{dataMap.natip}::inet, + #{dataMap.srcip_id}, + #{dataMap.cdnip}, + #{dataMap.natip}, #{dataMap.mail_sender}, #{dataMap.mail_receiver}, #{dataMap.vpn_mac}, @@ -439,8 +439,8 @@ #{dataMap.print_time}, #{dataMap.printer}, #{dataMap.printer_type}, - #{dataMap.print_pages}::int8, - #{dataMap.print_copies}::int8, + #{dataMap.print_pages}, + #{dataMap.print_copies}, #{dataMap.src_device}, #{dataMap.dst_device}, #{dataMap.src_file}, @@ -457,18 +457,18 @@ #{dataMap.env}, #{dataMap.brute_force_service}, #{dataMap.vuirs_name}, - #{dataMap.http_req_length}::int8, + #{dataMap.http_req_length}, #{dataMap.http_req_content_type}, - #{dataMap.tc_scan_port}::inet, - #{dataMap.tc_labels}::inet, + #{dataMap.tc_scan_port}, + #{dataMap.tc_labels}, #{dataMap.http_resp_content_type}, #{dataMap.dns_msg_type}, #{dataMap.dns_answer_length}, #{dataMap.dns_ioc}, - #{dataMap.tx_bytes}::double precision, - #{dataMap.rx_bytes}::double precision, - #{dataMap.all_bytes}::double precision, - #{dataMap.duration_time}::int8, + #{dataMap.tx_bytes}, + #{dataMap.rx_bytes}, + #{dataMap.all_bytes}, + #{dataMap.duration_time}, #{dataMap.mail_attach_name}, #{dataMap.mail_subject}, #{dataMap.mail_message}, @@ -488,27 +488,27 @@ #{dataMap.src_ip_apt}, #{dataMap.srcip_name}, #{dataMap.tc_client}, - #{dataMap.srcip_organization_id}::int8, + #{dataMap.srcip_organization_id}, #{dataMap.dest_ip_intranetip}, #{dataMap.dest_ip_ioc}, - #{dataMap.desip_id}::int8, + #{dataMap.desip_id}, #{dataMap.desip_name}, - #{dataMap.tc_hostip}::inet, - #{dataMap.desip_organization_id}::int8, + #{dataMap.tc_hostip}, + #{dataMap.desip_organization_id}, #{dataMap.origin_confidence}, #{dataMap.origin_malscore}, #{dataMap.attacker_icampaign}, - #{dataMap.attacker_host_asset_id}::int8, - #{dataMap.attacker_organization_id}::int8, - #{dataMap.victim_host_asset_id}::int8, - #{dataMap.victim_organization_id}::int8, + #{dataMap.attacker_host_asset_id}, + #{dataMap.attacker_organization_id}, + #{dataMap.victim_host_asset_id}, + #{dataMap.victim_organization_id}, #{dataMap.logout_time}, #{dataMap.http_req_line}, #{dataMap.desip_security_scope_id}, #{dataMap.srcip_security_scope_id}, - #{dataMap.http_resp_length}::int8, + #{dataMap.http_resp_length}, #{dataMap.tc_attack_type}, - #{dataMap.tc_realip}::inet, + #{dataMap.tc_realip}, #{dataMap.attacker_ip_lists}, #{dataMap.login_password}, #{dataMap.detail}, @@ -525,12 +525,12 @@ #{dataMap.login_abnormal_type}, #{dataMap.file_tag}, #{dataMap.file_platform}, - #{dataMap.target_ip}::inet, + #{dataMap.target_ip}, #{dataMap.collect_date}, - #{dataMap.tc_client_ip}::inet, - #{dataMap.tc_server_ip}::inet, - #{dataMap.tc_externalip}::inet, - #{dataMap.http_status_code}::int8, + #{dataMap.tc_client_ip}, + #{dataMap.tc_server_ip}, + #{dataMap.tc_externalip}, + #{dataMap.http_status_code}, #{dataMap.device_domian}, #{dataMap.src_ip_str}, #{dataMap.src_port_str}, @@ -576,28 +576,28 @@ #{dataMap.origin_agent_name}, #{dataMap.origin_work_group}, #{dataMap.origin_asset_group}, - #{dataMap.origin_local_port}::int8, - #{dataMap.origin_agent_ip}::inet, - #{dataMap.origin_internal_ip}::inet, - #{dataMap.origin_external_ip}::inet, - #{dataMap.origin_local_addr}::inet, - #{dataMap.agent_id}::int8, + #{dataMap.origin_local_port}, + #{dataMap.origin_agent_ip}, + #{dataMap.origin_internal_ip}, + #{dataMap.origin_external_ip}, + #{dataMap.origin_local_addr}, + #{dataMap.agent_id}, #{dataMap.agent_name}, #{dataMap.tc_title}, #{dataMap.log_id}, #{dataMap.event_date}, #{dataMap.event_time_ts}, - #{dataMap.event_level}::int , - #{dataMap.src_ip}::inet, - #{dataMap.src_port}::BIGINT , - #{dataMap.dest_ip}::inet, + #{dataMap.event_level}, + #{dataMap.src_ip}, + #{dataMap.src_port}, + #{dataMap.dest_ip}, #{dataMap.dest_port}::BIGINT, #{dataMap.event_time}, #{dataMap.attacker_country}, #{dataMap.src_mac}, #{dataMap.dest_mac}, #{dataMap.proto}, - #{dataMap.dev_id}::int8, + #{dataMap.dev_id}, #{dataMap.created_time}, #{dataMap.src_country}, #{dataMap.src_country_code}, @@ -621,20 +621,20 @@ #{dataMap.end_time}, #{dataMap.file_created_time}, #{dataMap.file_modified_time}, - #{dataMap.tc_miguan_scan_port}::inet, + #{dataMap.tc_miguan_scan_port}, #{dataMap.process_path}, #{dataMap.parent_process_path}, #{dataMap.gname}, #{dataMap.exe_name}, #{dataMap.exe_path}, #{dataMap.login_time}, - #{dataMap.login_times}::int8, + #{dataMap.login_times}, #{dataMap.check_item}, #{dataMap.check_type}, - #{dataMap.attacker_ip}::inet, - #{dataMap.attacker_port}::int8, - #{dataMap.victim_ip}::inet, - #{dataMap.victim_port}::int8, + #{dataMap.attacker_ip}, + #{dataMap.attacker_port}, + #{dataMap.victim_ip}, + #{dataMap.victim_port}, #{dataMap.attacker_city}, #{dataMap.attacker_lon}, #{dataMap.attacker_lat}, @@ -660,7 +660,7 @@ #{dataMap.file_ssdeep}, #{dataMap.victim_country_code}, #{dataMap.http_xff_ip}, - #{dataMap.tc_miguan_class}::inet, + #{dataMap.tc_miguan_class}, #{dataMap.pid}, #{dataMap.ppid}, #{dataMap.process_name}, @@ -687,35 +687,35 @@ #{dataMap.dest_city}, #{dataMap.dest_lon}, #{dataMap.dest_lat}, - #{dataMap.event_category}::int4, - #{dataMap.attack_result}::int4, - #{dataMap.probe_ip}::inet, - #{dataMap.device_ip}::inet, + #{dataMap.event_category}, + #{dataMap.attack_result}, + #{dataMap.probe_ip}, + #{dataMap.device_ip}, #{dataMap.device_manufacturer}, #{dataMap.device_name}, #{dataMap.product_name}, #{dataMap.__id}, - #{dataMap.__count}::int8, + #{dataMap.__count}, #{dataMap.__count_reason}, - #{dataMap.event_type}::int, + #{dataMap.event_type}, #{dataMap.protocol}, #{dataMap.shell_cmd}, #{dataMap.parent_name}, #{dataMap.host_file_path}, #{dataMap.uid}, - #{dataMap.fall}::int4, - #{dataMap.tc_miguan_server_ip}::inet, - #{dataMap.dev_type}::int4, - #{dataMap.collect_method}::int4, - #{dataMap.field_cate_id}::int4, - #{dataMap.device_type}::int4, - #{dataMap.tc_miguan_client_ip}::inet, - #{dataMap.tc_miguan_name}::inet, - #{dataMap.origin_total_packages}::int8, - #{dataMap.origin_total_bytes}::int8, - #{dataMap.origin_peak_packages_rate}::int8, - #{dataMap.origin_peak_bytes_rate}::int8, - #{dataMap.origin_peak_flows_rate}::int8, + #{dataMap.fall}, + #{dataMap.tc_miguan_server_ip}, + #{dataMap.dev_type}, + #{dataMap.collect_method}, + #{dataMap.field_cate_id}, + #{dataMap.device_type}, + #{dataMap.tc_miguan_client_ip}, + #{dataMap.tc_miguan_name}, + #{dataMap.origin_total_packages}, + #{dataMap.origin_total_bytes}, + #{dataMap.origin_peak_packages_rate}, + #{dataMap.origin_peak_bytes_rate}, + #{dataMap.origin_peak_flows_rate}, #{dataMap.apt_orgname}, #{dataMap.apt_orgmsg}, #{dataMap.mail_message_id}, @@ -725,18 +725,18 @@ #{dataMap.mail_url}, #{dataMap.mail_cc}, #{dataMap.algorithm}, - #{dataMap.miningpool_ip}::inet, + #{dataMap.miningpool_ip}, #{dataMap.process_md5}, #{dataMap.pprocess_md5}, #{dataMap.source_servername}, #{dataMap.origin_source_servername}, #{dataMap.mail_filename}, #{dataMap.dst_upload_appname}, - #{dataMap.target_port}::int8, + #{dataMap.target_port}, #{dataMap.gid}, #{dataMap.origin_uid}, #{dataMap.origin_gid}, - #{dataMap.target_ports}::int8, + #{dataMap.target_ports}, #{dataMap.tc_miguan_name1}, #{dataMap.tc_miguan_class1}, #{dataMap.etl_time}, @@ -744,7 +744,7 @@ #{dataMap.desip_security_scope}, #{dataMap.srcip_security_scope}, #{dataMap.collect_time_ts}, - #{dataMap.tc_miguan_scan_port1}::inet, + #{dataMap.tc_miguan_scan_port1}, #{dataMap.src_dev_name}, #{dataMap.collect_protocol}, #{dataMap.destination_system_type}, diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml index 9d60d7c..527387b 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml @@ -1,4 +1,4 @@ - + @@ -57,6 +57,7 @@ id, created_at, log_time, + device_id, device_ip, dest_ip, dest_port, @@ -103,6 +104,7 @@ id, created_at, log_time, + device_id, device_ip, dest_ip, dest_port, @@ -541,9 +543,9 @@ #{dataMap.container_name}, #{dataMap.container_id}, #{dataMap.http_resp_server}, - #{dataMap.srcip_id}::int8, - #{dataMap.cdnip}::inet, - #{dataMap.natip}::inet, + #{dataMap.srcip_id}, + #{dataMap.cdnip}, + #{dataMap.natip}, #{dataMap.mail_sender}, #{dataMap.mail_receiver}, #{dataMap.vpn_mac}, @@ -574,8 +576,8 @@ #{dataMap.print_time}, #{dataMap.printer}, #{dataMap.printer_type}, - #{dataMap.print_pages}::int8, - #{dataMap.print_copies}::int8, + #{dataMap.print_pages}, + #{dataMap.print_copies}, #{dataMap.src_device}, #{dataMap.dst_device}, #{dataMap.src_file}, @@ -592,18 +594,18 @@ #{dataMap.env}, #{dataMap.brute_force_service}, #{dataMap.vuirs_name}, - #{dataMap.http_req_length}::int8, + #{dataMap.http_req_length}, #{dataMap.http_req_content_type}, - #{dataMap.tc_scan_port}::inet, - #{dataMap.tc_labels}::inet, + #{dataMap.tc_scan_port}, + #{dataMap.tc_labels}, #{dataMap.http_resp_content_type}, #{dataMap.dns_msg_type}, #{dataMap.dns_answer_length}, #{dataMap.dns_ioc}, - #{dataMap.tx_bytes}::double precision, - #{dataMap.rx_bytes}::double precision, - #{dataMap.all_bytes}::double precision, - #{dataMap.duration_time}::int8, + #{dataMap.tx_bytes}, + #{dataMap.rx_bytes}, + #{dataMap.all_bytes}, + #{dataMap.duration_time}, #{dataMap.mail_attach_name}, #{dataMap.mail_subject}, #{dataMap.mail_message}, @@ -623,27 +625,27 @@ #{dataMap.src_ip_apt}, #{dataMap.srcip_name}, #{dataMap.tc_client}, - #{dataMap.srcip_organization_id}::int8, + #{dataMap.srcip_organization_id}, #{dataMap.dest_ip_intranetip}, #{dataMap.dest_ip_ioc}, - #{dataMap.desip_id}::int8, + #{dataMap.desip_id}, #{dataMap.desip_name}, - #{dataMap.tc_hostip}::inet, - #{dataMap.desip_organization_id}::int8, + #{dataMap.tc_hostip}, + #{dataMap.desip_organization_id}, #{dataMap.origin_confidence}, #{dataMap.origin_malscore}, #{dataMap.attacker_icampaign}, - #{dataMap.attacker_host_asset_id}::int8, - #{dataMap.attacker_organization_id}::int8, - #{dataMap.victim_host_asset_id}::int8, - #{dataMap.victim_organization_id}::int8, + #{dataMap.attacker_host_asset_id}, + #{dataMap.attacker_organization_id}, + #{dataMap.victim_host_asset_id}, + #{dataMap.victim_organization_id}, #{dataMap.logout_time}, #{dataMap.http_req_line}, #{dataMap.desip_security_scope_id}, #{dataMap.srcip_security_scope_id}, - #{dataMap.http_resp_length}::int8, + #{dataMap.http_resp_length}, #{dataMap.tc_attack_type}, - #{dataMap.tc_realip}::inet, + #{dataMap.tc_realip}, #{dataMap.attacker_ip_lists}, #{dataMap.login_password}, #{dataMap.detail}, @@ -660,12 +662,12 @@ #{dataMap.login_abnormal_type}, #{dataMap.file_tag}, #{dataMap.file_platform}, - #{dataMap.target_ip}::inet, + #{dataMap.target_ip}, #{dataMap.collect_date}, - #{dataMap.tc_client_ip}::inet, - #{dataMap.tc_server_ip}::inet, - #{dataMap.tc_externalip}::inet, - #{dataMap.http_status_code}::int8, + #{dataMap.tc_client_ip}, + #{dataMap.tc_server_ip}, + #{dataMap.tc_externalip}, + #{dataMap.http_status_code}, #{dataMap.device_domian}, #{dataMap.src_ip_str}, #{dataMap.src_port_str}, @@ -711,28 +713,28 @@ #{dataMap.origin_agent_name}, #{dataMap.origin_work_group}, #{dataMap.origin_asset_group}, - #{dataMap.origin_local_port}::int8, - #{dataMap.origin_agent_ip}::inet, - #{dataMap.origin_internal_ip}::inet, - #{dataMap.origin_external_ip}::inet, - #{dataMap.origin_local_addr}::inet, - #{dataMap.agent_id}::int8, + #{dataMap.origin_local_port}, + #{dataMap.origin_agent_ip}, + #{dataMap.origin_internal_ip}, + #{dataMap.origin_external_ip}, + #{dataMap.origin_local_addr}, + #{dataMap.agent_id}, #{dataMap.agent_name}, #{dataMap.tc_title}, #{dataMap.log_id}, #{dataMap.event_date}, #{dataMap.event_time_ts}, - #{dataMap.event_level}::int , - #{dataMap.src_ip}::inet, - #{dataMap.src_port}::BIGINT , - #{dataMap.dest_ip}::inet, + #{dataMap.event_level}, + #{dataMap.src_ip}, + #{dataMap.src_port}, + #{dataMap.dest_ip}, #{dataMap.dest_port}::BIGINT, #{dataMap.event_time}, #{dataMap.attacker_country}, #{dataMap.src_mac}, #{dataMap.dest_mac}, #{dataMap.proto}, - #{dataMap.dev_id}::int8, + #{dataMap.dev_id}, #{dataMap.created_time}, #{dataMap.src_country}, #{dataMap.src_country_code}, @@ -756,20 +758,20 @@ #{dataMap.end_time}, #{dataMap.file_created_time}, #{dataMap.file_modified_time}, - #{dataMap.tc_miguan_scan_port}::inet, + #{dataMap.tc_miguan_scan_port}, #{dataMap.process_path}, #{dataMap.parent_process_path}, #{dataMap.gname}, #{dataMap.exe_name}, #{dataMap.exe_path}, #{dataMap.login_time}, - #{dataMap.login_times}::int8, + #{dataMap.login_times}, #{dataMap.check_item}, #{dataMap.check_type}, - #{dataMap.attacker_ip}::inet, - #{dataMap.attacker_port}::int8, - #{dataMap.victim_ip}::inet, - #{dataMap.victim_port}::int8, + #{dataMap.attacker_ip}, + #{dataMap.attacker_port}, + #{dataMap.victim_ip}, + #{dataMap.victim_port}, #{dataMap.attacker_city}, #{dataMap.attacker_lon}, #{dataMap.attacker_lat}, @@ -795,7 +797,7 @@ #{dataMap.file_ssdeep}, #{dataMap.victim_country_code}, #{dataMap.http_xff_ip}, - #{dataMap.tc_miguan_class}::inet, + #{dataMap.tc_miguan_class}, #{dataMap.pid}, #{dataMap.ppid}, #{dataMap.process_name}, @@ -822,35 +824,35 @@ #{dataMap.dest_city}, #{dataMap.dest_lon}, #{dataMap.dest_lat}, - #{dataMap.event_category}::int4, - #{dataMap.attack_result}::int4, - #{dataMap.probe_ip}::inet, - #{dataMap.device_ip}::inet, + #{dataMap.event_category}, + #{dataMap.attack_result}, + #{dataMap.probe_ip}, + #{dataMap.device_ip}, #{dataMap.device_manufacturer}, #{dataMap.device_name}, #{dataMap.product_name}, #{dataMap.__id}, - #{dataMap.__count}::int8, + #{dataMap.__count}, #{dataMap.__count_reason}, - #{dataMap.event_type}::int, + #{dataMap.event_type}, #{dataMap.protocol}, #{dataMap.shell_cmd}, #{dataMap.parent_name}, #{dataMap.host_file_path}, #{dataMap.uid}, - #{dataMap.fall}::int4, - #{dataMap.tc_miguan_server_ip}::inet, - #{dataMap.dev_type}::int4, - #{dataMap.collect_method}::int4, - #{dataMap.field_cate_id}::int4, - #{dataMap.device_type}::int4, - #{dataMap.tc_miguan_client_ip}::inet, - #{dataMap.tc_miguan_name}::inet, - #{dataMap.origin_total_packages}::int8, - #{dataMap.origin_total_bytes}::int8, - #{dataMap.origin_peak_packages_rate}::int8, - #{dataMap.origin_peak_bytes_rate}::int8, - #{dataMap.origin_peak_flows_rate}::int8, + #{dataMap.fall}, + #{dataMap.tc_miguan_server_ip}, + #{dataMap.dev_type}, + #{dataMap.collect_method}, + #{dataMap.field_cate_id}, + #{dataMap.device_type}, + #{dataMap.tc_miguan_client_ip}, + #{dataMap.tc_miguan_name}, + #{dataMap.origin_total_packages}, + #{dataMap.origin_total_bytes}, + #{dataMap.origin_peak_packages_rate}, + #{dataMap.origin_peak_bytes_rate}, + #{dataMap.origin_peak_flows_rate}, #{dataMap.apt_orgname}, #{dataMap.apt_orgmsg}, #{dataMap.mail_message_id}, @@ -860,18 +862,18 @@ #{dataMap.mail_url}, #{dataMap.mail_cc}, #{dataMap.algorithm}, - #{dataMap.miningpool_ip}::inet, + #{dataMap.miningpool_ip}, #{dataMap.process_md5}, #{dataMap.pprocess_md5}, #{dataMap.source_servername}, #{dataMap.origin_source_servername}, #{dataMap.mail_filename}, #{dataMap.dst_upload_appname}, - #{dataMap.target_port}::int8, + #{dataMap.target_port}, #{dataMap.gid}, #{dataMap.origin_uid}, #{dataMap.origin_gid}, - #{dataMap.target_ports}::int8, + #{dataMap.target_ports}, #{dataMap.tc_miguan_name1}, #{dataMap.tc_miguan_class1}, #{dataMap.etl_time}, @@ -879,7 +881,7 @@ #{dataMap.desip_security_scope}, #{dataMap.srcip_security_scope}, #{dataMap.collect_time_ts}, - #{dataMap.tc_miguan_scan_port1}::inet, + #{dataMap.tc_miguan_scan_port1}, #{dataMap.src_dev_name}, #{dataMap.collect_protocol}, #{dataMap.destination_system_type}, diff --git a/haobang-security-dm/syslog-consumer/docker_run.txt b/haobang-security-dm/syslog-consumer/docker_run.txt index 0c1f8df..d059147 100644 --- a/haobang-security-dm/syslog-consumer/docker_run.txt +++ b/haobang-security-dm/syslog-consumer/docker_run.txt @@ -8,17 +8,17 @@ docker ps -a --2.docker imageļ (Dockerfile ǰĿ¼ docker build -t syslog-consumer:v1.X.X . - +docker build -t syslog-consumer-dm:v1.2.X . --3.ֹͣ ɾ -docker stop ct-syslog-consumer && docker rm ct-syslog-consumer +docker stop syslog-consumer-dm && docker rm syslog-consumer-dm --4.docker ļ -docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name ct-syslog-consumer -p 8089:8089 -v /home/syslog/logs:/app/logs --privileged=true syslog-consumer:v1.X.X +docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name syslog-consumer-dm -p 8089:8089 -v /home/syslog/logs:/app/logs --privileged=true syslog-consumer-dm:v1.X.X ڳCMD -docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name ct-syslog-consumer -p 8089:8089 -v /data/syslog/logs:/app/logs --privileged=true syslog-consumer:v1.X.X +docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name syslog-consumer-dm -p 8089:8089 -v /data/syslog/logs:/app/logs --privileged=true syslog-consumer-dm:v1.X.X -- -docker run -d --name ct-syslog-consumer -p 8089:8089 --privileged=true syslog-consumer:v1.X.X +docker run -d --name syslog-consumer -p 8089:8089 --privileged=true syslog-consumer:v1.X.X diff --git a/haobang-security-dm/syslog-consumer/pom.xml b/haobang-security-dm/syslog-consumer/pom.xml index 71b1d3c..86bdcb4 100644 --- a/haobang-security-dm/syslog-consumer/pom.xml +++ b/haobang-security-dm/syslog-consumer/pom.xml @@ -92,12 +92,21 @@ ${mybatis.version} - + + + + + + com.dameng + DmJdbcDriver18 + 8.1.2.141 + com.fasterxml.jackson.core @@ -193,11 +202,7 @@ - - com.dameng - DmJdbcDriver18 - 8.1.2.141 - + org.bouncycastle diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java index f00d35e..2d1ab30 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java @@ -183,7 +183,7 @@ public class LogNormalProcessor { } catch (Exception ex) { - logger.error("处理初始化异常："+ex.getMessage()); + logger.error("处理日志消息异常："+ex.getMessage()); System.out.println(ex.getMessage()); } @@ -195,6 +195,7 @@ public class LogNormalProcessor { { Cropper_paramsType cropperParams = new Cropper_paramsType(); try { + JSONObject jsonObject = new JSONObject( dmNormalizeRule.get("rule_content").toString()); if (jsonObject.isEmpty()) return null; diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayByteTypeHandler.java b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayByteTypeHandler.java index 484cb34..c100dde 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayByteTypeHandler.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayByteTypeHandler.java @@ -1,47 +1,127 @@ package com.Modules.etl.handler; - - import org.apache.ibatis.type.BaseTypeHandler; import org.apache.ibatis.type.JdbcType; import org.apache.ibatis.type.MappedJdbcTypes; import org.apache.ibatis.type.MappedTypes; -import java.sql.*; +import java.sql.CallableStatement; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; + +/** + * 字节数组类型处理器 - 达梦数据库兼容版本 + * + * 将 Java byte[][] 与数据库 TEXT 列进行互转。 + * 存储格式: JSON 数组字符串，每个元素为 Base64 编码，如 ["base64...","base64..."] + * 空数组: 存储为 "[]" + * null值: 存储为 NULL + * + * 原 PostgreSQL 版本使用 createArrayOf("bytea", ...) 创建原生 bytea 数组， + * 达梦数据库不兼容此 API，改为 TEXT 列存储 JSON 数组（Base64编码）。 + */ @MappedTypes(byte[][].class) -@MappedJdbcTypes(JdbcType.ARRAY) +@MappedJdbcTypes(JdbcType.VARCHAR) public class ArrayByteTypeHandler extends BaseTypeHandler { @Override public void setNonNullParameter(PreparedStatement ps, int i, byte[][] parameter, JdbcType jdbcType) throws SQLException { - Array array = ps.getConnection().createArrayOf("bytea", parameter); - ps.setArray(i, array); + ps.setString(i, toJsonArray(parameter)); } @Override public byte[][] getNullableResult(ResultSet rs, String columnName) throws SQLException { - return getArray(rs.getArray(columnName)); + return parseJsonArray(rs.getString(columnName)); } @Override public byte[][] getNullableResult(ResultSet rs, int columnIndex) throws SQLException { - return getArray(rs.getArray(columnIndex)); + return parseJsonArray(rs.getString(columnIndex)); } @Override public byte[][] getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { - return getArray(cs.getArray(columnIndex)); + return parseJsonArray(cs.getString(columnIndex)); } - private byte[][] getArray(Array array) throws SQLException { - if (array != null) { - Object[] objArray = (Object[]) array.getArray(); - byte[][] result = new byte[objArray.length][]; - for (int i = 0; i < objArray.length; i++) { - result[i] = (byte[]) objArray[i]; - } - return result; + private byte[][] parseJsonArray(String value) { + if (value == null || value.isEmpty() || "[]".equals(value.trim())) { + return new byte[0][]; } - return null; + if (!value.startsWith("[")) { + // 兼容旧的逗号分隔格式 + return parseCommaSeparated(value); + } + String content = value.substring(1, value.length() - 1).trim(); + if (content.isEmpty()) { + return new byte[0][]; + } + // 解析 JSON 字符串数组 + List result = new ArrayList<>(); + int pos = 0; + int len = content.length(); + while (pos < len) { + if (content.charAt(pos) == '"') { + int end = pos + 1; + while (end < len) { + if (content.charAt(end) == '"' && content.charAt(end - 1) != '\\') { + break; + } + end++; + } + String base64 = content.substring(pos + 1, end); + if (!base64.isEmpty()) { + result.add(Base64.getDecoder().decode(base64)); + } else { + result.add(new byte[0]); + } + pos = end + 1; + while (pos < len && (content.charAt(pos) == ',' || content.charAt(pos) == ' ')) { + pos++; + } + } else { + // skip non-string content + pos++; + } + } + return result.toArray(new byte[0][]); } -} \ No newline at end of file + + private byte[][] parseCommaSeparated(String value) { + String[] parts = value.split(",", -1); + byte[][] result = new byte[parts.length][]; + for (int i = 0; i < parts.length; i++) { + String part = parts[i].trim(); + if (part.isEmpty()) { + result[i] = new byte[0]; + } else { + result[i] = Base64.getDecoder().decode(part); + } + } + return result; + } + + static String toJsonArray(byte[][] array) { + if (array == null) { + return null; + } + if (array.length == 0) { + return "[]"; + } + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < array.length; i++) { + if (i > 0) sb.append(","); + sb.append("\""); + if (array[i] != null && array[i].length > 0) { + sb.append(Base64.getEncoder().encodeToString(array[i])); + } + sb.append("\""); + } + sb.append("]"); + return sb.toString(); + } +} diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayIntegerTypeHandler.java b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayIntegerTypeHandler.java index 0a40ce1..a70f327 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayIntegerTypeHandler.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayIntegerTypeHandler.java @@ -1,43 +1,106 @@ package com.Modules.etl.handler; - import org.apache.ibatis.type.BaseTypeHandler; import org.apache.ibatis.type.JdbcType; import org.apache.ibatis.type.MappedJdbcTypes; import org.apache.ibatis.type.MappedTypes; -import java.sql.*; -import java.util.Arrays; +import java.sql.CallableStatement; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +/** + * 整型数组类型处理器 - 达梦数据库兼容版本 + * + * 将 Java Integer[] 与数据库 TEXT 列进行互转。 + * 存储格式: JSON 数组字符串，如 [1,2,3] + * 空数组: 存储为 "[]" + * null值: 存储为 NULL + * + * 原 PostgreSQL 版本使用 createArrayOf("integer", ...) 创建原生 _int4 数组， + * 达梦数据库不兼容此 API，改为 TEXT 列存储 JSON 数组。 + */ @MappedTypes(Integer[].class) -@MappedJdbcTypes(JdbcType.ARRAY) +@MappedJdbcTypes(JdbcType.VARCHAR) public class ArrayIntegerTypeHandler extends BaseTypeHandler { @Override public void setNonNullParameter(PreparedStatement ps, int i, Integer[] parameter, JdbcType jdbcType) throws SQLException { - Array array = ps.getConnection().createArrayOf("integer", parameter); - ps.setArray(i, array); + ps.setString(i, toJsonArray(parameter)); } @Override public Integer[] getNullableResult(ResultSet rs, String columnName) throws SQLException { - return getArray(rs.getArray(columnName)); + return parseJsonArray(rs.getString(columnName)); } @Override public Integer[] getNullableResult(ResultSet rs, int columnIndex) throws SQLException { - return getArray(rs.getArray(columnIndex)); + return parseJsonArray(rs.getString(columnIndex)); } @Override public Integer[] getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { - return getArray(cs.getArray(columnIndex)); + return parseJsonArray(cs.getString(columnIndex)); } - private Integer[] getArray(Array array) throws SQLException { - if (array != null) { - Object[] objArray = (Object[]) array.getArray(); - return Arrays.copyOf(objArray, objArray.length, Integer[].class); + /** + * 将 JSON 字符串解析为 Integer[] + */ + private Integer[] parseJsonArray(String value) { + if (value == null || value.isEmpty() || "[]".equals(value.trim())) { + return new Integer[0]; } - return null; + if (!value.startsWith("[")) { + // 兼容旧的逗号分隔格式: "1,2,3" + return parseCommaSeparated(value); + } + String content = value.substring(1, value.length() - 1).trim(); + if (content.isEmpty()) { + return new Integer[0]; + } + String[] parts = content.split(","); + List result = new ArrayList<>(parts.length); + for (String part : parts) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) { + result.add(Integer.parseInt(trimmed)); + } + } + return result.toArray(new Integer[0]); } -} \ No newline at end of file + + private Integer[] parseCommaSeparated(String value) { + String[] parts = value.split(","); + List result = new ArrayList<>(parts.length); + for (String part : parts) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) { + result.add(Integer.parseInt(trimmed)); + } + } + return result.toArray(new Integer[0]); + } + + /** + * 将 Integer[] 序列化为 JSON 数组字符串 + */ + static String toJsonArray(Integer[] array) { + if (array == null) { + return null; + } + if (array.length == 0) { + return "[]"; + } + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < array.length; i++) { + if (i > 0) sb.append(","); + sb.append(array[i] != null ? array[i].toString() : "null"); + } + sb.append("]"); + return sb.toString(); + } +} diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayStringTypeHandler.java b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayStringTypeHandler.java index 174364f..0b64c99 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayStringTypeHandler.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/etl/handler/ArrayStringTypeHandler.java @@ -1,43 +1,120 @@ package com.Modules.etl.handler; - import org.apache.ibatis.type.BaseTypeHandler; import org.apache.ibatis.type.JdbcType; import org.apache.ibatis.type.MappedJdbcTypes; import org.apache.ibatis.type.MappedTypes; -import java.sql.*; -import java.util.Arrays; +import java.sql.CallableStatement; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +/** + * 字符串数组类型处理器 - 达梦数据库兼容版本 + * + * 将 Java String[] 与数据库 TEXT 列进行互转。 + * 存储格式: JSON 数组字符串，如 ["value1","value2","value3"] + * 空数组: 存储为 "[]" + * null值: 存储为 NULL + * + * 原 PostgreSQL 版本使用 createArrayOf("text", ...) 创建原生数组， + * 达梦数据库不兼容此 API，改为 TEXT 列存储 JSON 数组。 + */ @MappedTypes(String[].class) -@MappedJdbcTypes(JdbcType.ARRAY) +@MappedJdbcTypes(JdbcType.VARCHAR) public class ArrayStringTypeHandler extends BaseTypeHandler { @Override public void setNonNullParameter(PreparedStatement ps, int i, String[] parameter, JdbcType jdbcType) throws SQLException { - Array array = ps.getConnection().createArrayOf("text", parameter); - ps.setArray(i, array); + ps.setString(i, toJsonArray(parameter)); } @Override public String[] getNullableResult(ResultSet rs, String columnName) throws SQLException { - return getArray(rs.getArray(columnName)); + return parseJsonArray(rs.getString(columnName)); } @Override public String[] getNullableResult(ResultSet rs, int columnIndex) throws SQLException { - return getArray(rs.getArray(columnIndex)); + return parseJsonArray(rs.getString(columnIndex)); } @Override public String[] getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { - return getArray(cs.getArray(columnIndex)); + return parseJsonArray(cs.getString(columnIndex)); } - private String[] getArray(Array array) throws SQLException { - if (array != null) { - Object[] objArray = (Object[]) array.getArray(); - return Arrays.copyOf(objArray, objArray.length, String[].class); + /** + * 将 JSON 字符串解析为 String[] + */ + static String[] parseJsonArray(String value) { + if (value == null || value.isEmpty() || "[]".equals(value.trim())) { + return new String[0]; } - return null; + if (!value.startsWith("[") || !value.endsWith("]")) { + // 兼容旧的逗号分隔格式 + return value.split(",", -1); + } + String content = value.substring(1, value.length() - 1).trim(); + if (content.isEmpty()) { + return new String[0]; + } + List list = new ArrayList<>(); + int pos = 0; + int len = content.length(); + while (pos < len) { + if (content.charAt(pos) == '"') { + int end = pos + 1; + while (end < len) { + if (content.charAt(end) == '"' && (end == 0 || content.charAt(end - 1) != '\\')) { + break; + } + end++; + } + String item = content.substring(pos + 1, end); + item = item.replace("\\\"", "\"").replace("\\\\", "\\"); + list.add(item); + pos = end + 1; + // skip comma and whitespace + while (pos < len && (content.charAt(pos) == ',' || content.charAt(pos) == ' ')) { + pos++; + } + } else { + // unquoted value (comma-separated) + int end = pos; + while (end < len && content.charAt(end) != ',') { + end++; + } + list.add(content.substring(pos, end).trim()); + pos = end + 1; + } + } + return list.toArray(new String[0]); } -} \ No newline at end of file + + /** + * 将 String[] 序列化为 JSON 数组字符串 + */ + static String toJsonArray(String[] array) { + if (array == null) { + return null; + } + if (array.length == 0) { + return "[]"; + } + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < array.length; i++) { + if (i > 0) sb.append(","); + sb.append("\""); + if (array[i] != null) { + sb.append(array[i].replace("\\", "\\\\").replace("\"", "\\\"")); + } + sb.append("\""); + } + sb.append("]"); + return sb.toString(); + } +} diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlocking.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlocking.java index 6ce07ee..1c16f59 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlocking.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlocking.java @@ -1,6 +1,6 @@ package com.common.entity; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; /** * 联动设备表实体类（防火墙设备信息） @@ -17,9 +17,9 @@ public class DeviceInterlocking { private String tenantId; private Long createDept; private Long createBy; - private OffsetDateTime createTime; + private LocalDateTime createTime; private Long updateBy; - private OffsetDateTime updateTime; + private LocalDateTime updateTime; private String remark; private String authUsername; // 用户名 private String authPassword; // 密码 @@ -55,14 +55,14 @@ public class DeviceInterlocking { public Long getCreateBy() { return createBy; } public void setCreateBy(Long createBy) { this.createBy = createBy; } - public OffsetDateTime getCreateTime() { return createTime; } - public void setCreateTime(OffsetDateTime createTime) { this.createTime = createTime; } + public LocalDateTime getCreateTime() { return createTime; } + public void setCreateTime(LocalDateTime createTime) { this.createTime = createTime; } public Long getUpdateBy() { return updateBy; } public void setUpdateBy(Long updateBy) { this.updateBy = updateBy; } - public OffsetDateTime getUpdateTime() { return updateTime; } - public void setUpdateTime(OffsetDateTime updateTime) { this.updateTime = updateTime; } + public LocalDateTime getUpdateTime() { return updateTime; } + public void setUpdateTime(LocalDateTime updateTime) { this.updateTime = updateTime; } public String getRemark() { return remark; } public void setRemark(String remark) { this.remark = remark; } diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlockingCmd.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlockingCmd.java index 29343a0..c01ab29 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlockingCmd.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlockingCmd.java @@ -4,9 +4,8 @@ import com.Modules.etl.handler.ArrayIntegerTypeHandler; import com.Modules.etl.handler.ArrayStringTypeHandler; import org.apache.ibatis.type.JdbcType; import org.apache.ibatis.type.TypeHandler; -import org.apache.ibatis.type.ArrayTypeHandler; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; import java.util.Arrays; import java.util.List; @@ -25,8 +24,8 @@ public class DeviceInterlockingCmd { private String banType; // 封禁类型（1:白名单、0:黑名单） private String cmdStatus; // 指令状态（0:未执行、1:已完成、2:执行中） private Integer banDuration; // 封禁时长（秒，-1表示永久） - private OffsetDateTime createTime; - private OffsetDateTime updateTime; + private LocalDateTime createTime; + private LocalDateTime updateTime; private String tenantId; private Long createDept; private Long createBy; @@ -65,11 +64,11 @@ public class DeviceInterlockingCmd { public Integer getBanDuration() { return banDuration; } public void setBanDuration(Integer banDuration) { this.banDuration = banDuration; } - public OffsetDateTime getCreateTime() { return createTime; } - public void setCreateTime(OffsetDateTime createTime) { this.createTime = createTime; } + public LocalDateTime getCreateTime() { return createTime; } + public void setCreateTime(LocalDateTime createTime) { this.createTime = createTime; } - public OffsetDateTime getUpdateTime() { return updateTime; } - public void setUpdateTime(OffsetDateTime updateTime) { this.updateTime = updateTime; } + public LocalDateTime getUpdateTime() { return updateTime; } + public void setUpdateTime(LocalDateTime updateTime) { this.updateTime = updateTime; } public String getTenantId() { return tenantId; } public void setTenantId(String tenantId) { this.tenantId = tenantId; } diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlockingLog.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlockingLog.java index 516c829..c4147ee 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlockingLog.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceInterlockingLog.java @@ -1,6 +1,6 @@ package com.common.entity; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; /** * 封禁记录表实体类 @@ -12,15 +12,15 @@ public class DeviceInterlockingLog { private Long deviceInterlockingId; // 封禁设备ID private String banIp; // 封禁IP地址 private String deviceName; // 封禁设备名称 - private OffsetDateTime banTime; // 封禁时间 + private LocalDateTime banTime; // 封禁时间 private String banMethod; // 封禁方式（0.人工、1.自动化封禁） private Integer banResult; // 联动结果（成功：1、失败：0） private String tenantId; private Long createDept; private Long createBy; - private OffsetDateTime createTime; + private LocalDateTime createTime; private Long updateBy; - private OffsetDateTime updateTime; + private LocalDateTime updateTime; private String remark; private String respBody; // 响应body private String reqBody; // 请求body @@ -41,8 +41,8 @@ public class DeviceInterlockingLog { public String getDeviceName() { return deviceName; } public void setDeviceName(String deviceName) { this.deviceName = deviceName; } - public OffsetDateTime getBanTime() { return banTime; } - public void setBanTime(OffsetDateTime banTime) { this.banTime = banTime; } + public LocalDateTime getBanTime() { return banTime; } + public void setBanTime(LocalDateTime banTime) { this.banTime = banTime; } public String getBanMethod() { return banMethod; } public void setBanMethod(String banMethod) { this.banMethod = banMethod; } @@ -59,14 +59,14 @@ public class DeviceInterlockingLog { public Long getCreateBy() { return createBy; } public void setCreateBy(Long createBy) { this.createBy = createBy; } - public OffsetDateTime getCreateTime() { return createTime; } - public void setCreateTime(OffsetDateTime createTime) { this.createTime = createTime; } + public LocalDateTime getCreateTime() { return createTime; } + public void setCreateTime(LocalDateTime createTime) { this.createTime = createTime; } public Long getUpdateBy() { return updateBy; } public void setUpdateBy(Long updateBy) { this.updateBy = updateBy; } - public OffsetDateTime getUpdateTime() { return updateTime; } - public void setUpdateTime(OffsetDateTime updateTime) { this.updateTime = updateTime; } + public LocalDateTime getUpdateTime() { return updateTime; } + public void setUpdateTime(LocalDateTime updateTime) { this.updateTime = updateTime; } public String getRemark() { return remark; } public void setRemark(String remark) { this.remark = remark; } diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmMapper.java index b8a5ffc..f06eefb 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmMapper.java @@ -15,7 +15,7 @@ public interface AlarmMapper { "INSERT INTO alarm (", "id, created_at, alarm_name, alarm_level, alarm_type, ", "alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, ", - "device_id, comment,origin_log_ids,log_start_at, log_end_at, window_time, http_status, ", + "device_id, \"comment\",origin_log_ids,log_start_at, log_end_at, window_time, http_status, ", "attack_port, victim_port, attack_method, etl_time, log_count, ", "attack_chain_phase, disposition_advice, attack_direction, ", "judged_state, disposed_state, attack_result, fall, payload, dns_info, engine_type, " , @@ -52,7 +52,7 @@ public interface AlarmMapper { @Insert("INSERT INTO alarm (" + "id, created_at, alarm_name, alarm_level, alarm_type, " + "alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, " + - "device_id, comment,origin_log_ids, log_start_at, log_end_at, window_time, http_status, " + + "device_id, \"comment\",origin_log_ids, log_start_at, log_end_at, window_time, http_status, " + "attack_port, victim_port, attack_method, etl_time, log_count, " + "attack_chain_phase, disposition_advice, attack_direction, " + "judged_state, disposed_state, attack_result, fall, payload, dns_info, engine_type, " + diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitMapper.java index 00b1948..9a20855 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitMapper.java @@ -19,7 +19,7 @@ public interface AlarmVisitMapper { "INSERT INTO alarm_visit (", "id, created_at, alarm_name, alarm_level, alarm_type, ", "alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, ", - "device_id, comment,origin_log_ids,log_start_at, log_end_at,window_time, http_status, ", + "device_id, \"comment\",origin_log_ids,log_start_at, log_end_at,window_time, http_status, ", "attack_port, victim_port, attack_method, etl_time, log_count, ", "attack_chain_phase, disposition_advice, attack_direction, ", "judged_state, disposed_state, attack_result, fall, payload, dns_info, engine_type, " , @@ -56,7 +56,7 @@ public interface AlarmVisitMapper { @Insert("INSERT INTO alarm_visit (" + "id, created_at, alarm_name, alarm_level, alarm_type, " + "alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, " + - "device_id, comment,origin_log_ids, log_start_at, log_end_at, window_time,http_status, " + + "device_id, \"comment\",origin_log_ids, log_start_at, log_end_at, window_time,http_status, " + "attack_port, victim_port, attack_method, etl_time, log_count, " + "attack_chain_phase, disposition_advice, attack_direction, " + "judged_state, disposed_state, attack_result, fall, payload, dns_info,engine_type, " + diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectHeartbeatMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectHeartbeatMapper.java index b74af1b..a169f07 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectHeartbeatMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectHeartbeatMapper.java @@ -37,22 +37,37 @@ public interface DeviceCollectHeartbeatMapper { /** * 插入或更新（根据collect_id） + * 达梦数据库使用 MERGE INTO 实现 upsert */ - @Insert("INSERT INTO device_collect_heartbeat (" + + @Update("MERGE INTO device_collect_heartbeat t " + + "USING (SELECT " + + "#{collectId} AS collect_id, " + + "#{collectName} AS collect_name, " + + "#{deviceIp} AS device_ip, " + + "#{appVersion} AS app_version, " + + "#{lastHeartbeat} AS last_heartbeat, " + + "#{heartbeatCount} AS heartbeat_count, " + + "#{status} AS status, " + + "#{failCount} AS fail_count, " + + "#{updateTime} AS update_time " + + "FROM DUAL) s " + + "ON (t.collect_id = s.collect_id) " + + "WHEN MATCHED THEN UPDATE SET " + + "t.collect_name = s.collect_name, " + + "t.device_ip = s.device_ip, " + + "t.app_version = s.app_version, " + + "t.last_heartbeat = s.last_heartbeat, " + + "t.heartbeat_count = s.heartbeat_count, " + + "t.status = s.status, " + + "t.fail_count = s.fail_count, " + + "t.update_time = s.update_time " + + "WHEN NOT MATCHED THEN INSERT (" + "collect_id, collect_name, device_ip, app_version, last_heartbeat, " + - "heartbeat_count, status, fail_count, update_time " + + "heartbeat_count, status, fail_count, update_time" + ") VALUES (" + - "#{collectId}, #{collectName}, #{deviceIp}, #{appVersion}, #{lastHeartbeat}, " + - "#{heartbeatCount}, #{status}, #{failCount}, #{updateTime} " + - ") ON CONFLICT (collect_id) DO UPDATE SET " + - "collect_name = EXCLUDED.collect_name, " + - "device_ip = EXCLUDED.device_ip, " + - "app_version = EXCLUDED.app_version, " + - "last_heartbeat = EXCLUDED.last_heartbeat, " + - "heartbeat_count = EXCLUDED.heartbeat_count, " + - "status = EXCLUDED.status, " + - "fail_count = EXCLUDED.fail_count, " + - "update_time = EXCLUDED.update_time") + "s.collect_id, s.collect_name, s.device_ip, s.app_version, s.last_heartbeat, " + + "s.heartbeat_count, s.status, s.fail_count, s.update_time" + + ")") int upsert(DeviceCollectHeartbeat heartbeat); /** diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java index 3a05666..221223e 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java @@ -155,12 +155,12 @@ public interface DeviceCollectTaskMapper extends BaseMapper{ "" + "UPDATE device_collect_task SET " + " first_time = CASE " + - " WHEN first_time IS NULL AND #{task.firstTime}::TIMESTAMP IS NOT NULL THEN #{task.firstTime}::TIMESTAMP " + + " WHEN first_time IS NULL AND #{task.firstTime} IS NOT NULL THEN #{task.firstTime} " + " ELSE first_time " + " END, " + - " last_success_time = #{task.lastSuccessTime}::TIMESTAMP, " + - " last_failed_time = #{task.lastFailedTime}::TIMESTAMP, " + - " updated_at = #{task.updatedAt}::TIMESTAMP " + + " last_success_time = #{task.lastSuccessTime}, " + + " last_failed_time = #{task.lastFailedTime}, " + + " updated_at = #{task.updatedAt} " + "WHERE id = #{task.id}" + "" + "") @@ -171,12 +171,12 @@ public interface DeviceCollectTaskMapper extends BaseMapper{ */ @Update("UPDATE device_collect_task " + "SET first_time = CASE " + - " WHEN first_time IS NULL AND #{firstTime}::TIMESTAMP IS NOT NULL THEN #{firstTime}::TIMESTAMP " + + " WHEN first_time IS NULL AND #{firstTime} IS NOT NULL THEN #{firstTime} " + " ELSE first_time " + " END, " + - " last_success_time = #{lastSuccessTime}::TIMESTAMP, " + - " last_failed_time = #{lastFailTime}::TIMESTAMP, " + - " updated_at = #{updateTime}::TIMESTAMP " + + " last_success_time = #{lastSuccessTime}, " + + " last_failed_time = #{lastFailTime}, " + + " updated_at = #{updateTime} " + "WHERE id = #{deviceCollectId}") int updateTaskTime(@Param("deviceCollectId") String deviceCollectId, @Param("firstTime") LocalDateTime firstTime, diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceInterlockingCmdMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceInterlockingCmdMapper.java index 57bfe7e..a51f63c 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceInterlockingCmdMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceInterlockingCmdMapper.java @@ -61,8 +61,11 @@ public interface DeviceInterlockingCmdMapper { @Insert("INSERT INTO device_interlocking_cmd (probe_id, probe_ip, device_interlocking_id, device_interlocking_ip, " + "ban_ips, ban_method, ban_type, cmd_status, ban_duration, create_time, update_time, " + "tenant_id, create_dept, create_by, remark, ban_operation_type) " + - "VALUES (#{probeId}, #{probeIp}, ARRAY[:ids], ARRAY[:ips], ARRAY[:banIps], " + - "#{banMethod}, #{banType}, #{cmdStatus}, #{banDuration}, NOW(), NOW(), " + + "VALUES (#{probeId}, #{probeIp}, " + + "#{deviceInterlockingId, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, " + + "#{deviceInterlockingIp, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " + + "#{banIps, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " + + "#{banMethod}, #{banType}, #{cmdStatus}, #{banDuration}, SYSDATE, SYSDATE, " + "#{tenantId}, #{createDept}, #{createBy}, #{remark}, #{banOperationType})") @Options(useGeneratedKeys = true, keyProperty = "id") int insert(DeviceInterlockingCmd cmd); @@ -73,25 +76,25 @@ public interface DeviceInterlockingCmdMapper { * @param cmdStatus 新状态 * @return 影响行数 */ - @Update("UPDATE device_interlocking_cmd SET cmd_status = #{cmdStatus}, update_time = NOW() WHERE id = #{id}") + @Update("UPDATE device_interlocking_cmd SET cmd_status = #{cmdStatus}, update_time = SYSDATE WHERE id = #{id}") int updateStatus(@Param("id") Long id, @Param("cmdStatus") String cmdStatus); /** * 更新指令状态为执行中 */ - @Update("UPDATE device_interlocking_cmd SET cmd_status = '2', update_time = NOW() WHERE id = #{id}") + @Update("UPDATE device_interlocking_cmd SET cmd_status = '2', update_time = SYSDATE WHERE id = #{id}") int updateStatusToExecuting(@Param("id") Long id); /** * 更新指令状态为执行完成 */ - @Update("UPDATE device_interlocking_cmd SET cmd_status = '1', update_time = NOW() WHERE id = #{id}") + @Update("UPDATE device_interlocking_cmd SET cmd_status = '1', update_time = SYSDATE WHERE id = #{id}") int updateStatusToCompleted(@Param("id") Long id); /** * 更新指令状态为执行失败 */ - @Update("UPDATE device_interlocking_cmd SET cmd_status = '3', update_time = NOW() WHERE id = #{id}") + @Update("UPDATE device_interlocking_cmd SET cmd_status = '3', update_time = SYSDATE WHERE id = #{id}") int updateStatusToFailed(@Param("id") Long id); /** diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java index cf38883..6124c49 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java @@ -84,7 +84,7 @@ public interface DeviceReceiveLogMapper { */ @Select("SELECT device_collect_id, MAX(created_at) AS last_success_time " + "FROM device_receive_log " + - "WHERE push_success = true " + + "WHERE push_success = 1 " + "AND created_at >= CURRENT_DATE " + "GROUP BY device_collect_id") List selectDailySuccessTimes(); @@ -94,7 +94,7 @@ public interface DeviceReceiveLogMapper { */ @Select("SELECT device_collect_id, MAX(created_at) AS last_fail_time " + "FROM device_receive_log " + - "WHERE push_success = false " + + "WHERE push_success = 0 " + "AND created_at >= CURRENT_DATE " + "GROUP BY device_collect_id") List selectDailyFailTimes(); @@ -104,7 +104,7 @@ public interface DeviceReceiveLogMapper { */ @Select("SELECT device_collect_id, MIN(created_at) AS first_success_time " + "FROM device_receive_log " + - "WHERE push_success = true " + + "WHERE push_success = 1 " + "GROUP BY device_collect_id") List selectFirstSuccessTimes(); @@ -114,7 +114,7 @@ public interface DeviceReceiveLogMapper { @Select("SELECT device_collect_id, MIN(created_at) AS first_time, " + "MAX(created_at) AS last_success_time " + "FROM device_receive_log " + - "WHERE push_success = true " + + "WHERE push_success = 1 " + "AND created_at >= #{startTime} " + "AND created_at < #{endTime} " + "GROUP BY device_collect_id") diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java index e11ffaa..c9addc0 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java @@ -49,30 +49,29 @@ public interface SyslogNormalAlarmMapper { */ @Select("SELECT " + "to_char(log_time, 'YYYYMMDD') as log_date, " + - "ARRAY_AGG(DISTINCT host(src_ip)::text) as attack_ips, " + + "WM_CONCAT(DISTINCT src_ip) as attack_ips, " + "origin_event_name, " + "MAX(attack_result) as attack_result, " + "MIN(log_time) as min_log_time, " + "MAX(log_time) as max_log_time, " + "COUNT(1) as log_count, " + - "ARRAY_AGG(DISTINCT host(dest_ip)::text) as victim_ips, " + - "ARRAY_AGG(DISTINCT http_url) as victim_web_urls, " + - "ARRAY_AGG(DISTINCT device_id) as device_ids, " + - "ARRAY_AGG(DISTINCT id) as origin_log_ids, " + + "WM_CONCAT(DISTINCT dest_ip) as victim_ips, " + + "WM_CONCAT(DISTINCT http_url) as victim_web_urls, " + + "WM_CONCAT(DISTINCT device_id) as device_ids, " + + "WM_CONCAT(DISTINCT id) as origin_log_ids, " + "MAX(event_level) as max_event_level, " + "MIN(origin_event_type) AS first_event_type, " + "MAX(origin_event_type) as event_type, " + "MIN(event_type) as min_event_type, " + - "ARRAY_AGG(DISTINCT src_port::int4) as attack_ports, " + - "ARRAY_AGG(DISTINCT dest_port::int4) as victim_ports, " + - "ARRAY_AGG(DISTINCT http_resp_codes::text) as http_status_codes, " + - "ARRAY_AGG(DISTINCT payload::BYTEA) as payload_samples, " + - "ARRAY_AGG(DISTINCT http_req_header) as httpReqHeaders, " + - "ARRAY_AGG(DISTINCT http_req_body) as httpReqBodys, " + - "ARRAY_AGG(DISTINCT http_resp_header) as httpRespHeaders, " + - "ARRAY_AGG(DISTINCT http_resp_body) as httpRespBodys, " + - "MODE() WITHIN GROUP (ORDER BY dest_domain) as dns_info, " + - "STRING_AGG(DISTINCT COALESCE(host(dest_ip)::text, ''), ',') as victim_ips_str " + + "WM_CONCAT(DISTINCT src_port) as attack_ports, " + + "WM_CONCAT(DISTINCT dest_port) as victim_ports, " + + "WM_CONCAT(DISTINCT http_resp_codes) as http_status_codes, " + + "WM_CONCAT(DISTINCT payload) as payload_samples, " + + "WM_CONCAT(DISTINCT http_req_header) as httpReqHeaders, " + + "WM_CONCAT(DISTINCT http_req_body) as httpReqBodys, " + + "WM_CONCAT(DISTINCT http_resp_header) as httpRespHeaders, " + + "WM_CONCAT(DISTINCT http_resp_body) as httpRespBodys, " + + "LISTAGG(DISTINCT COALESCE(dest_ip, ''), ',') as victim_ips_str " + "FROM syslog_normal_alarm " + "WHERE log_time >= #{startTime} AND log_time < #{endTime} " + "AND event_level >= 1 AND src_ip NOT IN ('127.0.0.1', '127.0.0.2') " + diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalDataMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalDataMapper.java index 358218c..7ef7737 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalDataMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalDataMapper.java @@ -46,30 +46,29 @@ public interface SyslogNormalDataMapper { */ @Select("SELECT " + "to_char(log_time, 'YYYYMMDD') as log_date, " + - "ARRAY_AGG(DISTINCT host(src_ip)::text) as attack_ips, " + + "WM_CONCAT(DISTINCT src_ip) as attack_ips, " + "origin_event_name, " + "MAX(attack_result) as attack_result, " + "MIN(log_time) as min_log_time, " + "MAX(log_time) as max_log_time, " + "COUNT(1) as log_count, " + - "ARRAY_AGG(DISTINCT host(dest_ip)::text) as victim_ips, " + - "ARRAY_AGG(DISTINCT http_url) as victim_web_urls, " + - "ARRAY_AGG(DISTINCT device_id) as device_ids, " + - "ARRAY_AGG(DISTINCT id) as origin_log_ids, " + + "WM_CONCAT(DISTINCT dest_ip) as victim_ips, " + + "WM_CONCAT(DISTINCT http_url) as victim_web_urls, " + + "WM_CONCAT(DISTINCT device_id) as device_ids, " + + "WM_CONCAT(DISTINCT id) as origin_log_ids, " + "MAX(event_level) as max_event_level, " + "MIN(origin_event_type) AS first_event_type, " + "MAX(origin_event_type) as event_type, " + "MIN(event_type) as min_event_type, " + - "ARRAY_AGG(DISTINCT src_port::int4) as attack_ports, " + - "ARRAY_AGG(DISTINCT dest_port::int4) as victim_ports, " + - "ARRAY_AGG(DISTINCT http_resp_codes::text) as http_status_codes, " + - "ARRAY_AGG(DISTINCT payload::BYTEA) as payload_samples, " + - "ARRAY_AGG(DISTINCT http_req_header) as httpReqHeaders, " + - "ARRAY_AGG(DISTINCT http_req_body) as httpReqBodys, " + - "ARRAY_AGG(DISTINCT http_resp_header) as httpRespHeaders, " + - "ARRAY_AGG(DISTINCT http_resp_body) as httpRespBodys, " + - "MODE() WITHIN GROUP (ORDER BY dest_domain) as dns_info, " + - "STRING_AGG(DISTINCT COALESCE(host(dest_ip)::text, ''), ',') as victim_ips_str " + + "WM_CONCAT(DISTINCT src_port) as attack_ports, " + + "WM_CONCAT(DISTINCT dest_port) as victim_ports, " + + "WM_CONCAT(DISTINCT http_resp_codes) as http_status_codes, " + + "WM_CONCAT(DISTINCT payload) as payload_samples, " + + "WM_CONCAT(DISTINCT http_req_header) as httpReqHeaders, " + + "WM_CONCAT(DISTINCT http_req_body) as httpReqBodys, " + + "WM_CONCAT(DISTINCT http_resp_header) as httpRespHeaders, " + + "WM_CONCAT(DISTINCT http_resp_body) as httpRespBodys, " + + "LISTAGG(DISTINCT COALESCE(dest_ip, ''), ',') as victim_ips_str " + "FROM syslog_normal_data " + "WHERE log_time >= #{startTime} AND log_time < #{endTime} " + "AND http_resp_codes =200 and origin_event_type <> '' and origin_event_name='访问日志' AND src_ip NOT IN ('127.0.0.1', '127.0.0.2') " + diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/WecomNotificationMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/WecomNotificationMapper.java index abd6f61..24d81eb 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/WecomNotificationMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/WecomNotificationMapper.java @@ -18,12 +18,12 @@ public interface WecomNotificationMapper { "wecom_notification_time, tenant_id, create_dept, create_by, create_time, " + "update_by, update_time, remark, wecom_notification_status" + ") VALUES (" + - "nextval('seq_wecom_notification'), #{userId}, #{wecomNotificationName}, #{wecomNotificationIp}, " + + "seq_wecom_notification.NEXTVAL, #{userId}, #{wecomNotificationName}, #{wecomNotificationIp}, " + "#{wecomNotificationType}, #{wecomNotificationLevel}, #{wecomNotificationContent}, " + "#{wecomNotificationTime}, #{tenantId}, #{createDept}, #{createBy}, #{createTime}, " + "#{updateBy}, #{updateTime}, #{remark}, #{wecomNotificationStatus}" + ")") - @SelectKey(statement = "SELECT currval('seq_wecom_notification')", keyProperty = "wecomNotificationId", resultType = Long.class, before = false) + @SelectKey(statement = "SELECT seq_wecom_notification.currval", keyProperty = "wecomNotificationId", resultType = Long.class, before = false) int insert(WecomNotification notification); /** @@ -41,7 +41,7 @@ public interface WecomNotificationMapper { /** * 更新通知状态 */ - @Update("UPDATE wecom_notification SET wecom_notification_status = #{status}, update_time = NOW() " + + @Update("UPDATE wecom_notification SET wecom_notification_status = #{status}, update_time = SYSDATE " + "WHERE wecom_notification_id = #{wecomNotificationId}") int updateStatus(@Param("wecomNotificationId") Long wecomNotificationId, @Param("status") String status); } diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java index f048f14..b9ccaec 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java @@ -52,13 +52,10 @@ public class PartitionTableSchedule { logger.info("测试任务: 分区表创建完成"); } - - - /** * 每天检查第二天的分区表状态 - 凌晨2点执行 */ - @Scheduled(cron = "0 0 2 * * ?") + //@Scheduled(cron = "0 0 2 * * ?") public void checkTomorrowPartitionTable() { logger.info("开始检查第二天的分区表状态..."); @@ -84,7 +81,7 @@ public class PartitionTableSchedule { /** * 每周一检查未来7天的分区表状态 */ - @Scheduled(cron = "0 0 3 * * MON") + //@Scheduled(cron = "0 0 3 * * MON") public void checkNextWeekPartitionTables() { logger.info("开始检查未来7天的分区表状态..."); diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AccessLogAlertService.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AccessLogAlertService.java index 01af5f6..138a6e4 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AccessLogAlertService.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AccessLogAlertService.java @@ -272,6 +272,9 @@ public class AccessLogAlertService { logObject.put("_source.vlan_id", ""); logObject.put("_type", "skyeye-file"); + //补充syslog_normal_data.device_id 字段 + logObject.put("_source.device_id", logData.getDeviceId() != null ? logData.getDeviceId() : ""); + requestArray.add(logObject); } @@ -416,12 +419,25 @@ public class AccessLogAlertService { log.debug("算法：{},ID:{} ,AlarmNme:{} 没有返回 origin_log节点.",AlgorithmName, alarmVisit.getId(), alarmVisit.getAlarmName()); return false; } - alarmVisit.setAttackPort( new Integer[]{alarmResult.getInteger("_source.sport")} ); - alarmVisit.setVictimPort( new Integer[]{alarmResult.getInteger("_source.dport")} ); - alarmVisit.setAttackMethod(alarmResult.getString("_source.method") ); - String deviceIp= alarmResult.getString("_source.device_ip"); + // _source.sport/dport 在 JSON 中为字符串类型，需要用 getString() 读取后 parseInt + String sportStr = originLogObject.getString("_source.sport"); + if (sportStr != null && !sportStr.isEmpty()) { + alarmVisit.setAttackPort(new Integer[]{Integer.parseInt(sportStr)}); + } + String dportStr = originLogObject.getString("_source.dport"); + if (dportStr != null && !dportStr.isEmpty()) { + alarmVisit.setVictimPort(new Integer[]{Integer.parseInt(dportStr)}); + } + alarmVisit.setAttackMethod(originLogObject.getString("_source.method")); + String deviceIp = originLogObject.getString("_source.device_ip"); //alarmVisit.setDeviceId( new Integer[]{ getDeviceID(deviceIp)} ); - alarmVisit.setHttpStatus( alarmResult.getString("_source.status")); + + //补充alarm_visist.device_id + String deviceidStr = originLogObject.getString("_source.device_id"); + if (deviceidStr != null && !deviceidStr.isEmpty()) { + alarmVisit.setDeviceId(new Integer[]{Integer.parseInt(deviceidStr)}); + } + alarmVisit.setHttpStatus(originLogObject.getString("_source.status")); return true; } catch (Exception e) { log.error("算法：{} 补充原始记录日志字段异常。error:{} ",AlgorithmName,e.getMessage(), e ); diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AlarmHealthCheckService.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AlarmHealthCheckService.java index ab7ea69..46f4b2a 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AlarmHealthCheckService.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AlarmHealthCheckService.java @@ -64,9 +64,10 @@ public class AlarmHealthCheckService { */ public boolean checkAlarmTable() { String partitionDate = LocalDateTime.now().format(DATE_FORMATTER); - String tableName = "alarm_" + partitionDate; + String tableName = "alarm_p_" + partitionDate; - String sql = "SELECT COUNT(*) FROM " + tableName + " WHERE created_at >= NOW() - INTERVAL '" + alarmHoursThreshold + " hours'"; + // 达梦使用 SYSDATE 和 HOUR 关键字（PG 用 NOW() / INTERVAL 'N hours'） + String sql = "SELECT COUNT(*) FROM " + tableName + " WHERE created_at >= SYSDATE - INTERVAL '" + alarmHoursThreshold + "' HOUR"; try { Long count = jdbcTemplate.queryForObject(sql, Long.class); @@ -89,9 +90,10 @@ public class AlarmHealthCheckService { */ public boolean checkAlarmVisitTable() { String partitionDate = LocalDateTime.now().format(DATE_FORMATTER); - String tableName = "alarm_visit_" + partitionDate; + String tableName = "alarm_visit_p_" + partitionDate; - String sql = "SELECT COUNT(*) FROM " + tableName + " WHERE created_at >= NOW() - INTERVAL '" + alarmVisitHoursThreshold + " hours'"; + // 达梦使用 SYSDATE 和 HOUR 关键字（PG 用 NOW() / INTERVAL 'N hours'） + String sql = "SELECT COUNT(*) FROM " + tableName + " WHERE created_at >= SYSDATE - INTERVAL '" + alarmVisitHoursThreshold + "' HOUR"; try { Long count = jdbcTemplate.queryForObject(sql, Long.class); @@ -123,7 +125,6 @@ public class AlarmHealthCheckService { "建议: 请检查数据采集服务是否正常运行%n" + "状态: 需要人工介入检查", tableName, - LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")), hoursThreshold ); diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceInterlockingLogService.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceInterlockingLogService.java index b251092..af885b8 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceInterlockingLogService.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceInterlockingLogService.java @@ -4,7 +4,7 @@ import com.common.entity.DeviceInterlockingLog; import com.common.mapper.DeviceInterlockingLogMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; import java.util.List; @Service @@ -32,7 +32,7 @@ public class DeviceInterlockingLogService { */ public int insert(DeviceInterlockingLog log) { if (log.getBanTime() == null) { - log.setBanTime(OffsetDateTime.now()); + log.setBanTime(LocalDateTime.now()); } return logMapper.insert(log); } @@ -44,7 +44,7 @@ public class DeviceInterlockingLogService { if (logs != null && !logs.isEmpty()) { for (DeviceInterlockingLog log : logs) { if (log.getBanTime() == null) { - log.setBanTime(OffsetDateTime.now()); + log.setBanTime(LocalDateTime.now()); } } } diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java index f3713dc..75e48a6 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java @@ -75,14 +75,14 @@ public class DeviceStatsUpdateService { "SET last_receive_time = ?, " + " today_parse_count = ?, " + " today_non_log_count = ?, " + - " updated_at = NOW() " + + " updated_at = SYSDATE " + "WHERE id = ?"; private static final String UPDATE_DEVICE_COUNT_SQL = "UPDATE device_device " + "SET today_parse_count = 0, " + " today_non_log_count = 0, " + - " updated_at = NOW() " ; + " updated_at = SYSDATE " ; /** * 每5分钟执行一次设备统计更新（秒：0，分：*，时：*） @@ -141,9 +141,10 @@ public class DeviceStatsUpdateService { private List collectDeviceStats(String today) { // 构建动态表名 String receiveLogTable = "device_receive_log"; - String normalDataTable = "syslog_normal_data_" + today; - String normalAlarmTable = "syslog_normal_alarm_" + today; - String nonNormalTable = "syslog_non_normal_message_" + today; + // 达梦分区表命名规则：基表名_p_日期（达梦不支持纯数字分区名） + String normalDataTable = "syslog_normal_data_p_" + today; + String normalAlarmTable = "syslog_normal_alarm_p_" + today; + String nonNormalTable = "syslog_non_normal_message_p_" + today; String sql = String.format( COLLECT_DEVICE_STATS_SQL, receiveLogTable, diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DmNormalizeRuleService.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DmNormalizeRuleService.java index 58e7090..2bf5387 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DmNormalizeRuleService.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DmNormalizeRuleService.java @@ -77,7 +77,7 @@ public class DmNormalizeRuleService { List> ruleMap=dmNormalizeRuleMapper.selectByDeviceId(id); sqlSession.commit(); - return ruleMap; + return convertClobToString(ruleMap); } catch (Exception e) { logger.error("DmNormalizeRuleService MyBatisUtil getSqlSession 异常", e); @@ -95,7 +95,41 @@ public class DmNormalizeRuleService { { System.out.println("调用selectByDeviceIdAuto 方法,id:"+id); List> ruleMap=dmNormalizeRuleMapper.selectByDeviceId(id); - return ruleMap; + return convertClobToString(ruleMap); + } + + /** + * 将达梦 JDBC CLOB/NCLOB 对象转换为 String，避免缓存序列化报错 + * 达梦驱动返回的 TEXT/CLOB 列可能是 dm.jdbc.driver.DmdbNClob 等内部类型， + * toString() 只返回对象引用（如 DmdbNClob@xxx），必须通过 Clob 接口获取实际文本 + */ + private List> convertClobToString(List> list) { + if (list == null) return null; + for (Map map : list) { + if (map == null) continue; + for (Map.Entry entry : map.entrySet()) { + Object value = entry.getValue(); + if (value != null && value.getClass().getName().startsWith("dm.jdbc.")) { + try { + if (value instanceof java.sql.Clob) { + java.sql.Clob clob = (java.sql.Clob) value; + long length = clob.length(); + if (length > 0) { + entry.setValue(clob.getSubString(1, (int) length)); + } else { + entry.setValue(""); + } + } else { + entry.setValue(value.toString()); + } + } catch (Exception e) { + logger.warn("CLOB/NCLOB 转换 String 失败: " + e.getMessage()); + entry.setValue(null); + } + } + } + } + return list; } } \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/PartitionTableService.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/PartitionTableService.java index c20c17a..fa28f97 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/PartitionTableService.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/PartitionTableService.java @@ -34,7 +34,7 @@ public class PartitionTableService { try { LocalDate tomorrow = LocalDate.now().plusDays(1); - String tableName = "syslog_normal_data_" + tomorrow.format(DATE_FORMATTER); + String tableName = "syslog_normal_data_p_" + tomorrow.format(DATE_FORMATTER); status.setCheckDate(tomorrow); status.setTableName(tableName); @@ -100,7 +100,7 @@ public class PartitionTableService { for (int i = 1; i <= days; i++) { LocalDate checkDate = LocalDate.now().plusDays(i); - String tableName = "syslog_normal_data_" + checkDate.format(DATE_FORMATTER); + String tableName = "syslog_normal_data_p_" + checkDate.format(DATE_FORMATTER); PartitionTableStatus status = new PartitionTableStatus(); status.setCheckDate(checkDate); @@ -145,7 +145,7 @@ public class PartitionTableService { */ public boolean createDailyPartitionTable(LocalDate date) { try { - String tableName = "syslog_normal_data_" + date.format(DATE_FORMATTER); + String tableName = "syslog_normal_data_p_" + date.format(DATE_FORMATTER); String nextDay = date.plusDays(1).format(SQL_DATE_FORMATTER); String currentDay = date.format(SQL_DATE_FORMATTER); diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java index 03e9a81..c1b6407 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java @@ -734,7 +734,25 @@ public class RealtimeAnalysisEngine implements AnalysisEngine { return result; } // PostgreSQL数组以字符串形式返回，如 "{ip1,ip2,ip3}" + // 达梦 JSONB_AGG 返回 JSON 数组格式，如 "[41614, 8080]" String str = value.toString(); + if (str.startsWith("[") && str.endsWith("]")) { + str = str.substring(1, str.length() - 1).trim(); + if (str.isEmpty()) { + return new String[0]; + } + // 拆分 JSON 数组元素（兼容带引号和纯数字） + String[] parts = str.split(","); + String[] result = new String[parts.length]; + for (int i = 0; i < parts.length; i++) { + String part = parts[i].trim(); + if (part.startsWith("\"") && part.endsWith("\"")) { + part = part.substring(1, part.length() - 1); + } + result[i] = part; + } + return result; + } if (str.startsWith("{") && str.endsWith("}")) { str = str.substring(1, str.length() - 1); return str.split(","); diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java index eab92de..9925ad0 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java @@ -445,7 +445,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { String sizeUnit = convertTimeUnit(window.getTumbleWindowSizeUnit()); Integer size = window.getTumbleWindowSize(); - return String.format("TUMBLE(%s, INTERVAL '%d %s') AS window_start", timeCol, size, sizeUnit); + return String.format("TUMBLE(%s, INTERVAL '%d' %s) AS window_start", timeCol, size, sizeUnit); } /** @@ -456,7 +456,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { String sizeUnit = convertTimeUnit(window.getTumbleWindowSizeUnit()); Integer size = window.getTumbleWindowSize(); //return String.format("TUMBLE_START(%s, INTERVAL '%d %s') AS window_time", timeCol, size, sizeUnit); - return String.format("TUMBLE(%s, INTERVAL '%d %s') AS window_time", timeCol, size, sizeUnit); + return String.format("TUMBLE(%s, INTERVAL '%d' %s) AS window_time", timeCol, size, sizeUnit); } /** @@ -467,7 +467,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { String sizeUnit = convertTimeUnit(window.getTumbleWindowSizeUnit()); Integer size = window.getTumbleWindowSize(); - return String.format("TUMBLE(%s, INTERVAL '%d %s')", timeCol, size, sizeUnit); + return String.format("TUMBLE(%s, INTERVAL '%d' %s)", timeCol, size, sizeUnit); } /** @@ -480,7 +480,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { Integer size = window.getHopWindowSize(); Integer slide = window.getHopWindowSlide(); - return String.format("HOP(%s, INTERVAL '%d %s', INTERVAL '%d %s') AS window_start", + return String.format("HOP(%s, INTERVAL '%d' %s, INTERVAL '%d' %s) AS window_start", timeCol, size, sizeUnit, slide, slideUnit); } @@ -496,7 +496,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { //return String.format("HOP_START(%s, INTERVAL '%d %s', INTERVAL '%d %s') AS window_time", // timeCol, size, sizeUnit, slide, slideUnit); - return String.format("HOP(%s, INTERVAL '%d %s', INTERVAL '%d %s') AS window_time", + return String.format("HOP(%s, INTERVAL '%d' %s, INTERVAL '%d' %s) AS window_time", timeCol, size, sizeUnit, slide, slideUnit); } @@ -510,7 +510,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { Integer size = window.getHopWindowSize(); Integer slide = window.getHopWindowSlide(); - return String.format("HOP(%s, INTERVAL '%d %s', INTERVAL '%d %s')", + return String.format("HOP(%s, INTERVAL '%d' %s, INTERVAL '%d' %s)", timeCol, size, sizeUnit, slide, slideUnit); } @@ -522,7 +522,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { String sizeUnit = convertTimeUnit(window.getSessionWindowSizeUnit()); Integer size = window.getSessionWindowSize(); - return String.format("SESSION(%s, INTERVAL '%d %s') AS window_start", timeCol, size, sizeUnit); + return String.format("SESSION(%s, INTERVAL '%d' %s) AS window_start", timeCol, size, sizeUnit); } /** @@ -534,7 +534,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { Integer size = window.getSessionWindowSize(); //return String.format("SESSION_START(%s, INTERVAL '%d %s') AS window_time", timeCol, size, sizeUnit); - return String.format("SESSION(%s, INTERVAL '%d %s') AS window_time", timeCol, size, sizeUnit); + return String.format("SESSION(%s, INTERVAL '%d' %s) AS window_time", timeCol, size, sizeUnit); } /** @@ -545,7 +545,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { String sizeUnit = convertTimeUnit(window.getSessionWindowSizeUnit()); Integer size = window.getSessionWindowSize(); - return String.format("SESSION(%s, INTERVAL '%d %s')", timeCol, size, sizeUnit); + return String.format("SESSION(%s, INTERVAL '%d' %s)", timeCol, size, sizeUnit); } /** @@ -747,19 +747,19 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { return "AVG(" + columnName + ")"; case "DUPLICATESANDSPLICE": if (StringUtils.isNotBlank(argsStr)) { - return "STRING_AGG(DISTINCT " + columnName + ", '" + argsStr + "')"; + return "LISTAGG(DISTINCT " + columnName + ", '" + argsStr + "')"; } - return "STRING_AGG(DISTINCT " + columnName + ", ',')"; + return "LISTAGG(DISTINCT " + columnName + ", ',')"; case "CONCAT_AGG": if (StringUtils.isNotBlank(argsStr)) { - return "STRING_AGG(" + columnName + ", '" + argsStr + "')"; + return "LISTAGG(" + columnName + ", '" + argsStr + "')"; } - return "STRING_AGG(" + columnName + ", ',')"; + return "LISTAGG(" + columnName + ", ',')"; case "CONCAT_AGG_ID": if (StringUtils.isNotBlank(argsStr)) { - return "STRING_AGG(" + columnName + ", '" + argsStr + "')"; + return "LISTAGG(" + columnName + ", '" + argsStr + "')"; } - return "STRING_AGG(" + columnName + ", ',')"; + return "LISTAGG(" + columnName + ", ',')"; case "SPLIT_DISTINCT_CONCAT": if (StringUtils.isNotBlank(argsStr)) { String[] splitArgs = argsStr.split(","); @@ -767,23 +767,23 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { String separator = splitArgs[0].trim(); String delimiter = splitArgs[1].trim(); String limit = splitArgs[2].trim(); - return "STRING_AGG(DISTINCT REGEXP_SPLIT(" + columnName + ", '" + delimiter + "'), '" + separator + "') LIMIT " + limit; + return "LISTAGG(DISTINCT REGEXP_SPLIT(" + columnName + ", '" + delimiter + "'), '" + separator + "') LIMIT " + limit; } } return columnName; //自定添加方法 case "MODE_WITH_GROUP": - return "MODE() WITHIN GROUP (ORDER BY " + columnName + ")"; + + return "MAX( " + columnName + ") KEEP (DENSE_RANK FIRST ORDER BY dest_ip DESC)"; // 聚合函数（兼容旧代码） case "ARRAY_AGG": - - return "ARRAY_AGG(DISTINCT " + columnName + ")"; + return "JSONB_AGG(DISTINCT " + columnName + ")"; case "STRING_AGG": if (StringUtils.isNotBlank(argsStr)) { - return "STRING_AGG(" + columnSafeWrap(columnName) + ", " + argsStr + ")"; + return "LISTAGG(" + columnSafeWrap(columnName) + ", " + argsStr + ")"; } - return "STRING_AGG(DISTINCT " + columnName + ", ',')"; + return "LISTAGG(DISTINCT " + columnName + ", ',')"; // 时间函数 case "YEAR": @@ -847,7 +847,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { case "TO_CHAR": return "TO_CHAR(" + columnName + ", 'YYYYMMDD')"; case "HOST": - return "HOST(" + columnName + ")::text"; + return columnName; default: return functionName + "(" + columnName + ")"; } @@ -938,12 +938,7 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { * 列名安全包装（处理类型转换） */ private String columnSafeWrap(String columnName) { - if (columnName.toLowerCase().contains("ip")) { - return "host(" + columnName + ")::text"; - } - if (columnName.toLowerCase().contains("port")) { - return columnName + "::int4"; - } + // DM不需要类型转换，直接返回列名 return columnName; } diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/util/JsonbUtil.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/util/JsonbUtil.java index a3d0ca5..3b05f88 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/util/JsonbUtil.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/util/JsonbUtil.java @@ -22,6 +22,18 @@ public class JsonbUtil { return null; } + // 达梦数据库：TEXT/CLOB 列返回 dm.jdbc.driver.DmdbNClob 对象， + // toString() 只返回对象引用而非实际内容，必须通过 Clob 接口读取 + if (value instanceof java.sql.Clob) { + try { + java.sql.Clob clob = (java.sql.Clob) value; + long length = clob.length(); + value = length > 0 ? clob.getSubString(1, (int) length) : ""; + } catch (Exception e) { + value = value.toString(); + } + } + // 如果已经是字符串，直接返回 if (value instanceof String) { String strValue = (String) value; diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/config/CacheConfig.java b/haobang-security-dm/syslog-consumer/src/main/java/com/config/CacheConfig.java index 4daedcf..6ac6023 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/config/CacheConfig.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/config/CacheConfig.java @@ -1,8 +1,16 @@ package com.config; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.BeanDescription; +import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.jsontype.BasicPolymorphicTypeValidator; import com.fasterxml.jackson.databind.jsontype.PolymorphicTypeValidator; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.BeanSerializerModifier; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.EnableCaching; @@ -15,6 +23,7 @@ import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSeriali import org.springframework.data.redis.serializer.RedisSerializationContext; import org.springframework.data.redis.serializer.StringRedisSerializer; import com.fasterxml.jackson.databind.DeserializationFeature; +import java.io.IOException; import java.time.Duration; import java.util.Collections; import org.springframework.context.annotation.Primary; @@ -35,6 +44,33 @@ public class CacheConfig { // 禁用将日期序列化为时间戳 mapper.disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + // 注册达梦 JDBC 安全序列化器 — 拦截 dm.jdbc.* 类，直接返回 null 防止循环引用 StackOverflow + SimpleModule dmSafeModule = new SimpleModule("dm-safe"); + dmSafeModule.setSerializerModifier(new BeanSerializerModifier() { + @Override + public JsonSerializer modifySerializer(SerializationConfig config, + BeanDescription beanDesc, + JsonSerializer serializer) { + if (beanDesc.getBeanClass().getName().startsWith("dm.jdbc.")) { + return new JsonSerializer() { + @Override + public void serialize(Object value, JsonGenerator gen, + SerializerProvider serializers) throws IOException { + gen.writeNull(); + } + + @Override + public void serializeWithType(Object value, JsonGenerator gen, + SerializerProvider serializers, TypeSerializer typeSer) throws IOException { + gen.writeNull(); + } + }; + } + return serializer; + } + }); + mapper.registerModule(dmSafeModule); + // 启用类型信息，解决 LinkedHashMap 转换问题 PolymorphicTypeValidator ptv = BasicPolymorphicTypeValidator.builder() .allowIfSubType("com.common.entity.") // 允许你的实体类包 diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/config/RedisConfig.java b/haobang-security-dm/syslog-consumer/src/main/java/com/config/RedisConfig.java index 3bd7321..134af0f 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/config/RedisConfig.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/config/RedisConfig.java @@ -1,6 +1,14 @@ package com.config; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.BeanDescription; +import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.BeanSerializerModifier; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -9,6 +17,8 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; +import java.io.IOException; + @Configuration public class RedisConfig { @@ -23,6 +33,32 @@ public class RedisConfig { ObjectMapper mapper = new ObjectMapper(); mapper.registerModule(new JavaTimeModule()); + // 注册达梦 JDBC 安全序列化器 — 拦截 dm.jdbc.* 类，直接返回 null 防止循环引用 StackOverflow + SimpleModule dmSafeModule = new SimpleModule("dm-safe"); + dmSafeModule.setSerializerModifier(new BeanSerializerModifier() { + @Override + public JsonSerializer modifySerializer(SerializationConfig config, + BeanDescription beanDesc, + JsonSerializer serializer) { + if (beanDesc.getBeanClass().getName().startsWith("dm.jdbc.")) { + return new JsonSerializer() { + @Override + public void serialize(Object value, JsonGenerator gen, + SerializerProvider serializers) throws IOException { + gen.writeNull(); + } + + @Override + public void serializeWithType(Object value, JsonGenerator gen, + SerializerProvider serializers, TypeSerializer typeSer) throws IOException { + gen.writeNull(); + } + }; + } + return serializer; + } + }); + mapper.registerModule(dmSafeModule); mapper.activateDefaultTyping( mapper.getPolymorphicTypeValidator(), ObjectMapper.DefaultTyping.NON_FINAL diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/config/WebConfig.java b/haobang-security-dm/syslog-consumer/src/main/java/com/config/WebConfig.java index 5f52013..7330f38 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/config/WebConfig.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/config/WebConfig.java @@ -1,8 +1,16 @@ package com.config; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.BeanDescription; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.BeanSerializerModifier; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -10,6 +18,7 @@ import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import java.io.IOException; import java.util.List; @Configuration @@ -28,6 +37,33 @@ public class WebConfig implements WebMvcConfigurer { // 禁用将日期序列化为时间戳 mapper.disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + // 注册达梦 JDBC 安全序列化器 — 拦截 dm.jdbc.* 类，直接返回 null 防止循环引用 StackOverflow + SimpleModule dmSafeModule = new SimpleModule("dm-safe"); + dmSafeModule.setSerializerModifier(new BeanSerializerModifier() { + @Override + public JsonSerializer modifySerializer(SerializationConfig config, + BeanDescription beanDesc, + JsonSerializer serializer) { + if (beanDesc.getBeanClass().getName().startsWith("dm.jdbc.")) { + return new JsonSerializer() { + @Override + public void serialize(Object value, JsonGenerator gen, + SerializerProvider serializers) throws IOException { + gen.writeNull(); + } + + @Override + public void serializeWithType(Object value, JsonGenerator gen, + SerializerProvider serializers, TypeSerializer typeSer) throws IOException { + gen.writeNull(); + } + }; + } + return serializer; + } + }); + mapper.registerModule(dmSafeModule); + // 忽略未知属性 mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application-dev.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application-dev.properties index a468313..8794954 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application-dev.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application-dev.properties @@ -9,7 +9,6 @@ server.error.include-binding-errors=always #run.environment: dev|test|pro server.run.environment=dev - # Syslog Server Configuration syslog.tcp.port=514 syslog.udp.port=514 @@ -17,15 +16,15 @@ syslog.max.frame.length=65536 syslog.buffer.size=1000 syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # InfluxDB 2.7 Configuration -influxdb.url=http://192.168.222.131:8086 -influxdb.token=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA== -influxdb.org=yelang +influxdb.url=http://192.168.4.99:8087 +influxdb.token=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg== +influxdb.org=influxdb influxdb.bucket=yelangbucket influxdb.batch.size=1000 influxdb.flush.interval=1000 influxdb.retry.attempts=3 influxdb.retry.delay=1000 -# InfluxDB 2.7 ӳʱ +# InfluxDB 2.7 连接超时配置 influxdb.connection.timeout=30s influxdb.connection.read-timeout=30s influxdb.connection.write-timeout=60s @@ -36,16 +35,16 @@ app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys -spring.datasource.username=postgres -spring.datasource.password=TnLanWaidYSwTSG5 -spring.datasource.driver-class-name=org.postgresql.Driver - +spring.datasource.url=jdbc:dm://192.168.4.99:5237 +spring.datasource.username=SYSDBA +spring.datasource.password=caZ2TcmXNSW8L2Ap +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity -#mybatis handler +#mybatis handler 类 mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 mybatis.configuration.map-underscore-to-camel-case=true @@ -53,12 +52,12 @@ mybatis.type-handlers-package=com.Modules.etl.handler mybatis-plus.configuration.map-underscore-to-camel-case=true mybatis-plus.type-handlers-package=com.Modules.etl.handler # kafka Configuration -spring.kafka.consumer.bootstrap-servers=192.168.222.130:9092 -spring.kafka.consumer.group-id=test-group-app +spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092 +spring.kafka.consumer.group-id=test-group spring.kafka.consumer.auto-offset-reset=latest spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.auto-commit-interval=1000 -spring.kafka.consumer.topic=test-topic +spring.kafka.consumer.topic=agent-syslog-topic spring.kafka.consumer.max-poll-records=1000 spring.kafka.consumer.properties.max.poll.interval.ms=300000 @@ -71,23 +70,22 @@ spring.kafka.listener.ack-mode= manual spring.kafka.listener.concurrency= 2 spring.kafka.listener.type=batch - -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# +# 开发环境缓存配置 spring.redis.host=localhost spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password= spring.redis.database=0 spring.redis.timeout=2000 @@ -96,11 +94,11 @@ spring.redis.lettuce.pool.max-active=8 spring.redis.lettuce.pool.max-wait=-1 spring.redis.lettuce.pool.max-idle=8 spring.redis.lettuce.pool.min-idle=0 -# ʱ϶̣ +# 开发环境缓存时间较短，方便调试 spring.cache.redis.time-to-live=600000 -# Ӧô +# 应用处理器配置 app.processor.thread-pool.core-pool-size=10 app.processor.thread-pool.max-pool-size=20 app.processor.thread-pool.queue-capacity=2000 @@ -109,20 +107,20 @@ app.processor.batch-size=100 app.processor.process-timeout-ms=30000 -# Elasticsearch -# Elasticsearchӵַ +# 配置 Elasticsearch +# Elasticsearch连接地址 spring.elasticsearch.uris=http://192.168.1.174:9200 -# Elasticsearch û +# 配置 Elasticsearch 用户名 spring.elasticsearch.username=CONTAINER_NAME -# Elasticsearch +# 配置 Elasticsearch 密码 spring.elasticsearch.password=t2NZCiajmdazxBrF -# ӳʱʱ +# 连接超时时间 spring.elasticsearch.connection-timeout=10s -# Socket ʱʱ +# Socket 超时时间 spring.elasticsearch.socket-timeout=30s -# ETL +# ETL配置 etl.batch.page-size=1000 etl.batch.insert-batch-size=500 etl.schedule.cron=0 0 2 * * ? @@ -136,53 +134,54 @@ spring.datasource.hikari.minimum-idle=5 spring.datasource.hikari.connection-timeout=30000 spring.datasource.hikari.idle-timeout=600000 spring.datasource.hikari.max-lifetime=900000 -spring.datasource.hikari.connection-test-query=SELECT 1 +spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer spring.datasource.hikari.auto-commit=false -spring.datasource.hikari.schema=public -# +# 关联分析规则配置 analysis.realtime.enabled= true -# 룩 - Ĭ10 +# 检查间隔（秒） - 默认10秒 analysis.realtime.check-interval-seconds: 10 # ============================================ -# ̽API +# 探针联动API配置 # ============================================ -# API-KEY֤32λʹɵԿ +# API-KEY认证（32位，建议使用随机生成的密钥） interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 -# APIӿڻURLsyslog-serveã -interlocking.api.base-url=http://192.168.222.131:8089/xdrservice/interlocking +# API接口基础URL（供syslog-serve调用） +interlocking.api.base-url=http://localhost:8089/xdrservice/interlocking # ============================================ -# 澯 +# 告警健康检查配置 # ============================================ -# 澯ֵСʱ -alarm.health-check.alarm-hours=2 -# 澯־ֵСʱ -alarm.health-check.alarm-visit-hours=4 -# ǷöʱѲ +# 告警表无数据阈值（小时） +alarm.health-check.alarm-hours=4 +# 告警日志表无数据阈值（小时） +alarm.health-check.alarm-visit-hours=2 +# 是否启用定时巡检 alarm.health-check.enabled=true - # ============================================ -# ̽ +# 探针心跳检测配置 # ============================================ -# Ƿ +# 是否启用心跳检测 probe.heartbeat.enabled=true -# ֵ̽ӣʱδյжΪ +# 探针离线阈值（分钟），超过此时间未收到心跳则判定为离线 probe.heartbeat.offline-threshold-minutes=10 -# ״̬CronʽĬÿ10ӣ +# 状态检查Cron表达式（默认每10分钟） probe.status.check.cron=0 */10 * * * ? -# ̽⻧ID +# 探针租户ID probe.heartbeat.tenant-id=000000 -# ʷ +# 心跳历史保留天数 probe.heartbeat.history.keep-days=10 -# Ƿʷ +# 是否启用历史清理 probe.heartbeat.history.cleanup-enabled=true -# ʷCronʽĬÿ賿1㣩 -probe.history.cleanup.cron=0 0 1 * * ? \ No newline at end of file +# 历史清理Cron表达式（默认每天凌晨1点） +probe.history.cleanup.cron=0 0 1 * * ? + +# Jackson 配置 — 修复达梦 JDBC 驱动循环引用导致 JSON 序列化 StackOverflow +spring.jackson.serialization.fail-on-self-references=false \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application-pre-nw.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application-pre-nw.properties index e0060c9..0e5812c 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application-pre-nw.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application-pre-nw.properties @@ -26,7 +26,7 @@ influxdb.batch.size=1000 influxdb.flush.interval=1000 influxdb.retry.attempts=3 influxdb.retry.delay=1000 -# InfluxDB 2.7 ӳʱ +# InfluxDB 2.7 连接超时配置 influxdb.connection.timeout=30s influxdb.connection.read-timeout=30s influxdb.connection.write-timeout=60s @@ -37,15 +37,15 @@ app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://10.150 81.209:5432/ecosys -spring.datasource.username=postgres +spring.datasource.url=jdbc:dm://192.163.4.99:5237/DM_ecosys +spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap -spring.datasource.driver-class-name=org.postgresql.Driver - +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity -#mybatis handler +#mybatis handler 类 mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 mybatis.configuration.map-underscore-to-camel-case=true @@ -72,23 +72,23 @@ spring.kafka.listener.concurrency= 2 spring.kafka.listener.type=batch -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# +# 生产环境缓存配置 spring.redis.host=192.168.4.26 spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password=123456 spring.redis.database=0 spring.redis.timeout=5000 @@ -99,10 +99,10 @@ spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 -# Ӧô +# 应用处理器配置 app.processor.thread-pool.core-pool-size=10 app.processor.thread-pool.max-pool-size=20 app.processor.thread-pool.queue-capacity=2000 @@ -111,20 +111,20 @@ app.processor.batch-size=100 app.processor.process-timeout-ms=30000 -# Elasticsearch -# Elasticsearchӵַ +# 配置 Elasticsearch +# Elasticsearch连接地址 spring.elasticsearch.uris=http://192.168.1.174:9200 -# Elasticsearch û +# 配置 Elasticsearch 用户名 spring.elasticsearch.username=CONTAINER_NAME -# Elasticsearch +# 配置 Elasticsearch 密码 spring.elasticsearch.password=t2NZCiajmdazxBrF -# ӳʱʱ +# 连接超时时间 spring.elasticsearch.connection-timeout=10s -# Socket ʱʱ +# Socket 超时时间 spring.elasticsearch.socket-timeout=30s -# ETL +# ETL配置 etl.batch.page-size=1000 etl.batch.insert-batch-size=500 etl.schedule.cron=0 0 2 * * ? @@ -138,51 +138,53 @@ spring.datasource.hikari.minimum-idle=5 spring.datasource.hikari.connection-timeout=30000 spring.datasource.hikari.idle-timeout=600000 spring.datasource.hikari.max-lifetime=900000 -spring.datasource.hikari.connection-test-query=SELECT 1 +spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer spring.datasource.hikari.auto-commit=false -spring.datasource.hikari.schema=public -# +# 关联分析规则配置 analysis.realtime.enabled= true -# 룩 - Ĭ10 +# 检查间隔（秒） - 默认10秒 analysis.realtime.check-interval-seconds: 10 # ============================================ -# ̽API +# 探针联动API配置 # ============================================ -# API-KEY֤32λʹɵԿ +# API-KEY认证（32位，建议使用随机生成的密钥） interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 -# APIӿڻURLsyslog-serveã +# API接口基础URL（供syslog-serve调用） interlocking.api.base-url=http://10.150 81.210:8089/xdrservice/interlocking # ============================================ -# 澯 +# 告警健康检查配置 # ============================================ -# 澯ֵСʱ +# 告警表无数据阈值（小时） alarm.health-check.alarm-hours=2 -# 澯־ֵСʱ +# 告警日志表无数据阈值（小时） alarm.health-check.alarm-visit-hours=4 -# ǷöʱѲ +# 是否启用定时巡检 alarm.health-check.enabled=true # ============================================ -# ̽ +# 探针心跳检测配置 # ============================================ -# Ƿ +# 是否启用心跳检测 probe.heartbeat.enabled=true -# ֵ̽ӣʱδյжΪ +# 探针离线阈值（分钟），超过此时间未收到心跳则判定为离线 probe.heartbeat.offline-threshold-minutes=10 -# ״̬CronʽĬÿ10ӣ +# 状态检查Cron表达式（默认每10分钟） probe.status.check.cron=0 */10 * * * ? -# ̽⻧ID +# 探针租户ID probe.heartbeat.tenant-id=000000 -# ʷ +# 心跳历史保留天数 probe.heartbeat.history.keep-days=10 -# Ƿʷ +# 是否启用历史清理 probe.heartbeat.history.cleanup-enabled=true -# ʷCronʽĬÿ賿1㣩 -probe.history.cleanup.cron=0 0 1 * * ? \ No newline at end of file +# 历史清理Cron表达式（默认每天凌晨1点） +probe.history.cleanup.cron=0 0 1 * * ? + +# Jackson 配置 — 修复达梦 JDBC 驱动循环引用导致 JSON 序列化 StackOverflow +spring.jackson.serialization.fail-on-self-references=false \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application-prod-zc.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application-prod-zc.properties index 597fdfa..fe53bf5 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application-prod-zc.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application-prod-zc.properties @@ -25,7 +25,7 @@ influxdb.batch.size=1000 influxdb.flush.interval=1000 influxdb.retry.attempts=3 influxdb.retry.delay=1000 -# InfluxDB 2.7 ӳʱ +# InfluxDB 2.7 连接超时配置 influxdb.connection.timeout=30s influxdb.connection.read-timeout=30s influxdb.connection.write-timeout=60s @@ -36,16 +36,17 @@ app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://10.11.2.141:5432/ecosys -spring.datasource.username=ecosys -spring.datasource.password=wsYDPjrpNZPrkPrR -spring.datasource.driver-class-name=org.postgresql.Driver +spring.datasource.url=jdbc:dm://192.168.4.99:5237 +spring.datasource.username=SYSDBA +spring.datasource.password=caZ2TcmXNSW8L2Ap +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity -#mybatis handler +#mybatis handler 类 mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 mybatis.configuration.map-underscore-to-camel-case=true @@ -73,23 +74,23 @@ spring.kafka.listener.ack-mode= manual spring.kafka.listener.concurrency= 2 spring.kafka.listener.type=batch -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# +# 生产环境缓存配置 spring.redis.host=10.11.2.142 spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password=redis_edP6N6 spring.redis.database=0 spring.redis.timeout=5000 @@ -99,10 +100,10 @@ spring.redis.lettuce.pool.max-active=20 spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 -# Ӧô +# 应用处理器配置 app.processor.thread-pool.core-pool-size=10 app.processor.thread-pool.max-pool-size=20 app.processor.thread-pool.queue-capacity=2000 @@ -110,20 +111,20 @@ app.processor.thread-pool.keep-alive-seconds=60 app.processor.batch-size=100 app.processor.process-timeout-ms=30000 -# Elasticsearch -# Elasticsearchӵַ +# 配置 Elasticsearch +# Elasticsearch连接地址 spring.elasticsearch.uris=http://192.168.1.174:9200 -# Elasticsearch û +# 配置 Elasticsearch 用户名 spring.elasticsearch.username=CONTAINER_NAME -# Elasticsearch +# 配置 Elasticsearch 密码 spring.elasticsearch.password=t2NZCiajmdazxBrF -# ӳʱʱ +# 连接超时时间 spring.elasticsearch.connection-timeout=10s -# Socket ʱʱ +# Socket 超时时间 spring.elasticsearch.socket-timeout=30s -# ETL +# ETL配置 etl.batch.page-size=1000 etl.batch.insert-batch-size=500 etl.schedule.cron=0 0 2 * * ? @@ -137,15 +138,14 @@ spring.datasource.hikari.minimum-idle=5 spring.datasource.hikari.connection-timeout=30000 spring.datasource.hikari.idle-timeout=600000 spring.datasource.hikari.max-lifetime=900000 -spring.datasource.hikari.connection-test-query=SELECT 1 +spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer spring.datasource.hikari.auto-commit=false -spring.datasource.hikari.schema=public -# +# 关联分析规则配置 analysis.realtime.enabled= true -# 룩 - Ĭ10 +# 检查间隔（秒） - 默认10秒 analysis.realtime.check-interval-seconds: 10 \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application-prod.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application-prod.properties index ec93600..ffb5c5e 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application-prod.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application-prod.properties @@ -18,15 +18,15 @@ syslog.buffer.size=1000 syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # InfluxDB 2.7 Configuration -influxdb.url=http://192.168.4.26:8087 -influxdb.token=LFjXZyRxTf1V84oN-wwjhSjS4qIK-ZMoHzQJB67ir3qHNSBVJbMcTkPuNmM0cNxvzFEDWLYNzrz1VJKMitY5hw== +influxdb.url=http://192.168.4.99:8087 +influxdb.token=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg== influxdb.org=influxdb influxdb.bucket=yelangbucket influxdb.batch.size=1000 influxdb.flush.interval=1000 influxdb.retry.attempts=3 influxdb.retry.delay=1000 -# InfluxDB 2.7 ӳʱ +# InfluxDB 2.7 连接超时配置 influxdb.connection.timeout=30s influxdb.connection.read-timeout=30s influxdb.connection.write-timeout=60s @@ -37,15 +37,15 @@ app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://192.168.4.26:5432/ecosys -spring.datasource.username=postgres +spring.datasource.url=jdbc:dm://192.168.4.99:5237 +spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap -spring.datasource.driver-class-name=org.postgresql.Driver - +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity -#mybatis handler +#mybatis handler 类 mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 mybatis.configuration.map-underscore-to-camel-case=true @@ -54,7 +54,7 @@ mybatis-plus.configuration.map-underscore-to-camel-case=true mybatis-plus.type-handlers-package=com.Modules.etl.handler # kafka Configuration -spring.kafka.consumer.bootstrap-servers=192.168.4.26:9092 +spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092 spring.kafka.consumer.group-id=agent-syslog-group spring.kafka.consumer.auto-offset-reset=latest spring.kafka.consumer.enable-auto-commit=false @@ -72,24 +72,24 @@ spring.kafka.listener.concurrency= 2 spring.kafka.listener.type=batch -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# -spring.redis.host=192.168.4.26 +# 生产环境缓存配置 +spring.redis.host=192.168.4.99 spring.redis.port=6379 -# 루û룬ʡԣ -spring.redis.password=123456 +# 密码（如果没有设置密码，可以省略） +spring.redis.password=redis_GdGWte spring.redis.database=0 spring.redis.timeout=5000 #spring.redis.password=${REDIS_PASSWORD:default_prod_password} @@ -99,10 +99,10 @@ spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 -# Ӧô +# 应用处理器配置 app.processor.thread-pool.core-pool-size=10 app.processor.thread-pool.max-pool-size=20 app.processor.thread-pool.queue-capacity=2000 @@ -111,20 +111,20 @@ app.processor.batch-size=100 app.processor.process-timeout-ms=30000 -# Elasticsearch -# Elasticsearchӵַ +# 配置 Elasticsearch +# Elasticsearch连接地址 spring.elasticsearch.uris=http://192.168.1.174:9200 -# Elasticsearch û +# 配置 Elasticsearch 用户名 spring.elasticsearch.username=CONTAINER_NAME -# Elasticsearch +# 配置 Elasticsearch 密码 spring.elasticsearch.password=t2NZCiajmdazxBrF -# ӳʱʱ +# 连接超时时间 spring.elasticsearch.connection-timeout=10s -# Socket ʱʱ +# Socket 超时时间 spring.elasticsearch.socket-timeout=30s -# ETL +# ETL配置 etl.batch.page-size=1000 etl.batch.insert-batch-size=500 etl.schedule.cron=0 0 2 * * ? @@ -138,51 +138,53 @@ spring.datasource.hikari.minimum-idle=5 spring.datasource.hikari.connection-timeout=30000 spring.datasource.hikari.idle-timeout=600000 spring.datasource.hikari.max-lifetime=900000 -spring.datasource.hikari.connection-test-query=SELECT 1 +spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer spring.datasource.hikari.auto-commit=false -spring.datasource.hikari.schema=public -# +# 关联分析规则配置 analysis.realtime.enabled= true -# 룩 - Ĭ10 +# 检查间隔（秒） - 默认10秒 analysis.realtime.check-interval-seconds: 10 # ============================================ -# ̽API +# 探针联动API配置 # ============================================ -# API-KEY֤32λʹɵԿ +# API-KEY认证（32位，建议使用随机生成的密钥） interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 -# APIӿڻURLsyslog-serveã -interlocking.api.base-url=http://localhost:8089/xdrservice/interlocking +# API接口基础URL（供syslog-serve调用） +interlocking.api.base-url=http://192.168.4.99:8089/xdrservice/interlocking # ============================================ -# 澯 +# 告警健康检查配置 # ============================================ -# 澯ֵСʱ +# 告警表无数据阈值（小时） alarm.health-check.alarm-hours=2 -# 澯־ֵСʱ +# 告警日志表无数据阈值（小时） alarm.health-check.alarm-visit-hours=4 -# ǷöʱѲ +# 是否启用定时巡检 alarm.health-check.enabled=true # ============================================ -# ̽ +# 探针心跳检测配置 # ============================================ -# Ƿ +# 是否启用心跳检测 probe.heartbeat.enabled=true -# ֵ̽ӣʱδյжΪ +# 探针离线阈值（分钟），超过此时间未收到心跳则判定为离线 probe.heartbeat.offline-threshold-minutes=10 -# ״̬CronʽĬÿ10ӣ +# 状态检查Cron表达式（默认每10分钟） probe.status.check.cron=0 */10 * * * ? -# ̽⻧ID +# 探针租户ID probe.heartbeat.tenant-id=000000 -# ʷ +# 心跳历史保留天数 probe.heartbeat.history.keep-days=10 -# Ƿʷ +# 是否启用历史清理 probe.heartbeat.history.cleanup-enabled=true -# ʷCronʽĬÿ賿1㣩 -probe.history.cleanup.cron=0 0 1 * * ? \ No newline at end of file +# 历史清理Cron表达式（默认每天凌晨1点） +probe.history.cleanup.cron=0 0 1 * * ? + +# Jackson 配置 — 修复达梦 JDBC 驱动循环引用导致 JSON 序列化 StackOverflow +spring.jackson.serialization.fail-on-self-references=false \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application-test.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application-test.properties index 4cfb03b..0274086 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application-test.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application-test.properties @@ -33,10 +33,11 @@ app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://192.168.4.32:5432/ecosys -spring.datasource.username=user_eSER8N -spring.datasource.password=password_QCYKj6 -spring.datasource.driver-class-name=org.postgresql.Driver +spring.datasource.url=jdbc:dm://192.163.4.99:5237/DM_ecosys +spring.datasource.username=SYSDBA +spring.datasource.password=caZ2TcmXNSW8L2Ap +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml @@ -54,23 +55,23 @@ spring.kafka.consumer.topic=agent-syslog-topic -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# +# 生产环境缓存配置 spring.redis.host=192.168.4.32 spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password=redis_edP6N6 spring.redis.database=0 spring.redis.timeout=5000 @@ -81,5 +82,5 @@ spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application.properties index 57af480..ffb5c5e 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application.properties @@ -7,7 +7,8 @@ server.error.include-message=always server.error.include-binding-errors=always #run.environment: dev|test|pro -server.run.environment=dev +server.run.environment=pro + # Syslog Server Configuration syslog.tcp.port=514 @@ -15,16 +16,17 @@ syslog.udp.port=514 syslog.max.frame.length=65536 syslog.buffer.size=1000 syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a + # InfluxDB 2.7 Configuration -influxdb.url=http://192.168.222.131:8086 -influxdb.token=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA== -influxdb.org=yelang +influxdb.url=http://192.168.4.99:8087 +influxdb.token=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg== +influxdb.org=influxdb influxdb.bucket=yelangbucket influxdb.batch.size=1000 influxdb.flush.interval=1000 influxdb.retry.attempts=3 influxdb.retry.delay=1000 -# InfluxDB 2.7 ӳʱ +# InfluxDB 2.7 连接超时配置 influxdb.connection.timeout=30s influxdb.connection.read-timeout=30s influxdb.connection.write-timeout=60s @@ -35,71 +37,72 @@ app.metrics.enabled=true #database Configuration -spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys -spring.datasource.username=postgres -spring.datasource.password=TnLanWaidYSwTSG5 -spring.datasource.driver-class-name=org.postgresql.Driver - +spring.datasource.url=jdbc:dm://192.168.4.99:5237 +spring.datasource.username=SYSDBA +spring.datasource.password=caZ2TcmXNSW8L2Ap +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity - -#mybatis handler +#mybatis handler 类 mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 mybatis.configuration.map-underscore-to-camel-case=true mybatis.type-handlers-package=com.Modules.etl.handler mybatis-plus.configuration.map-underscore-to-camel-case=true mybatis-plus.type-handlers-package=com.Modules.etl.handler + # kafka Configuration -spring.kafka.consumer.bootstrap-servers=192.168.222.130:9092 -spring.kafka.consumer.group-id=test-group-app +spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092 +spring.kafka.consumer.group-id=agent-syslog-group spring.kafka.consumer.auto-offset-reset=latest spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.auto-commit-interval=1000 -spring.kafka.consumer.topic=test-topic +spring.kafka.consumer.topic=agent-syslog-topic + spring.kafka.consumer.max-poll-records=1000 spring.kafka.consumer.properties.max.poll.interval.ms=300000 spring.kafka.consumer.properties.session.timeout.ms=45000 -#spring.kafka.consumer.key-deserializer: org.apache.kafka.common.serialization.StringDeserializer -#spring.kafka.consumer.value-deserializer: org.apache.kafka.common.serialization.StringDeserializer spring.kafka.consumer.fetch-min-size= 1048576 spring.kafka.listener.ack-mode= manual spring.kafka.listener.concurrency= 2 spring.kafka.listener.type=batch -# ʱ +# 定时任务配置 spring.task.scheduling.pool.size=10 -# ־ +# 日志配置 logging.level.com.common.schedule=INFO logging.level.com.common.service=INFO -# +# 分区表检查配置 partition.check.tomorrow.enabled=true partition.check.future.days=7 partition.auto.create=true -# -spring.redis.host=localhost + +# 生产环境缓存配置 +spring.redis.host=192.168.4.99 spring.redis.port=6379 -# 루û룬ʡԣ -spring.redis.password= +# 密码（如果没有设置密码，可以省略） +spring.redis.password=redis_GdGWte spring.redis.database=0 -spring.redis.timeout=2000 +spring.redis.timeout=5000 +#spring.redis.password=${REDIS_PASSWORD:default_prod_password} -spring.redis.lettuce.pool.max-active=8 -spring.redis.lettuce.pool.max-wait=-1 -spring.redis.lettuce.pool.max-idle=8 -spring.redis.lettuce.pool.min-idle=0 -# ʱ϶̣ -spring.cache.redis.time-to-live=600000 +spring.redis.lettuce.pool.max-active=20 +spring.redis.lettuce.pool.max-wait=5000 +spring.redis.lettuce.pool.max-idle=10 +spring.redis.lettuce.pool.min-idle=5 +# 生产环境缓存时间较长 +spring.cache.redis.time-to-live=3600000 -# Ӧô +# 应用处理器配置 app.processor.thread-pool.core-pool-size=10 app.processor.thread-pool.max-pool-size=20 app.processor.thread-pool.queue-capacity=2000 @@ -108,20 +111,20 @@ app.processor.batch-size=100 app.processor.process-timeout-ms=30000 -# Elasticsearch -# Elasticsearchӵַ +# 配置 Elasticsearch +# Elasticsearch连接地址 spring.elasticsearch.uris=http://192.168.1.174:9200 -# Elasticsearch û +# 配置 Elasticsearch 用户名 spring.elasticsearch.username=CONTAINER_NAME -# Elasticsearch +# 配置 Elasticsearch 密码 spring.elasticsearch.password=t2NZCiajmdazxBrF -# ӳʱʱ +# 连接超时时间 spring.elasticsearch.connection-timeout=10s -# Socket ʱʱ +# Socket 超时时间 spring.elasticsearch.socket-timeout=30s -# ETL +# ETL配置 etl.batch.page-size=1000 etl.batch.insert-batch-size=500 etl.schedule.cron=0 0 2 * * ? @@ -135,52 +138,53 @@ spring.datasource.hikari.minimum-idle=5 spring.datasource.hikari.connection-timeout=30000 spring.datasource.hikari.idle-timeout=600000 spring.datasource.hikari.max-lifetime=900000 -spring.datasource.hikari.connection-test-query=SELECT 1 +spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer spring.datasource.hikari.auto-commit=false -spring.datasource.hikari.schema=public -# + +# 关联分析规则配置 analysis.realtime.enabled= true -# 룩 - Ĭ10 +# 检查间隔（秒） - 默认10秒 analysis.realtime.check-interval-seconds: 10 - - # ============================================ -# ̽API +# 探针联动API配置 # ============================================ -# API-KEY֤32λʹɵԿ +# API-KEY认证（32位，建议使用随机生成的密钥） interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 -# APIӿڻURLsyslog-serveã -interlocking.api.base-url=http://localhost:8089/xdrservice/interlocking +# API接口基础URL（供syslog-serve调用） +interlocking.api.base-url=http://192.168.4.99:8089/xdrservice/interlocking # ============================================ -# 澯 +# 告警健康检查配置 # ============================================ -# 澯ֵСʱ -alarm.health-check.alarm-hours=4 -# 澯־ֵСʱ -alarm.health-check.alarm-visit-hours=2 -# ǷöʱѲ +# 告警表无数据阈值（小时） +alarm.health-check.alarm-hours=2 +# 告警日志表无数据阈值（小时） +alarm.health-check.alarm-visit-hours=4 +# 是否启用定时巡检 alarm.health-check.enabled=true # ============================================ -# ̽ +# 探针心跳检测配置 # ============================================ -# Ƿ +# 是否启用心跳检测 probe.heartbeat.enabled=true -# ֵ̽ӣʱδյжΪ +# 探针离线阈值（分钟），超过此时间未收到心跳则判定为离线 probe.heartbeat.offline-threshold-minutes=10 -# ״̬CronʽĬÿ10ӣ +# 状态检查Cron表达式（默认每10分钟） probe.status.check.cron=0 */10 * * * ? -# ̽⻧ID +# 探针租户ID probe.heartbeat.tenant-id=000000 -# ʷ +# 心跳历史保留天数 probe.heartbeat.history.keep-days=10 -# Ƿʷ +# 是否启用历史清理 probe.heartbeat.history.cleanup-enabled=true -# ʷCronʽĬÿ賿1㣩 -probe.history.cleanup.cron=0 0 1 * * ? \ No newline at end of file +# 历史清理Cron表达式（默认每天凌晨1点） +probe.history.cleanup.cron=0 0 1 * * ? + +# Jackson 配置 — 修复达梦 JDBC 驱动循环引用导致 JSON 序列化 StackOverflow +spring.jackson.serialization.fail-on-self-references=false \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/logback.xml b/haobang-security-dm/syslog-consumer/src/main/resources/logback.xml index 596186a..ef4c9d3 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/logback.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/logback.xml @@ -7,9 +7,9 @@ - logs/syslog-consumer.log + logs/syslog-consumer-dm.log - logs/syslog-consumer.%d{yyyy-MM-dd}.log + logs/syslog-consumer-dm.%d{yyyy-MM-dd}.log 1 diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml index 9402d7e..c09d908 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisAnalysisRuleMapper.xml @@ -1,4 +1,4 @@ - + @@ -53,7 +53,7 @@ SELECT FROM analysis_analysis_rule - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND del_flag = '0' @@ -62,7 +62,7 @@ SELECT FROM analysis_analysis_rule - WHERE rule_id = #{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id = #{ruleId, jdbcType=OTHER} AND del_flag = '0' @@ -70,11 +70,11 @@ UPDATE analysis_analysis_rule SET task_status = #{taskStatus}, - update_time = NOW() + update_time = SYSDATE ,update_by = #{updateBy} - WHERE rule_id = #{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id = #{ruleId, jdbcType=OTHER} diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisFieldMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisFieldMapper.xml index 9416648..45df145 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisFieldMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisFieldMapper.xml @@ -1,4 +1,4 @@ - + @@ -42,7 +42,7 @@ SELECT FROM analysis_field - WHERE rule_id = #{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id = #{ruleId, jdbcType=OTHER} AND del_flag = '0' ORDER BY id ASC @@ -52,7 +52,7 @@ SELECT FROM analysis_field - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND del_flag = '0' AND type IN ('measure', 'calc') ORDER BY id ASC @@ -63,7 +63,7 @@ SELECT FROM analysis_field - WHERE rule_id = #{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id = #{ruleId, jdbcType=OTHER} AND del_flag = '0' AND type = 'dimension' ORDER BY id ASC diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisFilterMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisFilterMapper.xml index b8f9dd6..c98190c 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisFilterMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisFilterMapper.xml @@ -1,4 +1,4 @@ - + @@ -13,9 +13,9 @@ - + - + @@ -42,7 +42,7 @@ SELECT FROM analysis_filter - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND del_flag = '0' diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByColumnMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByColumnMapper.xml index 64048ae..a5de79f 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByColumnMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByColumnMapper.xml @@ -1,4 +1,4 @@ - + @@ -44,7 +44,7 @@ create_time, update_time, create_by, update_by, remark, tenant_id, rule_id, group_id, field_id, sort FROM analysis_group_by_column - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND del_flag = '0' ORDER BY sort ASC diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByHavingMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByHavingMapper.xml index ca3cdb1..8cd682a 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByHavingMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByHavingMapper.xml @@ -1,4 +1,4 @@ - + @@ -13,9 +13,9 @@ - + - + @@ -53,7 +53,7 @@ h.update_time, h.create_by, h.update_by, h.remark, h.tenant_id FROM analysis_group_by_having h INNER JOIN analysis_group_by g ON h.group_by_id = g.id - WHERE g.rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE g.rule_id =#{ruleId, jdbcType=OTHER} AND h.del_flag = '0' AND g.del_flag = '0' ORDER BY h.id ASC diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByMapper.xml index 1e426e8..f9d0bba 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisGroupByMapper.xml @@ -1,10 +1,10 @@ - + - + @@ -33,7 +33,7 @@ SELECT FROM analysis_group_by - WHERE rule_id = #{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id = #{ruleId, jdbcType=OTHER} AND del_flag = '0' diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisTaskHistoryMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisTaskHistoryMapper.xml index 10eb841..99dccbe 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisTaskHistoryMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisTaskHistoryMapper.xml @@ -1,4 +1,4 @@ - + @@ -33,7 +33,7 @@ INSERT INTO analysis_task_history ( ) VALUES ( - #{id}, #{ruleId}::uuid, #{startTime}, #{endTime}, #{durationTime}, #{progressPercent}, + #{id}, #{ruleId}, #{startTime}, #{endTime}, #{durationTime}, #{progressPercent}, #{inputCount}, #{outputCount}, #{status}, #{createDept}, #{delFlag}, #{createTime}, #{updateTime}, #{createBy}, #{updateBy}, #{remark}, #{tenantId} ) @@ -48,7 +48,7 @@ input_count = #{inputCount}, output_count = #{outputCount}, status = #{status}, - update_time = NOW() + update_time = SYSDATE ,remark = #{remark} @@ -60,7 +60,7 @@ SELECT FROM analysis_task_history - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND del_flag = '0' ORDER BY create_time DESC @@ -73,7 +73,7 @@ SELECT FROM analysis_task_history - WHERE rule_id =#{ruleId, jdbcType=OTHER}::uuid + WHERE rule_id =#{ruleId, jdbcType=OTHER} AND status = #{status} AND del_flag = '0' ORDER BY create_time DESC diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisWhereConditionMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisWhereConditionMapper.xml index 541bebb..8b65090 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisWhereConditionMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/AnalysisWhereConditionMapper.xml @@ -1,4 +1,4 @@ - + @@ -29,7 +29,7 @@ SELECT FROM analysis_where_condition - WHERE rule_id = #{ruleId}::uuid + WHERE rule_id = #{ruleId} AND del_flag = '0' ORDER BY seq_num ASC @@ -39,7 +39,7 @@ SELECT FROM analysis_where_condition - WHERE rule_id = #{ruleId}::uuid + WHERE rule_id = #{ruleId} AND (parent_cond_id IS NULL OR parent_cond_id = 0) AND del_flag = '0' ORDER BY seq_num ASC diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceDeviceMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceDeviceMapper.xml index e6f8c47..97558fc 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceDeviceMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceDeviceMapper.xml @@ -46,11 +46,11 @@ - id, created_at::timestamp , updated_at::timestamp, deleted_at::timestamp, name, ip, device_group, device_type, - vendor, product_name, organization_id, last_receive_time::timestamp, agent_id, detail_id, - control_agent_id, license_start_time::timestamp, license_end_time::timestamp, is_monitoring, + id, created_at, updated_at, deleted_at, name, ip, device_group, device_type, + vendor, product_name, organization_id, last_receive_time, agent_id, detail_id, + control_agent_id, license_start_time, license_end_time, is_monitoring, security_scope_id, owner_id, ssh_config_id, status, created_by_id, decode_type, - miss_policy, tenant_id, create_time::timestamp, update_time::timestamp, create_by, update_by, del_flag, + miss_policy, tenant_id, create_time, update_time, create_by, update_by, del_flag, manager_name, today_parse_count, today_non_log_count, create_dept, device_collect_id diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceReceiveLogMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceReceiveLogMapper.xml index e07532e..d4ff680 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceReceiveLogMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceReceiveLogMapper.xml @@ -26,10 +26,10 @@ receive_time_str, syslog_message ) VALUES ( - COALESCE(#{createdAt}, NOW() AT TIME ZONE 'utc'), + COALESCE(#{createdAt}, SYSDATE), #{deviceCollectId}, #{deviceId}, - #{deviceIp}::inet, + #{deviceIp}, #{receiveTime}, #{receiveTimeStr}, #{syslogMessage} @@ -49,10 +49,10 @@ ) VALUES ( - COALESCE(#{item.createdAt}, NOW() AT TIME ZONE 'utc'), + COALESCE(#{item.createdAt}, SYSDATE), #{item.deviceCollectId}, #{item.deviceId}, - #{item.deviceIp}::inet, + #{item.deviceIp}, #{item.receiveTime}, #{item.receiveTimeStr}, #{item.syslogMessage} @@ -80,10 +80,10 @@ ORDER BY receive_time DESC - + @@ -105,7 +105,7 @@ AND device_collect_id = #{deviceCollectId} - AND device_ip >>= #{deviceIp}::inet + AND device_ip = #{deviceIp} AND receive_time >= #{receiveTime} @@ -128,7 +128,7 @@ AND device_collect_id = #{deviceCollectId} - AND device_ip >>= #{deviceIp}::inet + AND device_ip = #{deviceIp} AND receive_time >= #{receiveTime} diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DmColumnMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DmColumnMapper.xml index c72ee92..2eadc56 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DmColumnMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DmColumnMapper.xml @@ -1,4 +1,4 @@ - + @@ -65,13 +65,13 @@ @@ -80,13 +80,13 @@ @@ -137,7 +137,7 @@ UPDATE syslog_non_normal_message SET del_flag = '1', - update_time = NOW() + update_time = SYSDATE WHERE id IN #{item.id} diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalAlarmMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalAlarmMapper.xml index 004a2b3..7d22699 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalAlarmMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalAlarmMapper.xml @@ -1,4 +1,4 @@ - + @@ -406,9 +406,9 @@ #{dataMap.container_name}, #{dataMap.container_id}, #{dataMap.http_resp_server}, - #{dataMap.srcip_id}::int8, - #{dataMap.cdnip}::inet, - #{dataMap.natip}::inet, + #{dataMap.srcip_id}, + #{dataMap.cdnip}, + #{dataMap.natip}, #{dataMap.mail_sender}, #{dataMap.mail_receiver}, #{dataMap.vpn_mac}, @@ -439,8 +439,8 @@ #{dataMap.print_time}, #{dataMap.printer}, #{dataMap.printer_type}, - #{dataMap.print_pages}::int8, - #{dataMap.print_copies}::int8, + #{dataMap.print_pages}, + #{dataMap.print_copies}, #{dataMap.src_device}, #{dataMap.dst_device}, #{dataMap.src_file}, @@ -457,18 +457,18 @@ #{dataMap.env}, #{dataMap.brute_force_service}, #{dataMap.vuirs_name}, - #{dataMap.http_req_length}::int8, + #{dataMap.http_req_length}, #{dataMap.http_req_content_type}, - #{dataMap.tc_scan_port}::inet, - #{dataMap.tc_labels}::inet, + #{dataMap.tc_scan_port}, + #{dataMap.tc_labels}, #{dataMap.http_resp_content_type}, #{dataMap.dns_msg_type}, #{dataMap.dns_answer_length}, #{dataMap.dns_ioc}, - #{dataMap.tx_bytes}::double precision, - #{dataMap.rx_bytes}::double precision, - #{dataMap.all_bytes}::double precision, - #{dataMap.duration_time}::int8, + #{dataMap.tx_bytes}, + #{dataMap.rx_bytes}, + #{dataMap.all_bytes}, + #{dataMap.duration_time}, #{dataMap.mail_attach_name}, #{dataMap.mail_subject}, #{dataMap.mail_message}, @@ -488,27 +488,27 @@ #{dataMap.src_ip_apt}, #{dataMap.srcip_name}, #{dataMap.tc_client}, - #{dataMap.srcip_organization_id}::int8, + #{dataMap.srcip_organization_id}, #{dataMap.dest_ip_intranetip}, #{dataMap.dest_ip_ioc}, - #{dataMap.desip_id}::int8, + #{dataMap.desip_id}, #{dataMap.desip_name}, - #{dataMap.tc_hostip}::inet, - #{dataMap.desip_organization_id}::int8, + #{dataMap.tc_hostip}, + #{dataMap.desip_organization_id}, #{dataMap.origin_confidence}, #{dataMap.origin_malscore}, #{dataMap.attacker_icampaign}, - #{dataMap.attacker_host_asset_id}::int8, - #{dataMap.attacker_organization_id}::int8, - #{dataMap.victim_host_asset_id}::int8, - #{dataMap.victim_organization_id}::int8, + #{dataMap.attacker_host_asset_id}, + #{dataMap.attacker_organization_id}, + #{dataMap.victim_host_asset_id}, + #{dataMap.victim_organization_id}, #{dataMap.logout_time}, #{dataMap.http_req_line}, #{dataMap.desip_security_scope_id}, #{dataMap.srcip_security_scope_id}, - #{dataMap.http_resp_length}::int8, + #{dataMap.http_resp_length}, #{dataMap.tc_attack_type}, - #{dataMap.tc_realip}::inet, + #{dataMap.tc_realip}, #{dataMap.attacker_ip_lists}, #{dataMap.login_password}, #{dataMap.detail}, @@ -525,12 +525,12 @@ #{dataMap.login_abnormal_type}, #{dataMap.file_tag}, #{dataMap.file_platform}, - #{dataMap.target_ip}::inet, + #{dataMap.target_ip}, #{dataMap.collect_date}, - #{dataMap.tc_client_ip}::inet, - #{dataMap.tc_server_ip}::inet, - #{dataMap.tc_externalip}::inet, - #{dataMap.http_status_code}::int8, + #{dataMap.tc_client_ip}, + #{dataMap.tc_server_ip}, + #{dataMap.tc_externalip}, + #{dataMap.http_status_code}, #{dataMap.device_domian}, #{dataMap.src_ip_str}, #{dataMap.src_port_str}, @@ -576,28 +576,28 @@ #{dataMap.origin_agent_name}, #{dataMap.origin_work_group}, #{dataMap.origin_asset_group}, - #{dataMap.origin_local_port}::int8, - #{dataMap.origin_agent_ip}::inet, - #{dataMap.origin_internal_ip}::inet, - #{dataMap.origin_external_ip}::inet, - #{dataMap.origin_local_addr}::inet, - #{dataMap.agent_id}::int8, + #{dataMap.origin_local_port}, + #{dataMap.origin_agent_ip}, + #{dataMap.origin_internal_ip}, + #{dataMap.origin_external_ip}, + #{dataMap.origin_local_addr}, + #{dataMap.agent_id}, #{dataMap.agent_name}, #{dataMap.tc_title}, #{dataMap.log_id}, #{dataMap.event_date}, #{dataMap.event_time_ts}, - #{dataMap.event_level}::int , - #{dataMap.src_ip}::inet, + #{dataMap.event_level}, + #{dataMap.src_ip}, #{dataMap.src_port}::BIGINT , - #{dataMap.dest_ip}::inet, + #{dataMap.dest_ip}, #{dataMap.dest_port}::BIGINT, #{dataMap.event_time}, #{dataMap.attacker_country}, #{dataMap.src_mac}, #{dataMap.dest_mac}, #{dataMap.proto}, - #{dataMap.dev_id}::int8, + #{dataMap.dev_id}, #{dataMap.created_time}, #{dataMap.src_country}, #{dataMap.src_country_code}, @@ -621,20 +621,20 @@ #{dataMap.end_time}, #{dataMap.file_created_time}, #{dataMap.file_modified_time}, - #{dataMap.tc_miguan_scan_port}::inet, + #{dataMap.tc_miguan_scan_port}, #{dataMap.process_path}, #{dataMap.parent_process_path}, #{dataMap.gname}, #{dataMap.exe_name}, #{dataMap.exe_path}, #{dataMap.login_time}, - #{dataMap.login_times}::int8, + #{dataMap.login_times}, #{dataMap.check_item}, #{dataMap.check_type}, - #{dataMap.attacker_ip}::inet, - #{dataMap.attacker_port}::int8, - #{dataMap.victim_ip}::inet, - #{dataMap.victim_port}::int8, + #{dataMap.attacker_ip}, + #{dataMap.attacker_port}, + #{dataMap.victim_ip}, + #{dataMap.victim_port}, #{dataMap.attacker_city}, #{dataMap.attacker_lon}, #{dataMap.attacker_lat}, @@ -660,7 +660,7 @@ #{dataMap.file_ssdeep}, #{dataMap.victim_country_code}, #{dataMap.http_xff_ip}, - #{dataMap.tc_miguan_class}::inet, + #{dataMap.tc_miguan_class}, #{dataMap.pid}, #{dataMap.ppid}, #{dataMap.process_name}, @@ -687,35 +687,35 @@ #{dataMap.dest_city}, #{dataMap.dest_lon}, #{dataMap.dest_lat}, - #{dataMap.event_category}::int4, - #{dataMap.attack_result}::int4, - #{dataMap.probe_ip}::inet, - #{dataMap.device_ip}::inet, + #{dataMap.event_category}, + #{dataMap.attack_result}, + #{dataMap.probe_ip}, + #{dataMap.device_ip}, #{dataMap.device_manufacturer}, #{dataMap.device_name}, #{dataMap.product_name}, #{dataMap.__id}, - #{dataMap.__count}::int8, + #{dataMap.__count}, #{dataMap.__count_reason}, - #{dataMap.event_type}::int, + #{dataMap.event_type}, #{dataMap.protocol}, #{dataMap.shell_cmd}, #{dataMap.parent_name}, #{dataMap.host_file_path}, #{dataMap.uid}, - #{dataMap.fall}::int4, - #{dataMap.tc_miguan_server_ip}::inet, - #{dataMap.dev_type}::int4, - #{dataMap.collect_method}::int4, - #{dataMap.field_cate_id}::int4, - #{dataMap.device_type}::int4, - #{dataMap.tc_miguan_client_ip}::inet, - #{dataMap.tc_miguan_name}::inet, - #{dataMap.origin_total_packages}::int8, - #{dataMap.origin_total_bytes}::int8, - #{dataMap.origin_peak_packages_rate}::int8, - #{dataMap.origin_peak_bytes_rate}::int8, - #{dataMap.origin_peak_flows_rate}::int8, + #{dataMap.fall}, + #{dataMap.tc_miguan_server_ip}, + #{dataMap.dev_type}, + #{dataMap.collect_method}, + #{dataMap.field_cate_id}, + #{dataMap.device_type}, + #{dataMap.tc_miguan_client_ip}, + #{dataMap.tc_miguan_name}, + #{dataMap.origin_total_packages}, + #{dataMap.origin_total_bytes}, + #{dataMap.origin_peak_packages_rate}, + #{dataMap.origin_peak_bytes_rate}, + #{dataMap.origin_peak_flows_rate}, #{dataMap.apt_orgname}, #{dataMap.apt_orgmsg}, #{dataMap.mail_message_id}, @@ -725,18 +725,18 @@ #{dataMap.mail_url}, #{dataMap.mail_cc}, #{dataMap.algorithm}, - #{dataMap.miningpool_ip}::inet, + #{dataMap.miningpool_ip}, #{dataMap.process_md5}, #{dataMap.pprocess_md5}, #{dataMap.source_servername}, #{dataMap.origin_source_servername}, #{dataMap.mail_filename}, #{dataMap.dst_upload_appname}, - #{dataMap.target_port}::int8, + #{dataMap.target_port}, #{dataMap.gid}, #{dataMap.origin_uid}, #{dataMap.origin_gid}, - #{dataMap.target_ports}::int8, + #{dataMap.target_ports}, #{dataMap.tc_miguan_name1}, #{dataMap.tc_miguan_class1}, #{dataMap.etl_time}, @@ -744,7 +744,7 @@ #{dataMap.desip_security_scope}, #{dataMap.srcip_security_scope}, #{dataMap.collect_time_ts}, - #{dataMap.tc_miguan_scan_port1}::inet, + #{dataMap.tc_miguan_scan_port1}, #{dataMap.src_dev_name}, #{dataMap.collect_protocol}, #{dataMap.destination_system_type}, diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml index 9d60d7c..bf7ceac 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml @@ -1,4 +1,4 @@ - + @@ -57,6 +57,7 @@ id, created_at, log_time, + device_id, device_ip, dest_ip, dest_port, @@ -541,9 +542,9 @@ #{dataMap.container_name}, #{dataMap.container_id}, #{dataMap.http_resp_server}, - #{dataMap.srcip_id}::int8, - #{dataMap.cdnip}::inet, - #{dataMap.natip}::inet, + #{dataMap.srcip_id}, + #{dataMap.cdnip}, + #{dataMap.natip}, #{dataMap.mail_sender}, #{dataMap.mail_receiver}, #{dataMap.vpn_mac}, @@ -574,8 +575,8 @@ #{dataMap.print_time}, #{dataMap.printer}, #{dataMap.printer_type}, - #{dataMap.print_pages}::int8, - #{dataMap.print_copies}::int8, + #{dataMap.print_pages}, + #{dataMap.print_copies}, #{dataMap.src_device}, #{dataMap.dst_device}, #{dataMap.src_file}, @@ -592,18 +593,18 @@ #{dataMap.env}, #{dataMap.brute_force_service}, #{dataMap.vuirs_name}, - #{dataMap.http_req_length}::int8, + #{dataMap.http_req_length}, #{dataMap.http_req_content_type}, - #{dataMap.tc_scan_port}::inet, - #{dataMap.tc_labels}::inet, + #{dataMap.tc_scan_port}, + #{dataMap.tc_labels}, #{dataMap.http_resp_content_type}, #{dataMap.dns_msg_type}, #{dataMap.dns_answer_length}, #{dataMap.dns_ioc}, - #{dataMap.tx_bytes}::double precision, - #{dataMap.rx_bytes}::double precision, - #{dataMap.all_bytes}::double precision, - #{dataMap.duration_time}::int8, + #{dataMap.tx_bytes}, + #{dataMap.rx_bytes}, + #{dataMap.all_bytes}, + #{dataMap.duration_time}, #{dataMap.mail_attach_name}, #{dataMap.mail_subject}, #{dataMap.mail_message}, @@ -623,27 +624,27 @@ #{dataMap.src_ip_apt}, #{dataMap.srcip_name}, #{dataMap.tc_client}, - #{dataMap.srcip_organization_id}::int8, + #{dataMap.srcip_organization_id}, #{dataMap.dest_ip_intranetip}, #{dataMap.dest_ip_ioc}, - #{dataMap.desip_id}::int8, + #{dataMap.desip_id}, #{dataMap.desip_name}, - #{dataMap.tc_hostip}::inet, - #{dataMap.desip_organization_id}::int8, + #{dataMap.tc_hostip}, + #{dataMap.desip_organization_id}, #{dataMap.origin_confidence}, #{dataMap.origin_malscore}, #{dataMap.attacker_icampaign}, - #{dataMap.attacker_host_asset_id}::int8, - #{dataMap.attacker_organization_id}::int8, - #{dataMap.victim_host_asset_id}::int8, - #{dataMap.victim_organization_id}::int8, + #{dataMap.attacker_host_asset_id}, + #{dataMap.attacker_organization_id}, + #{dataMap.victim_host_asset_id}, + #{dataMap.victim_organization_id}, #{dataMap.logout_time}, #{dataMap.http_req_line}, #{dataMap.desip_security_scope_id}, #{dataMap.srcip_security_scope_id}, - #{dataMap.http_resp_length}::int8, + #{dataMap.http_resp_length}, #{dataMap.tc_attack_type}, - #{dataMap.tc_realip}::inet, + #{dataMap.tc_realip}, #{dataMap.attacker_ip_lists}, #{dataMap.login_password}, #{dataMap.detail}, @@ -660,12 +661,12 @@ #{dataMap.login_abnormal_type}, #{dataMap.file_tag}, #{dataMap.file_platform}, - #{dataMap.target_ip}::inet, + #{dataMap.target_ip}, #{dataMap.collect_date}, - #{dataMap.tc_client_ip}::inet, - #{dataMap.tc_server_ip}::inet, - #{dataMap.tc_externalip}::inet, - #{dataMap.http_status_code}::int8, + #{dataMap.tc_client_ip}, + #{dataMap.tc_server_ip}, + #{dataMap.tc_externalip}, + #{dataMap.http_status_code}, #{dataMap.device_domian}, #{dataMap.src_ip_str}, #{dataMap.src_port_str}, @@ -711,28 +712,28 @@ #{dataMap.origin_agent_name}, #{dataMap.origin_work_group}, #{dataMap.origin_asset_group}, - #{dataMap.origin_local_port}::int8, - #{dataMap.origin_agent_ip}::inet, - #{dataMap.origin_internal_ip}::inet, - #{dataMap.origin_external_ip}::inet, - #{dataMap.origin_local_addr}::inet, - #{dataMap.agent_id}::int8, + #{dataMap.origin_local_port}, + #{dataMap.origin_agent_ip}, + #{dataMap.origin_internal_ip}, + #{dataMap.origin_external_ip}, + #{dataMap.origin_local_addr}, + #{dataMap.agent_id}, #{dataMap.agent_name}, #{dataMap.tc_title}, #{dataMap.log_id}, #{dataMap.event_date}, #{dataMap.event_time_ts}, - #{dataMap.event_level}::int , - #{dataMap.src_ip}::inet, + #{dataMap.event_level}, + #{dataMap.src_ip}, #{dataMap.src_port}::BIGINT , - #{dataMap.dest_ip}::inet, + #{dataMap.dest_ip}, #{dataMap.dest_port}::BIGINT, #{dataMap.event_time}, #{dataMap.attacker_country}, #{dataMap.src_mac}, #{dataMap.dest_mac}, #{dataMap.proto}, - #{dataMap.dev_id}::int8, + #{dataMap.dev_id}, #{dataMap.created_time}, #{dataMap.src_country}, #{dataMap.src_country_code}, @@ -756,20 +757,20 @@ #{dataMap.end_time}, #{dataMap.file_created_time}, #{dataMap.file_modified_time}, - #{dataMap.tc_miguan_scan_port}::inet, + #{dataMap.tc_miguan_scan_port}, #{dataMap.process_path}, #{dataMap.parent_process_path}, #{dataMap.gname}, #{dataMap.exe_name}, #{dataMap.exe_path}, #{dataMap.login_time}, - #{dataMap.login_times}::int8, + #{dataMap.login_times}, #{dataMap.check_item}, #{dataMap.check_type}, - #{dataMap.attacker_ip}::inet, - #{dataMap.attacker_port}::int8, - #{dataMap.victim_ip}::inet, - #{dataMap.victim_port}::int8, + #{dataMap.attacker_ip}, + #{dataMap.attacker_port}, + #{dataMap.victim_ip}, + #{dataMap.victim_port}, #{dataMap.attacker_city}, #{dataMap.attacker_lon}, #{dataMap.attacker_lat}, @@ -795,7 +796,7 @@ #{dataMap.file_ssdeep}, #{dataMap.victim_country_code}, #{dataMap.http_xff_ip}, - #{dataMap.tc_miguan_class}::inet, + #{dataMap.tc_miguan_class}, #{dataMap.pid}, #{dataMap.ppid}, #{dataMap.process_name}, @@ -822,35 +823,35 @@ #{dataMap.dest_city}, #{dataMap.dest_lon}, #{dataMap.dest_lat}, - #{dataMap.event_category}::int4, - #{dataMap.attack_result}::int4, - #{dataMap.probe_ip}::inet, - #{dataMap.device_ip}::inet, + #{dataMap.event_category}, + #{dataMap.attack_result}, + #{dataMap.probe_ip}, + #{dataMap.device_ip}, #{dataMap.device_manufacturer}, #{dataMap.device_name}, #{dataMap.product_name}, #{dataMap.__id}, - #{dataMap.__count}::int8, + #{dataMap.__count}, #{dataMap.__count_reason}, - #{dataMap.event_type}::int, + #{dataMap.event_type}, #{dataMap.protocol}, #{dataMap.shell_cmd}, #{dataMap.parent_name}, #{dataMap.host_file_path}, #{dataMap.uid}, - #{dataMap.fall}::int4, - #{dataMap.tc_miguan_server_ip}::inet, - #{dataMap.dev_type}::int4, - #{dataMap.collect_method}::int4, - #{dataMap.field_cate_id}::int4, - #{dataMap.device_type}::int4, - #{dataMap.tc_miguan_client_ip}::inet, - #{dataMap.tc_miguan_name}::inet, - #{dataMap.origin_total_packages}::int8, - #{dataMap.origin_total_bytes}::int8, - #{dataMap.origin_peak_packages_rate}::int8, - #{dataMap.origin_peak_bytes_rate}::int8, - #{dataMap.origin_peak_flows_rate}::int8, + #{dataMap.fall}, + #{dataMap.tc_miguan_server_ip}, + #{dataMap.dev_type}, + #{dataMap.collect_method}, + #{dataMap.field_cate_id}, + #{dataMap.device_type}, + #{dataMap.tc_miguan_client_ip}, + #{dataMap.tc_miguan_name}, + #{dataMap.origin_total_packages}, + #{dataMap.origin_total_bytes}, + #{dataMap.origin_peak_packages_rate}, + #{dataMap.origin_peak_bytes_rate}, + #{dataMap.origin_peak_flows_rate}, #{dataMap.apt_orgname}, #{dataMap.apt_orgmsg}, #{dataMap.mail_message_id}, @@ -860,18 +861,18 @@ #{dataMap.mail_url}, #{dataMap.mail_cc}, #{dataMap.algorithm}, - #{dataMap.miningpool_ip}::inet, + #{dataMap.miningpool_ip}, #{dataMap.process_md5}, #{dataMap.pprocess_md5}, #{dataMap.source_servername}, #{dataMap.origin_source_servername}, #{dataMap.mail_filename}, #{dataMap.dst_upload_appname}, - #{dataMap.target_port}::int8, + #{dataMap.target_port}, #{dataMap.gid}, #{dataMap.origin_uid}, #{dataMap.origin_gid}, - #{dataMap.target_ports}::int8, + #{dataMap.target_ports}, #{dataMap.tc_miguan_name1}, #{dataMap.tc_miguan_class1}, #{dataMap.etl_time}, @@ -879,7 +880,7 @@ #{dataMap.desip_security_scope}, #{dataMap.srcip_security_scope}, #{dataMap.collect_time_ts}, - #{dataMap.tc_miguan_scan_port1}::inet, + #{dataMap.tc_miguan_scan_port1}, #{dataMap.src_dev_name}, #{dataMap.collect_protocol}, #{dataMap.destination_system_type}, diff --git a/haobang-security-dm/syslog-serve/docker_run.txt b/haobang-security-dm/syslog-serve/docker_run.txt index 31dfaac..423a31f 100644 --- a/haobang-security-dm/syslog-serve/docker_run.txt +++ b/haobang-security-dm/syslog-serve/docker_run.txt @@ -8,13 +8,15 @@ cd /opt/syslog/docker/serve --docker build -f /opt/syslog/docker/serve/Dockerfile -t syslog-serve:v1.0 --docker imageļ (Dockerfile ǰĿ¼ docker build -t syslog-serve:v1.X.X . +docker build -t syslog-serve-dm:v1.2.X . --2.鿴 docker ps -a --3.docker imageļ (Dockerfile ǰĿ¼ docker build -t syslog-serve:v1.X.X . - +--dameng +docker build -t syslog-serve-dm:v1.2.X . --4.ֹͣ ɾ docker stop ct-syslog-serve && docker rm ct-syslog-serve @@ -22,8 +24,8 @@ docker rmi --5.docker ļ -docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name ct-syslog-serve -p 514:514 -p 8189:8189 -v /home/syslog/logs:/app/logs --privileged=true syslog-serve:v1.X.X -docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name ct-syslog-serve -p 514:514/udp -p 514:514/tcp -p 8189:8189 -v /home/syslog/logs:/app/logs --privileged=true syslog-serve:v1.X.X +docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name syslog-serve-dm -p 514:514 -p 8189:8189 -v /home/syslog/logs:/app/logs --privileged=true syslog-serve-dm:v1.X.X +docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name syslog-serve-dm -p 514:514/udp -p 514:514/tcp -p 8189:8189 -v /home/syslog/logs:/app/logs --privileged=true syslog-serve-dm:v1.X.X ZC CMD docker run --restart unless-stopped -e TZ=Asia/Shanghai -d --name ct-syslog-serve -p 514:514 -p 8189:8189 -v /data/syslog/logs:/app/logs --privileged=true syslog-serve:v1.X.X diff --git a/haobang-security-dm/syslog-serve/pom.xml b/haobang-security-dm/syslog-serve/pom.xml index 1b962fa..0a0f98d 100644 --- a/haobang-security-dm/syslog-serve/pom.xml +++ b/haobang-security-dm/syslog-serve/pom.xml @@ -122,12 +122,21 @@ ${mybatis.version} - + + + + + + com.dameng + DmJdbcDriver18 + 8.1.2.141 + com.fasterxml.jackson.core diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/config/CacheConfig.java b/haobang-security-dm/syslog-serve/src/main/java/com/config/CacheConfig.java index 7599012..e2408cd 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/config/CacheConfig.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/config/CacheConfig.java @@ -13,11 +13,21 @@ import org.springframework.context.annotation.Configuration; import java.time.Duration; import java.util.Collections; import org.springframework.cache.annotation.EnableCaching; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.BeanDescription; +import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.BeanSerializerModifier; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.fasterxml.jackson.databind.jsontype.BasicPolymorphicTypeValidator; import com.fasterxml.jackson.databind.jsontype.PolymorphicTypeValidator; +import java.io.IOException; + @Configuration @EnableCaching // 启用缓存 public class CacheConfig { @@ -35,6 +45,33 @@ public class CacheConfig { // 禁用将日期序列化为时间戳 mapper.disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + // 注册达梦 JDBC 安全序列化器 — 拦截 dm.jdbc.* 类，直接返回 null 防止循环引用 StackOverflow + SimpleModule dmSafeModule = new SimpleModule("dm-safe"); + dmSafeModule.setSerializerModifier(new BeanSerializerModifier() { + @Override + public JsonSerializer modifySerializer(SerializationConfig config, + BeanDescription beanDesc, + JsonSerializer serializer) { + if (beanDesc.getBeanClass().getName().startsWith("dm.jdbc.")) { + return new JsonSerializer() { + @Override + public void serialize(Object value, JsonGenerator gen, + SerializerProvider serializers) throws IOException { + gen.writeNull(); + } + + @Override + public void serializeWithType(Object value, JsonGenerator gen, + SerializerProvider serializers, TypeSerializer typeSer) throws IOException { + gen.writeNull(); + } + }; + } + return serializer; + } + }); + mapper.registerModule(dmSafeModule); + // 启用类型信息，解决 LinkedHashMap 转换问题 PolymorphicTypeValidator ptv = BasicPolymorphicTypeValidator.builder() .allowIfSubType("com.common.entity.") // 允许你的实体类包 @@ -50,7 +87,6 @@ public class CacheConfig { ObjectMapper.DefaultTyping.NON_FINAL, com.fasterxml.jackson.annotation.JsonTypeInfo.As.PROPERTY ); - return mapper; } diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/config/RedisConfig.java b/haobang-security-dm/syslog-serve/src/main/java/com/config/RedisConfig.java index fb208c1..857f510 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/config/RedisConfig.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/config/RedisConfig.java @@ -1,4 +1,13 @@ package com.config; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.BeanDescription; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.BeanSerializerModifier; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -7,7 +16,7 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; -import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; @Configuration public class RedisConfig { @@ -22,6 +31,32 @@ public class RedisConfig { ObjectMapper mapper = new ObjectMapper(); mapper.registerModule(new JavaTimeModule()); + // 注册达梦 JDBC 安全序列化器 — 拦截 dm.jdbc.* 类，直接返回 null 防止循环引用 StackOverflow + SimpleModule dmSafeModule = new SimpleModule("dm-safe"); + dmSafeModule.setSerializerModifier(new BeanSerializerModifier() { + @Override + public JsonSerializer modifySerializer(SerializationConfig config, + BeanDescription beanDesc, + JsonSerializer serializer) { + if (beanDesc.getBeanClass().getName().startsWith("dm.jdbc.")) { + return new JsonSerializer() { + @Override + public void serialize(Object value, JsonGenerator gen, + SerializerProvider serializers) throws IOException { + gen.writeNull(); + } + + @Override + public void serializeWithType(Object value, JsonGenerator gen, + SerializerProvider serializers, TypeSerializer typeSer) throws IOException { + gen.writeNull(); + } + }; + } + return serializer; + } + }); + mapper.registerModule(dmSafeModule); mapper.activateDefaultTyping( mapper.getPolymorphicTypeValidator(), ObjectMapper.DefaultTyping.NON_FINAL diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppConfig.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppConfig.java index 730d0c5..50b08be 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppConfig.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppConfig.java @@ -4,6 +4,7 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import java.io.File; import com.typesafe.config.ConfigValueFactory; +import java.nio.charset.Charset; import java.util.Map; public class AppConfig { @@ -101,6 +102,17 @@ public class AppConfig { return config.getInt("syslog.buffer.size"); } + /** + * syslog 消息字符编码，默认 GBK（国内安全设备普遍使用 GBK） + * 配置示例: syslog.charset=GBK + */ + public static Charset getSyslogCharset() { + if (config.hasPath("syslog.charset")) { + return Charset.forName(config.getString("syslog.charset")); + } + return Charset.forName("GBK"); + } + // app service 配置 public static String getAppServieDeviceId() { return config.getString("app.service.device_id"); diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/interlocking/InterlockingService.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/interlocking/InterlockingService.java index 3cb3e97..9ff226c 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/interlocking/InterlockingService.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/interlocking/InterlockingService.java @@ -54,9 +54,9 @@ public class InterlockingService { /** * 定时任务：检查并执行待处理的封禁指令 - * 每30秒执行一次 + * 每60秒执行一次 */ - @Scheduled(fixedDelay = 30000) + @Scheduled(fixedDelay = 60000) public void processPendingCommands() { if (!interlockingEnabled) { return; diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java b/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java index e5a0416..2803269 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogMessageHandler.java @@ -7,6 +7,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; +import java.nio.charset.Charset; import com.kafka.kafkaProducer; import com.Modules.Device.DeviceProcess; import com.haobang.util.Sm4Util; @@ -16,6 +17,7 @@ import com.haobang.config.AppConfig; */ public class SyslogMessageHandler extends SimpleChannelInboundHandler { private static final Logger logger = LoggerFactory.getLogger(SyslogMessageHandler.class); + private static final Charset SYSLOG_CHARSET = AppConfig.getSyslogCharset(); @Override protected void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception { @@ -27,7 +29,7 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler { if (msg instanceof DatagramPacket) { // UDP 消息处理 DatagramPacket packet = (DatagramPacket) msg; - message = packet.content().toString(io.netty.util.CharsetUtil.UTF_8); + message = packet.content().toString(SYSLOG_CHARSET); source_ip=packet.sender().getAddress().getHostAddress(); source_port=packet.sender().getPort(); source = packet.sender().getAddress().getHostAddress() + ":" + packet.sender().getPort(); diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogTcpChannelInitializer.java b/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogTcpChannelInitializer.java index 44ec9f3..696b992 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogTcpChannelInitializer.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/netty/SyslogTcpChannelInitializer.java @@ -9,13 +9,14 @@ import io.netty.handler.codec.DelimiterBasedFrameDecoder; import io.netty.handler.codec.Delimiters; import io.netty.handler.codec.string.StringDecoder; import io.netty.handler.codec.string.StringEncoder; -import io.netty.util.CharsetUtil; +import java.nio.charset.Charset; /** * TCP 通道初始化器 */ public class SyslogTcpChannelInitializer extends ChannelInitializer { private static final int MAX_FRAME_LENGTH = AppConfig.getSyslogMaxFrameLength(); + private static final Charset SYSLOG_CHARSET = AppConfig.getSyslogCharset(); @Override protected void initChannel(SocketChannel ch) throws Exception { @@ -26,9 +27,9 @@ public class SyslogTcpChannelInitializer extends ChannelInitializer { + private static final Charset SYSLOG_CHARSET = AppConfig.getSyslogCharset(); + @Override protected void initChannel(DatagramChannel ch) throws Exception { ChannelPipeline pipeline = ch.pipeline(); - // 添加字符串解码器和编码器 - pipeline.addLast("decoder", new StringDecoder(CharsetUtil.UTF_8)); - pipeline.addLast("encoder", new StringEncoder(CharsetUtil.UTF_8)); + // 添加字符串解码器和编码器（使用可配置的字符集） + pipeline.addLast("decoder", new StringDecoder(SYSLOG_CHARSET)); + pipeline.addLast("encoder", new StringEncoder(SYSLOG_CHARSET)); // 添加业务处理器 pipeline.addLast("handler", new SyslogMessageHandler()); diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties index 83c2e14..c316fea 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties @@ -11,6 +11,8 @@ syslog.tcp.port=514 syslog.udp.port=514 syslog.max.frame.length=262144 syslog.buffer.size=1000 +# syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 +syslog.charset=GBK syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # APP Service Configuration @@ -18,15 +20,17 @@ app.service.device_id=1 app.service.device_name=honeypot app.service.vendor=changting app.service.product_name=diting -#ɼ̽ID +#采集探针ID app.service.device_collect_id=${DEVICE_COLLECT_ID:1} +app.service.version=${APP_SERVICE_VER:V1.0.0-20260527} +app.service.device_collect_name=${DEVICE_COLLECT_NAME:DevCollect-01} # syslog message data_type app.service.data_type=json # kafka Configuration -spring.kafka.producer.bootstrap-servers=192.168.222.130:9092 +spring.kafka.producer.bootstrap-servers=192.168.4.99:9092 spring.kafka.producer.topic =${KAFKA_PRODUCER_TOPIC:agent-syslog-topic} -# kafka Configuration Ż +# kafka Configuration 新增优化配置 spring.kafka.producer.properties.retries=10 spring.kafka.producer.properties.retry.backoff.ms=500 spring.kafka.producer.properties.connections.max.idle.ms=600000 @@ -35,20 +39,21 @@ spring.kafka.producer.properties.request.timeout.ms=30000 spring.kafka.producer.properties.delivery.timeout.ms=120000 #database Configuration -spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys -spring.datasource.username=postgres -spring.datasource.password=TnLanWaidYSwTSG5 -spring.datasource.driver-class-name=org.postgresql.Driver +spring.datasource.url=jdbc:dm://192.168.4.99:5237 +spring.datasource.username=SYSDBA +spring.datasource.password=caZ2TcmXNSW8L2Ap +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity mybatis.configuration.map-underscore-to-camel-case=true -# -spring.redis.host=192.168.222.131 +# 开发环境缓存配置 +spring.redis.host=localhost spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password= spring.redis.database=0 spring.redis.timeout=2000 @@ -57,44 +62,44 @@ spring.redis.lettuce.pool.max-active=8 spring.redis.lettuce.pool.max-wait=-1 spring.redis.lettuce.pool.max-idle=8 spring.redis.lettuce.pool.min-idle=0 -# ʱ϶̣ +# 开发环境缓存时间较短，方便调试 spring.cache.redis.time-to-live=600000 -# API +# 黑名单API配置 blacklist.api.url= https://103.43.84.11/api/v3/Objects/Blacklist blacklist.api.username=apt-admin103 blacklist.api.password=C9W2xYgfc%SN1 -# API +# 白名单API配置 whitelist.api.url=https://103.43.84.11/api/v3/Policies/GlobalWhitelist # ============================================ -# ̽ +# 探针联动配置 # ============================================ -# Ƿ +# 是否启用联动功能 interlocking.enabled=true -# syslog-consumer APIURLȫƽ̨ӳַ -interlocking.api.base-url=http://192.168.222.131:8089/xdrservice/interlocking -# API-KEY֤32λsyslog-consumerһ£ +# syslog-consumer API基础URL（安全平台外网映射地址） +interlocking.api.base-url=http://localhost:8089/xdrservice/interlocking +# API-KEY认证（32位，需与syslog-consumer配置一致） interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 -# ʱִм룩Ĭ30 +# 定时任务执行间隔（毫秒），默认30秒 interlocking.schedule.interval=30000 # ============================================ -# ̽ +# 探针心跳配置 # ============================================ -# Ƿ +# 是否启用心跳 probe.heartbeat.enabled=true -# ͼ룩 +# 心跳发送间隔（秒） probe.heartbeat.interval-seconds=60 -# ͳʼӳ٣룩 +# 心跳发送初始延迟（毫秒） probe.heartbeat.initial-delay-ms=5000 -# ƽ̨սӿURL -probe.platform.api-url=http://192.168.222.131:8089/xdrservice/interlocking/probe/heartbeat -# ƽ̨API Key +# 平台端心跳接收接口URL +probe.platform.api-url=http://localhost:8089/xdrservice/interlocking/probe/heartbeat +# 平台API Key probe.platform.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 # ============================================ -# ʱ +# 定时任务配置 # ============================================ spring.task.scheduling.pool.size=5 diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties index 9574fec..08fb189 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties @@ -11,6 +11,8 @@ syslog.tcp.port=514 syslog.udp.port=514 syslog.max.frame.length=262144 syslog.buffer.size=1000 +# syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 +syslog.charset=GBK # APP Service Configuration app.service.device_id=1 @@ -19,7 +21,7 @@ app.service.vendor=changting app.service.product_name=diting # syslog message data_type app.service.data_type=json -#ɼ̽ID +#采集探针ID #app.service.device_collect_id=${DEVICE_COLLECT_ID:2} app.service.device_collect_id=${DEVICE_COLLECT_ID:1} @@ -28,20 +30,21 @@ spring.kafka.producer.bootstrap-servers=10.11.2.142:9092 spring.kafka.producer.topic =agent-syslog-topic #database Configuration -spring.datasource.url=jdbc:postgresql://10.11.2.141:5432/ecosys -spring.datasource.username=ecosys -spring.datasource.password=wsYDPjrpNZPrkPrR -spring.datasource.driver-class-name=org.postgresql.Driver +spring.datasource.url=jdbc:dm://192.163.4.99:5237/DM_ecosys +spring.datasource.username=SYSDBA +spring.datasource.password=caZ2TcmXNSW8L2Ap +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=public # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity mybatis.configuration.map-underscore-to-camel-case=true -# +# 生产环境缓存配置 spring.redis.host=10.11.2.142 spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password=redis_edP6N6 spring.redis.database=0 spring.redis.timeout=5000 @@ -52,5 +55,5 @@ spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 \ No newline at end of file diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties index 7ef9192..6da6d00 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties @@ -11,6 +11,8 @@ syslog.tcp.port=514 syslog.udp.port=514 syslog.max.frame.length=262144 syslog.buffer.size=1000 +# syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 +syslog.charset=GBK syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # APP Service Configuration app.service.device_id=1 @@ -19,15 +21,15 @@ app.service.vendor=changting app.service.product_name=diting # syslog message data_type app.service.data_type=json -#ɼ̽ID +#采集探针ID app.service.device_collect_id=${DEVICE_COLLECT_ID:1} -app.service.version=${APP_SERVICE_VER:V1.0.0-20260509} +app.service.version=${APP_SERVICE_VER:V1.0.0-20260527} app.service.device_collect_name=${DEVICE_COLLECT_NAME:DevCollect-01} # kafka Configuration -spring.kafka.producer.bootstrap-servers=192.168.4.26:9092 +spring.kafka.producer.bootstrap-servers=192.168.4.99:9092 spring.kafka.producer.topic =agent-01-syslog-topic -# kafka Configuration Ż +# kafka Configuration 新增优化配置 spring.kafka.producer.properties.retries=10 spring.kafka.producer.properties.retry.backoff.ms=500 spring.kafka.producer.properties.connections.max.idle.ms=600000 @@ -35,20 +37,21 @@ spring.kafka.producer.properties.socket.keepalive.enable=true spring.kafka.producer.properties.request.timeout.ms=30000 spring.kafka.producer.properties.delivery.timeout.ms=120000 #database Configuration -spring.datasource.url=jdbc:postgresql://192.168.4.26:5432/ecosys -spring.datasource.username=postgres +spring.datasource.url=jdbc:dm://192.163.4.99:5237 +spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap -spring.datasource.driver-class-name=org.postgresql.Driver +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity mybatis.configuration.map-underscore-to-camel-case=true -# -spring.redis.host=192.168.4.26 +# 生产环境缓存配置 +spring.redis.host=192.168.4.99 spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password=123456 spring.redis.database=0 spring.redis.timeout=5000 @@ -59,42 +62,42 @@ spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 -# API +# 黑名单API配置 blacklist.api.url= https://103.43.84.11/api/v3/Objects/Blacklist blacklist.api.username=apt-admin103 blacklist.api.password=C9W2xYgfc%SN1 -# API +# 白名单API配置 whitelist.api.url=https://103.43.84.11/api/v3/Policies/GlobalWhitelist # ============================================ -# ̽ +# 探针联动配置 # ============================================ -# Ƿ +# 是否启用联动功能 interlocking.enabled=true -# syslog-consumer APIURLȫƽ̨ӳַ -interlocking.api.base-url=http://192.168.4.26:8089/xdrservice/interlocking -# API-KEY֤32λsyslog-consumerһ£ +# syslog-consumer API基础URL（安全平台外网映射地址） +interlocking.api.base-url=http://192.168.4.99:8089/xdrservice/interlocking +# API-KEY认证（32位，需与syslog-consumer配置一致） interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 -# ʱִм룩Ĭ30 +# 定时任务执行间隔（毫秒），默认30秒 interlocking.schedule.interval=30000 # ============================================ -# ̽ +# 探针心跳配置 # ============================================ -# Ƿ +# 是否启用心跳 probe.heartbeat.enabled=true -# ͼ룩 +# 心跳发送间隔（秒） probe.heartbeat.interval-seconds=60 -# ͳʼӳ٣룩 +# 心跳发送初始延迟（毫秒） probe.heartbeat.initial-delay-ms=5000 -# ƽ̨սӿURL -probe.platform.api-url=http://192.168.4.26:8089/xdrservice/interlocking/probe/heartbeat +# 平台端心跳接收接口URL +probe.platform.api-url=http://192.168.4.99:8089/xdrservice/interlocking/probe/heartbeat # ============================================ -# ʱ +# 定时任务配置 # ============================================ spring.task.scheduling.pool.size=5 diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-test.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-test.properties index 434691b..d7c4daf 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-test.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-test.properties @@ -11,6 +11,8 @@ syslog.tcp.port=514 syslog.udp.port=514 syslog.max.frame.length=262144 syslog.buffer.size=1000 +# syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 +syslog.charset=GBK syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # APP Service Configuration app.service.device_id=1 @@ -19,14 +21,14 @@ app.service.vendor=changting app.service.product_name=diting # syslog message data_type app.service.data_type=json -#ɼ̽ID +#采集探针ID app.service.device_collect_id=${DEVICE_COLLECT_ID:7} app.service.version=${APP_SERVICE_VER:V1.0.0-20260509} # kafka Configuration spring.kafka.producer.bootstrap-servers=${KAFKA_PRODUCER_SERVERS:120.238.245.132:32718} spring.kafka.producer.topic =${KAFKA_PRODUCER_TOPIC:agent-syslog-topic} -# kafka Configuration Ż +# kafka Configuration 新增优化配置 spring.kafka.producer.properties.retries=10 spring.kafka.producer.properties.retry.backoff.ms=500 spring.kafka.producer.properties.connections.max.idle.ms=600000 @@ -34,20 +36,21 @@ spring.kafka.producer.properties.socket.keepalive.enable=true spring.kafka.producer.properties.request.timeout.ms=30000 spring.kafka.producer.properties.delivery.timeout.ms=120000 #database Configuration -spring.datasource.url=jdbc:postgresql://120.238.245.132:31777/ecosys -spring.datasource.username=postgres +spring.datasource.url=jdbc:dm://192.163.4.99:5237 +spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap -spring.datasource.driver-class-name=org.postgresql.Driver +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity mybatis.configuration.map-underscore-to-camel-case=true -# +# 生产环境缓存配置 spring.redis.host=localhost spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password= spring.redis.database=0 spring.redis.timeout=5000 @@ -58,27 +61,27 @@ spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 -# API +# 黑名单API配置 blacklist.api.url= https://103.43.84.11/api/v3/Objects/Blacklist blacklist.api.username=apt-admin103 blacklist.api.password=C9W2xYgfc%SN1 -# API +# 白名单API配置 whitelist.api.url=https://103.43.84.11/api/v3/Policies/GlobalWhitelist whitelist.api.username=apt-admin103 whitelist.api.password=C9W2xYgfc%SN1 # ============================================ -# ̽ +# 探针联动配置 # ============================================ -# Ƿ +# 是否启用联动功能 interlocking.enabled=true -# syslog-consumer APIURLȫƽ̨ӳַ +# syslog-consumer API基础URL（安全平台外网映射地址） interlocking.api.base-url=http://120.238.245.132:32720/xdrservice/interlocking -# API-KEY֤32λsyslog-consumerһ£ +# API-KEY认证（32位，需与syslog-consumer配置一致） interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 -# ʱִм룩Ĭ30 +# 定时任务执行间隔（毫秒），默认30秒 interlocking.schedule.interval=30000 diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application.properties b/haobang-security-dm/syslog-serve/src/main/resources/application.properties index 7ef9192..6da6d00 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application.properties @@ -11,6 +11,8 @@ syslog.tcp.port=514 syslog.udp.port=514 syslog.max.frame.length=262144 syslog.buffer.size=1000 +# syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 +syslog.charset=GBK syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # APP Service Configuration app.service.device_id=1 @@ -19,15 +21,15 @@ app.service.vendor=changting app.service.product_name=diting # syslog message data_type app.service.data_type=json -#ɼ̽ID +#采集探针ID app.service.device_collect_id=${DEVICE_COLLECT_ID:1} -app.service.version=${APP_SERVICE_VER:V1.0.0-20260509} +app.service.version=${APP_SERVICE_VER:V1.0.0-20260527} app.service.device_collect_name=${DEVICE_COLLECT_NAME:DevCollect-01} # kafka Configuration -spring.kafka.producer.bootstrap-servers=192.168.4.26:9092 +spring.kafka.producer.bootstrap-servers=192.168.4.99:9092 spring.kafka.producer.topic =agent-01-syslog-topic -# kafka Configuration Ż +# kafka Configuration 新增优化配置 spring.kafka.producer.properties.retries=10 spring.kafka.producer.properties.retry.backoff.ms=500 spring.kafka.producer.properties.connections.max.idle.ms=600000 @@ -35,20 +37,21 @@ spring.kafka.producer.properties.socket.keepalive.enable=true spring.kafka.producer.properties.request.timeout.ms=30000 spring.kafka.producer.properties.delivery.timeout.ms=120000 #database Configuration -spring.datasource.url=jdbc:postgresql://192.168.4.26:5432/ecosys -spring.datasource.username=postgres +spring.datasource.url=jdbc:dm://192.163.4.99:5237 +spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap -spring.datasource.driver-class-name=org.postgresql.Driver +spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver +spring.datasource.hikari.schema=\"PUBLIC\" # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity mybatis.configuration.map-underscore-to-camel-case=true -# -spring.redis.host=192.168.4.26 +# 生产环境缓存配置 +spring.redis.host=192.168.4.99 spring.redis.port=6379 -# 루û룬ʡԣ +# 密码（如果没有设置密码，可以省略） spring.redis.password=123456 spring.redis.database=0 spring.redis.timeout=5000 @@ -59,42 +62,42 @@ spring.redis.lettuce.pool.max-wait=5000 spring.redis.lettuce.pool.max-idle=10 spring.redis.lettuce.pool.min-idle=5 -# ʱϳ +# 生产环境缓存时间较长 spring.cache.redis.time-to-live=3600000 -# API +# 黑名单API配置 blacklist.api.url= https://103.43.84.11/api/v3/Objects/Blacklist blacklist.api.username=apt-admin103 blacklist.api.password=C9W2xYgfc%SN1 -# API +# 白名单API配置 whitelist.api.url=https://103.43.84.11/api/v3/Policies/GlobalWhitelist # ============================================ -# ̽ +# 探针联动配置 # ============================================ -# Ƿ +# 是否启用联动功能 interlocking.enabled=true -# syslog-consumer APIURLȫƽ̨ӳַ -interlocking.api.base-url=http://192.168.4.26:8089/xdrservice/interlocking -# API-KEY֤32λsyslog-consumerһ£ +# syslog-consumer API基础URL（安全平台外网映射地址） +interlocking.api.base-url=http://192.168.4.99:8089/xdrservice/interlocking +# API-KEY认证（32位，需与syslog-consumer配置一致） interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 -# ʱִм룩Ĭ30 +# 定时任务执行间隔（毫秒），默认30秒 interlocking.schedule.interval=30000 # ============================================ -# ̽ +# 探针心跳配置 # ============================================ -# Ƿ +# 是否启用心跳 probe.heartbeat.enabled=true -# ͼ룩 +# 心跳发送间隔（秒） probe.heartbeat.interval-seconds=60 -# ͳʼӳ٣룩 +# 心跳发送初始延迟（毫秒） probe.heartbeat.initial-delay-ms=5000 -# ƽ̨սӿURL -probe.platform.api-url=http://192.168.4.26:8089/xdrservice/interlocking/probe/heartbeat +# 平台端心跳接收接口URL +probe.platform.api-url=http://192.168.4.99:8089/xdrservice/interlocking/probe/heartbeat # ============================================ -# ʱ +# 定时任务配置 # ============================================ spring.task.scheduling.pool.size=5 diff --git a/haobang-security-dm/syslog-serve/src/main/resources/logback.xml b/haobang-security-dm/syslog-serve/src/main/resources/logback.xml index 0b4ad03..f90779a 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/logback.xml +++ b/haobang-security-dm/syslog-serve/src/main/resources/logback.xml @@ -14,10 +14,10 @@ - logs/syslog-serve.log + logs/syslog-serve-dm.log - logs/syslog-serve.%d{yyyy-MM-dd}.log + logs/syslog-serve-dm.%d{yyyy-MM-dd}.log 1 diff --git a/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceDeviceMapper.xml b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceDeviceMapper.xml index e6f8c47..97558fc 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceDeviceMapper.xml +++ b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceDeviceMapper.xml @@ -46,11 +46,11 @@ - id, created_at::timestamp , updated_at::timestamp, deleted_at::timestamp, name, ip, device_group, device_type, - vendor, product_name, organization_id, last_receive_time::timestamp, agent_id, detail_id, - control_agent_id, license_start_time::timestamp, license_end_time::timestamp, is_monitoring, + id, created_at, updated_at, deleted_at, name, ip, device_group, device_type, + vendor, product_name, organization_id, last_receive_time, agent_id, detail_id, + control_agent_id, license_start_time, license_end_time, is_monitoring, security_scope_id, owner_id, ssh_config_id, status, created_by_id, decode_type, - miss_policy, tenant_id, create_time::timestamp, update_time::timestamp, create_by, update_by, del_flag, + miss_policy, tenant_id, create_time, update_time, create_by, update_by, del_flag, manager_name, today_parse_count, today_non_log_count, create_dept, device_collect_id diff --git a/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceReceiveLogMapper.xml b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceReceiveLogMapper.xml index 28163b7..d86df55 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceReceiveLogMapper.xml +++ b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceReceiveLogMapper.xml @@ -28,10 +28,10 @@ syslog_message, push_success ) VALUES ( - COALESCE(#{createdAt}, NOW() AT TIME ZONE 'utc'), + COALESCE(#{createdAt}, SYSDATE), #{deviceCollectId}, #{deviceId}, - #{deviceIp}::inet, + #{deviceIp}, #{receiveTime}, #{receiveTimeStr}, #{syslogMessage}, @@ -53,10 +53,10 @@ ) VALUES ( - COALESCE(#{item.createdAt}, NOW() AT TIME ZONE 'utc'), + COALESCE(#{item.createdAt}, SYSDATE), #{item.deviceCollectId}, #{item.deviceId}, - #{item.deviceIp}::inet, + #{item.deviceIp}, #{item.receiveTime}, #{item.receiveTimeStr}, #{item.syslogMessage}, @@ -85,10 +85,10 @@ ORDER BY receive_time DESC - + @@ -110,7 +110,7 @@ AND device_collect_id = #{deviceCollectId} - AND device_ip >>= #{deviceIp}::inet + AND device_ip = #{deviceIp} AND receive_time >= #{receiveTime} @@ -136,7 +136,7 @@ AND device_collect_id = #{deviceCollectId} - AND device_ip >>= #{deviceIp}::inet + AND device_ip = #{deviceIp} AND receive_time >= #{receiveTime} diff --git a/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceUnknownMapper.xml b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceUnknownMapper.xml index 846e881..c610ec9 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceUnknownMapper.xml +++ b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceUnknownMapper.xml @@ -31,10 +31,10 @@ network_protocol, source_method ) VALUES ( - COALESCE(#{createdAt}, NOW() AT TIME ZONE 'utc'), + COALESCE(#{createdAt}, SYSDATE), #{deviceCollectId}, #{deviceCollectName}, - #{deviceIp}::inet, + #{deviceIp}, #{firstTime}, #{lastTime}, #{organizationId}, @@ -58,10 +58,10 @@ ) VALUES ( - COALESCE(#{device.createdAt}, NOW() AT TIME ZONE 'utc'), + COALESCE(#{device.createdAt}, SYSDATE), #{device.deviceCollectId}, #{device.deviceCollectName}, - #{device.deviceIp}::inet, + #{device.deviceIp}, #{device.firstTime}, #{device.lastTime}, #{device.organizationId}, @@ -78,7 +78,7 @@ created_at, device_collect_id, device_collect_name, - device_ip::text as device_ip, + device_ip, first_time, last_time, organization_id, @@ -95,14 +95,14 @@ created_at, device_collect_id, device_collect_name, - device_ip::text as device_ip, + device_ip, first_time, last_time, organization_id, network_protocol, source_method FROM device_unknown - WHERE device_ip = #{deviceIp}::inet + WHERE device_ip = #{deviceIp} ORDER BY last_time DESC @@ -113,7 +113,7 @@ created_at, device_collect_id, device_collect_name, - device_ip::text as device_ip, + device_ip, first_time, last_time, organization_id, @@ -131,7 +131,7 @@ created_at, device_collect_id, device_collect_name, - device_ip::text as device_ip, + device_ip, first_time, last_time, organization_id, @@ -148,7 +148,7 @@ created_at, device_collect_id, device_collect_name, - device_ip::text as device_ip, + device_ip, first_time, last_time, organization_id, @@ -167,7 +167,7 @@ created_at, device_collect_id, device_collect_name, - device_ip::text as device_ip, + device_ip, first_time, last_time, organization_id, @@ -179,10 +179,10 @@ AND device_collect_id = #{deviceCollectId} - AND device_collect_name ILIKE CONCAT('%', #{deviceCollectName}, '%') + AND UPPER(device_collect_name) LIKE UPPER(CONCAT('%', #{deviceCollectName}, '%')) - AND device_ip = #{deviceIp}::inet + AND device_ip = #{deviceIp} AND organization_id = #{organizationId} @@ -214,7 +214,7 @@ device_collect_name = #{deviceCollectName}, - device_ip = #{deviceIp}::inet, + device_ip = #{deviceIp}, first_time = #{firstTime}, @@ -268,10 +268,10 @@ AND device_collect_id = #{deviceCollectId} - AND device_collect_name ILIKE CONCAT('%', #{deviceCollectName}, '%') + AND UPPER(device_collect_name) LIKE UPPER(CONCAT('%', #{deviceCollectName}, '%')) - AND device_ip = #{deviceIp}::inet + AND device_ip = #{deviceIp} AND organization_id = #{organizationId}