nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1、修改算子运算结果后入库BUG

Browse Source 
2、根据设备ID获取设备IP、设备厂商、设备名称 信息补全标准化表
3、完善告警表字段内容：syslog_normal_alarm: http_url ->alarm: victim_web_url
This commit is contained in:
nanChen
2026-01-26 15:20:46 +08:00
parent 6603c6f4a1
commit cf6b89ea94
23 changed files with 1941 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java
View File

@@ -36,6 +36,9 @@ import com.common.service.LogDataFilterService;
import com.common.service.LogDataCompleteService;
import com.common.service.DeviceCollectTaskService;
import com.common.entity.DeviceCollectTask;
import com.common.entity.DeviceDevice;
import com.common.util.TimeConversionUtils;
import com.common.service.DeviceDeviceService;
public class LogNormalProcessor {
private static final Logger logger = LoggerFactory.getLogger(LogNormalProcessor.class);
@@ -65,7 +68,8 @@ public class LogNormalProcessor {
private LogDataCompleteService logDataCompleteService= SpringContextUtil.getBean(LogDataCompleteService.class);
@Autowired
private DeviceCollectTaskService deviceCollectTaskService= SpringContextUtil.getBean(DeviceCollectTaskService.class);
@Autowired
private DeviceDeviceService deviceDeviceService= SpringContextUtil.getBean(DeviceDeviceService.class);
@Autowired
SyslogNonNormalMessage syslogNonNormalMessage=new SyslogNonNormalMessage();
@@ -433,15 +437,26 @@ public class LogNormalProcessor {
//匹配并获取映射枚举值
normalColumMap.put("dest_field_value",getMappingValue(action_param ,entry.getValue().toString() ));
}
else if(((HashMap<String, Object>)map.get("action")).get("type").equals("time\""))
else if(((HashMap<String, Object>)map.get("action")).get("type").equals("time"))
{
//time 类型
normalColumMap.put("dest_field_value",entry.getValue() );
//normalColumMap.put("dest_field_value",entry.getValue() );
normalColumMap.put("action_param",((HashMap<String, Object>)map.get("action")).get("param") );
HashMap<String, Object> action_param=(HashMap<String, Object>)((HashMap<String, Object>)map.get("action")).get("param") ;
//匹配时间格式并转成换整型格式
try {
long longTime = TimeConversionUtils.convertToMillis(entry.getValue().toString(), action_param.get("timezone").toString());
normalColumMap.put("dest_field_value", longTime);
} catch (Exception e) {
logger.error("时间类型转换错误，源值：" + entry.getValue().toString() + ",java_date_format:" + action_param.get("java_date_format").toString());
e.printStackTrace();
}
}
columnlist.add((HashMap<String, Object>)normalColumMap);
//System.out.println( "normalColumMap: " +normalColumMap);
break;
//存在源字段配置多个目标字段，使用continue,而不是break
continue;
}
}
}
@@ -499,6 +514,9 @@ public class LogNormalProcessor {
return ;
}
Map<String, Object> columnMap = logColumnMap;
//补全设备信息字段
CompletionDeviceInfo(columnMap, deviceId);
columnMap.put("device_id", deviceId);
columnMap.put("log_time", logtime);
columnMap.put("id", UUID.randomUUID().toString());
@@ -519,6 +537,30 @@ public class LogNormalProcessor {
}
}
/**
* 补全设备信息字段
* @param columnMap
* @param deviceID
*/
public void CompletionDeviceInfo(Map<String, Object> columnMap ,Long deviceID)
{
try {
DeviceDevice devInfo = deviceDeviceService.getByIdSafely( Long.valueOf(deviceID).intValue() );
if (devInfo != null) {
columnMap.put("device_ip", devInfo.getIp());
columnMap.put("device_manufacturer", devInfo.getVendor());
columnMap.put("device_name", devInfo.getName());
}
} catch (Exception e) {
logger.error("CompletionDeviceInfo 失败！ID："+deviceID );
}
}
/**
* 保存非标日志记录
* @param deviceId
* @param logtime
*/
public void SaveNonNormalMessage(long deviceId , DateTime logtime)
{
try {

376
haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/DeviceDevice.java Normal file
View File

@@ -0,0 +1,376 @@
package com.common.entity;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
public class DeviceDevice {
public Integer id;
public LocalDateTime createdAt;
public LocalDateTime updatedAt;
public LocalDateTime deletedAt;
public String name;
public String ip;
public Integer deviceGroup;
public Integer deviceType;
public String vendor;
public String productName;
public Integer organizationId;
public LocalDateTime lastReceiveTime;
public Integer agentId;
public Integer detailId;
public Integer controlAgentId;
public LocalDateTime licenseStartTime;
public LocalDateTime licenseEndTime;
public Boolean isMonitoring;
public Long securityScopeId;
public Long ownerId;
public Long sshConfigId;
public Short status;
public Long createdById;
public Integer decodeType;
public Integer missPolicy;
public String tenantId;
public LocalDateTime createTime;
public LocalDateTime updateTime;
public Long createBy;
public Long updateBy;
public String delFlag;
public String managerName;
public Integer todayParseCount;
public Integer todayNonLogCount;
public Long createDept;
public Integer deviceCollectId;
// Getter and Setter methods
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public void setCreatedAt(LocalDateTime createdAt) {
this.createdAt = createdAt;
}
public LocalDateTime getUpdatedAt() {
return updatedAt;
}
public void setUpdatedAt(LocalDateTime updatedAt) {
this.updatedAt = updatedAt;
}
public LocalDateTime getDeletedAt() {
return deletedAt;
}
public void setDeletedAt(LocalDateTime deletedAt) {
this.deletedAt = deletedAt;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getIp() {
return ip;
}
public void setIp(String ip) {
this.ip = ip;
}
public Integer getDeviceGroup() {
return deviceGroup;
}
public void setDeviceGroup(Integer deviceGroup) {
this.deviceGroup = deviceGroup;
}
public Integer getDeviceType() {
return deviceType;
}
public void setDeviceType(Integer deviceType) {
this.deviceType = deviceType;
}
public String getVendor() {
return vendor;
}
public void setVendor(String vendor) {
this.vendor = vendor;
}
public String getProductName() {
return productName;
}
public void setProductName(String productName) {
this.productName = productName;
}
public Integer getOrganizationId() {
return organizationId;
}
public void setOrganizationId(Integer organizationId) {
this.organizationId = organizationId;
}
public LocalDateTime getLastReceiveTime() {
return lastReceiveTime;
}
public void setLastReceiveTime(LocalDateTime lastReceiveTime) {
this.lastReceiveTime = lastReceiveTime;
}
public Integer getAgentId() {
return agentId;
}
public void setAgentId(Integer agentId) {
this.agentId = agentId;
}
public Integer getDetailId() {
return detailId;
}
public void setDetailId(Integer detailId) {
this.detailId = detailId;
}
public Integer getControlAgentId() {
return controlAgentId;
}
public void setControlAgentId(Integer controlAgentId) {
this.controlAgentId = controlAgentId;
}
public LocalDateTime getLicenseStartTime() {
return licenseStartTime;
}
public void setLicenseStartTime(LocalDateTime licenseStartTime) {
this.licenseStartTime = licenseStartTime;
}
public LocalDateTime getLicenseEndTime() {
return licenseEndTime;
}
public void setLicenseEndTime(LocalDateTime licenseEndTime) {
this.licenseEndTime = licenseEndTime;
}
public Boolean getIsMonitoring() {
return isMonitoring;
}
public void setIsMonitoring(Boolean isMonitoring) {
this.isMonitoring = isMonitoring;
}
public Long getSecurityScopeId() {
return securityScopeId;
}
public void setSecurityScopeId(Long securityScopeId) {
this.securityScopeId = securityScopeId;
}
public Long getOwnerId() {
return ownerId;
}
public void setOwnerId(Long ownerId) {
this.ownerId = ownerId;
}
public Long getSshConfigId() {
return sshConfigId;
}
public void setSshConfigId(Long sshConfigId) {
this.sshConfigId = sshConfigId;
}
public Short getStatus() {
return status;
}
public void setStatus(Short status) {
this.status = status;
}
public Long getCreatedById() {
return createdById;
}
public void setCreatedById(Long createdById) {
this.createdById = createdById;
}
public Integer getDecodeType() {
return decodeType;
}
public void setDecodeType(Integer decodeType) {
this.decodeType = decodeType;
}
public Integer getMissPolicy() {
return missPolicy;
}
public void setMissPolicy(Integer missPolicy) {
this.missPolicy = missPolicy;
}
public String getTenantId() {
return tenantId;
}
public void setTenantId(String tenantId) {
this.tenantId = tenantId;
}
public LocalDateTime getCreateTime() {
return createTime;
}
public void setCreateTime(LocalDateTime createTime) {
this.createTime = createTime;
}
public LocalDateTime getUpdateTime() {
return updateTime;
}
public void setUpdateTime(LocalDateTime updateTime) {
this.updateTime = updateTime;
}
public Long getCreateBy() {
return createBy;
}
public void setCreateBy(Long createBy) {
this.createBy = createBy;
}
public Long getUpdateBy() {
return updateBy;
}
public void setUpdateBy(Long updateBy) {
this.updateBy = updateBy;
}
public String getDelFlag() {
return delFlag;
}
public void setDelFlag(String delFlag) {
this.delFlag = delFlag;
}
public String getManagerName() {
return managerName;
}
public void setManagerName(String managerName) {
this.managerName = managerName;
}
public Integer getTodayParseCount() {
return todayParseCount;
}
public void setTodayParseCount(Integer todayParseCount) {
this.todayParseCount = todayParseCount;
}
public Integer getTodayNonLogCount() {
return todayNonLogCount;
}
public void setTodayNonLogCount(Integer todayNonLogCount) {
this.todayNonLogCount = todayNonLogCount;
}
public Long getCreateDept() {
return createDept;
}
public void setCreateDept(Long createDept) {
this.createDept = createDept;
}
public Integer getDeviceCollectId() {
return deviceCollectId;
}
public void setDeviceCollectId(Integer deviceCollectId) {
this.deviceCollectId = deviceCollectId;
}
@Override
public String toString() {
return "DeviceDevice{" +
"id=" + id +
", createdAt=" + createdAt +
", updatedAt=" + updatedAt +
", deletedAt=" + deletedAt +
", name='" + name + '\'' +
", ip='" + ip + '\'' +
", deviceGroup=" + deviceGroup +
", deviceType=" + deviceType +
", vendor='" + vendor + '\'' +
", productName='" + productName + '\'' +
", organizationId=" + organizationId +
", lastReceiveTime=" + lastReceiveTime +
", agentId=" + agentId +
", detailId=" + detailId +
", controlAgentId=" + controlAgentId +
", licenseStartTime=" + licenseStartTime +
", licenseEndTime=" + licenseEndTime +
", isMonitoring=" + isMonitoring +
", securityScopeId=" + securityScopeId +
", ownerId=" + ownerId +
", sshConfigId=" + sshConfigId +
", status=" + status +
", createdById=" + createdById +
", decodeType=" + decodeType +
", missPolicy=" + missPolicy +
", tenantId='" + tenantId + '\'' +
", createTime=" + createTime +
", updateTime=" + updateTime +
", createBy=" + createBy +
", updateBy=" + updateBy +
", delFlag='" + delFlag + '\'' +
", managerName='" + managerName + '\'' +
", todayParseCount=" + todayParseCount +
", todayNonLogCount=" + todayNonLogCount +
", createDept=" + createDept +
", deviceCollectId=" + deviceCollectId +
'}';
}
}

1
haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/GroupedSyslogData.java
View File

@@ -14,6 +14,7 @@ public class GroupedSyslogData {
private LocalDateTime maxLogTime;
private Long logCount;
private String[] victimIps;
private String[] victimWebUrls;
private Integer[] deviceIds;
private String[] originLogIds;
private Integer maxEventLevel;

23
haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/RuleHitTimeDTO.java Normal file
View File

@@ -0,0 +1,23 @@
package com.common.entity;
import lombok.Data;
import java.time.LocalDateTime;
@Data
public class RuleHitTimeDTO {
/**
* 泛化规则ID
*/
private Long normalizeRuleId;
/**
* 最大命中时间（从标准化表中统计）
*/
private LocalDateTime maxLogTime;
/**
* 数据来源表
*/
private String sourceTable;
}

14
haobang-security-xdr/syslog-consumer/src/main/java/com/common/mapper/AlarmMapper.java
View File

@@ -14,11 +14,11 @@ public interface AlarmMapper {
@Insert({"<script>",
"INSERT INTO alarm (",
"id, created_at, alarm_name, alarm_level, alarm_type, ",
"alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, ",
"alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, ",
"device_id, comment,origin_log_ids,log_start_at, log_end_at, http_status, ",
"attack_port, victim_port, attack_method, etl_time, log_count, ",
"attack_chain_phase, disposition_advice, attack_direction, ",
"judged_state, disposed_state, attack_result, fall, payload, " ,
"judged_state, disposed_state, attack_result, fall, payload, engine_type, " ,
"http_req_header , http_req_body,http_resp_header , http_resp_body ",
") VALUES ",
"<foreach collection='list' item='item' separator=','>",
@@ -26,6 +26,7 @@ public interface AlarmMapper {
"#{item.alarmType}, #{item.alarmMajorType}, #{item.alarmMinorType}, #{item.alarmAreaId}, ",
"#{item.attackIp, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
"#{item.victimIp, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
"#{item.victimWebUrl, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
"#{item.deviceId, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, ",
"#{item.comment}, " ,
"#{item.originLogIds, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
@@ -36,7 +37,7 @@ public interface AlarmMapper {
"#{item.attackChainPhase, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, ",
"#{item.dispositionAdvice}, #{item.attackDirection}, ",
"#{item.judgedState}, #{item.disposedState}, #{item.attackResult}, #{item.fall}, ",
"#{item.payload}, ",
"#{item.payload}, #{item.engineType}, ",
"#{item.httpReqHeaders, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
"#{item.httpReqBodys, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
"#{item.httpRespHeaders, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
@@ -50,17 +51,18 @@ public interface AlarmMapper {
*/
@Insert("INSERT INTO alarm (" +
"id, created_at, alarm_name, alarm_level, alarm_type, " +
"alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, " +
"alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, " +
"device_id, comment,origin_log_ids, log_start_at, log_end_at, http_status, " +
"attack_port, victim_port, attack_method, etl_time, log_count, " +
"attack_chain_phase, disposition_advice, attack_direction, " +
"judged_state, disposed_state, attack_result, fall, payload, " +
"judged_state, disposed_state, attack_result, fall, payload, engine_type, " +
"http_req_header , http_req_body,http_resp_header , http_resp_body " +
") VALUES (" +
"#{id}, #{createdAt}, #{alarmName}, #{alarmLevel}, " +
"#{alarmType}, #{alarmMajorType}, #{alarmMinorType}, #{alarmAreaId}, " +
"#{attackIp, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
"#{victimIp, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
"#{victimWebUrl, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, "+
"#{deviceId, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, " +
"#{comment}, " +
"#{originLogIds, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
@@ -70,7 +72,7 @@ public interface AlarmMapper {
"#{attackMethod}, #{etlTime}, #{logCount}, " +
"#{attackChainPhase, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, " +
"#{dispositionAdvice}, #{attackDirection}, " +
"#{judgedState}, #{disposedState}, #{attackResult}, #{fall}, #{payload}, " +
"#{judgedState}, #{disposedState}, #{attackResult}, #{fall}, #{payload}, #{engineType}, " +
"#{httpReqHeaders, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
"#{httpReqBodys, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
"#{httpRespHeaders, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +

6
haobang-security-xdr/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitMapper.java
View File

@@ -18,7 +18,7 @@ public interface AlarmVisitMapper {
@Insert({"<script>",
"INSERT INTO alarm_visit (",
"id, created_at, alarm_name, alarm_level, alarm_type, ",
"alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, ",
"alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, ",
"device_id, comment,origin_log_ids,log_start_at, log_end_at, http_status, ",
"attack_port, victim_port, attack_method, etl_time, log_count, ",
"attack_chain_phase, disposition_advice, attack_direction, ",
@@ -30,6 +30,7 @@ public interface AlarmVisitMapper {
"#{item.alarmType}, #{item.alarmMajorType}, #{item.alarmMinorType}, #{item.alarmAreaId}, ",
"#{item.attackIp, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
"#{item.victimIp, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
"#{item.victimWebUrl, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
"#{item.deviceId, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, ",
"#{item.comment}, " ,
"#{item.originLogIds, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
@@ -54,7 +55,7 @@ public interface AlarmVisitMapper {
*/
@Insert("INSERT INTO alarm_visit (" +
"id, created_at, alarm_name, alarm_level, alarm_type, " +
"alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, " +
"alarm_major_type, alarm_minor_type,alarm_area_id, attack_ip, victim_ip, victim_web_url, " +
"device_id, comment,origin_log_ids, log_start_at, log_end_at, http_status, " +
"attack_port, victim_port, attack_method, etl_time, log_count, " +
"attack_chain_phase, disposition_advice, attack_direction, " +
@@ -65,6 +66,7 @@ public interface AlarmVisitMapper {
"#{alarmType}, #{alarmMajorType}, #{alarmMinorType}, #{alarmAreaId}, " +
"#{attackIp, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
"#{victimIp, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
"#{victimWebUrl, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, "+
"#{deviceId, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, " +
"#{comment}, " +
"#{originLogIds, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +

93
haobang-security-xdr/syslog-consumer/src/main/java/com/common/mapper/DeviceDeviceMapper.java Normal file
View File

@@ -0,0 +1,93 @@
package com.common.mapper;
import com.common.entity.DeviceDevice;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
import java.util.Map;
@Mapper
public interface DeviceDeviceMapper {
/**
* 根据ID查询设备
*/
DeviceDevice selectById(Integer id);
/**
* 查询所有设备
*/
List<DeviceDevice> selectAll();
/**
* 根据IP地址查询设备
*/
List<DeviceDevice> selectByIp(String ip);
/**
* 根据设备名称模糊查询
*/
List<DeviceDevice> selectByNameLike(String name);
/**
* 根据设备组查询
*/
List<DeviceDevice> selectByDeviceGroup(Integer deviceGroup);
/**
* 根据设备类型查询
*/
List<DeviceDevice> selectByDeviceType(Integer deviceType);
/**
* 根据组织ID查询
*/
List<DeviceDevice> selectByOrganizationId(Integer organizationId);
/**
* 根据状态查询设备
*/
List<DeviceDevice> selectByStatus(Short status);
/**
* 多条件组合查询
*/
List<DeviceDevice> selectByCondition(DeviceDevice condition);
/**
* 动态条件查询
*/
List<DeviceDevice> selectByMap(Map<String, Object> params);
/**
* 分页查询
*/
List<DeviceDevice> selectByPage(@Param("offset") int offset, @Param("limit") int limit);
/**
* 统计设备数量
*/
Long count();
/**
* 根据条件统计数量
*/
Long countByCondition(DeviceDevice condition);
/**
* 查询监控中的设备
*/
List<DeviceDevice> selectMonitoringDevices();
/**
* 查询未删除的设备（del_flag = '0'）
*/
List<DeviceDevice> selectActiveDevices();
/**
* 根据厂商查询设备
*/
List<DeviceDevice> selectByVendor(String vendor);
}

46
haobang-security-xdr/syslog-consumer/src/main/java/com/common/mapper/NormalizeRuleStatsMapper.java Normal file
View File

@@ -0,0 +1,46 @@
package com.common.mapper;
import com.common.entity.RuleHitTimeDTO;
import org.apache.ibatis.annotations.*;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
@Mapper
public interface NormalizeRuleStatsMapper {
@Select("SELECT normalize_rule_id AS normalizeRuleId, " +
" MAX(log_time) AS maxLogTime " +
"FROM syslog_normal_data " +
"WHERE created_at >= #{startDate} " +
"GROUP BY normalize_rule_id")
List<RuleHitTimeDTO> selectMaxHitTimeFromNormalData(
@Param("startDate") LocalDateTime startDate
);
@Select("SELECT normalize_rule_id AS normalizeRuleId, " +
" MAX(log_time) AS maxLogTime " +
"FROM syslog_normal_alarm " +
"WHERE created_at >= #{startDate} " +
"GROUP BY normalize_rule_id")
List<RuleHitTimeDTO> selectMaxHitTimeFromNormalAlarm(
@Param("startDate") LocalDateTime startDate
);
@Select("SELECT id FROM dm_normalize_rule " +
"WHERE del_flag = '0' " +
"ORDER BY id")
List<Long> selectActiveRuleIds();
@Update("UPDATE dm_normalize_rule " +
"SET updated_at = NOW(), " +
" first_data_saved_at = #{hitTime} " +
"WHERE id = #{ruleId} " +
" AND (first_data_saved_at IS NULL OR first_data_saved_at < #{hitTime})")
int updateRuleHitTime(
@Param("ruleId") Long ruleId,
@Param("hitTime") LocalDateTime hitTime
);
}

4
haobang-security-xdr/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java
View File

@@ -49,16 +49,18 @@ public interface SyslogNormalAlarmMapper {
*/
@Select("SELECT " +
"to_char(log_time, 'YYYYMMDD') as log_date, " +
" MIN(origin_event_type) AS first_event_type, " +
"ARRAY_AGG(DISTINCT host(src_ip)::text) as attack_ips, " +
"origin_event_name, " +
"MAX(attack_result) as attack_result, " +
"MIN(log_time) as min_log_time, " +
"MAX(log_time) as max_log_time, " +
"COUNT(1) as log_count, " +
"ARRAY_AGG(DISTINCT host(dest_ip)::text) as victim_ips, " +
"ARRAY_AGG(DISTINCT http_url) as victim_web_urls, " +
"ARRAY_AGG(DISTINCT device_id) as device_ids, " +
"ARRAY_AGG(DISTINCT id) as origin_log_ids, " +
"MAX(event_level) as max_event_level, " +
"MIN(origin_event_type) AS first_event_type, " +
"MAX(origin_event_type) as event_type, " +
"MIN(event_type) as min_event_type, " +
"ARRAY_AGG(DISTINCT src_port::int4) as attack_ports, " +

4
haobang-security-xdr/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalDataMapper.java
View File

@@ -46,16 +46,18 @@ public interface SyslogNormalDataMapper {
*/
@Select("SELECT " +
"to_char(log_time, 'YYYYMMDD') as log_date, " +
" MIN(origin_event_type) AS first_event_type, " +
"ARRAY_AGG(DISTINCT host(src_ip)::text) as attack_ips, " +
"origin_event_name, " +
"MAX(attack_result) as attack_result, " +
"MIN(log_time) as min_log_time, " +
"MAX(log_time) as max_log_time, " +
"COUNT(1) as log_count, " +
"ARRAY_AGG(DISTINCT host(dest_ip)::text) as victim_ips, " +
"ARRAY_AGG(DISTINCT http_url) as victim_web_urls, " +
"ARRAY_AGG(DISTINCT device_id) as device_ids, " +
"ARRAY_AGG(DISTINCT id) as origin_log_ids, " +
"MAX(event_level) as max_event_level, " +
"MIN(origin_event_type) AS first_event_type, " +
"MAX(origin_event_type) as event_type, " +
"MIN(event_type) as min_event_type, " +
"ARRAY_AGG(DISTINCT src_port::int4) as attack_ports, " +

20
haobang-security-xdr/syslog-consumer/src/main/java/com/common/schedule/ETLOrchestrator.java
View File

@@ -13,6 +13,7 @@ import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import com.common.service.DeviceReceiveLogService;
import lombok.RequiredArgsConstructor;
import com.common.service.NormalizeRuleHitTimeService;
@Slf4j
@Component
@RequiredArgsConstructor
@@ -26,18 +27,23 @@ public class ETLOrchestrator {
@Autowired
private DeviceReceiveLogService deviceReceiveLogService;
@Autowired
private NormalizeRuleHitTimeService normalizeRuleHitTimeService;
/**
* 定时任务 - 从每小时第1分钟开始，5分钟间隔执行
*/
@Scheduled(cron = "0 1/5 * * * ?")
public void scheduledETL() {
//log.info("开始定时ETL任务");
long startTime = System.currentTimeMillis();
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
LocalDateTime[] currentWindow=TimeWindowCalculator.getPrevious5MinuteWindow();
String strStartTime= currentWindow[0].format(formatter);
String strEndTime= currentWindow[1].format(formatter);
log.info("ETL任务开始执行，开始时间：{},结束时间：{}",strStartTime,strEndTime );
//泛化标准数据告警降噪任务
try {
//retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcess24HoursGroupedData());
retryHandler.executeWithRetry(() -> dataExtractor.extractAndProcessQueryHoursGroupedData(strStartTime,strEndTime ));
@@ -49,6 +55,14 @@ public class ETLOrchestrator {
} catch (Exception e) {
log.error("定时ETL任务执行失败", e);
}
//泛化规则最新命中时间更新任务
try {
normalizeRuleHitTimeService.updateRuleHitTimeTask();
} catch (Exception e) {
log.error("泛化规则最新命中时间更新任务执行失败", e);
}
}
/**
@@ -89,8 +103,8 @@ public class ETLOrchestrator {
/**
* 每天凌晨3点清理2天前的数据
*/
//@Scheduled(cron = "0 0 3 * * ?")
@Scheduled(cron = "0 * * * * ?")
@Scheduled(cron = "0 0 3 * * ?")
//@Scheduled(cron = "0 * * * * ?")
public void cleanupOldLogs() {
try {
LocalDateTime cutoffTime = LocalDateTime.now().minusDays(2);

2
haobang-security-xdr/syslog-consumer/src/main/java/com/common/schedule/ScheduledTask.java
View File

@@ -25,7 +25,7 @@ public class ScheduledTask {
/**
* 每天1点同步昨天的数据
*/
//c
public void syncYesterdayData() {
log.info("开始执行昨天数据同步任务");
LocalDate yesterday = LocalDate.now().minusDays(1);

38
haobang-security-xdr/syslog-consumer/src/main/java/com/common/service/AccessLogAlertService.java
View File

@@ -85,7 +85,26 @@ public class AccessLogAlertService {
log.error("加载算法配置失败: {}", e.getMessage(), e);
}
}
/**
* 批量处理开关 - 避免重复处理
*/
private AtomicBoolean processing = new AtomicBoolean(false);
/**
* 安全的定时任务入口
*/
@Scheduled(cron = "0 */2 * * * ?")
public void safeProcessTask() {
if (processing.compareAndSet(false, true)) {
try {
processAccessLogAlert();
} finally {
processing.set(false);
}
} else {
log.warn("上一个任务仍在执行中，跳过本次执行");
}
}
/**
* 定时任务入口 - 每2分钟执行一次
*/
@@ -372,26 +391,7 @@ public class AccessLogAlertService {
}
}
/**
* 批量处理开关 - 避免重复处理
*/
private AtomicBoolean processing = new AtomicBoolean(false);
/**
* 安全的定时任务入口
*/
@Scheduled(cron = "0 */2 * * * ?")
public void safeProcessTask() {
if (processing.compareAndSet(false, true)) {
try {
processAccessLogAlert();
} finally {
processing.set(false);
}
} else {
log.warn("上一个任务仍在执行中，跳过本次执行");
}
}
}

15
haobang-security-xdr/syslog-consumer/src/main/java/com/common/service/DataTransformer.java
View File

@@ -57,13 +57,13 @@ public class DataTransformer {
.createdAt(LocalDateTime.now())
.alarmName( groupedData.getOriginEventName())
.alarmLevel(convertAlarmLevel(groupedData.getMaxEventLevel()))
.alarmType(getAlarmType(groupedData.getFirstEventType(),groupedData.getEventType()))
.alarmType(getAlarmType(groupedData.getFirstEventType(),groupedData.getMinEventType()))
.alarmMajorType(groupedData.getEventType())
.alarmMinorType(groupedData.getMinEventType())
.alarmAreaId(0)
.attackIp(groupedData.getAttackIps())
.victimIp(groupedData.getVictimIps())
.victimWebUrl(null)
.victimWebUrl(groupedData.getVictimWebUrls())
.attackChainPhase(new Integer[]{-1})
.deviceId(groupedData.getDeviceIds())
.tag(null)
@@ -139,7 +139,7 @@ public class DataTransformer {
.alarmAreaId(0)
.attackIp(groupedData.getAttackIps())
.victimIp(groupedData.getVictimIps())
.victimWebUrl(null)
.victimWebUrl(groupedData.getVictimWebUrls())
.attackChainPhase(new Integer[]{-1})
.deviceId(groupedData.getDeviceIds())
.tag(null)
@@ -203,10 +203,13 @@ public class DataTransformer {
}
private String getAlarmType(String firstEventType,String eventType) {
if (firstEventType.equals(""))
// 如果 firstEventType 有效则使用它
if (firstEventType != null && !firstEventType.isEmpty()) {
return firstEventType;
else
return eventType;
}
// 否则使用 eventType（如果也为 null 则返回空字符串）
return eventType != null ? eventType : "";
}
/**
* 确定attack_result的值

34
haobang-security-xdr/syslog-consumer/src/main/java/com/common/service/DeviceDeviceService.java Normal file
View File

@@ -0,0 +1,34 @@
package com.common.service;
import com.common.entity.DeviceDevice;
import java.util.List;
import java.util.Map;
public interface DeviceDeviceService {
DeviceDevice getByIdSafely(Integer id);
List<DeviceDevice> getByIpSafely(String ip);
DeviceDevice getById(Integer id);
List<DeviceDevice> getAll();
List<DeviceDevice> getByIp(String ip);
List<DeviceDevice> getByNameLike(String name);
List<DeviceDevice> getByCondition(DeviceDevice condition);
List<DeviceDevice> getByMap(Map<String, Object> params);
List<DeviceDevice> getByPage(int pageNum, int pageSize);
Long getCount();
List<DeviceDevice> getMonitoringDevices();
List<DeviceDevice> getActiveDevices();
}

27
haobang-security-xdr/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java
View File

@@ -78,6 +78,12 @@ public class DeviceStatsUpdateService {
" updated_at = NOW() " +
"WHERE id = ?";
private static final String UPDATE_DEVICE_COUNT_SQL =
"UPDATE device_device " +
"SET today_parse_count = 0, " +
" today_non_log_count = 0, " +
" updated_at = NOW() " ;
/**
* 每分钟执行一次统计更新（秒：0，分：*，时：*）
*/
@@ -110,6 +116,25 @@ public class DeviceStatsUpdateService {
}
}
/**
* 每天执行一次设备统计数值清零更新（秒：1，分：0，时：0）
*/
@Scheduled(cron = "1 0 0 * * ?")
@Transactional
public void updateDeviceCount() {
long startTime = System.currentTimeMillis();
log.info("开始执行设备统计数值清零更新任务...");
try {
int devCount= jdbcTemplate.update(UPDATE_DEVICE_COUNT_SQL);
long endTime = System.currentTimeMillis();
log.info("设备统计数值清零更新完成，处理设备数：{} ，耗时：{}ms",devCount, (endTime - startTime));
} catch (Exception e) {
log.error("设备统计数值清零更新任务执行失败", e);
throw e;
}
}
/**
* 收集设备统计信息
*/
@@ -119,7 +144,6 @@ public class DeviceStatsUpdateService {
String normalDataTable = "syslog_normal_data_" + today;
String normalAlarmTable = "syslog_normal_alarm_" + today;
String nonNormalTable = "syslog_non_normal_message_" + today;
String sql = String.format(
COLLECT_DEVICE_STATS_SQL,
receiveLogTable,
@@ -178,7 +202,6 @@ public class DeviceStatsUpdateService {
// 使用批量更新方法
updateService.batchUpdateDeviceTaskTimes();
long endTime = System.currentTimeMillis();
log.info("设备采集探针任务时间更新完成，耗时: {}ms", endTime - startTime);

248
haobang-security-xdr/syslog-consumer/src/main/java/com/common/service/NormalizeRuleHitTimeService.java Normal file
View File

@@ -0,0 +1,248 @@
package com.common.service;
import com.common.entity.RuleHitTimeDTO;
import com.common.mapper.NormalizeRuleStatsMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@Slf4j
@Service
public class NormalizeRuleHitTimeService {
@Autowired
private NormalizeRuleStatsMapper normalizeRuleStatsMapper;
// 用于缓存最新的命中时间，避免重复查询
private final Map<Long, LocalDateTime> ruleLatestHitCache = new ConcurrentHashMap<>();
// 记录上次执行时间
private LocalDateTime lastExecutionTime;
public void updateRuleHitTimeTask() {
long startTime = System.currentTimeMillis();
log.info("开始执行泛化规则命中时间更新任务，时间：{}", LocalDateTime.now());
try {
// 执行更新逻辑
int updatedCount = updateNormalizeRuleHitTime();
long endTime = System.currentTimeMillis();
log.info("泛化规则命中时间更新任务完成，更新规则数：{}，耗时：{}ms",
updatedCount, (endTime - startTime));
// 更新上次执行时间
lastExecutionTime = LocalDateTime.now();
} catch (Exception e) {
log.error("泛化规则命中时间更新任务执行失败", e);
}
}
/**
* 更新泛化规则命中时间
* @return 更新的规则数量
*/
public int updateNormalizeRuleHitTime() {
// 1. 获取当天开始时间（用于过滤当天数据）
LocalDateTime todayStart = LocalDateTime.of(LocalDate.now(), LocalTime.MIN);
// 2. 从两个标准化表中统计最新命中时间
List<RuleHitTimeDTO> normalDataStats = normalizeRuleStatsMapper
.selectMaxHitTimeFromNormalData(todayStart);
List<RuleHitTimeDTO> alarmDataStats = normalizeRuleStatsMapper
.selectMaxHitTimeFromNormalAlarm(todayStart);
log.info("从 syslog_normal_data 表统计到 {} 条规则命中记录",
normalDataStats.size());
log.info("从 syslog_normal_alarm 表统计到 {} 条规则命中记录",
alarmDataStats.size());
// 3. 合并两个统计结果，取每个规则的最新命中时间
Map<Long, LocalDateTime> ruleMaxHitTimeMap = mergeRuleHitTimes(
normalDataStats, alarmDataStats
);
// 4. 获取所有启用状态的规则ID
List<Long> activeRuleIds = normalizeRuleStatsMapper.selectActiveRuleIds();
log.info("当前启用状态的规则数量：{}", activeRuleIds.size());
// 5. 批量更新规则的最新命中时间
int updatedCount = batchUpdateRuleHitTime(activeRuleIds, ruleMaxHitTimeMap);
// 6. 更新缓存
updateRuleHitCache(ruleMaxHitTimeMap);
return updatedCount;
}
/**
* 合并两个表的统计结果，取每个规则的最新命中时间
*/
private Map<Long, LocalDateTime> mergeRuleHitTimes(
List<RuleHitTimeDTO> normalDataStats,
List<RuleHitTimeDTO> alarmDataStats
) {
Map<Long, LocalDateTime> resultMap = new HashMap<>();
// 处理 syslog_normal_data 表统计结果
if (!CollectionUtils.isEmpty(normalDataStats)) {
for (RuleHitTimeDTO dto : normalDataStats) {
if (dto.getNormalizeRuleId() != null && dto.getMaxLogTime() != null) {
LocalDateTime currentMax = resultMap.get(dto.getNormalizeRuleId());
if (currentMax == null || dto.getMaxLogTime().isAfter(currentMax)) {
resultMap.put(dto.getNormalizeRuleId(), dto.getMaxLogTime());
}
}
}
}
// 处理 syslog_normal_alarm 表统计结果
if (!CollectionUtils.isEmpty(alarmDataStats)) {
for (RuleHitTimeDTO dto : alarmDataStats) {
if (dto.getNormalizeRuleId() != null && dto.getMaxLogTime() != null) {
LocalDateTime currentMax = resultMap.get(dto.getNormalizeRuleId());
if (currentMax == null || dto.getMaxLogTime().isAfter(currentMax)) {
resultMap.put(dto.getNormalizeRuleId(), dto.getMaxLogTime());
}
}
}
}
log.info("合并后需要更新的规则数量：{}", resultMap.size());
return resultMap;
}
/**
* 批量更新规则命中时间
*/
private int batchUpdateRuleHitTime(
List<Long> activeRuleIds,
Map<Long, LocalDateTime> ruleMaxHitTimeMap
) {
if (CollectionUtils.isEmpty(activeRuleIds)) {
return 0;
}
int updatedCount = 0;
int batchSize = 1000; // 每批处理1000条
int totalBatches = (activeRuleIds.size() + batchSize - 1) / batchSize;
log.info("开始批量更新，规则总数：{}，分批数：{}", activeRuleIds.size(), totalBatches);
for (int i = 0; i < totalBatches; i++) {
int fromIndex = i * batchSize;
int toIndex = Math.min(fromIndex + batchSize, activeRuleIds.size());
List<Long> batchRuleIds = activeRuleIds.subList(fromIndex, toIndex);
// 处理当前批次
int batchUpdated = processBatchUpdate(batchRuleIds, ruleMaxHitTimeMap);
updatedCount += batchUpdated;
log.debug("第 {} 批处理完成，更新 {} 条规则", i + 1, batchUpdated);
// 每处理完一批，稍作停顿，避免数据库压力过大
if (i < totalBatches - 1) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.warn("批量更新任务被中断");
}
}
}
return updatedCount;
}
/**
* 处理单个批次的更新
*/
private int processBatchUpdate(
List<Long> batchRuleIds,
Map<Long, LocalDateTime> ruleMaxHitTimeMap
) {
int batchUpdated = 0;
for (Long ruleId : batchRuleIds) {
LocalDateTime maxHitTime = ruleMaxHitTimeMap.get(ruleId);
if (maxHitTime != null) {
// 检查缓存中是否有更晚的时间
LocalDateTime cachedTime = ruleLatestHitCache.get(ruleId);
if (cachedTime == null || maxHitTime.isAfter(cachedTime)) {
// 执行更新
int updated = normalizeRuleStatsMapper.updateRuleHitTime(ruleId, maxHitTime);
if (updated > 0) {
batchUpdated++;
}
}
}
}
return batchUpdated;
}
/**
* 更新命中时间缓存
*/
private void updateRuleHitCache(Map<Long, LocalDateTime> ruleMaxHitTimeMap) {
ruleMaxHitTimeMap.forEach((ruleId, hitTime) -> {
LocalDateTime cachedTime = ruleLatestHitCache.get(ruleId);
if (cachedTime == null || hitTime.isAfter(cachedTime)) {
ruleLatestHitCache.put(ruleId, hitTime);
}
});
// 清理过期的缓存（超过1天）
cleanupExpiredCache();
}
/**
* 清理过期缓存
*/
private void cleanupExpiredCache() {
LocalDateTime oneDayAgo = LocalDateTime.now().minusDays(1);
Iterator<Map.Entry<Long, LocalDateTime>> iterator = ruleLatestHitCache.entrySet().iterator();
int removedCount = 0;
while (iterator.hasNext()) {
Map.Entry<Long, LocalDateTime> entry = iterator.next();
if (entry.getValue().isBefore(oneDayAgo)) {
iterator.remove();
removedCount++;
}
}
if (removedCount > 0) {
log.debug("清理了 {} 条过期缓存记录", removedCount);
}
}
/**
* 获取规则最新命中时间（缓存中）
*/
public LocalDateTime getLatestHitTime(Long ruleId) {
return ruleLatestHitCache.get(ruleId);
}
/**
* 手动触发更新（可用于测试或紧急更新）
*/
public int triggerManualUpdate() {
log.info("手动触发泛化规则命中时间更新");
return updateNormalizeRuleHitTime();
}
}

103
haobang-security-xdr/syslog-consumer/src/main/java/com/common/service/impl/DeviceDeviceServiceImpl.java Normal file
View File

@@ -0,0 +1,103 @@
package com.common.service.impl;
import com.common.entity.DeviceDevice;
import com.common.mapper.DeviceDeviceMapper;
import com.common.service.DeviceDeviceService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.Cacheable;
import com.common.util.SafeCacheUtil;
@CacheConfig(cacheNames = "device")
@Service
public class DeviceDeviceServiceImpl implements DeviceDeviceService {
@Autowired
private DeviceDeviceMapper deviceDeviceMapper;
@Autowired
private SafeCacheUtil safeCacheUtil;
/**
* 使用安全的缓存方法
*/
@Cacheable( key = "'device:id:' +#id")
@Override
public DeviceDevice getByIdSafely(Integer id) {
String cacheKey = "device:id:" + id;
return safeCacheUtil.getSafe(cacheKey, DeviceDevice.class,
() -> deviceDeviceMapper.selectById(id));
}
/**
* 使用安全的列表缓存方法
*/
@Cacheable( key = "'device:ip:' + #ip")
@Override
public List<DeviceDevice> getByIpSafely(String ip) {
String cacheKey = "device:ip:" + ip;
return safeCacheUtil.getSafeList(cacheKey, DeviceDevice.class,
() -> deviceDeviceMapper.selectByIp(ip));
}
@Cacheable( key = "'device:id:' +#id")
@Override
public DeviceDevice getById(Integer id) {
System.out.println("exec deviceDeviceMapper.selectById :" + id.toString());
return deviceDeviceMapper.selectById(id);
}
@Override
public List<DeviceDevice> getAll() {
return deviceDeviceMapper.selectAll();
}
@Cacheable( key = "'device:ip:' + #ip")
@Override
public List<DeviceDevice> getByIp(String ip) {
return deviceDeviceMapper.selectByIp(ip);
}
@Override
public List<DeviceDevice> getByNameLike(String name) {
return deviceDeviceMapper.selectByNameLike(name);
}
@Override
public List<DeviceDevice> getByCondition(DeviceDevice condition) {
return deviceDeviceMapper.selectByCondition(condition);
}
@Override
public List<DeviceDevice> getByMap(Map<String, Object> params) {
return deviceDeviceMapper.selectByMap(params);
}
@Override
public List<DeviceDevice> getByPage(int pageNum, int pageSize) {
int offset = (pageNum - 1) * pageSize;
return deviceDeviceMapper.selectByPage(offset, pageSize);
}
@Override
public Long getCount() {
return deviceDeviceMapper.count();
}
@Override
public List<DeviceDevice> getMonitoringDevices() {
return deviceDeviceMapper.selectMonitoringDevices();
}
@Override
public List<DeviceDevice> getActiveDevices() {
return deviceDeviceMapper.selectActiveDevices();
}
}

109
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/SafeCacheUtil.java Normal file
View File

@@ -0,0 +1,109 @@
package com.common.util;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import java.util.LinkedHashMap;
import java.util.List ;
import java.util.stream.Collectors;
@Component
public class SafeCacheUtil {
@Autowired
private RedisTemplate<String, Object> redisTemplate;
@Autowired
private ObjectMapper objectMapper;
/**
* 安全的缓存获取方法
*/
@SuppressWarnings("unchecked")
public <T> T getSafe(String key, Class<T> clazz, Supplier<T> loader) {
try {
Object cached = redisTemplate.opsForValue().get(key);
if (cached == null) {
// 缓存不存在，从数据源加载
T value = loader.get();
if (value != null) {
redisTemplate.opsForValue().set(key, value, 30, TimeUnit.MINUTES);
}
return value;
}
// 类型匹配，直接返回
if (clazz.isInstance(cached)) {
return (T) cached;
}
// 类型不匹配，尝试转换
if (cached instanceof LinkedHashMap) {
return objectMapper.convertValue(cached, clazz);
}
// 无法转换，重新加载
T value = loader.get();
if (value != null) {
redisTemplate.opsForValue().set(key, value, 30, TimeUnit.MINUTES);
}
return value;
} catch (Exception e) {
// 缓存出错，降级到数据源
return loader.get();
}
}
/**
* 安全的列表缓存获取
*/
@SuppressWarnings("unchecked")
public <T> List<T> getSafeList(String key, Class<T> elementClass, Supplier<List<T>> loader) {
try {
Object cached = redisTemplate.opsForValue().get(key);
if (cached == null) {
List<T> value = loader.get();
if (value != null && !value.isEmpty()) {
redisTemplate.opsForValue().set(key, value, 30, TimeUnit.MINUTES);
}
return value;
}
// 已经是正确的类型
if (cached instanceof List &&
!((List<?>) cached).isEmpty() &&
elementClass.isInstance(((List<?>) cached).get(0))) {
return (List<T>) cached;
}
// 需要转换
if (cached instanceof List) {
List<LinkedHashMap> rawList = (List<LinkedHashMap>) cached;
List<T> convertedList = rawList.stream()
.map(item -> objectMapper.convertValue(item, elementClass))
.collect(Collectors.toList());
// 更新缓存为正确格式
redisTemplate.opsForValue().set(key, convertedList, 30, TimeUnit.MINUTES);
return convertedList;
}
// 无法处理，重新加载
List<T> value = loader.get();
if (value != null && !value.isEmpty()) {
redisTemplate.opsForValue().set(key, value, 30, TimeUnit.MINUTES);
}
return value;
} catch (Exception e) {
return loader.get();
}
}
}

528
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/TimeConversionUtils.java Normal file
View File

@@ -0,0 +1,528 @@
package com.common.util;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.*;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
/**
* 时间转换工具类 - 支持多种时间格式
* 支持格式包括：
* 1. 2025-11-01T15:23:41.188078+0800 (6位微秒，+0800时区)
* 2. 2025-11-01T15:23:41.188+0800 (3位毫秒，+0800时区)
* 3. 2025-11-01T15:23:41+0800 (无小数秒，+0800时区)
* 4. 2025-11-01T15:23:41.188078+08:00 (6位微秒，带冒号时区)
* 5. 2025-11-01T15:23:41.188Z (UTC时间，Z表示)
* 6. yyyy-MM-dd'T'HH:mm:ss.SSS'Z' (您原始格式)
* 7. 多种其他常见格式
*/
public class TimeConversionUtils {
// 缓存SimpleDateFormat对象（非线程安全，使用ThreadLocal）
private static final ConcurrentHashMap<String, ThreadLocal<SimpleDateFormat>> SDF_CACHE = new ConcurrentHashMap<>();
// 缓存DateTimeFormatter对象（线程安全）
private static final ConcurrentHashMap<String, DateTimeFormatter> DTF_CACHE = new ConcurrentHashMap<>();
// 预定义的常用格式列表
private static final List<DateTimeFormatter> PREDEFINED_FORMATTERS = new ArrayList<>();
static {
// 初始化预定义格式
initPredefinedFormatters();
}
/**
* 初始化预定义的多种时间格式
*/
private static void initPredefinedFormatters() {
// 1. 6位微秒，+0800时区格式
PREDEFINED_FORMATTERS.add(new DateTimeFormatterBuilder()
.appendPattern("yyyy-MM-dd'T'HH:mm:ss")
.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true)
.appendPattern("xx")
.toFormatter());
// 2. 3位毫秒，+0800时区格式
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSxx"));
// 3. 无小数秒，+0800时区格式
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssxx"));
// 4. 6位微秒，带冒号时区格式
PREDEFINED_FORMATTERS.add(new DateTimeFormatterBuilder()
.appendPattern("yyyy-MM-dd'T'HH:mm:ss")
.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true)
.appendPattern("xxx")
.toFormatter());
// 5. 3位毫秒，带冒号时区格式
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSxxx"));
// 6. 无小数秒，带冒号时区格式
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssxxx"));
// 7. UTC时间，Z表示，6位微秒
PREDEFINED_FORMATTERS.add(new DateTimeFormatterBuilder()
.appendPattern("yyyy-MM-dd'T'HH:mm:ss")
.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true)
.appendPattern("'Z'")
.toFormatter());
// 8. UTC时间，Z表示，3位毫秒
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
// 9. 无时区信息，6位微秒
PREDEFINED_FORMATTERS.add(new DateTimeFormatterBuilder()
.appendPattern("yyyy-MM-dd'T'HH:mm:ss")
.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true)
.toFormatter());
// 10. 无时区信息，3位毫秒
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS"));
// 11. 标准ISO格式
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ISO_ZONED_DATE_TIME);
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ISO_LOCAL_DATE_TIME);
// 12. 其他常见格式
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS"));
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ofPattern("yyyy年MM月dd日 HH:mm:ss"));
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ofPattern("dd/MM/yyyy HH:mm:ss"));
PREDEFINED_FORMATTERS.add(DateTimeFormatter.ofPattern("MM/dd/yyyy HH:mm:ss"));
}
/**
* 主转换方法 - 智能解析多种时间格式
*
* @param dateString 时间字符串
* @param timezone 时区（CST/UTC），当时间字符串中不包含时区信息时使用
* @return 毫秒级时间戳
* @throws ParseException 解析失败时抛出异常
*/
public static long convertToMillis(String dateString, String timezone) throws ParseException {
// 1. 首先尝试使用预定义的格式智能解析
try {
return smartParseToMillis(dateString, timezone);
} catch (Exception e1) {
// 2. 如果智能解析失败，尝试正则表达式匹配
try {
return parseWithRegex(dateString, timezone);
} catch (Exception e2) {
// 3. 如果都失败，尝试SimpleDateFormat的通用解析
return parseWithSimpleDateFormat(dateString, timezone);
}
}
}
/**
* 智能解析 - 尝试所有预定义格式
*/
private static long smartParseToMillis(String dateString, String timezone) throws ParseException {
// 移除空格和非法字符
dateString = cleanDateString(dateString);
// 尝试所有预定义格式
for (DateTimeFormatter formatter : PREDEFINED_FORMATTERS) {
try {
return parseWithFormatter(dateString, formatter, timezone);
} catch (DateTimeParseException e) {
// 尝试下一个格式
continue;
}
}
throw new ParseException("无法解析时间字符串: " + dateString, 0);
}
/**
* 使用指定的DateTimeFormatter解析
*/
private static long parseWithFormatter(String dateString, DateTimeFormatter formatter, String timezone)
throws DateTimeParseException {
try {
// 尝试解析为OffsetDateTime（包含时区）
OffsetDateTime odt = OffsetDateTime.parse(dateString, formatter);
return odt.toInstant().toEpochMilli();
} catch (DateTimeParseException e1) {
try {
// 尝试解析为ZonedDateTime
ZonedDateTime zdt = ZonedDateTime.parse(dateString, formatter);
return zdt.toInstant().toEpochMilli();
} catch (DateTimeParseException e2) {
try {
// 尝试解析为LocalDateTime，然后附加时区
LocalDateTime ldt = LocalDateTime.parse(dateString, formatter);
ZonedDateTime zdt = ldt.atZone(getZoneId(timezone));
return zdt.toInstant().toEpochMilli();
} catch (DateTimeParseException e3) {
throw e1; // 抛出原始异常
}
}
}
}
/**
* 使用正则表达式解析（处理特殊格式）
*/
private static long parseWithRegex(String dateString, String timezone) throws ParseException {
// 定义常见时间格式的正则表达式
Map<Pattern, String> regexPatterns = new LinkedHashMap<>();
// 匹配 2025-11-01T15:23:41.188078+0800 格式
regexPatterns.put(
Pattern.compile("^(\\d{4}-\\d{2}-\\d{2})T(\\d{2}:\\d{2}:\\d{2})\\.(\\d{1,6})([+-]\\d{4})$"),
"MICROSECONDS_WITH_OFFSET"
);
// 匹配 2025-11-01T15:23:41.188078+08:00 格式
regexPatterns.put(
Pattern.compile("^(\\d{4}-\\d{2}-\\d{2})T(\\d{2}:\\d{2}:\\d{2})\\.(\\d{1,6})([+-]\\d{2}:\\d{2})$"),
"MICROSECONDS_WITH_COLON_OFFSET"
);
// 尝试匹配并解析
for (Map.Entry<Pattern, String> entry : regexPatterns.entrySet()) {
java.util.regex.Matcher matcher = entry.getKey().matcher(dateString);
if (matcher.matches()) {
return parseWithRegexMatcher(matcher, entry.getValue(), timezone);
}
}
throw new ParseException("正则表达式无法匹配时间字符串: " + dateString, 0);
}
/**
* 解析正则表达式匹配的结果
*/
private static long parseWithRegexMatcher(java.util.regex.Matcher matcher, String patternType, String timezone)
throws ParseException {
String datePart = matcher.group(1);
String timePart = matcher.group(2);
String fractionPart = matcher.group(3);
String offsetPart = matcher.group(4);
// 处理微秒/毫秒部分
int nanoseconds;
if (fractionPart.length() <= 3) {
// 毫秒
nanoseconds = Integer.parseInt(fractionPart) * 1_000_000;
} else {
// 微秒或纳秒
if (fractionPart.length() > 6) {
fractionPart = fractionPart.substring(0, 6); // 最多保留6位
}
while (fractionPart.length() < 6) {
fractionPart = fractionPart + "0";
}
nanoseconds = Integer.parseInt(fractionPart) * 1000; // 微秒转纳秒
}
try {
// 构建LocalDateTime
LocalDateTime ldt = LocalDateTime.parse(datePart + "T" + timePart);
// 处理时区
ZonedDateTime zdt;
if (offsetPart != null && !offsetPart.isEmpty()) {
// 有时区偏移
String offsetStr = offsetPart.replace(":", "");
if (offsetStr.length() == 5) { // +0800 或 -0500
int hours = Integer.parseInt(offsetStr.substring(0, 3));
int minutes = Integer.parseInt(offsetStr.substring(3));
ZoneOffset offset = ZoneOffset.ofHoursMinutes(hours, minutes);
zdt = ldt.atZone(offset);
} else if (offsetStr.equalsIgnoreCase("Z")) {
zdt = ldt.atZone(ZoneOffset.UTC);
} else {
// 默认使用提供的时区
zdt = ldt.atZone(getZoneId(timezone));
}
} else {
// 无时区信息，使用提供的时区
zdt = ldt.atZone(getZoneId(timezone));
}
// 添加纳秒部分
zdt = zdt.withNano(nanoseconds);
return zdt.toInstant().toEpochMilli();
} catch (Exception e) {
throw new ParseException("正则解析失败: " + e.getMessage(), 0);
}
}
/**
* 使用SimpleDateFormat解析（兼容Java 7）
*/
private static long parseWithSimpleDateFormat(String dateString, String timezone) throws ParseException {
// 清理字符串
dateString = cleanDateString(dateString);
// 准备多种格式尝试
String[] patterns = {
"yyyy-MM-dd'T'HH:mm:ss.SSSXXX",
"yyyy-MM-dd'T'HH:mm:ssXXX",
"yyyy-MM-dd HH:mm:ss.SSS",
"yyyy-MM-dd HH:mm:ss",
"yyyy/MM/dd HH:mm:ss",
"dd/MM/yyyy HH:mm:ss",
"MM/dd/yyyy HH:mm:ss"
};
for (String pattern : patterns) {
try {
SimpleDateFormat sdf = getCachedSimpleDateFormat(pattern, timezone);
Date date = sdf.parse(dateString);
return date.getTime();
} catch (ParseException e) {
continue;
}
}
throw new ParseException("无法解析时间字符串: " + dateString, 0);
}
/**
* 清理时间字符串
*/
private static String cleanDateString(String dateString) {
if (dateString == null) {
return "";
}
// 移除前后空格
String cleaned = dateString.trim();
// 替换中文括号等
cleaned = cleaned.replace("（", "(").replace("）", ")");
// 处理常见的错误格式
cleaned = cleaned.replace("T ", "T"); // 移除T后面的空格
return cleaned;
}
/**
* 获取缓存的SimpleDateFormat
*/
private static SimpleDateFormat getCachedSimpleDateFormat(String pattern, String timezone) {
String key = pattern + "_" + timezone;
ThreadLocal<SimpleDateFormat> threadLocalSdf = SDF_CACHE.computeIfAbsent(key, k ->
ThreadLocal.withInitial(() -> {
SimpleDateFormat sdf = new SimpleDateFormat(pattern);
sdf.setTimeZone(getTimeZone(timezone));
return sdf;
})
);
return threadLocalSdf.get();
}
/**
* 获取缓存的DateTimeFormatter
*/
private static DateTimeFormatter getCachedDateTimeFormatter(String pattern) {
return DTF_CACHE.computeIfAbsent(pattern, DateTimeFormatter::ofPattern);
}
/**
* 获取ZoneId
*/
private static ZoneId getZoneId(String timezone) {
if (timezone == null) {
return ZoneId.systemDefault();
}
switch (timezone.toUpperCase()) {
case "CST":
return ZoneId.of("Asia/Shanghai");
case "UTC":
return ZoneId.of("UTC");
case "GMT":
return ZoneId.of("GMT");
case "EST":
return ZoneId.of("America/New_York");
case "PST":
return ZoneId.of("America/Los_Angeles");
default:
try {
return ZoneId.of(timezone);
} catch (Exception e) {
return ZoneId.systemDefault();
}
}
}
/**
* 获取TimeZone
*/
private static TimeZone getTimeZone(String timezone) {
if (timezone == null) {
return TimeZone.getDefault();
}
switch (timezone.toUpperCase()) {
case "CST":
return TimeZone.getTimeZone("Asia/Shanghai");
case "UTC":
return TimeZone.getTimeZone("UTC");
case "GMT":
return TimeZone.getTimeZone("GMT");
case "EST":
return TimeZone.getTimeZone("America/New_York");
case "PST":
return TimeZone.getTimeZone("America/Los_Angeles");
default:
TimeZone tz = TimeZone.getTimeZone(timezone);
if ("GMT".equals(tz.getID()) && !"GMT".equalsIgnoreCase(timezone)) {
return TimeZone.getDefault();
}
return tz;
}
}
/**
* 检测Java版本是否支持java.time
*/
public static boolean isJava8OrHigher() {
try {
Class.forName("java.time.LocalDateTime");
return true;
} catch (ClassNotFoundException e) {
return false;
}
}
/**
* 根据Action JSON结构调用
* 保持与原接口的兼容性
*/
public static long processTimeConversion(String dateString, ActionParam param) throws Exception {
if (param == null) {
throw new IllegalArgumentException("ActionParam不能为空");
}
return convertToMillis(dateString, param.timezone);
}
/**
* 添加自定义格式到预定义列表中
*/
public static void addCustomFormatter(String pattern) {
try {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
PREDEFINED_FORMATTERS.add(0, formatter); // 添加到开头，优先尝试
} catch (Exception e) {
throw new IllegalArgumentException("无效的时间格式模式: " + pattern, e);
}
}
/**
* 添加自定义格式到预定义列表中（带优先级别）
*/
public static void addCustomFormatter(String pattern, int priority) {
try {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
if (priority < 0) {
PREDEFINED_FORMATTERS.add(formatter);
} else {
PREDEFINED_FORMATTERS.add(Math.min(priority, PREDEFINED_FORMATTERS.size()), formatter);
}
} catch (Exception e) {
throw new IllegalArgumentException("无效的时间格式模式: " + pattern, e);
}
}
/**
* 获取支持的所有格式
*/
public static List<String> getSupportedFormats() {
List<String> formats = new ArrayList<>();
for (DateTimeFormatter formatter : PREDEFINED_FORMATTERS) {
formats.add(formatter.toString());
}
return formats;
}
// 模拟Action JSON结构的类
public static class ActionParam {
public String javaDateFormat;
public String goDateFormat;
public String timezone;
public ActionParam() {}
public ActionParam(String javaDateFormat, String timezone) {
this.javaDateFormat = javaDateFormat;
this.timezone = timezone;
}
}
/**
* 测试方法
*/
public static void main(String[] args) {
// 测试各种时间格式
List<TestData> testCases = new ArrayList<>();
// 添加测试用例
testCases.add(new TestData("2025-11-01T15:23:41.188078+0800", "CST"));
testCases.add(new TestData("2025-11-01T15:23:41.188+0800", "CST"));
testCases.add(new TestData("2025-11-01T15:23:41+0800", "CST"));
testCases.add(new TestData("2025-11-01T07:23:41.188078Z", "UTC"));
testCases.add(new TestData("2025-11-01T15:23:41.188078+08:00", "CST"));
testCases.add(new TestData("2025-11-01T15:23:41", "CST")); // 无时区
testCases.add(new TestData("2025-11-01 15:23:41", "CST"));
testCases.add(new TestData("2025/11/01 15:23:41.123", "CST"));
testCases.add(new TestData("2025-11-01T15:23:41.123456", "UTC"));
testCases.add(new TestData("2025年11月01日 15:23:41", "CST"));
testCases.add(new TestData("01/11/2025 15:23:41", "CST"));
testCases.add(new TestData("11/01/2025 15:23:41", "CST"));
System.out.println("=== 时间格式转换测试 ===\n");
for (TestData test : testCases) {
try {
long timestamp = convertToMillis(test.dateString, test.timezone);
System.out.println(String.format("✓ 成功: %-40s | 时区: %-5s | 时间戳: %d",
test.dateString, test.timezone, timestamp));
} catch (Exception e) {
System.out.println(String.format("✗ 失败: %-40s | 时区: %-5s | 错误: %s",
test.dateString, test.timezone, e.getMessage()));
}
}
// 测试Action JSON参数
System.out.println("\n=== Action JSON参数测试 ===");
try {
ActionParam param = new ActionParam("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", "CST");
long timestamp = processTimeConversion("2025-11-01T15:23:41.188078+0800", param);
System.out.println("Action JSON转换结果: " + timestamp);
} catch (Exception e) {
e.printStackTrace();
}
}
static class TestData {
String dateString;
String timezone;
TestData(String dateString, String timezone) {
this.dateString = dateString;
this.timezone = timezone;
}
}
}

238
haobang-security-xdr/syslog-consumer/src/main/resources/mapper/DeviceDeviceMapper.xml Normal file
View File

@@ -0,0 +1,238 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.common.mapper.DeviceDeviceMapper">
<!-- 基础结果映射 -->
<resultMap id="BaseResultMap" type="com.common.entity.DeviceDevice">
<id column="id" property="id" />
<result column="created_at" property="createdAt" />
<result column="updated_at" property="updatedAt" />
<result column="deleted_at" property="deletedAt" />
<result column="name" property="name" />
<result column="ip" property="ip" />
<result column="device_group" property="deviceGroup" />
<result column="device_type" property="deviceType" />
<result column="vendor" property="vendor" />
<result column="product_name" property="productName" />
<result column="organization_id" property="organizationId" />
<result column="last_receive_time" property="lastReceiveTime" />
<result column="agent_id" property="agentId" />
<result column="detail_id" property="detailId" />
<result column="control_agent_id" property="controlAgentId" />
<result column="license_start_time" property="licenseStartTime" />
<result column="license_end_time" property="licenseEndTime" />
<result column="is_monitoring" property="isMonitoring" />
<result column="security_scope_id" property="securityScopeId" />
<result column="owner_id" property="ownerId" />
<result column="ssh_config_id" property="sshConfigId" />
<result column="status" property="status" />
<result column="created_by_id" property="createdById" />
<result column="decode_type" property="decodeType" />
<result column="miss_policy" property="missPolicy" />
<result column="tenant_id" property="tenantId" />
<result column="create_time" property="createTime" />
<result column="update_time" property="updateTime" />
<result column="create_by" property="createBy" />
<result column="update_by" property="updateBy" />
<result column="del_flag" property="delFlag" />
<result column="manager_name" property="managerName" />
<result column="today_parse_count" property="todayParseCount" />
<result column="today_non_log_count" property="todayNonLogCount" />
<result column="create_dept" property="createDept" />
<result column="device_collect_id" property="deviceCollectId" />
</resultMap>
<!-- 基础查询列 -->
<sql id="Base_Column_List">
id, created_at::timestamp , updated_at::timestamp, deleted_at::timestamp, name, ip, device_group, device_type,
vendor, product_name, organization_id, last_receive_time::timestamp, agent_id, detail_id,
control_agent_id, license_start_time::timestamp, license_end_time::timestamp, is_monitoring,
security_scope_id, owner_id, ssh_config_id, status, created_by_id, decode_type,
miss_policy, tenant_id, create_time::timestamp, update_time::timestamp, create_by, update_by, del_flag,
manager_name, today_parse_count, today_non_log_count, create_dept, device_collect_id
</sql>
<!-- 根据ID查询 -->
<select id="selectById" parameterType="java.lang.Integer" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
WHERE id = #{id}
</select>
<!-- 查询所有设备 -->
<select id="selectAll" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
</select>
<!-- 根据IP查询 -->
<select id="selectByIp" parameterType="java.lang.String" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
WHERE ip = #{ip} and del_flag='0'
</select>
<!-- 根据名称模糊查询 -->
<select id="selectByNameLike" parameterType="java.lang.String" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
WHERE name LIKE CONCAT('%', #{name}, '%')
</select>
<!-- 根据设备组查询 -->
<select id="selectByDeviceGroup" parameterType="java.lang.Integer" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
WHERE device_group = #{deviceGroup}
</select>
<!-- 根据设备类型查询 -->
<select id="selectByDeviceType" parameterType="java.lang.Integer" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
WHERE device_type = #{deviceType}
</select>
<!-- 根据组织ID查询 -->
<select id="selectByOrganizationId" parameterType="java.lang.Integer" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
WHERE organization_id = #{organizationId}
</select>
<!-- 根据状态查询 -->
<select id="selectByStatus" parameterType="java.lang.Short" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
WHERE status = #{status}
</select>
<!-- 多条件组合查询 -->
<select id="selectByCondition" parameterType="com.common.entity.DeviceDevice" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
<where>
<if test="name != null and name != ''">
AND name LIKE CONCAT('%', #{name}, '%')
</if>
<if test="ip != null and ip != ''">
AND ip = #{ip}
</if>
<if test="deviceGroup != null">
AND device_group = #{deviceGroup}
</if>
<if test="deviceType != null">
AND device_type = #{deviceType}
</if>
<if test="vendor != null and vendor != ''">
AND vendor = #{vendor}
</if>
<if test="organizationId != null">
AND organization_id = #{organizationId}
</if>
<if test="status != null">
AND status = #{status}
</if>
<if test="isMonitoring != null">
AND is_monitoring = #{isMonitoring}
</if>
</where>
ORDER BY created_at DESC
</select>
<!-- 动态条件查询 -->
<select id="selectByMap" parameterType="java.util.Map" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
<where>
<if test="name != null">
AND name LIKE CONCAT('%', #{name}, '%')
</if>
<if test="ip != null">
AND ip = #{ip}
</if>
<if test="deviceGroup != null">
AND device_group = #{deviceGroup}
</if>
<if test="status != null">
AND status = #{status}
</if>
<if test="vendor != null">
AND vendor = #{vendor}
</if>
<if test="startTime != null">
AND created_at >= #{startTime}
</if>
<if test="endTime != null">
AND created_at &lt;= #{endTime}
</if>
</where>
</select>
<!-- 分页查询 -->
<select id="selectByPage" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
ORDER BY id
LIMIT #{limit} OFFSET #{offset}
</select>
<!-- 统计总数 -->
<select id="count" resultType="java.lang.Long">
SELECT COUNT(*) FROM device_device
</select>
<!-- 根据条件统计 -->
<select id="countByCondition" parameterType="com.common.entity.DeviceDevice" resultType="java.lang.Long">
SELECT COUNT(*) FROM device_device
<where>
<if test="name != null and name != ''">
AND name LIKE CONCAT('%', #{name}, '%')
</if>
<if test="status != null">
AND status = #{status}
</if>
<if test="isMonitoring != null">
AND is_monitoring = #{isMonitoring}
</if>
</where>
</select>
<!-- 查询监控中的设备 -->
<select id="selectMonitoringDevices" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
WHERE is_monitoring = true
</select>
<!-- 查询未删除的设备 -->
<select id="selectActiveDevices" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
WHERE del_flag = '0'
</select>
<!-- 根据厂商查询 -->
<select id="selectByVendor" parameterType="java.lang.String" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List" />
FROM device_device
WHERE vendor = #{vendor}
</select>
</mapper>

6
haobang-security-xdr/syslog-consumer/src/main/resources/mapper/SyslogNormalAlarmMapper.xml
View File

@@ -467,7 +467,7 @@
<if test="dataMap.tx_bytes != null">#{dataMap.tx_bytes}::double precision,</if>
<if test="dataMap.rx_bytes != null">#{dataMap.rx_bytes}::double precision,</if>
<if test="dataMap.all_bytes != null">#{dataMap.all_bytes}::double precision,</if>
<if test="dataMap.duration_time != null">#{dataMap.duration_time},</if>
<if test="dataMap.duration_time != null">#{dataMap.duration_time}::int8,</if>
<if test="dataMap.mail_attach_name != null">#{dataMap.mail_attach_name},</if>
<if test="dataMap.mail_subject != null">#{dataMap.mail_subject},</if>
<if test="dataMap.mail_message != null">#{dataMap.mail_message},</if>
@@ -627,7 +627,7 @@
<if test="dataMap.exe_name != null">#{dataMap.exe_name},</if>
<if test="dataMap.exe_path != null">#{dataMap.exe_path},</if>
<if test="dataMap.login_time != null">#{dataMap.login_time},</if>
<if test="dataMap.login_times != null">#{dataMap.login_times},</if>
<if test="dataMap.login_times != null">#{dataMap.login_times}::int8,</if>
<if test="dataMap.check_item != null">#{dataMap.check_item},</if>
<if test="dataMap.check_type != null">#{dataMap.check_type},</if>
<if test="dataMap.attacker_ip != null">#{dataMap.attacker_ip}::inet,</if>
@@ -687,7 +687,7 @@
<if test="dataMap.dest_lon != null">#{dataMap.dest_lon},</if>
<if test="dataMap.dest_lat != null">#{dataMap.dest_lat},</if>
<if test="dataMap.event_category != null">#{dataMap.event_category},</if>
<if test="dataMap.attack_result != null">#{dataMap.attack_result},</if>
<if test="dataMap.attack_result != null">#{dataMap.attack_result}::int,</if>
<if test="dataMap.probe_ip != null">#{dataMap.probe_ip}::inet,</if>
<if test="dataMap.device_ip != null">#{dataMap.device_ip}::inet,</if>
<if test="dataMap.device_manufacturer != null">#{dataMap.device_manufacturer},</if>

4
haobang-security-xdr/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml
View File

@@ -603,7 +603,7 @@
<if test="dataMap.tx_bytes != null">#{dataMap.tx_bytes}::double precision,</if>
<if test="dataMap.rx_bytes != null">#{dataMap.rx_bytes}::double precision,</if>
<if test="dataMap.all_bytes != null">#{dataMap.all_bytes}::double precision,</if>
<if test="dataMap.duration_time != null">#{dataMap.duration_time},</if>
<if test="dataMap.duration_time != null">#{dataMap.duration_time}::int8,</if>
<if test="dataMap.mail_attach_name != null">#{dataMap.mail_attach_name},</if>
<if test="dataMap.mail_subject != null">#{dataMap.mail_subject},</if>
<if test="dataMap.mail_message != null">#{dataMap.mail_message},</if>
@@ -763,7 +763,7 @@
<if test="dataMap.exe_name != null">#{dataMap.exe_name},</if>
<if test="dataMap.exe_path != null">#{dataMap.exe_path},</if>
<if test="dataMap.login_time != null">#{dataMap.login_time},</if>
<if test="dataMap.login_times != null">#{dataMap.login_times},</if>
<if test="dataMap.login_times != null">#{dataMap.login_times}::int8,</if>
<if test="dataMap.check_item != null">#{dataMap.check_item},</if>
<if test="dataMap.check_type != null">#{dataMap.check_type},</if>
<if test="dataMap.attacker_ip != null">#{dataMap.attacker_ip}::inet,</if>