diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java index 5f7c89c..cd0f502 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java @@ -36,6 +36,9 @@ import com.common.service.LogDataFilterService; import com.common.service.LogDataCompleteService; import com.common.service.DeviceCollectTaskService; import com.common.entity.DeviceCollectTask; +import com.common.entity.DeviceDevice; +import com.common.util.TimeConversionUtils; +import com.common.service.DeviceDeviceService; public class LogNormalProcessor { private static final Logger logger = LoggerFactory.getLogger(LogNormalProcessor.class); @@ -65,7 +68,8 @@ public class LogNormalProcessor { private LogDataCompleteService logDataCompleteService= SpringContextUtil.getBean(LogDataCompleteService.class); @Autowired private DeviceCollectTaskService deviceCollectTaskService= SpringContextUtil.getBean(DeviceCollectTaskService.class); - + @Autowired + private DeviceDeviceService deviceDeviceService= SpringContextUtil.getBean(DeviceDeviceService.class); @Autowired SyslogNonNormalMessage syslogNonNormalMessage=new SyslogNonNormalMessage(); @@ -433,15 +437,26 @@ public class LogNormalProcessor { //匹配并获取映射枚举值 normalColumMap.put("dest_field_value",getMappingValue(action_param ,entry.getValue().toString() )); } - else if(((HashMap)map.get("action")).get("type").equals("time\"")) + else if(((HashMap)map.get("action")).get("type").equals("time")) { //time 类型 - normalColumMap.put("dest_field_value",entry.getValue() ); + //normalColumMap.put("dest_field_value",entry.getValue() ); + normalColumMap.put("action_param",((HashMap)map.get("action")).get("param") ); + HashMap action_param=(HashMap)((HashMap)map.get("action")).get("param") ; + //匹配时间格式并转成换整型格式 + try { + long longTime = TimeConversionUtils.convertToMillis(entry.getValue().toString(), action_param.get("timezone").toString()); + normalColumMap.put("dest_field_value", longTime); + } catch (Exception e) { + logger.error("时间类型转换错误,源值:" + entry.getValue().toString() + ",java_date_format:" + action_param.get("java_date_format").toString()); + e.printStackTrace(); + } + } columnlist.add((HashMap)normalColumMap); - //System.out.println( "normalColumMap: " +normalColumMap); - break; + //存在源字段配置多个目标字段,使用continue,而不是break + continue; } } } @@ -499,6 +514,9 @@ public class LogNormalProcessor { return ; } Map columnMap = logColumnMap; + //补全设备信息字段 + CompletionDeviceInfo(columnMap, deviceId); + columnMap.put("device_id", deviceId); columnMap.put("log_time", logtime); columnMap.put("id", UUID.randomUUID().toString()); @@ -519,6 +537,30 @@ public class LogNormalProcessor { } } + /** + * 补全设备信息字段 + * @param columnMap + * @param deviceID + */ + public void CompletionDeviceInfo(Map columnMap ,Long deviceID) + { + try { + DeviceDevice devInfo = deviceDeviceService.getByIdSafely( Long.valueOf(deviceID).intValue() ); + if (devInfo != null) { + columnMap.put("device_ip", devInfo.getIp()); + columnMap.put("device_manufacturer", devInfo.getVendor()); + columnMap.put("device_name", devInfo.getName()); + } + } catch (Exception e) { + logger.error("CompletionDeviceInfo 失败!ID:"+deviceID ); + } + } + + /** + * 保存非标日志记录 + * @param deviceId + * @param logtime + */ public void SaveNonNormalMessage(long deviceId , DateTime logtime) { try { diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/DeviceDevice.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/DeviceDevice.java new file mode 100644 index 0000000..4025256 --- /dev/null +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/DeviceDevice.java @@ -0,0 +1,376 @@ +package com.common.entity; + + +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +public class DeviceDevice { + public Integer id; + public LocalDateTime createdAt; + public LocalDateTime updatedAt; + public LocalDateTime deletedAt; + public String name; + public String ip; + public Integer deviceGroup; + public Integer deviceType; + public String vendor; + public String productName; + public Integer organizationId; + public LocalDateTime lastReceiveTime; + public Integer agentId; + public Integer detailId; + public Integer controlAgentId; + public LocalDateTime licenseStartTime; + public LocalDateTime licenseEndTime; + public Boolean isMonitoring; + public Long securityScopeId; + public Long ownerId; + public Long sshConfigId; + public Short status; + public Long createdById; + public Integer decodeType; + public Integer missPolicy; + public String tenantId; + public LocalDateTime createTime; + public LocalDateTime updateTime; + public Long createBy; + public Long updateBy; + public String delFlag; + public String managerName; + public Integer todayParseCount; + public Integer todayNonLogCount; + public Long createDept; + public Integer deviceCollectId; + + + // Getter and Setter methods + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public LocalDateTime getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(LocalDateTime createdAt) { + this.createdAt = createdAt; + } + + public LocalDateTime getUpdatedAt() { + return updatedAt; + } + + public void setUpdatedAt(LocalDateTime updatedAt) { + this.updatedAt = updatedAt; + } + + public LocalDateTime getDeletedAt() { + return deletedAt; + } + + public void setDeletedAt(LocalDateTime deletedAt) { + this.deletedAt = deletedAt; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getIp() { + return ip; + } + + public void setIp(String ip) { + this.ip = ip; + } + + public Integer getDeviceGroup() { + return deviceGroup; + } + + public void setDeviceGroup(Integer deviceGroup) { + this.deviceGroup = deviceGroup; + } + + public Integer getDeviceType() { + return deviceType; + } + + public void setDeviceType(Integer deviceType) { + this.deviceType = deviceType; + } + + public String getVendor() { + return vendor; + } + + public void setVendor(String vendor) { + this.vendor = vendor; + } + + public String getProductName() { + return productName; + } + + public void setProductName(String productName) { + this.productName = productName; + } + + public Integer getOrganizationId() { + return organizationId; + } + + public void setOrganizationId(Integer organizationId) { + this.organizationId = organizationId; + } + + public LocalDateTime getLastReceiveTime() { + return lastReceiveTime; + } + + public void setLastReceiveTime(LocalDateTime lastReceiveTime) { + this.lastReceiveTime = lastReceiveTime; + } + + public Integer getAgentId() { + return agentId; + } + + public void setAgentId(Integer agentId) { + this.agentId = agentId; + } + + public Integer getDetailId() { + return detailId; + } + + public void setDetailId(Integer detailId) { + this.detailId = detailId; + } + + public Integer getControlAgentId() { + return controlAgentId; + } + + public void setControlAgentId(Integer controlAgentId) { + this.controlAgentId = controlAgentId; + } + + public LocalDateTime getLicenseStartTime() { + return licenseStartTime; + } + + public void setLicenseStartTime(LocalDateTime licenseStartTime) { + this.licenseStartTime = licenseStartTime; + } + + public LocalDateTime getLicenseEndTime() { + return licenseEndTime; + } + + public void setLicenseEndTime(LocalDateTime licenseEndTime) { + this.licenseEndTime = licenseEndTime; + } + + public Boolean getIsMonitoring() { + return isMonitoring; + } + + public void setIsMonitoring(Boolean isMonitoring) { + this.isMonitoring = isMonitoring; + } + + public Long getSecurityScopeId() { + return securityScopeId; + } + + public void setSecurityScopeId(Long securityScopeId) { + this.securityScopeId = securityScopeId; + } + + public Long getOwnerId() { + return ownerId; + } + + public void setOwnerId(Long ownerId) { + this.ownerId = ownerId; + } + + public Long getSshConfigId() { + return sshConfigId; + } + + public void setSshConfigId(Long sshConfigId) { + this.sshConfigId = sshConfigId; + } + + public Short getStatus() { + return status; + } + + public void setStatus(Short status) { + this.status = status; + } + + public Long getCreatedById() { + return createdById; + } + + public void setCreatedById(Long createdById) { + this.createdById = createdById; + } + + public Integer getDecodeType() { + return decodeType; + } + + public void setDecodeType(Integer decodeType) { + this.decodeType = decodeType; + } + + public Integer getMissPolicy() { + return missPolicy; + } + + public void setMissPolicy(Integer missPolicy) { + this.missPolicy = missPolicy; + } + + public String getTenantId() { + return tenantId; + } + + public void setTenantId(String tenantId) { + this.tenantId = tenantId; + } + + public LocalDateTime getCreateTime() { + return createTime; + } + + public void setCreateTime(LocalDateTime createTime) { + this.createTime = createTime; + } + + public LocalDateTime getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(LocalDateTime updateTime) { + this.updateTime = updateTime; + } + + public Long getCreateBy() { + return createBy; + } + + public void setCreateBy(Long createBy) { + this.createBy = createBy; + } + + public Long getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(Long updateBy) { + this.updateBy = updateBy; + } + + public String getDelFlag() { + return delFlag; + } + + public void setDelFlag(String delFlag) { + this.delFlag = delFlag; + } + + public String getManagerName() { + return managerName; + } + + public void setManagerName(String managerName) { + this.managerName = managerName; + } + + public Integer getTodayParseCount() { + return todayParseCount; + } + + public void setTodayParseCount(Integer todayParseCount) { + this.todayParseCount = todayParseCount; + } + + public Integer getTodayNonLogCount() { + return todayNonLogCount; + } + + public void setTodayNonLogCount(Integer todayNonLogCount) { + this.todayNonLogCount = todayNonLogCount; + } + + public Long getCreateDept() { + return createDept; + } + + public void setCreateDept(Long createDept) { + this.createDept = createDept; + } + + public Integer getDeviceCollectId() { + return deviceCollectId; + } + + public void setDeviceCollectId(Integer deviceCollectId) { + this.deviceCollectId = deviceCollectId; + } + + @Override + public String toString() { + return "DeviceDevice{" + + "id=" + id + + ", createdAt=" + createdAt + + ", updatedAt=" + updatedAt + + ", deletedAt=" + deletedAt + + ", name='" + name + '\'' + + ", ip='" + ip + '\'' + + ", deviceGroup=" + deviceGroup + + ", deviceType=" + deviceType + + ", vendor='" + vendor + '\'' + + ", productName='" + productName + '\'' + + ", organizationId=" + organizationId + + ", lastReceiveTime=" + lastReceiveTime + + ", agentId=" + agentId + + ", detailId=" + detailId + + ", controlAgentId=" + controlAgentId + + ", licenseStartTime=" + licenseStartTime + + ", licenseEndTime=" + licenseEndTime + + ", isMonitoring=" + isMonitoring + + ", securityScopeId=" + securityScopeId + + ", ownerId=" + ownerId + + ", sshConfigId=" + sshConfigId + + ", status=" + status + + ", createdById=" + createdById + + ", decodeType=" + decodeType + + ", missPolicy=" + missPolicy + + ", tenantId='" + tenantId + '\'' + + ", createTime=" + createTime + + ", updateTime=" + updateTime + + ", createBy=" + createBy + + ", updateBy=" + updateBy + + ", delFlag='" + delFlag + '\'' + + ", managerName='" + managerName + '\'' + + ", todayParseCount=" + todayParseCount + + ", todayNonLogCount=" + todayNonLogCount + + ", createDept=" + createDept + + ", deviceCollectId=" + deviceCollectId + + '}'; + } + +} \ No newline at end of file diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/GroupedSyslogData.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/GroupedSyslogData.java index 8a69423..1540f48 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/GroupedSyslogData.java +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/GroupedSyslogData.java @@ -14,6 +14,7 @@ public class GroupedSyslogData { private LocalDateTime maxLogTime; private Long logCount; private String[] victimIps; + private String[] victimWebUrls; private Integer[] deviceIds; private String[] originLogIds; private Integer maxEventLevel; diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/RuleHitTimeDTO.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/RuleHitTimeDTO.java new file mode 100644 index 0000000..4d7ebb8 --- /dev/null +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/entity/RuleHitTimeDTO.java @@ -0,0 +1,23 @@ +package com.common.entity; + + +import lombok.Data; +import java.time.LocalDateTime; + +@Data +public class RuleHitTimeDTO { + /** + * 泛化规则ID + */ + private Long normalizeRuleId; + + /** + * 最大命中时间(从标准化表中统计) + */ + private LocalDateTime maxLogTime; + + /** + * 数据来源表 + */ + private String sourceTable; +} \ No newline at end of file diff --git a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/mapper/AlarmMapper.java b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/mapper/AlarmMapper.java index 61adaf9..944154d 100644 --- a/haobang-security-xdr/syslog-consumer/src/main/java/com/common/mapper/AlarmMapper.java +++ b/haobang-security-xdr/syslog-consumer/src/main/java/com/common/mapper/AlarmMapper.java @@ -14,11 +14,11 @@ public interface AlarmMapper { @Insert({"