nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

初次提交代码

Browse Source
This commit is contained in:
nanChen
2026-01-11 15:33:22 +08:00
commit 6603c6f4a1
455 changed files with 32175 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

323
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/AlgorithmResultParser.java Normal file
View File

@@ -0,0 +1,323 @@
package com.common.util;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.common.entity.SyslogNormalData;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@Slf4j
@Component
public class AlgorithmResultParser {
private static final DateTimeFormatter[] DATE_FORMATTERS = {
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"),
DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss"),
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"),
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"),
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS")
};
/**
* 构建示例JSON对应的newLogs
*/
public List<SyslogNormalData> buildNewLogsFromExample() {
String exampleJson = "[\n" +
" {\n" +
" \"_index\": \"es:skyeye-weblog-2025.09.30\",\n" +
" \"access_time\": \"2025-09-30 12:05:00\",\n" +
" \"dip\": \"10.20.30.51\",\n" +
" \"dname\": \"网页木马流量\",\n" +
" \"dtype\": \"疑似木马活动\",\n" +
" \"host\": \"\",\n" +
" \"log_id\": \"hidden-002\",\n" +
" \"origin_field\": \"匹配到文件名: shell.php:.jpg 与 匹配到恶意请求: exec=dir\",\n" +
" \"reason\": \"源IP 203.0.113.51 访问 目的IP 10.20.30.51 异常，入度=0, 出度=0, 独立访客=1，匹配到文件名: shell.php:.jpg，匹配到恶意请求: exec=dir\",\n" +
" \"referer\": \"\",\n" +
" \"sip\": \"203.0.113.51\",\n" +
" \"status_code\": 200,\n" +
" \"url\": \"/admin/shell.php:.jpg\"\n" +
" }\n" +
"]";
return parseJsonToLogs(exampleJson);
}
/**
* 解析JSON字符串为SyslogNormalData列表
*/
public List<SyslogNormalData> parseJsonToLogs(String jsonStr) {
List<SyslogNormalData> logs = new ArrayList<>();
try {
JSONArray jsonArray = JSON.parseArray(jsonStr);
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject jsonObj = jsonArray.getJSONObject(i);
SyslogNormalData logData = convertJsonObject(jsonObj);
if (logData != null) {
logs.add(logData);
}
}
log.info("成功解析 {} 条日志数据", logs.size());
} catch (Exception e) {
log.error("解析JSON失败: {}", e.getMessage(), e);
}
return logs;
}
/**
* 转换单个JSON对象
*/
private SyslogNormalData convertJsonObject(JSONObject jsonObj) {
try {
SyslogNormalData logData = new SyslogNormalData();
// 基础字段
logData.setId(getString(jsonObj, "log_id", UUID.randomUUID().toString()));
logData.setSyslogUuid(UUID.randomUUID().toString());
logData.setLogId(getString(jsonObj, "log_id"));
// 时间字段
LocalDateTime now = LocalDateTime.now();
logData.setCreatedAt(now);
logData.setEventDate(now);
String accessTime = getString(jsonObj, "access_time");
if (accessTime != null && !accessTime.isEmpty()) {
LocalDateTime logTime = parseDateTime(accessTime);
logData.setLogTime(logTime != null ? logTime : now);
} else {
logData.setLogTime(now);
}
// IP字段
logData.setSrcIp(getString(jsonObj, "sip"));
logData.setDestIp(getString(jsonObj, "dip"));
logData.setSrcIpStr(getString(jsonObj, "sip"));
logData.setDestIpStr(getString(jsonObj, "dip"));
// HTTP字段
logData.setHttpUrl(getString(jsonObj, "url"));
logData.setHttpHost(getString(jsonObj, "host"));
logData.setHttpReferer(getString(jsonObj, "referer"));
logData.setHttpStatusCode(getLong(jsonObj, "status_code", 200L));
logData.setHttpMethod("GET");
// 安全检测字段
logData.setAttackResult(1);
logData.setEventCategory(1);
logData.setEventType(1);
logData.setEventLevel(3);
logData.setEngineType(getString(jsonObj, "dtype"));
logData.setOriginRuleName(getString(jsonObj, "dname"));
logData.setOriginEventName(getString(jsonObj, "dname"));
logData.setDescription(getString(jsonObj, "reason"));
logData.setOriginAttackResult(getString(jsonObj, "origin_field"));
// 协议
logData.setProto("http");
logData.setSyslogTopic("algorithm_detection");
// 从origin_field提取额外信息
String originField = getString(jsonObj, "origin_field");
if (originField != null) {
extractAdditionalInfo(logData, originField);
}
// 设置索引信息
logData.setId(getString(jsonObj, "_index"));
return logData;
} catch (Exception e) {
log.error("转换JSON对象失败: {}", e.getMessage());
return null;
}
}
/**
* 从origin_field提取额外信息
*/
private void extractAdditionalInfo(SyslogNormalData logData, String originField) {
// 提取文件名
if (originField.contains("文件名:")) {
String[] parts = originField.split("文件名:");
if (parts.length > 1) {
String fileInfo = parts[1].split("与")[0].trim();
logData.setFileName(fileInfo);
logData.setFileType(determineFileType(fileInfo));
if (fileInfo.toLowerCase().contains("shell")) {
logData.setWebshellType("疑似Webshell");
logData.setBackdoorType("网页后门");
}
}
}
// 提取恶意请求
if (originField.contains("恶意请求:")) {
String[] parts = originField.split("恶意请求:");
if (parts.length > 1) {
String maliciousRequest = parts[1].trim();
logData.setCmdline(maliciousRequest);
logData.setShellCmdline(maliciousRequest);
logData.setDetail(maliciousRequest);
// 检查是否是命令执行
if (maliciousRequest.contains("exec=") ||
maliciousRequest.contains("cmd=") ||
maliciousRequest.contains("system(")) {
logData.setOriginAttackAction("命令执行");
}
}
}
}
/**
* 根据文件名确定文件类型
*/
private String determineFileType(String fileName) {
if (fileName == null) return "";
fileName = fileName.toLowerCase();
if (fileName.endsWith(".php") || fileName.contains(".php:")) {
return "php";
} else if (fileName.endsWith(".jsp")) {
return "jsp";
} else if (fileName.endsWith(".asp") || fileName.endsWith(".aspx")) {
return "asp";
} else if (fileName.endsWith(".jpg") || fileName.endsWith(".jpeg")) {
return "image";
} else if (fileName.endsWith(".png")) {
return "image";
} else if (fileName.endsWith(".gif")) {
return "image";
} else if (fileName.endsWith(".bmp")) {
return "image";
} else if (fileName.endsWith(".exe")) {
return "executable";
} else if (fileName.endsWith(".dll")) {
return "library";
} else if (fileName.endsWith(".bat") || fileName.endsWith(".cmd")) {
return "batch";
} else if (fileName.endsWith(".sh")) {
return "shell";
} else if (fileName.endsWith(".py")) {
return "python";
} else if (fileName.endsWith(".js")) {
return "javascript";
} else if (fileName.endsWith(".html") || fileName.endsWith(".htm")) {
return "html";
} else if (fileName.endsWith(".css")) {
return "css";
} else if (fileName.endsWith(".xml")) {
return "xml";
} else if (fileName.endsWith(".json")) {
return "json";
} else if (fileName.endsWith(".txt")) {
return "text";
} else if (fileName.endsWith(".pdf")) {
return "pdf";
} else if (fileName.endsWith(".doc") || fileName.endsWith(".docx")) {
return "document";
} else if (fileName.endsWith(".xls") || fileName.endsWith(".xlsx")) {
return "spreadsheet";
} else if (fileName.endsWith(".zip") || fileName.endsWith(".rar") ||
fileName.endsWith(".7z") || fileName.endsWith(".tar") ||
fileName.endsWith(".gz")) {
return "archive";
} else {
return "unknown";
}
}
/**
* 解析时间字符串
*/
private LocalDateTime parseDateTime(String timeStr) {
if (timeStr == null || timeStr.isEmpty()) {
return LocalDateTime.now();
}
for (DateTimeFormatter formatter : DATE_FORMATTERS) {
try {
return LocalDateTime.parse(timeStr, formatter);
} catch (Exception e) {
// 继续尝试下一个格式
}
}
// 如果所有格式都失败，尝试其他格式
try {
// 尝试处理带有时区的时间格式
if (timeStr.contains("T")) {
return LocalDateTime.parse(timeStr,
DateTimeFormatter.ISO_LOCAL_DATE_TIME);
}
// 尝试处理日期部分
if (timeStr.length() >= 10) {
String datePart = timeStr.substring(0, 10);
return LocalDateTime.parse(datePart + "T00:00:00");
}
} catch (Exception e) {
log.warn("无法解析时间字符串: {}, 使用当前时间", timeStr);
}
return LocalDateTime.now();
}
/**
* 安全获取字符串字段
*/
private String getString(JSONObject jsonObj, String key) {
return getString(jsonObj, key, "");
}
private String getString(JSONObject jsonObj, String key, String defaultValue) {
try {
String value = jsonObj.getString(key);
return value != null ? value : defaultValue;
} catch (Exception e) {
return defaultValue;
}
}
/**
* 安全获取Long字段
*/
private Long getLong(JSONObject jsonObj, String key, Long defaultValue) {
try {
Long value = jsonObj.getLong(key);
return value != null ? value : defaultValue;
} catch (Exception e) {
return defaultValue;
}
}
/**
* 安全获取Integer字段
*/
private Integer getInteger(JSONObject jsonObj, String key, Integer defaultValue) {
try {
Integer value = jsonObj.getInteger(key);
return value != null ? value : defaultValue;
} catch (Exception e) {
return defaultValue;
}
}
}

351
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/JsonParser.java Normal file
View File

@@ -0,0 +1,351 @@
package com.common.util;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.JsonNodeType;
import com.common.entity.*;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.*;
import java.util.Map.Entry;
import org.json.JSONObject;
import org.json.JSONArray;
import com.fasterxml.jackson.core.type.TypeReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class JsonParser {
private static final ObjectMapper objectMapper = new ObjectMapper();
private static final Logger logger = LoggerFactory.getLogger(JsonParser.class);
public static void main(String[] args) {
String logMessage = "<15>Sep 24 12:06:59 LAPTOP-ARDUR3N0 <15>2025-09-24T11:52:26Z 5f46d3be75e1 supermario 128 honeypot_event - {\"source\":\"honeypot\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}";
System.out.println(logMessage);
try {
XdrHoneypot xdrHoneypot = parseLogMessageToXdrHoneypot(logMessage);
System.out.println("解析成功！");
System.out.println(xdrHoneypot);
} catch (Exception e) {
System.err.println("解析失败: " + e.getMessage());
e.printStackTrace();
}
// 多层嵌套 JSON
String complexJson = "{\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}";
System.out.println("=== 扁平化 Map 解析结果 ===");
Map<String, Object> flatMap = parseJsonToFlatMap(complexJson);
flatMap.forEach((key, value) -> System.out.println(key + " = " + value));
System.out.println("\n=== 嵌套结构 Map 解析结果 ===");
Map<String, Object> nestedMap = parseJsonToNestedMap(complexJson);
printMap(nestedMap);
System.out.println("\n=== 特定值访问示例 ===");
// 访问特定值
System.out.println("extra.payload.format: " + flatMap.get("extra.payload.format"));
System.out.println("extra.payload.format: " + flatMap.get("extra.payload.name.cn"));
}
/**
* 从日志消息中解析 JSON 并转换为 XdrHoneypot 对象
*/
public static XdrHoneypot parseLogMessageToXdrHoneypot(String logMessage) throws Exception {
// 1. 提取 JSON 部分
String jsonString = extractJsonFromLogMessage(logMessage);
// 2. 解析 JSON
JsonNode jsonNode = objectMapper.readTree(jsonString);
// 3. 创建并填充 XdrHoneypot 对象
XdrHoneypot xdrHoneypot = new XdrHoneypot();
// 设置字段值
xdrHoneypot.setVcsource(getStringValue(jsonNode, "source"));
xdrHoneypot.setDstartTime(getStringValue(jsonNode, "start_time"));
xdrHoneypot.setDtime(getStringValue(jsonNode, "time"));
xdrHoneypot.setRiskLevel(getStringValue(jsonNode, "risk_level"));
xdrHoneypot.setVcconnection(getStringValue(jsonNode, "connection"));
xdrHoneypot.setFileInfo(getStringValue(jsonNode, "file_info"));
xdrHoneypot.setExtra(getExtraAsString(jsonNode.get("extra")));
xdrHoneypot.setVctype(getStringValue(jsonNode, "type"));
xdrHoneypot.setAgentSn(getStringValue(jsonNode, "agent_sn"));
xdrHoneypot.setAgentName(getStringValue(jsonNode, "agent_name"));
xdrHoneypot.setHoneypotId(getStringValue(jsonNode, "honeypot_id"));
xdrHoneypot.setHoneypotName(getStringValue(jsonNode, "honeypot_name"));
xdrHoneypot.setSrcIp(getStringValue(jsonNode, "src_ip"));
xdrHoneypot.setSrcPort(getStringValue(jsonNode, "src_port"));
xdrHoneypot.setSrcMac(getStringValue(jsonNode, "src_mac"));
xdrHoneypot.setDestIp(getStringValue(jsonNode, "dest_ip"));
xdrHoneypot.setDestPort(getStringValue(jsonNode, "dest_port"));
xdrHoneypot.setProxyIp(getStringValue(jsonNode, "proxy_ip"));
xdrHoneypot.setNode(getStringValue(jsonNode, "node"));
xdrHoneypot.setCreateTime(LocalDateTime.now());
return xdrHoneypot;
}
/**
* 从日志消息中提取 JSON 部分
*/
public static String extractJsonFromLogMessage(String logMessage) {
// 查找第一个 { 和最后一个 } 的位置
int startIndex = logMessage.indexOf('{');
int endIndex = logMessage.lastIndexOf('}');
if (startIndex == -1 || endIndex == -1 || endIndex <= startIndex) {
throw new IllegalArgumentException("日志消息中未找到有效的 JSON 内容");
}
return logMessage.substring(startIndex, endIndex + 1);
}
/**
* 安全地获取字符串值
*/
private static String getStringValue(JsonNode jsonNode, String fieldName) {
JsonNode fieldNode = jsonNode.get(fieldName);
if (fieldNode == null || fieldNode.isNull()) {
return null;
}
if (fieldNode.isTextual()) {
return fieldNode.asText();
} else {
// 对于非文本类型（如数字），转换为字符串
return fieldNode.toString();
}
}
/**
* 将 extra 字段转换为 JSON 字符串
*/
private static String getExtraAsString(JsonNode extraNode) {
if (extraNode == null || extraNode.isNull()) {
return null;
}
try {
return objectMapper.writeValueAsString(extraNode);
} catch (Exception e) {
// 如果序列化失败，返回原始字符串
return extraNode.toString();
}
}
/**
* 解析 JSON 字符串为扁平化的 Map
* @param jsonStr JSON 字符串
* @return 扁平化的 Map，key 使用点号表示嵌套路径
*/
public static LinkedHashMap<String, Object> parseJsonToFlatMap(String jsonStr) {
try {
LinkedHashMap<String, Object> resultMap = new LinkedHashMap<>();
if(!isValidJson(jsonStr))
{
System.out.println("parseJsonToFlatMap() json str:"+jsonStr);
System.out.println("isValidJson(string) is false");
return null;
}
TypeReference<Map<String, Object>> typeRef = new TypeReference<Map<String, Object>>() {};
Object jsonObject = objectMapper.readValue(jsonStr, typeRef);
//Object jsonObject = objectMapper.readValue(jsonStr.trim(), Object.class);
flattenJson("", jsonObject, resultMap);
return resultMap;
} catch (Exception e) {
//System.out.println("Exception ex.message:"+ e.getMessage());
logger.error("parseJsonToFlatMap ex.message:{}", e.getMessage());
throw new RuntimeException("parseJsonToFlatMap JSON 解析失败", e);
}
}
/**
* 递归扁平化 JSON 对象
*/
private static void flattenJson(String currentPath, Object jsonObject, Map<String, Object> resultMap) {
try {
if (jsonObject instanceof Map) {
// 处理 JSON 对象
Map<?, ?> map = (Map<?, ?>) jsonObject;
for (Entry<?, ?> entry : map.entrySet()) {
String key = entry.getKey().toString();
String newPath = currentPath.isEmpty() ? key : currentPath + "." + key;
flattenJson(newPath, entry.getValue(), resultMap);
}
} else if (jsonObject instanceof List) {
// 处理 JSON 数组
List<?> list = (List<?>) jsonObject;
for (int i = 0; i < list.size(); i++) {
String newPath = currentPath + "[" + i + "]";
flattenJson(newPath, list.get(i), resultMap);
}
} else {
// 基本类型值
resultMap.put(currentPath, jsonObject);
}
} catch (Exception e) {
System.out.println("Exception ex.message:"+ e.getMessage());
throw new RuntimeException("flattenJson 处理异常", e);
}
}
/**
* 保持嵌套结构的 JSON 解析
*/
public static Map<String, Object> parseJsonToNestedMap(String jsonStr) {
try {
return objectMapper.readValue(jsonStr, Map.class);
} catch (Exception e) {
throw new RuntimeException("parseJsonToNestedMap() JSON 解析失败", e);
}
}
/**
* 打印 Map 内容
*/
public static void printMap(Map<String, Object> map) {
printMap("", map, 0);
}
private static void printMap(String prefix, Map<String, Object> map, int indent) {
// String indentStr = " ".repeat(indent);
StringBuilder sb = new StringBuilder();
for (int i = 0; i < indent; i++) {
sb.append(" ");
}
String indentStr = sb.toString();
for (Entry<String, Object> entry : map.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
String fullKey = prefix.isEmpty() ? key : prefix + "." + key;
if (value instanceof Map) {
System.out.println(indentStr + key + ":");
printMap(fullKey, (Map<String, Object>) value, indent + 1);
} else if (value instanceof List) {
System.out.println(indentStr + key + ":");
printList(fullKey, (List<Object>) value, indent + 1);
} else {
System.out.println(indentStr + key + " = " + value);
}
}
}
private static void printList(String prefix, List<Object> list, int indent) {
//String indentStr = " ".repeat(indent);
StringBuilder sb = new StringBuilder();
for (int i = 0; i < indent; i++) {
sb.append(" ");
}
String indentStr = sb.toString();
for (int i = 0; i < list.size(); i++) {
Object value = list.get(i);
String itemKey = prefix + "[" + i + "]";
if (value instanceof Map) {
System.out.println(indentStr + "[" + i + "]:");
printMap(itemKey, (Map<String, Object>) value, indent + 1);
} else if (value instanceof List) {
System.out.println(indentStr + "[" + i + "]:");
printList(itemKey, (List<Object>) value, indent + 1);
} else {
System.out.println(indentStr + "[" + i + "] = " + value);
}
}
}
/**
* 将 JSONObject 转换为 Map
*/
public static LinkedHashMap<String, Object> jsonToMap(String jsonString) {
try {
JSONObject jsonObject = new JSONObject(jsonString.trim());
return toMap(jsonObject);
} catch (Exception e) {
throw new RuntimeException("jsonToMap 转换失败: " + e.getMessage(), e);
}
}
/**
* 递归转换 JSONObject 到 Map
*/
private static LinkedHashMap<String, Object> toMap(JSONObject jsonObject) {
LinkedHashMap<String, Object> map = new LinkedHashMap<>();
Iterator<String> keys = jsonObject.keys();
while (keys.hasNext()) {
String key = keys.next();
Object value = jsonObject.get(key);
if (value instanceof JSONObject) {
// 嵌套对象
value = toMap((JSONObject) value);
} else if (value instanceof JSONArray) {
// 数组
value = toList((JSONArray) value);
}
map.put(key, value);
}
return map;
}
/**
* 递归转换 JSONArray 到 List
*/
private static List<Object> toList(JSONArray array) {
List<Object> list = new ArrayList<>();
for (int i = 0; i < array.length(); i++) {
Object value = array.get(i);
if (value instanceof JSONObject) {
value = toMap((JSONObject) value);
} else if (value instanceof JSONArray) {
value = toList((JSONArray) value);
}
list.add(value);
}
return list;
}
/**
* 预处理 JSON 字符串
*/
public static Optional<String> preprocessJson(String jsonString) {
if (jsonString == null) {
return Optional.empty();
}
String trimmed = jsonString.trim();
// 处理空字符串
if (trimmed.isEmpty()) {
return Optional.empty();
}
// 处理 BOM
if (trimmed.startsWith("\uFEFF")) {
trimmed = trimmed.substring(1).trim();
if (trimmed.isEmpty()) {
return Optional.empty();
}
}
return Optional.of(trimmed);
}
/**
* 检查 JSON 字符串是否有效
*/
public static boolean isValidJson(String jsonString) {
return preprocessJson(jsonString).isPresent();
}
}

135
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/KeyValueParser.java Normal file
View File

@@ -0,0 +1,135 @@
package com.common.util;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class KeyValueParser {
/**
* 解析键值对字符串
* @param input 输入字符串
* @return 解析后的键值对Map
*/
public static Map<String, String> parseKeyValueString(String input) {
Map<String, String> result = new HashMap<>();
// 移除首尾的方括号（如果存在）
String cleanedInput = input.trim();
if (cleanedInput.startsWith("[") && cleanedInput.endsWith("]")) {
cleanedInput = cleanedInput.substring(1, cleanedInput.length() - 1);
}
// 使用正则表达式匹配键值对
// 匹配模式：键=值，值可以包含任意字符（包括空格和标点）
Pattern pattern = Pattern.compile("(\\w+)=([^=]+?)(?=\\s+\\w+=|$)");
Matcher matcher = pattern.matcher(cleanedInput);
while (matcher.find()) {
String key = matcher.group(1);
String value = matcher.group(2).trim();
result.put(key, value);
}
return result;
}
/**
* 简单的分割解析方法（适用于格式规整的情况）
* @param input 输入字符串
* @return 解析后的键值对Map
*/
public static Map<String, String> parseBySplit(String input) {
Map<String, String> result = new HashMap<>();
// 移除首尾的方括号
String cleanedInput = input.trim();
if (cleanedInput.startsWith("[") && cleanedInput.endsWith("]")) {
cleanedInput = cleanedInput.substring(1, cleanedInput.length() - 1);
}
// 按空格分割，但保留值中的空格
String[] pairs = cleanedInput.split("\\s+(?=\\w+=)");
for (String pair : pairs) {
int equalsIndex = pair.indexOf('=');
if (equalsIndex > 0) {
String key = pair.substring(0, equalsIndex);
String value = pair.substring(equalsIndex + 1);
result.put(key, value);
}
}
return result;
}
/**
* 从文本中提取第一个 [] 包裹的键值对字符串并转换为 Map
*
* @param text 包含键值对的原始文本
* @return 包含键值对的 Map
* @throws IllegalArgumentException 如果格式不正确
*/
public static Map<String, String> parseKeyValuePairs(String text) {
if (text == null || text.trim().isEmpty()) {
throw new IllegalArgumentException("Input text cannot be null or empty");
}
// 查找第一个 [ 和对应的 ]
int start = text.indexOf('[');
int end = text.indexOf(']', start + 1);
if (start == -1 || end == -1) {
throw new IllegalArgumentException("No valid [] pair found in text");
}
// 提取 [] 内的内容
String content = text.substring(start + 1, end).trim();
if (content.isEmpty()) {
return new HashMap<>(); // 返回空Map
}
// 分割键值对
String[] pairs = content.split("\\s+");
Map<String, String> result = new HashMap<>();
for (String pair : pairs) {
// 分割键和值
int equalSign = pair.indexOf('=');
if (equalSign == -1) {
throw new IllegalArgumentException("Invalid key-value pair format: " + pair);
}
String key = pair.substring(0, equalSign).trim();
String value = pair.substring(equalSign + 1).trim();
if (key.isEmpty()) {
throw new IllegalArgumentException("Key cannot be empty in pair: " + pair);
}
result.put(key, value);
}
return result;
}
public static void main(String[] args) {
String input = "[name=非工作时间访问 riskLevel=中 riskMain=账号 appName=蜜罐系统-企信外网 riskDesc=账号在2025-04-27，通过47.94.211.49客户端IP在非常用访问时间2025-04-27 15:38:54，进行了业务访问]";
System.out.println("=== 使用正则表达式解析 ===");
Map<String, String> result1 = parseKeyValueString(input);
for (Map.Entry<String, String> entry : result1.entrySet()) {
System.out.println(entry.getKey() + ": " + entry.getValue());
}
System.out.println("\n=== 使用分割方法解析 ===");
Map<String, String> result2 = parseBySplit(input);
for (Map.Entry<String, String> entry : result2.entrySet()) {
System.out.println(entry.getKey() + ": " + entry.getValue());
}
System.out.println("\n=== 获取特定字段 ===");
System.out.println("应用名称: " + result1.get("appName"));
System.out.println("风险等级: " + result1.get("riskLevel"));
System.out.println("风险描述: " + result1.get("riskDesc"));
}
}

145
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/KvTextParser.java Normal file
View File

@@ -0,0 +1,145 @@
package com.common.util;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.common.entity.RuleContent.kv_paramsType;
@Component
public class KvTextParser {
/**
* 解析键值对类型的文本数据
* @param text 要解析的文本
* @param params 解析参数配置
* @return 解析后的Map对象
*/
public LinkedHashMap<String, Object> parseKvText(String text, kv_paramsType params) {
LinkedHashMap<String, Object> result = new LinkedHashMap<>();
if (text == null || text.trim().isEmpty()) {
return result;
}
// 获取分隔符配置
String internal = params.getInternal(); // KV内部分隔符，如"="
String external = params.getExternal(); // KV之间分隔符，如空格
String lTrim = params.getL_trim(); // 左修剪字符
String rTrim = params.getR_trim(); // 右修剪字符
// 转义分隔符，用于正则表达式
String escapedExternal = Pattern.quote(external);
String escapedInternal = Pattern.quote(internal);
try {
// 使用外部KV分隔符分割字符串
String[] kvPairs = text.split(escapedExternal);
for (String kvPair : kvPairs) {
if (kvPair == null || kvPair.trim().isEmpty()) {
continue;
}
// 使用内部分隔符分割键值对
String[] parts = kvPair.split(escapedInternal, 2); // 限制分割为2部分
if (parts.length == 2) {
String key = parts[0].trim();
String value = parts[1].trim();
// 应用修剪
key = applyTrim(key, lTrim, rTrim);
value = applyTrim(value, lTrim, rTrim);
// 移除可能的引号
value = removeQuotes(value);
result.put(key, value);
} else if (parts.length == 1) {
// 只有key没有value的情况
String key = applyTrim(parts[0].trim(), lTrim, rTrim);
result.put(key, "");
}
}
} catch (Exception e) {
throw new RuntimeException("解析键值对文本时发生错误: " + e.getMessage(), e);
}
return result;
}
/**
* 应用字符串修剪
*/
private String applyTrim(String str, String lTrim, String rTrim) {
if (str == null) return "";
String result = str;
// 左修剪
if (lTrim != null && !lTrim.isEmpty()) {
result = result.replaceAll("^[" + Pattern.quote(lTrim) + "]+", "");
}
// 右修剪
if (rTrim != null && !rTrim.isEmpty()) {
result = result.replaceAll("[" + Pattern.quote(rTrim) + "]+$", "");
}
return result;
}
/**
* 移除字符串两端的引号
*/
private String removeQuotes(String str) {
if (str == null || str.length() < 2) {
return str;
}
// 检查是否被引号包围
if ((str.startsWith("\"") && str.endsWith("\"")) ||
(str.startsWith("'") && str.endsWith("'"))) {
return str.substring(1, str.length() - 1);
}
return str;
}
/**
* 使用正则表达式解析更复杂的键值对格式
* 适用于包含特殊字符的值
*/
public LinkedHashMap<String, Object> parseKvTextWithRegex(String text, kv_paramsType params) {
// Map<String, String> result = new HashMap<>();
LinkedHashMap<String, Object> result = new LinkedHashMap<>();
if (text == null || text.trim().isEmpty()) {
return result;
}
String internal = Pattern.quote(params.getInternal());
String external = Pattern.quote(params.getExternal());
// 构建正则表达式模式
// 匹配格式: key=value 或 key="包含空格的value"
String patternStr = "(\\w+)" + internal + "(\"[^\"]*\"|[^" + external + "]*)";
Pattern pattern = Pattern.compile(patternStr);
Matcher matcher = pattern.matcher(text);
while (matcher.find()) {
String key = matcher.group(1);
String value = matcher.group(2);
// 移除引号并应用修剪
value = removeQuotes(value);
value = applyTrim(value, params.getL_trim(), params.getR_trim());
result.put(key, value);
}
return result;
}
}

89
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/MyBatisUtil.java Normal file
View File

@@ -0,0 +1,89 @@
package com.common.util;
import org.springframework.beans.factory.annotation.Autowired;
import org.apache.ibatis.io.Resources;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.util.function.Function;
import org.springframework.stereotype.Component;
/**
* MyBatis 工具类
*/
@Component
public class MyBatisUtil {
private static final Logger logger = LoggerFactory.getLogger(MyBatisUtil.class);
private static SqlSessionFactory sqlSessionFactory;
@Autowired
private SqlSessionFactory autoWiredSqlSessionFactory;
static {
init();
}
private static void init() {
try {
String resource = "mybatis-config.xml";
InputStream inputStream = Resources.getResourceAsStream(resource);
sqlSessionFactory = new SqlSessionFactoryBuilder().build(inputStream);
logger.info("MyBatis 初始化成功");
} catch (IOException e) {
logger.error("初始化MyBatis失败", e);
throw new RuntimeException("初始化MyBatis失败", e);
}
}
public static SqlSession getSqlSession() {
return sqlSessionFactory.openSession();
}
public static SqlSessionFactory getSqlSessionFactory() {
return sqlSessionFactory;
}
/**
* 获取 SqlSession（自动提交事务）
*/
public static SqlSession getSqlSessionAutoCommit() {
return sqlSessionFactory.openSession(true);
}
/**
* 执行查询操作（自动管理资源）
*/
public static <T> T executeQuery(Function<SqlSession, T> function) {
SqlSession sqlSession = getSqlSession();
try {
return function.apply(sqlSession);
} finally {
sqlSession.close();
}
}
/**
* 执行更新操作（自动提交事务和管理资源）
*/
public static <T> T executeUpdate(Function<SqlSession, T> function) {
SqlSession sqlSession = getSqlSession();
try {
T result = function.apply(sqlSession);
sqlSession.commit();
return result;
} catch (Exception e) {
sqlSession.rollback();
throw new RuntimeException("数据库操作失败", e);
} finally {
sqlSession.close();
}
}
/**
* 重新初始化（用于配置热更新）
*/
public static void reload() {
init();
}
}

510
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/NestedJsonParserUtil.java Normal file
View File

@@ -0,0 +1,510 @@
package com.common.util;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.core.JsonParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
public class NestedJsonParserUtil {
private static final Logger logger = LoggerFactory.getLogger(NestedJsonParserUtil.class);
// 配置ObjectMapper，启用容错特性
private static final ObjectMapper objectMapper = new ObjectMapper()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false)
.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, true)
.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true)
.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)
.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true)
.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true);
/**
* 安全的JSON字符串转Map方法（支持嵌套结构）
*/
public static LinkedHashMap<String, Object> safeParseJson(String jsonStr) {
return safeParseJson(jsonStr, new LinkedHashMap<>());
}
/**
* 安全的JSON字符串转Map方法（支持嵌套结构和自定义选项）
*/
public static LinkedHashMap<String, Object> safeParseJson(String jsonStr, LinkedHashMap<String, Object> options) {
if (jsonStr == null || jsonStr.trim().isEmpty()) {
logger.warn("JSON字符串为空或null");
//return Collections.emptyMap();
return null;
}
// 获取选项,嵌套3层
int maxDepth = (Integer) options.getOrDefault("maxDepth", 3);
boolean allowUnquoted = (Boolean) options.getOrDefault("allowUnquoted", true);
boolean allowSingleQuotes = (Boolean) options.getOrDefault("allowSingleQuotes", true);
try {
// 1. 预处理JSON字符串
// String processedJson = preprocessJsonString(jsonStr, allowUnquoted, allowSingleQuotes);
String processedJson =jsonStr;
System.out.println("processedJson:"+processedJson);
// 2. 深度验证嵌套结构
//validateNestedStructure(processedJson, maxDepth);
//ObjectMapper mapper = new ObjectMapper(); // 这里使用Jackson
//System.out.println( mapper.readTree(processedJson));
//TypeReference<Map<String, String>> typeRef1 = new TypeReference<Map<String, String>>() {};
//Map<String, String> result1 = objectMapper.readValue(processedJson, typeRef1);
// 3. 尝试解析
TypeReference<Map<String, Object>> typeRef = new TypeReference<Map<String, Object>>() {};
Map<String, Object> result = objectMapper.readValue(processedJson, typeRef);
// 4. 后处理：清理和验证解析结果
return postProcessParsedData(result, maxDepth);
} catch (JsonProcessingException e) {
logger.error("JSON解析失败json:{} 错误消息： {}",jsonStr, e.getMessage());
// 5. 尝试修复并重新解析
try {
String fixedJson = tryFixNestedJson(jsonStr, maxDepth);
if (!fixedJson.equals(jsonStr)) {
TypeReference<Map<String, Object>> typeRef = new TypeReference<Map<String, Object>>() {};
Map<String, Object> result = objectMapper.readValue(fixedJson, typeRef);
return postProcessParsedData(result, maxDepth);
}
} catch (Exception ex) {
logger.error("JSON修复后解析仍然失败", ex);
}
return null;
//return Collections.emptyMap();
} catch (Exception e) {
logger.error("JSON解析发生未知异常", e);
return null;
}
}
/**
* 解析JSON为JsonNode（更灵活的嵌套处理）
*/
public static JsonNode safeParseJsonNode(String jsonStr) {
if (jsonStr == null || jsonStr.trim().isEmpty()) {
return objectMapper.createObjectNode();
}
try {
String processedJson = preprocessJsonString(jsonStr, true, true);
return objectMapper.readTree(processedJson);
} catch (Exception e) {
logger.error("解析JSON为JsonNode失败", e);
try {
String fixedJson = tryFixNestedJson(jsonStr, 10);
return objectMapper.readTree(fixedJson);
} catch (Exception ex) {
logger.error("修复后解析JSON Node仍然失败", ex);
return objectMapper.createObjectNode();
}
}
}
/**
* 预处理JSON字符串（增强版）
*/
private static String preprocessJsonString(String jsonStr, boolean allowUnquoted, boolean allowSingleQuotes) {
if (jsonStr == null) {
return "{}";
}
String processed = jsonStr.trim();
// 移除BOM字符
if (processed.startsWith("\uFEFF")) {
processed = processed.substring(1);
}
// 处理不同编码的空白字符
processed = processed.replaceAll("[\\u00A0\\u2007\\u202F]", " ");
// 确保字符串以{开头，以}结尾
if (!processed.startsWith("{") && !processed.startsWith("[")) {
// 尝试检测是否是JSONP格式
if (processed.contains("(") && processed.contains(")")) {
processed = extractJsonFromJsonp(processed);
} else {
// 如果不是JSON对象或数组，尝试包装成对象
processed = "{\"data\": " + processed + "}";
}
}
// 根据选项处理引号
if (!allowSingleQuotes) {
processed = convertSingleQuotesToDouble(processed);
}
return processed;
}
/**
* 从JSONP中提取JSON
*/
private static String extractJsonFromJsonp(String jsonp) {
try {
int start = jsonp.indexOf('(');
int end = jsonp.lastIndexOf(')');
if (start != -1 && end != -1 && end > start) {
return jsonp.substring(start + 1, end);
}
} catch (Exception e) {
logger.warn("JSONP提取失败", e);
}
return "{}";
}
/**
* 验证嵌套结构深度
*/
private static void validateNestedStructure(String jsonStr, int maxDepth) {
int depth = calculateJsonDepth(jsonStr);
if (depth > maxDepth) {
throw new IllegalArgumentException("JSON嵌套深度超过限制: " + depth + " > " + maxDepth);
}
}
/**
* 计算JSON最大嵌套深度
*/
private static int calculateJsonDepth(String jsonStr) {
int maxDepth = 0;
int currentDepth = 0;
for (int i = 0; i < jsonStr.length(); i++) {
char c = jsonStr.charAt(i);
if (c == '{' || c == '[') {
currentDepth++;
maxDepth = Math.max(maxDepth, currentDepth);
} else if (c == '}' || c == ']') {
currentDepth--;
}
if (currentDepth < 0) {
throw new IllegalArgumentException("calculateJsonDepth() JSON括号不匹配");
}
}
if (currentDepth != 0) {
throw new IllegalArgumentException("calculateJsonDepth() JSON括号不匹配");
}
return maxDepth;
}
/**
* 尝试修复嵌套JSON
*/
private static String tryFixNestedJson(String jsonStr, int maxDepth) {
if (jsonStr == null) {
return "{}";
}
String fixed = jsonStr.trim();
try {
// 修复步骤序列
fixed = fixJsonpWrapper(fixed);
fixed = fixUnescapedQuotesInNestedJson(fixed);
fixed = fixSingleQuotes(fixed);
fixed = fixUnescapedBackslashes(fixed);
fixed = fixTrailingCommas(fixed);
fixed = fixMissingQuotesInNestedJson(fixed);
fixed = fixNumericIssues(fixed);
fixed = fixBooleanIssues(fixed);
fixed = fixNullIssues(fixed);
fixed = fixArrayIssues(fixed);
logger.info("尝试修复嵌套JSON字符串");
return fixed;
} catch (Exception e) {
logger.warn("嵌套JSON修复过程中发生异常", e);
return jsonStr;
}
}
/**
* 修复JSONP包装
*/
private static String fixJsonpWrapper(String json) {
if (json.matches("^[a-zA-Z_$][a-zA-Z0-9_$]*\\s*\\(") && json.contains(")")) {
return extractJsonFromJsonp(json);
}
return json;
}
/**
* 修复嵌套JSON中的未转义引号
*/
private static String fixUnescapedQuotesInNestedJson(String json) {
StringBuilder sb = new StringBuilder();
boolean inString = false;
char prevChar = 0;
int braceCount = 0;
int bracketCount = 0;
for (int i = 0; i < json.length(); i++) {
char c = json.charAt(i);
// 跟踪嵌套级别
if (c == '{') braceCount++;
if (c == '}') braceCount--;
if (c == '[') bracketCount++;
if (c == ']') bracketCount--;
// 处理字符串状态
if (c == '"' && prevChar != '\\') {
inString = !inString;
sb.append(c);
} else if (inString && c == '"' && prevChar == '\\') {
sb.append(c);
} else if (inString && c == '"') {
// 在字符串中遇到未转义的双引号，需要转义
sb.append("\\\"");
} else {
sb.append(c);
}
prevChar = c;
}
return sb.toString();
}
/**
* 修复嵌套JSON中缺失的引号
*/
private static String fixMissingQuotesInNestedJson(String json) {
// 使用更复杂的正则表达式来修复嵌套对象中的键
Pattern pattern = Pattern.compile("([{,{]\\s*)([a-zA-Z_$][a-zA-Z0-9_$]*)(\\s*:\\s*)");
Matcher matcher = pattern.matcher(json);
StringBuffer result = new StringBuffer();
while (matcher.find()) {
String replacement = matcher.group(1) + "\"" + matcher.group(2) + "\"" + matcher.group(3);
matcher.appendReplacement(result, Matcher.quoteReplacement(replacement));
}
matcher.appendTail(result);
return result.toString();
}
/**
* 修复数字问题
*/
private static String fixNumericIssues(String json) {
// 修复前导零的数字
json = json.replaceAll(":\\s*0+(\\d+)", ":$1");
// 修复缺失小数点的数字
json = json.replaceAll(":\\s*(\\d+)\\.(\\s*[,\\}])", ":$1.0$2");
return json;
}
/**
* 修复布尔值问题
*/
private static String fixBooleanIssues(String json) {
json = json.replaceAll(":\\s*true", ":true");
json = json.replaceAll(":\\s*false", ":false");
json = json.replaceAll(":\\s*TRUE", ":true");
json = json.replaceAll(":\\s*FALSE", ":false");
return json;
}
/**
* 修复null值问题
*/
private static String fixNullIssues(String json) {
json = json.replaceAll(":\\s*null", ":null");
json = json.replaceAll(":\\s*NULL", ":null");
json = json.replaceAll(":\\s*Null", ":null");
return json;
}
/**
* 修复数组问题
*/
private static String fixArrayIssues(String json) {
// 修复数组中的尾随逗号
//json = json.replaceAll(",(\s*])", "$1");
// json =json.replaceAll(",(\s[}\])])", "$1");
return json;
}
/**
* 单引号转双引号
*/
private static String fixSingleQuotes(String json) {
return convertSingleQuotesToDouble(json);
}
/**
* 单引号转双引号（智能转换，不转换字符串内的单引号）
*/
private static String convertSingleQuotesToDouble(String json) {
StringBuilder sb = new StringBuilder();
boolean inDoubleString = false;
boolean inSingleString = false;
char prevChar = 0;
for (int i = 0; i < json.length(); i++) {
char c = json.charAt(i);
if (c == '"' && prevChar != '\\') {
inDoubleString = !inDoubleString;
sb.append(c);
} else if (c == '\'' && prevChar != '\\' && !inDoubleString) {
if (!inSingleString) {
sb.append('"');
inSingleString = true;
} else {
sb.append('"');
inSingleString = false;
}
} else {
sb.append(c);
}
prevChar = c;
}
return sb.toString();
}
/**
* 修复未转义的反斜杠
*/
private static String fixUnescapedBackslashes(String json) {
// 简单的反斜杠转义，注意不要破坏已经转义的内容
return json.replace("\\", "\\\\").replace("\\\\\"", "\\\"");
}
/**
* 修复尾随逗号
*/
private static String fixTrailingCommas(String json) {
// 移除对象和数组中的尾随逗号
//json = json.replaceAll(",(\s*[}\]])", "$1");
return json;
}
/**
* 后处理解析的数据
*/
private static LinkedHashMap<String, Object> postProcessParsedData(Map<String, Object> data, int maxDepth) {
if (data == null) {
//return Collections.emptyMap();
return null;
}
// 深度清理和验证
return cleanNestedData(data, 0, maxDepth);
}
/**
* 清理嵌套数据
*/
@SuppressWarnings("unchecked")
private static LinkedHashMap<String, Object> cleanNestedData(Map<String, Object> data, int currentDepth, int maxDepth) {
if (currentDepth > maxDepth) {
logger.warn("数据嵌套过深，进行截断");
// return Collections.emptyMap();
return null;
}
LinkedHashMap<String, Object> cleaned = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : data.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
// 清理键
String cleanedKey = key.trim();
// 清理值
Object cleanedValue = cleanValue(value, currentDepth + 1, maxDepth);
cleaned.put(cleanedKey, cleanedValue);
}
return cleaned;
}
/**
* 清理值
*/
@SuppressWarnings("unchecked")
private static Object cleanValue(Object value, int currentDepth, int maxDepth) {
if (value == null) {
return null;
}
if (value instanceof Map) {
return cleanNestedData((Map<String, Object>) value, currentDepth, maxDepth);
} else if (value instanceof List) {
List<Object> cleanedList = new ArrayList<>();
for (Object item : (List<?>) value) {
cleanedList.add(cleanValue(item, currentDepth + 1, maxDepth));
}
return cleanedList;
} else if (value instanceof String) {
return ((String) value).trim();
}
return value;
}
public static void main(String[] args) {
// 测试复杂的嵌套JSON
String complexJson = "{\"source\":\"portrait\",\"uuid\":\"1a26ac6e-2d77-4ada-b560-1abbcae1de98\",\"host\":{\"cpuConcurrency\":8,\"fonts\":[\"Rockwell\",\"Calibri\",\"Gadugi\",\"Leelawadee UI\",\"Bahnschrift\",\"DengXian\",\"Roboto\",\"DejaVu Sans Mono\",\"Open Sans\",\"Source Han Serif CN\"],\"hasUnity\":false,\"language\":\"zh-CN\",\"memory\":0,\"os\":\"Windows 10.0\",\"render\":\"ANGLE (Intel, Intel(R) UHD Graphics 620 Direct3D9Ex vs_3_0 ps_3_0, igdumdim32.dll-30.0.101.1338)\",\"screenResolution\":[1366,768],\"timezone\":\"Asia/Shanghai\",\"touchSupport\":true},\"network\":{\"externalIP\":{\"ip\":\"60.190.198.14\"},\"internalIP\":{\"ip\":\"\"},\"realIP\":{\"ip\":\"60.190.198.14\"}},\"browser\":{\"arch\":\"\",\"bitness\":\"\",\"canvasFingerprint\":\"7031cc506eaded347eb1b596677ec7be\",\"canvas_fp\":\"7031cc506eaded347eb1b596677ec7be\",\"canvas_fp2\":\"7031cc506eaded347eb1b596677ec7be\",\"chrome_ext\":[\"Google Office\"],\"fp2\":\"559732dbe9bafced9536c77a6c020f88\",\"is_private\":false,\"mobile\":false,\"model\":\"\",\"name\":\"Chrome\",\"os\":\"Windows 10.0\",\"tid\":\"s:16951889730ae4d6af8-b3b4b-5ede70.22c7306819e72dd14e3e5c5644e49ed42a44d857ca55f6df0acd0460f510f15f\",\"version\":\"94.0.4606.71\",\"versionNumber\":94,\"webgl_fp\":\"487f7b22f68312d2c1bbc93b1aea445b\",\"webgl_fp2\":\"487f7b22f68312d2c1bbc93b1aea445b\"},\"social\":{},\"extra\":{\"version\":\"1.1\"},\"node\":\"AQSE\"}";
//String complexJson ="{\"sn\":\"8cf9a388-578e-4b30-ac4b-098a46dde642\",\"name\":\"茂名市住房和城乡建设局蜜罐探针\",\"send\":true,\"host\":\"172.25.142.16\",\"type\":\"agent_connect\",\"event_type_display_name\":{\"en\":\"Agent Connect Event\",\"cn\":\"探针连接建立\"},\"node\":\"WRx3\"}";
// 测试有问题的嵌套JSON
String problematicJson = "{\"source\":\"portrait\",\"uuid\":\"1a26ac6e-2d77-4ada-b560-1abbcae1de98\",\"host\":{\"cpuConcurrency\":8,\"fonts\":[\"Rockwell\",\"Calibri\",\"Gadugi\",\"Leelawadee UI\",\"Bahnschrift\",\"DengXian\",\"Roboto\",\"DejaVu Sans Mono\",\"Open Sans\",\"Source Han Serif CN\"],\"hasUnity\":false,\"language\":\"zh-CN\",\"memory\":0,\"os\":\"Windows 10.0\",\"render\":\"ANGLE (Intel, Intel(R) UHD Graphics 620 Direct3D9Ex vs_3_0 ps_3_0, igdumdim32.dll-30.0.101.1338)\",\"screenResolution\":[1366,768],\"timezone\":\"Asia/Shanghai\",\"touchSupport\":true},\"network\":{\"externalIP\":{\"ip\":\"60.190.198.14\"},\"internalIP\":{\"ip\":\"\"},\"realIP\":{\"ip\":\"60.190.198.14\"}},\"browser\":{\"arch\":\"\",\"bitness\":\"\",\"canvasFingerprint\":\"7031cc506eaded347eb1b596677ec7be\",\"canvas_fp\":\"7031cc506eaded347eb1b596677ec7be\",\"canvas_fp2\":\"7031cc506eaded347eb1b596677ec7be\",\"chrome_ext\":[\"Google Office\"],\"fp2\":\"559732dbe9bafced9536c77a6c020f88\",\"is_private\":false,\"mobile\":false,\"model\":\"\",\"name\":\"Chrome\",\"os\":\"Windows 10.0\",\"tid\":\"s:16951889730ae4d6af8-b3b4b-5ede70.22c7306819e72dd14e3e5c5644e49ed42a44d857ca55f6df0acd0460f510f15f\",\"version\":\"94.0.4606.71\",\"versionNumber\":94,\"webgl_fp\":\"487f7b22f68312d2c1bbc93b1aea445b\",\"webgl_fp2\":\"487f7b22f68312d2c1bbc93b1aea445b\"},\"social\":{},\"extra\":{\"version\":\"1.1\"},\"node\":\"AQSE\"}";
// 测试解析
System.out.println("=== 正常嵌套JSON解析 ===");
LinkedHashMap<String, Object> result1 = NestedJsonParserUtil.safeParseJson(complexJson);
System.out.println("解析结果: " + result1);
System.out.println("\\n=== 有问题嵌套JSON解析 ===");
LinkedHashMap<String, Object> result2 = NestedJsonParserUtil.safeParseJson(problematicJson);
System.out.println("解析结果: " + result2);
/**
System.out.println("\\n=== 嵌套值获取 ===");
String userName = NestedJsonUtils.getNestedString(result1, "user.name");
Integer userAge = NestedJsonUtils.getNestedInteger(result1, "user.profile.age");
String city = NestedJsonUtils.getNestedString(result1, "user.profile.address.city");
System.out.println("用户名: " + userName);
System.out.println("年龄: " + userAge);
System.out.println("城市: " + city);
**/
System.out.println("\\n=== 扁平化处理 ===");
LinkedHashMap<String, Object> flattened = NestedJsonUtils.flattenNestedJson(result1);
System.out.println("扁平化结果: " + flattened);
}
}

169
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/NestedJsonUtils.java Normal file
View File

@@ -0,0 +1,169 @@
package com.common.util;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.core.JsonParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
/**
* 嵌套JSON专门处理工具
*/
public class NestedJsonUtils {
private static final Logger logger = LoggerFactory.getLogger(NestedJsonUtils.class);
/**
* 安全地获取嵌套值
*/
public static Object getNestedValue(Map<String, Object> data, String path) {
return getNestedValue(data, path, null);
}
/**
* 安全地获取嵌套值（带默认值）
*/
@SuppressWarnings("unchecked")
public static Object getNestedValue(Map<String, Object> data, String path, Object defaultValue) {
if (data == null || path == null || path.trim().isEmpty()) {
return defaultValue;
}
String[] keys = path.split("\\.");
Object current = data;
for (String key : keys) {
if (current instanceof Map) {
current = ((Map<String, Object>) current).get(key);
} else {
return defaultValue;
}
if (current == null) {
return defaultValue;
}
}
return current != null ? current : defaultValue;
}
/**
* 安全地获取嵌套字符串
*/
public static String getNestedString(Map<String, Object> data, String path) {
return getNestedString(data, path, null);
}
public static String getNestedString(Map<String, Object> data, String path, String defaultValue) {
Object value = getNestedValue(data, path, defaultValue);
return value != null ? value.toString() : defaultValue;
}
/**
* 安全地获取嵌套整数
*/
public static Integer getNestedInteger(Map<String, Object> data, String path) {
return getNestedInteger(data, path, null);
}
public static Integer getNestedInteger(Map<String, Object> data, String path, Integer defaultValue) {
Object value = getNestedValue(data, path, defaultValue);
if (value instanceof Integer) {
return (Integer) value;
} else if (value instanceof String) {
try {
return Integer.parseInt((String) value);
} catch (NumberFormatException e) {
logger.warn("无法将值转换为整数: {}", value);
return defaultValue;
}
} else if (value instanceof Number) {
return ((Number) value).intValue();
}
return defaultValue;
}
/**
* 安全地获取嵌套Map
*/
@SuppressWarnings("unchecked")
public static Map<String, Object> getNestedMap(Map<String, Object> data, String path) {
Object value = getNestedValue(data, path, Collections.emptyMap());
if (value instanceof Map) {
return (Map<String, Object>) value;
}
return Collections.emptyMap();
}
/**
* 安全地获取嵌套列表
*/
@SuppressWarnings("unchecked")
public static List<Object> getNestedList(Map<String, Object> data, String path) {
Object value = getNestedValue(data, path, Collections.emptyList());
if (value instanceof List) {
return (List<Object>) value;
}
return Collections.emptyList();
}
/**
* 扁平化嵌套JSON
*/
public static LinkedHashMap<String, Object> flattenNestedJson(LinkedHashMap<String, Object> data) {
return flattenNestedJson(data, null);
}
/**
* 扁平化嵌套JSON（带前缀）
*/
@SuppressWarnings("unchecked")
public static LinkedHashMap<String, Object> flattenNestedJson(Map<String, Object> data, String prefix) {
LinkedHashMap<String, Object> flattened = new LinkedHashMap<>();
String currentPrefix = prefix != null ? prefix + "." : "";
for (Map.Entry<String, Object> entry : data.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
String fullKey = currentPrefix + key;
if (value instanceof Map) {
flattened.putAll(flattenNestedJson((Map<String, Object>) value, fullKey));
} else if (value instanceof List) {
// 处理列表，可以按索引展开或保持为列表
flattened.put(fullKey, value);
} else {
flattened.put(fullKey, value);
}
}
return flattened;
}
/**
* 验证嵌套JSON结构
*/
public static boolean validateNestedStructure(Map<String, Object> data, String schema) {
// 项目中可以使用JSON Schema验证库
try {
// 这里可以实现自定义的结构验证逻辑
return data != null && !data.isEmpty();
} catch (Exception e) {
logger.error("嵌套结构验证失败", e);
return false;
}
}
}

204
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/RegexTextParser.java Normal file
View File

@@ -0,0 +1,204 @@
package com.common.util;
import org.springframework.stereotype.Component;
import java.util.LinkedHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* 正则表达式文本解析工具类
*/
@Component
public class RegexTextParser {
/**
* 使用正则表达式解析文本数据
* @param text 原始文本
* @param regex 正则表达式（必须包含捕获组）
* @return LinkedHashMap对象，key为序号字符串，value为实际值
*/
public static LinkedHashMap<String, Object> parseWithRegex(String text, String regex) {
LinkedHashMap<String, Object> resultMap = new LinkedHashMap<>();
if (text == null || text.trim().isEmpty()) {
return resultMap;
}
if (regex == null || regex.isEmpty()) {
throw new IllegalArgumentException("正则表达式不能为空");
}
try {
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(text);
if (matcher.find()) {
// 获取所有捕获组（从group(1)开始，group(0)是整个匹配）
int groupCount = matcher.groupCount();
if (groupCount == 0) {
// 如果没有捕获组，将整个匹配作为第一个元素
resultMap.put("1", matcher.group(0));
} else {
// 遍历所有捕获组
for (int i = 1; i <= groupCount; i++) {
String value = matcher.group(i);
resultMap.put(String.valueOf(i), value != null ? value.trim() : "");
}
}
}
return resultMap;
} catch (Exception e) {
throw new RuntimeException("正则表达式解析失败: " + e.getMessage(), e);
}
}
/**
* 使用正则表达式解析文本数据（支持多次匹配）
* @param text 原始文本
* @param regex 正则表达式
* @param matchAll 是否匹配所有结果
* @return LinkedHashMap对象，key为匹配序号，value为实际值
*/
public static LinkedHashMap<String, Object> parseWithRegex(String text, String regex, boolean matchAll) {
LinkedHashMap<String, Object> resultMap = new LinkedHashMap<>();
if (text == null || text.trim().isEmpty()) {
return resultMap;
}
if (regex == null || regex.isEmpty()) {
throw new IllegalArgumentException("正则表达式不能为空");
}
if (!matchAll) {
return parseWithRegex(text, regex);
}
try {
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(text);
int matchCount = 0;
while (matcher.find()) {
matchCount++;
int groupCount = matcher.groupCount();
if (groupCount == 0) {
// 如果没有捕获组，将整个匹配作为一个元素
resultMap.put(String.valueOf(matchCount), matcher.group(0));
} else {
// 对于每个匹配，将捕获组按顺序存储
for (int i = 1; i <= groupCount; i++) {
String key = matchCount + "_" + i; // 格式：匹配序号_组号
String value = matcher.group(i);
resultMap.put(key, value != null ? value.trim() : "");
}
}
}
return resultMap;
} catch (Exception e) {
throw new RuntimeException("正则表达式解析失败: " + e.getMessage(), e);
}
}
/**
* 使用正则表达式解析文本数据（带类型转换）
* @param text 原始文本
* @param regex 正则表达式
* @param valueType 值类型
* @return LinkedHashMap对象
*/
public static LinkedHashMap<String, Object> parseWithRegex(String text, String regex, ValueType valueType) {
LinkedHashMap<String, Object> resultMap = parseWithRegex(text, regex);
// 应用类型转换
resultMap.replaceAll((key, value) -> convertValue(value.toString(), valueType));
return resultMap;
}
/**
* 使用正则表达式分割文本（类似String.split但返回LinkedHashMap）
* @param text 原始文本
* @param regex 分割正则表达式
* @return LinkedHashMap对象
*/
public static LinkedHashMap<String, Object> splitWithRegex(String text, String regex) {
LinkedHashMap<String, Object> resultMap = new LinkedHashMap<>();
if (text == null || text.trim().isEmpty()) {
return resultMap;
}
if (regex == null || regex.isEmpty()) {
throw new IllegalArgumentException("分割正则表达式不能为空");
}
try {
String[] parts = text.split(regex, -1); // 使用-1保留空字符串
for (int i = 0; i < parts.length; i++) {
resultMap.put(String.valueOf(i + 1), parts[i].trim());
}
return resultMap;
} catch (Exception e) {
throw new RuntimeException("正则表达式分割失败: " + e.getMessage(), e);
}
}
/**
* 提取引号内的内容（专用方法，处理示例中的情况）
* @param text 原始文本
* @return LinkedHashMap对象
*/
public static LinkedHashMap<String, Object> extractQuotedContent(String text) {
// 正则表达式匹配双引号内的内容，忽略转义引号
String regex = "\"([^\"]*)\"";
return parseWithRegex(text, regex, true);
}
/**
* 值类型枚举
*/
public enum ValueType {
STRING, INTEGER, LONG, DOUBLE, BOOLEAN, DATE
}
/**
* 值类型转换
*/
private static Object convertValue(String value, ValueType valueType) {
if (value == null || value.isEmpty()) {
return null;
}
try {
switch (valueType) {
case INTEGER:
return Integer.parseInt(value);
case LONG:
return Long.parseLong(value);
case DOUBLE:
return Double.parseDouble(value);
case BOOLEAN:
return Boolean.parseBoolean(value) || "1".equals(value) || "true".equalsIgnoreCase(value);
case DATE:
// 简单日期解析，实际项目中可以使用DateTimeFormatter
return java.sql.Timestamp.valueOf(value);
case STRING:
default:
return value;
}
} catch (Exception e) {
// 转换失败时返回原始字符串
return value;
}
}
}

34
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/SpringContextUtil.java Normal file
View File

@@ -0,0 +1,34 @@
package com.common.util;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
@Component
public class SpringContextUtil implements ApplicationContextAware {
private static ApplicationContext applicationContext;
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
SpringContextUtil.applicationContext = applicationContext;
}
public static ApplicationContext getApplicationContext() {
return applicationContext;
}
public static <T> T getBean(Class<T> clazz) {
return applicationContext.getBean(clazz);
}
public static <T> T getBean(String name, Class<T> clazz) {
return applicationContext.getBean(name, clazz);
}
// 专门获取 Mapper 的方法
public static <T> T getMapper(Class<T> mapperClass) {
return applicationContext.getBean(mapperClass);
}
}

285
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/StringExtractorUtil.java Normal file
View File

@@ -0,0 +1,285 @@
package com.common.util;
public class StringExtractorUtil {
/**
* 简化的字符串截取方法
* @param text 原始文本
* @param startChar 开始字符
* @param startOffset 开始偏移量（负数表示向前）
* @param endChar 结束字符
* @param endOccurrence 结束字符第几次出现
* @return 截取的字符串
*/
public static String extract(String text, char startChar, int startOffset,
char endChar, int endOccurrence) {
if (text == null) return "";
int startIndex = text.indexOf(startChar);
if (startIndex == -1) return "";
// 计算实际开始位置
int actualStart = Math.max(0, startIndex + startOffset);
if (actualStart >= text.length()) return "";
// 查找结束字符
int currentIndex = actualStart;
int count = 0;
while (currentIndex < text.length()) {
if (text.charAt(currentIndex) == endChar) {
count++;
if (count == endOccurrence) {
return text.substring(actualStart, currentIndex);
}
}
currentIndex++;
}
return ""; // 未找到足够的结束字符
}
/**
* 从文本中截取字符串，从指定起始字符串的第0位开始，直到文本末尾
* @param text 原始文本
* @param startString 起始字符串
* @return 截取后的字符串，如果起始字符串不存在则返回空字符串
*/
public static String extractFromStartToEnd(String text, String startString) {
if (text == null || startString == null) {
return "";
}
// 查找起始字符串的位置
int startIndex = text.indexOf(startString);
if (startIndex == -1) {
return ""; // 起始字符串不存在
}
// 从起始字符串的第0位开始截取到文本末尾
return text.substring(startIndex);
}
/**
* 从文本中截取字符串，从指定起始字符串的指定位置开始，直到文本末尾
* @param text 原始文本
* @param startString 起始字符串
* @param startOffset 起始偏移量（从起始字符串开始计算）
* @return 截取后的字符串
*/
public static String extractFromStartToEnd(String text, String startString, int startOffset) {
if (text == null || startString == null) {
return "";
}
int startIndex = text.indexOf(startString);
if (startIndex == -1) {
return "";
}
// 计算实际开始位置
int actualStartIndex = startIndex + startOffset;
// 确保开始位置在有效范围内
if (actualStartIndex < 0) {
actualStartIndex = 0;
} else if (actualStartIndex >= text.length()) {
return "";
}
return text.substring(actualStartIndex);
}
/**
* 从文本中截取字符串，从指定起始字符串开始，到指定结束字符串为止
* @param text 原始文本
* @param startString 起始字符串
* @param endString 结束字符串
* @return 截取后的字符串
*/
public static String extractBetweenStrings(String text, String startString, String endString) {
if (text == null || startString == null || endString == null) {
return "";
}
int startIndex = text.indexOf(startString);
if (startIndex == -1) {
return "";
}
// 从起始字符串位置开始查找结束字符串
int endIndex = text.indexOf(endString, startIndex + startString.length());
if (endIndex == -1) {
// 如果找不到结束字符串，则截取到文本末尾
return text.substring(startIndex);
}
return text.substring(startIndex, endIndex + endString.length());
}
/**
* 高级版本：支持多个起始字符串选项和容错处理
* @param text 原始文本
* @param possibleStartStrings 可能的起始字符串数组（按优先级排序）
* @return 截取后的字符串
*/
public static String extractWithFallback(String text, String[] possibleStartStrings) {
if (text == null || possibleStartStrings == null || possibleStartStrings.length == 0) {
return "";
}
// 尝试每个可能的起始字符串
for (String startString : possibleStartStrings) {
int startIndex = text.indexOf(startString);
if (startIndex != -1) {
return text.substring(startIndex);
}
}
return ""; // 所有起始字符串都不存在
}
/**
* 字符串截取方法
* @param text 原始文本
* @param startStr 开始字符串
* @param startOffset 开始偏移量（负数表示向前）
* @param endStr 结束字符串
* @param endOffset 结束偏移量（负数表示向前）
* @return 截取的字符串
*/
public static String extract(String text, String startStr, int startOffset,
String endStr, int endOffset) {
if (text == null || text.isEmpty()) {
return "";
}
int startIndex = 0;
int endIndex = text.length();
// 处理开始位置
if (startStr != null && !startStr.isEmpty()) {
int foundStartIndex = text.indexOf(startStr);
if (foundStartIndex == -1) {
return ""; // 未找到开始字符串
}
startIndex = foundStartIndex + startStr.length() + startOffset;
startIndex = Math.max(0, Math.min(startIndex, text.length()));
} else {
// 没有开始字符串，直接使用偏移量
startIndex = Math.max(0, startOffset);
startIndex = Math.min(startIndex, text.length());
}
// 处理结束位置
if (endStr != null && !endStr.isEmpty()) {
int foundEndIndex = text.indexOf(endStr, startIndex);
if (foundEndIndex == -1) {
return ""; // 未找到结束字符串
}
endIndex = foundEndIndex + endOffset;
endIndex = Math.max(0, Math.min(endIndex, text.length()));
} else if (endOffset != 0) {
// 没有结束字符串但有结束偏移量
endIndex = startIndex + endOffset;
endIndex = Math.max(0, Math.min(endIndex, text.length()));
}
// 如果结束字符串为空且结束偏移量为0，则endIndex保持为文本长度（截取到最后）
// 验证位置有效性
if (startIndex >= endIndex || startIndex >= text.length()) {
return "";
}
return text.substring(startIndex, endIndex);
}
public static void main(String[] args) {
// 测试示例
String text = "这是一段前置文本，然后是我们需要的内容，这是后续文本。";
String startString = "然后";
System.out.println("=== 基本用法：从起始字符串第0位开始到文本末尾 ===");
String result1 = extractFromStartToEnd(text, startString);
System.out.println("结果1: " + result1);
System.out.println("\n=== 使用偏移量：从起始字符串第2位开始 ===");
String result2 = extractFromStartToEnd(text, startString, 2);
System.out.println("结果2: " + result2);
System.out.println("\n=== 在两个字符串之间截取 ===");
String result3 = extractBetweenStrings(text, "然后", "后续");
System.out.println("结果3: " + result3);
System.out.println("\n=== 使用多个可能的起始字符串 ===");
String[] startOptions = {"不存在的字符串", "然后", "这是"};
String result4 = extractWithFallback(text, startOptions);
System.out.println("结果4: " + result4);
System.out.println("\n=== 处理边界情况 ===");
// 测试起始字符串不存在的情况
String result5 = extractFromStartToEnd(text, "不存在的字符串");
System.out.println("起始字符串不存在: '" + result5 + "'");
// 测试空文本
String result6 = extractFromStartToEnd("", startString);
System.out.println("空文本: '" + result6 + "'");
// 测试实际应用场景
System.out.println("\n=== 实际应用示例 ===");
String logText = "2024-01-15 10:30:25 [INFO] User login successful. User ID: 12345, Session: abcdef";
String logStart = "[INFO]";
String logResult = extractFromStartToEnd(logText, logStart);
System.out.println("日志内容: " + logResult);
// 只提取消息部分（去掉日志级别）
String messageResult = extractFromStartToEnd(logText, logStart, logStart.length());
System.out.println("纯消息: " + messageResult.trim());
}
/**
* 简化的文本截取方法
* @param text 原始文本
* @param startString 起始字符串
* @return 从起始字符串开始到文本末尾的内容
*/
public static String extract(String text, String startString) {
// 参数检查
if (text == null || startString == null) {
return "";
}
// 查找起始字符串
int startIndex = text.indexOf(startString);
// 如果找到起始字符串，则截取从该位置到末尾的内容
if (startIndex != -1) {
return text.substring(startIndex);
}
return ""; // 起始字符串不存在
}
public static void main22(String[] args) {
// 示例使用
String exampleText = "前置内容，这是起始点，这是我们需要的内容，继续到文本结束。";
String start = "这是起始点";
String result = extract(exampleText, start);
System.out.println("截取结果: " + result);
// 如果起始字符串不存在
String noResult = extract(exampleText, "不存在的字符串");
System.out.println("起始不存在: '" + noResult + "'");
String example = "前置文本{这是要截取的内容}后置文本";
// 从 { 前1位开始，到第一个 } 结束
// String result = extract(example, '{', 0, '}', 0);
//System.out.println("截取结果: " + result); // 输出: 前置文本{
}
}

355
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/SyslogParser.java Normal file
View File

@@ -0,0 +1,355 @@
package com.common.util;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.common.entity.SyslogMessage;
import com.common.entity.RFC3164Message;
import com.common.entity.RFC5424Message;
import com.influx.SyslogToInfluxApp;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.util.Locale;
public class SyslogParser {
private static final Logger logger = LoggerFactory.getLogger(SyslogToInfluxApp.class);
// RFC 5424 正则表达式
private static final Pattern RFC5424_PATTERN = Pattern.compile(
"^<(?<pri>\\d{1,3})>(?<version>\\d+) " +
"(?<timestamp>\\S+) " +
"(?<host>\\S+) " +
"(?<app>\\S+) " +
"(?<pid>\\S+) " +
"(?<msgid>\\S+) " +
"(?<sd>(?:-|\\[.*\\]))" +
"(?: (?<message>.+))?$"
);
// RFC 3164 正则表达式
private static final Pattern RFC3164_PATTERN = Pattern.compile(
"^<(?<pri>\\d{1,3})>" +
"(?<timestamp>[A-Z][a-z]{2}\\s+\\d{1,2}\\s+\\d{2}:\\d{2}:\\d{2}) " +
"(?<host>\\S+) " +
"(?<content>.+)$"
);
// RFC 3164 时间戳格式
private static final DateTimeFormatter RFC3164_FORMATTER =
DateTimeFormatter.ofPattern("MMM dd HH:mm:ss");
/**
* 解析 Syslog 消息，自动检测格式
*/
public static SyslogMessage parse(String logLine) {
if (logLine == null || logLine.trim().isEmpty()) {
throw new IllegalArgumentException("Log line cannot be null or empty");
}
// 尝试解析为 RFC 5424 格式
try {
RFC5424Message msg5424 = parseRFC5424(logLine);
if (msg5424 != null) {
return msg5424;
}
} catch (Exception e) {
// 继续尝试 RFC 3164
}
// 尝试解析为 RFC 3164 格式
try {
RFC3164Message msg3164 = parseRFC3164(logLine);
if (msg3164 != null) {
return msg3164;
}
} catch (Exception e) {
// 解析失败
}
// 解析非标数据格式，提取字符【{】自后的内容
try {
String content = extractJsonStringNew(logLine);
RFC5424Message msgClass =new RFC5424Message();
if (content != null) {
msgClass.setMessage(content.toString());
return msgClass;
}
} catch (Exception e) {
// 解析失败
}
throw new IllegalArgumentException("Unable to parse syslog message: " + logLine);
}
/**
* 解析 RFC 5424 格式
*/
public static RFC5424Message parseRFC5424(String logLine) {
Matcher matcher = RFC5424_PATTERN.matcher(logLine);
if (!matcher.find()) {
return null;
}
RFC5424Message message = new RFC5424Message();
// 解析优先级
int pri = Integer.parseInt(matcher.group("pri"));
message.setPriority(pri);
// 解析版本
message.setVersion(Integer.parseInt(matcher.group("version")));
// 解析时间戳 (RFC 3339 格式)
String timestampStr = matcher.group("timestamp");
try {
message.setTimestamp(OffsetDateTime.parse(timestampStr));
} catch (DateTimeParseException e) {
throw new IllegalArgumentException("Invalid RFC5424 timestamp: " + timestampStr, e);
}
// 解析其他字段
message.setHostname(matcher.group("host"));
message.setAppName(matcher.group("app"));
message.setProcId(matcher.group("pid"));
message.setMsgId(matcher.group("msgid"));
// 解析结构化数据
String sd = matcher.group("sd");
if (!"-".equals(sd)) {
message.setStructuredData(parseStructuredData(sd));
}
// 解析消息内容
String msg = matcher.group("message");
if (msg != null) {
message.setMessage(msg.trim());
}
return message;
}
/**
* 解析 RFC 3164 格式
*/
public static RFC3164Message parseRFC3164(String logLine) {
Matcher matcher = RFC3164_PATTERN.matcher(logLine);
if (!matcher.find()) {
return null;
}
RFC3164Message message = new RFC3164Message();
// 解析优先级
int pri = Integer.parseInt(matcher.group("pri"));
message.setPriority(pri);
// 解析时间戳 (需要特殊处理，因为没有年份)
String timestampStr = matcher.group("timestamp");
message.setTimestamp(parseRFC3164Timestamp(timestampStr));
// 解析主机名
message.setHostname(matcher.group("host"));
// 解析内容和标签
String content = matcher.group("content");
parseRFC3164Content(content, message);
return message;
}
/**
* 解析 RFC 3164 时间戳
*/
private static OffsetDateTime parseRFC3164Timestamp(String timestampStr) {
try {
// 添加当前年份，因为 RFC3164 时间戳不包含年份
String currentYear = String.valueOf(java.time.Year.now().getValue());
String fullTimestamp = currentYear + " " + timestampStr;
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy MMM dd HH:mm:ss", Locale.US);
LocalDateTime localDateTime = LocalDateTime.parse(fullTimestamp, formatter);
// DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy MMM dd HH:mm:ss", Locale.US);
// 假设为系统默认时区
return localDateTime.atOffset(ZoneOffset.systemDefault().getRules().getOffset(localDateTime));
} catch (DateTimeParseException e) {
throw new IllegalArgumentException("Invalid RFC3164 timestamp: " + timestampStr, e);
}
}
/**
* 解析 RFC 3164 内容部分
*/
private static void parseRFC3164Content(String content, RFC3164Message message) {
// 尝试匹配标签格式: app[pid]:
Pattern tagPattern = Pattern.compile("^(?<tag>\\S+?)(?:\\[(?<pid>\\d+)\\])?:\\s*(?<message>.*)$");
Matcher tagMatcher = tagPattern.matcher(content);
if (tagMatcher.find()) {
message.setTag(tagMatcher.group("tag"));
String pid = tagMatcher.group("pid");
if (pid != null) {
message.setProcId(pid);
}
message.setMessage(tagMatcher.group("message").trim());
} else {
// 如果没有标签格式，整个内容作为消息
message.setTag("");
message.setMessage(content.trim());
}
// 对于 RFC3164，appName 通常与 tag 相同
message.setAppName(message.getTag());
}
/**
* 解析结构化数据
*/
private static Map<String, Map<String, String>> parseStructuredData(String sd) {
Map<String, Map<String, String>> structuredData = new HashMap<>();
// 简单的结构化数据解析，实际应用可能需要更复杂的解析
Pattern sdPattern = Pattern.compile("\\[(?<id>[^\\]]+)\\]");
Pattern paramPattern = Pattern.compile("(?<key>[^=\\s]+)=\"(?<value>[^\"]*)\"");
Matcher sdMatcher = sdPattern.matcher(sd);
while (sdMatcher.find()) {
String sdElement = sdMatcher.group("id");
Map<String, String> params = new HashMap<>();
Matcher paramMatcher = paramPattern.matcher(sdElement);
while (paramMatcher.find()) {
params.put(paramMatcher.group("key"), paramMatcher.group("value"));
}
// 使用第一个参数作为键，或者生成一个键
String key = params.containsKey("sdId") ? params.get("sdId") :
"element" + structuredData.size();
structuredData.put(key, params);
}
return structuredData;
}
/**
* 提取 JSON 字符串（不解析）
*/
public static Optional<String> extractJsonString(String syslogMessage) {
try {
if (syslogMessage == null) return Optional.empty();
int jsonStart = syslogMessage.indexOf('{');
if (jsonStart == -1) return Optional.empty();
return Optional.of(syslogMessage.substring(jsonStart));
} catch (Exception e) {
return Optional.empty();
}
}
/**
* 提取 JSON 字符串（不解析）
*/
public static String extractJsonStringNew (String syslogMessage) {
try {
if (syslogMessage == null) return "";
int jsonStart = syslogMessage.indexOf('{');
if (jsonStart == -1) return "";
return syslogMessage.substring(jsonStart);
} catch (Exception e) {
return "";
}
}
/**
* 截取字符串中第一个 ']' 之后的所有内容
*
* @param text 原始字符串
* @return 第一个 ']' 之后的内容，如果没有 ']' 则返回空字符串
*/
public static String substringAfterFirstCloseBracket(String text) {
if (text == null) {
return "";
}
int closeBracketIndex = text.indexOf(']');
if (closeBracketIndex == -1) {
return ""; // 没有找到 ']'，返回空字符串
}
// 返回 ']' 之后的内容（+1 是为了跳过 ']' 本身）
return text.substring(closeBracketIndex + 1);
}
/**
* 截取字符串中第一个指定字符之前的所有内容
*
* @param text 原始字符串
* @param delimiterChar 分隔字符
* @return 分隔字符之前的内容
*/
public static String substringBeforeFirstChar(String text, char delimiterChar) {
if (text == null) {
return "";
}
int delimiterIndex = text.indexOf(delimiterChar);
if (delimiterIndex == -1) {
return text;
}
return text.substring(0, delimiterIndex+1);
}
/**
* 从文本中提取第一个 [] 包裹的键值对字符串并转换为 Map
*
* @param text 包含键值对的原始文本
* @return 包含键值对的 Map
* @throws IllegalArgumentException 如果格式不正确
*/
public static Map<String, String> parseKeyValuePairs(String text) {
if (text == null || text.trim().isEmpty()) {
throw new IllegalArgumentException("Input text cannot be null or empty");
}
// 查找第一个 [ 和对应的 ]
int start = text.indexOf('[');
int end = text.indexOf(']', start + 1);
if (start == -1 || end == -1) {
throw new IllegalArgumentException("No valid [] pair found in text");
}
// 提取 [] 内的内容
String content = text.substring(start + 1, end).trim();
if (content.isEmpty()) {
return new HashMap<>(); // 返回空Map
}
// 分割键值对
String[] pairs = content.split("\\s+");
Map<String, String> result = new HashMap<>();
for (String pair : pairs) {
// 分割键和值
int equalSign = pair.indexOf('=');
if (equalSign == -1) {
throw new IllegalArgumentException("Invalid key-value pair format: " + pair);
}
String key = pair.substring(0, equalSign).trim();
String value = pair.substring(equalSign + 1).trim();
if (key.isEmpty()) {
throw new IllegalArgumentException("Key cannot be empty in pair: " + pair);
}
result.put(key, value);
}
return result;
}
}

115
haobang-security-xdr/syslog-consumer/src/main/java/com/common/util/TextParserUtil.java Normal file
View File

@@ -0,0 +1,115 @@
package com.common.util;
import org.springframework.stereotype.Component;
import java.util.LinkedHashMap;
import java.util.Map;
/**
* 分隔符文本解析工具类
*/
@Component
public class TextParserUtil {
/**
* 解析分隔符类型的文本数据
* @param text 原始文本
* @param sepKey 分隔符（字符或字符串）
* @return LinkedHashMap对象，key为序号字符串，value为实际值
*/
public static LinkedHashMap<String, Object> parseSeparatedText(String text, String sepKey) {
LinkedHashMap<String, Object> resultMap = new LinkedHashMap<>();
if (text == null || text.trim().isEmpty()) {
return resultMap;
}
if (sepKey == null || sepKey.isEmpty()) {
// 如果分隔符为空，将整个文本作为第一个元素
resultMap.put("1", text.trim());
return resultMap;
}
// 使用分隔符分割文本
String[] parts = text.split(sepKey, -1); // 使用-1保留空字符串
// 将分割后的部分放入LinkedHashMap，键为字符串形式的序号
for (int i = 0; i < parts.length; i++) {
resultMap.put(String.valueOf(i + 1), parts[i].trim());
}
return resultMap;
}
/**
* 解析分隔符文本并过滤空值
* @param text 原始文本
* @param sepKey 分隔符
* @param removeEmpty 是否移除空值
* @return LinkedHashMap对象
*/
public static LinkedHashMap<String, Object> parseSeparatedText(String text, String sepKey, boolean removeEmpty) {
LinkedHashMap<String, Object> resultMap = parseSeparatedText(text, sepKey);
if (removeEmpty) {
resultMap.entrySet().removeIf(entry ->
entry.getValue() == null ||
entry.getValue().toString().isEmpty());
}
return resultMap;
}
/**
* 解析分隔符文本并转换为特定类型
* @param text 原始文本
* @param sepKey 分隔符
* @param valueType 值类型（STRING, INTEGER, LONG, DOUBLE, BOOLEAN）
* @return LinkedHashMap对象
*/
public static LinkedHashMap<String, Object> parseSeparatedTextWithType(
String text, String sepKey, ValueType valueType) {
LinkedHashMap<String, Object> resultMap = parseSeparatedText(text, sepKey);
if (valueType != ValueType.STRING) {
resultMap.replaceAll((key, value) -> convertValue(value.toString(), valueType));
}
return resultMap;
}
/**
* 值类型枚举
*/
public enum ValueType {
STRING, INTEGER, LONG, DOUBLE, BOOLEAN
}
/**
* 值类型转换
*/
private static Object convertValue(String value, ValueType valueType) {
if (value == null || value.isEmpty()) {
return null;
}
try {
switch (valueType) {
case INTEGER:
return Integer.parseInt(value);
case LONG:
return Long.parseLong(value);
case DOUBLE:
return Double.parseDouble(value);
case BOOLEAN:
return Boolean.parseBoolean(value) || "1".equals(value);
case STRING:
default:
return value;
}
} catch (NumberFormatException e) {
// 转换失败时返回原始字符串
return value;
}
}
}