nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(syslog-consumer-rule): 修复现场日志解析并补充升级工具

Browse Source 
- 支持设备头字段值包含空格，避免 WAF 等名称被误拆为无效键值对
- 将空字符串 cropper offset 按未配置处理，兼容历史标准化规则
- 增加设备头解析和空 offset 回归测试
- 增加统计物化 Topic 创建脚本
- 增加容器内 JAR 热替换脚本及操作说明
This commit is contained in:
王绍宏
2026-09-12 11:40:57 +08:00
parent c1472b9a95
commit 3b6c9e4714
10 changed files with 321 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java
+9 -6
View File
@@ -920,12 +920,15 @@ public class LogNormalProcessor {
try {
JSONObject jsonObject = new JSONObject(dmNormalizeRule.get("rule_content").toString());
if (jsonObject.isEmpty()) return null;
cropperParams.sethead_key(jsonObject.getJSONObject("cropper_params").get("head_key").toString());
cropperParams.sethead_offset(Integer.parseInt(
jsonObject.getJSONObject("cropper_params").get("head_offset").toString()));
cropperParams.settail_key(jsonObject.getJSONObject("cropper_params").get("tail_key").toString());
cropperParams.settail_offset(Integer.parseInt(
jsonObject.getJSONObject("cropper_params").get("tail_offset").toString()));
JSONObject cropper = jsonObject.getJSONObject("cropper_params");
String headOffset = cropper.get("head_offset").toString().trim();
String tailOffset = cropper.get("tail_offset").toString().trim();
// 历史规则可能把 offset 保存为空字符串；此时等同于未配置截取规则，直接使用默认解析。
if (headOffset.isEmpty() || tailOffset.isEmpty()) return null;
cropperParams.sethead_key(cropper.get("head_key").toString());
cropperParams.sethead_offset(Integer.parseInt(headOffset));
cropperParams.settail_key(cropper.get("tail_key").toString());
cropperParams.settail_offset(Integer.parseInt(tailOffset));
return cropperParams;
} catch (Exception ex) {
logger.error("getCropperParams：" + ex.getMessage());
haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/compiled/CompiledRuleCompiler.java
+6
View File
@@ -98,10 +98,16 @@ public class CompiledRuleCompiler {
private CompiledCropper compileCropper(JsonNode node, RuleSource source) {
if (node == null || node.isNull()) return null;
if (!node.isObject()) throw invalid(source, "cropper_params 必须是对象");
// 达梦历史规则可能把 offset 保存为空字符串；与旧执行器保持一致，忽略整段截取配置。
if (emptyText(node.get("head_offset")) || emptyText(node.get("tail_offset"))) return null;
return new CompiledCropper(textOrEmpty(node.get("head_key")), integer(node, "head_offset", source),
textOrEmpty(node.get("tail_key")), integer(node, "tail_offset", source));
}
private static boolean emptyText(JsonNode node) {
return node != null && node.isTextual() && node.textValue().trim().isEmpty();
}
private KvConfig compileKv(JsonNode node, String dataType, RuleSource source) {
if (!"kv".equals(dataType)) {
return node != null && node.isObject()
haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/KeyValueParser.java
+2 -41
View File
@@ -71,46 +71,7 @@ public class KeyValueParser {
* @throws IllegalArgumentException 如果格式不正确
*/
public static Map<String, String> parseKeyValuePairs(String text) {
if (text == null || text.trim().isEmpty()) {
throw new IllegalArgumentException("Input text cannot be null or empty");
}
// 查找第一个 [ 和对应的 ]
int start = text.indexOf('[');
int end = text.indexOf(']', start + 1);
if (start == -1 || end == -1) {
throw new IllegalArgumentException("No valid [] pair found in text");
}
// 提取 [] 内的内容
String content = text.substring(start + 1, end).trim();
if (content.isEmpty()) {
return new HashMap<>(); // 返回空Map
}
// 分割键值对
String[] pairs = content.split("\\s+");
Map<String, String> result = new HashMap<>();
for (String pair : pairs) {
// 分割键和值
int equalSign = pair.indexOf('=');
if (equalSign == -1) {
throw new IllegalArgumentException("Invalid key-value pair format: " + pair);
}
String key = pair.substring(0, equalSign).trim();
String value = pair.substring(equalSign + 1).trim();
if (key.isEmpty()) {
throw new IllegalArgumentException("Key cannot be empty in pair: " + pair);
}
result.put(key, value);
}
return result;
return SyslogParser.parseKeyValuePairs(text);
}
public static void main(String[] args) {
String input = "[name=非工作时间访问 riskLevel=中 riskMain=账号 appName=蜜罐系统-企信外网 riskDesc=账号在2025-04-27，通过47.94.211.49客户端IP在非常用访问时间2025-04-27 15:38:54，进行了业务访问]";
@@ -132,4 +93,4 @@ public class KeyValueParser {
System.out.println("风险等级: " + result1.get("riskLevel"));
System.out.println("风险描述: " + result1.get("riskDesc"));
}
}
}
haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/util/SyslogParser.java
+28 -18
View File
@@ -42,6 +42,13 @@ public class SyslogParser {
"(?<content>.+)$"
);
/**
* syslog-serve 的设备头使用空格分隔 key=value，但历史格式没有对 value 中的空格做转义。
* 因此不能直接按空白切分，只能把“空白 + 合法 key=”识别为下一个字段的开始。
*/
private static final Pattern DEVICE_HEADER_KEY_PATTERN =
Pattern.compile("(?<!\\S)([A-Za-z_][A-Za-z0-9_.-]*)=");
// RFC 3164 时间戳格式
private static final DateTimeFormatter RFC3164_FORMATTER =
DateTimeFormatter.ofPattern("MMM dd HH:mm:ss");
@@ -328,28 +335,31 @@ public class SyslogParser {
return new HashMap<>(); // 返回空Map
}
// 分割键值对
String[] pairs = content.split("\\s+");
Map<String, String> result = new HashMap<>();
Map<String, String> result = new LinkedHashMap<>();
Matcher matcher = DEVICE_HEADER_KEY_PATTERN.matcher(content);
String previousKey = null;
int previousValueStart = -1;
for (String pair : pairs) {
// 分割键和值
int equalSign = pair.indexOf('=');
if (equalSign == -1) {
throw new IllegalArgumentException("Invalid key-value pair format: " + pair);
while (matcher.find()) {
if (previousKey == null) {
String invalidPrefix = content.substring(0, matcher.start()).trim();
if (!invalidPrefix.isEmpty()) {
throw new IllegalArgumentException("Invalid key-value pair format: " + invalidPrefix);
}
} else {
String value = content.substring(previousValueStart, matcher.start()).trim();
result.put(previousKey, value);
}
String key = pair.substring(0, equalSign).trim();
String value = pair.substring(equalSign + 1).trim();
if (key.isEmpty()) {
throw new IllegalArgumentException("Key cannot be empty in pair: " + pair);
}
result.put(key, value);
previousKey = matcher.group(1);
previousValueStart = matcher.end();
}
if (previousKey == null) {
throw new IllegalArgumentException("Invalid key-value pair format: " + content);
}
result.put(previousKey, content.substring(previousValueStart).trim());
return result;
}
}
haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/CompiledRuleExecutionEquivalenceTest.java
+16
View File
@@ -23,6 +23,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.Mockito.mock;
class CompiledRuleExecutionEquivalenceTest {
@@ -61,6 +62,21 @@ class CompiledRuleExecutionEquivalenceTest {
}
}
@Test
void blankCropperOffsetsAreTreatedAsNoCropperConfiguration() {
RuleSource source = new RuleSource(9L, "blank-offset", "blank-offset", "json", "{"
+ "\"cropper_params\":{\"head_key\":\"{\",\"head_offset\":\"\","
+ "\"tail_key\":\"\",\"tail_offset\":\"\"},\"mappers\":[]}", true, "0");
String message = "<14> host {\"status\":\"ok\"}";
assertNull(legacy.getCropperParams(source.asLegacyMap()));
assertEquals("ok", legacy.getMessageToMap(source.asLegacyMap(), message, "json").get("status"));
CompiledRule compiled = compiler.compile(source);
assertNull(compiled.getCropper());
assertEquals("ok", compiled.parse(message).getFields().get("status"));
}
private static final class Fixture {
private final String type;
private final String message;
haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/util/SyslogParserHeaderTest.java
+54
View File
@@ -0,0 +1,54 @@
package com.common.util;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
class SyslogParserHeaderTest {
@Test
void shouldKeepSpacesInsideDeviceHeaderValues() {
String message = "[receive_time=20260911221200000 event_time=20260911221159000 "
+ "device_id=12 device_name=边界防护 WAF vendor=某某 安全厂商 "
+ "data_type=json device_collect_id=2]{\"message\":\"ok\"}";
Map<String, String> result = SyslogParser.parseKeyValuePairs(message);
assertEquals("20260911221200000", result.get("receive_time"));
assertEquals("20260911221159000", result.get("event_time"));
assertEquals("12", result.get("device_id"));
assertEquals("边界防护 WAF", result.get("device_name"));
assertEquals("某某 安全厂商", result.get("vendor"));
assertEquals("json", result.get("data_type"));
assertEquals("2", result.get("device_collect_id"));
}
@Test
void shouldKeepEmptyAndEqualsContainingValues() {
Map<String, String> result = SyslogParser.parseKeyValuePairs(
"[device_id=12 device_name= vendor=WAF=主节点 data_type=json]");
assertEquals("", result.get("device_name"));
assertEquals("WAF=主节点", result.get("vendor"));
}
@Test
void shouldStillParseLegacySingleWordValues() {
Map<String, String> result = SyslogParser.parseKeyValuePairs(
"[device_id=1 device_name=honeypot vendor=changting data_type=json]");
assertEquals("honeypot", result.get("device_name"));
assertEquals("changting", result.get("vendor"));
}
@Test
void shouldRejectHeaderWithoutKeyValuePair() {
assertThrows(IllegalArgumentException.class,
() -> SyslogParser.parseKeyValuePairs("[WAF]"));
assertThrows(IllegalArgumentException.class,
() -> SyslogParser.parseKeyValuePairs("[WAF device_id=12]"));
}
}