feat(syslog): 增加 WAL 与分 IP 接入诊断
- 每分钟保存 WAL 全局状态和分 IP 接收统计,历史保留 7 天 - 记录接收、WAL 落盘、Kafka ACK、过滤、背压及丢弃数量 - 增加 wal_max_pending_bytes 和 wal_admission_unavailable 指标 - 增加 TCP pending 当前值及进程内峰值统计 - 网络异常后同一 IP 恢复接收时打印恢复日志 - 提供当前状态及按日期查询诊断历史的接口 - 连接关闭时清理 TCP pending 计数并停止无效重试 - 增加相关单元测试和现场排查文档
This commit is contained in:
+6
-5
@@ -170,10 +170,10 @@ public class CompiledRuleCompiler {
|
||||
}
|
||||
|
||||
private List<CompiledFilterGroup> compileFilters(JsonNode node, RuleSource source) {
|
||||
node = decodeEmbeddedJson(node, source, "filters_params");
|
||||
if (node == null || node.isNull()) return Collections.emptyList();
|
||||
// 旧执行器遇到 []、空字符串等非对象结构时捕获异常并按“无过滤条件”继续。
|
||||
// 字符串即使内容是 JSON 对象也不会被旧执行器二次解析,必须同样忽略。
|
||||
// 编译路径必须保持相同语义,不能让历史脏格式阻塞整条消息。
|
||||
// Fastjson 的 JSONObject.getJSONObject 会二次解析合法的内嵌 JSON 字符串,
|
||||
// 因此编译路径也必须支持该历史存储格式。其余非对象结构仍按旧执行器语义忽略。
|
||||
if (!node.isObject()) return Collections.emptyList();
|
||||
JsonNode groups = node.get("expressions");
|
||||
if (groups == null || groups.isNull()) return Collections.emptyList();
|
||||
@@ -191,9 +191,10 @@ public class CompiledRuleCompiler {
|
||||
}
|
||||
|
||||
private List<CompiledCompletion> compileCompletions(JsonNode node, RuleSource source) {
|
||||
node = decodeEmbeddedJson(node, source, "complete_columns");
|
||||
if (node == null || node.isNull()) return Collections.emptyList();
|
||||
// 旧补全服务对 {}、空字符串等非数组结构捕获异常后按“无补全配置”继续。
|
||||
// 字符串即使内容是 JSON 数组也不会被旧执行器二次解析,必须同样忽略。
|
||||
// Fastjson 的 JSONObject.getJSONArray 会二次解析合法的内嵌 JSON 字符串,
|
||||
// 现场规则普遍沿用这种格式;其余非数组结构仍按旧执行器语义忽略。
|
||||
if (!node.isArray()) return Collections.emptyList();
|
||||
List<CompiledCompletion> result = new ArrayList<>();
|
||||
for (JsonNode completion : node) {
|
||||
|
||||
+6
-4
@@ -61,6 +61,7 @@ class LogNormalProcessorCompiledEquivalenceTest {
|
||||
assertEquals(legacyResult.getRows(), compiledResult.getRows());
|
||||
assertEquals(1, compiledResult.getRows().size());
|
||||
assertEquals(31L, compiledResult.getRows().get(0).get("normalize_rule_id"));
|
||||
assertEquals("CN", compiledResult.getRows().get(0).get("country"));
|
||||
assertEquals(1, metrics.getParseMisses());
|
||||
assertEquals(1, metrics.getParseHits());
|
||||
assertEquals(2, metrics.getCompiledMisses());
|
||||
@@ -98,10 +99,11 @@ class LogNormalProcessorCompiledEquivalenceTest {
|
||||
+ "\"mappers\":["
|
||||
+ "{\"origin_field\":\"src_ip\",\"dest_field\":\"src_ip\",\"action\":{\"type\":\"equal\",\"param\":{}}},"
|
||||
+ "{\"origin_field\":\"level\",\"dest_field\":\"severity\",\"action\":{\"type\":\"mapping\",\"param\":{\"mapping\":{\"high\":3}}}}],"
|
||||
+ "\"filters_params\":{\"expressions\":[]},"
|
||||
+ "\"complete_columns\":[{\"column_name\":\"country\",\"value\":\"CN\","
|
||||
+ "\"filters_relation\":\"and\",\"conditions\":[{\"column_name\":\"src_ip\","
|
||||
+ "\"op\":\"regexp\",\"value\":\"10\\\\..*\"}]}]}";
|
||||
+ "\"filters_params\":\"{\\\"expressions\\\":[]}\","
|
||||
+ "\"complete_columns\":\"[{\\\"column_name\\\":\\\"country\\\","
|
||||
+ "\\\"value\\\":\\\"CN\\\",\\\"filters_relation\\\":\\\"and\\\","
|
||||
+ "\\\"conditions\\\":[{\\\"column_name\\\":\\\"src_ip\\\","
|
||||
+ "\\\"op\\\":\\\"regexp\\\",\\\"value\\\":\\\"10\\\\\\\\..*\\\"}]}]\"}";
|
||||
return new RuleSource(id, name, name, "json", content, true, "0");
|
||||
}
|
||||
|
||||
|
||||
+13
-3
@@ -110,7 +110,7 @@ class CompiledRuleCompilerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void decodesStringMappersButIgnoresStringFiltersAndCompletionsLikeLegacy() {
|
||||
void decodesStringMappersFiltersAndCompletionsLikeLegacyFastjsonAccessors() {
|
||||
RuleSource legacyShape = source(1996235894300086273L, "embedded-json", "json", "{"
|
||||
+ "\"mappers\":\"[{\\\"origin_field\\\":\\\"src\\\","
|
||||
+ "\\\"dest_field\\\":\\\"dest\\\","
|
||||
@@ -124,8 +124,18 @@ class CompiledRuleCompilerTest {
|
||||
CompiledRule compiled = compiler.compile(legacyShape);
|
||||
|
||||
assertEquals(1, compiled.getMapperByOriginField().get("src").size());
|
||||
assertTrue(compiled.getFilters().isEmpty());
|
||||
assertTrue(compiled.getCompletions().isEmpty());
|
||||
assertEquals(1, compiled.getFilters().size());
|
||||
assertEquals(1, compiled.getCompletions().size());
|
||||
|
||||
Map<String, Object> allowed = new java.util.LinkedHashMap<>();
|
||||
allowed.put("dest", "allowed");
|
||||
assertFalse(compiled.isFiltered(allowed));
|
||||
compiled.complete(allowed);
|
||||
assertEquals("CN", allowed.get("country"));
|
||||
|
||||
Map<String, Object> blocked = new java.util.LinkedHashMap<>();
|
||||
blocked.put("dest", "blocked");
|
||||
assertTrue(compiled.isFiltered(blocked));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user