feat(syslog): 增加 WAL 与分 IP 接入诊断

- 每分钟保存 WAL 全局状态和分 IP 接收统计,历史保留 7 天
- 记录接收、WAL 落盘、Kafka ACK、过滤、背压及丢弃数量
- 增加 wal_max_pending_bytes 和 wal_admission_unavailable 指标
- 增加 TCP pending 当前值及进程内峰值统计
- 网络异常后同一 IP 恢复接收时打印恢复日志
- 提供当前状态及按日期查询诊断历史的接口
- 连接关闭时清理 TCP pending 计数并停止无效重试
- 增加相关单元测试和现场排查文档
This commit is contained in:
王绍宏
2026-09-16 17:32:08 +08:00
parent faa9ffe2d9
commit 8f1a8b6283
30 changed files with 1722 additions and 91 deletions
@@ -170,10 +170,10 @@ public class CompiledRuleCompiler {
}
private List<CompiledFilterGroup> compileFilters(JsonNode node, RuleSource source) {
node = decodeEmbeddedJson(node, source, "filters_params");
if (node == null || node.isNull()) return Collections.emptyList();
// 旧执行器遇到 []、空字符串等非对象结构时捕获异常并按“无过滤条件”继续。
// 字符串即使内容是 JSON 对象也不会被旧执行器二次解析,必须同样忽略。
// 编译路径必须保持相同语义,不能让历史脏格式阻塞整条消息。
// Fastjson 的 JSONObject.getJSONObject 会二次解析合法的内嵌 JSON 字符串,
// 因此编译路径也必须支持该历史存储格式。其余非对象结构仍按旧执行器语义忽略。
if (!node.isObject()) return Collections.emptyList();
JsonNode groups = node.get("expressions");
if (groups == null || groups.isNull()) return Collections.emptyList();
@@ -191,9 +191,10 @@ public class CompiledRuleCompiler {
}
private List<CompiledCompletion> compileCompletions(JsonNode node, RuleSource source) {
node = decodeEmbeddedJson(node, source, "complete_columns");
if (node == null || node.isNull()) return Collections.emptyList();
// 旧补全服务对 {}、空字符串等非数组结构捕获异常后按“无补全配置”继续。
// 字符串即使内容是 JSON 数组也不会被旧执行器二次解析,必须同样忽略。
// Fastjson 的 JSONObject.getJSONArray 会二次解析合法的内嵌 JSON 字符串,
// 现场规则普遍沿用这种格式;其余非数组结构仍按旧执行器语义忽略。
if (!node.isArray()) return Collections.emptyList();
List<CompiledCompletion> result = new ArrayList<>();
for (JsonNode completion : node) {
@@ -61,6 +61,7 @@ class LogNormalProcessorCompiledEquivalenceTest {
assertEquals(legacyResult.getRows(), compiledResult.getRows());
assertEquals(1, compiledResult.getRows().size());
assertEquals(31L, compiledResult.getRows().get(0).get("normalize_rule_id"));
assertEquals("CN", compiledResult.getRows().get(0).get("country"));
assertEquals(1, metrics.getParseMisses());
assertEquals(1, metrics.getParseHits());
assertEquals(2, metrics.getCompiledMisses());
@@ -98,10 +99,11 @@ class LogNormalProcessorCompiledEquivalenceTest {
+ "\"mappers\":["
+ "{\"origin_field\":\"src_ip\",\"dest_field\":\"src_ip\",\"action\":{\"type\":\"equal\",\"param\":{}}},"
+ "{\"origin_field\":\"level\",\"dest_field\":\"severity\",\"action\":{\"type\":\"mapping\",\"param\":{\"mapping\":{\"high\":3}}}}],"
+ "\"filters_params\":{\"expressions\":[]},"
+ "\"complete_columns\":[{\"column_name\":\"country\",\"value\":\"CN\","
+ "\"filters_relation\":\"and\",\"conditions\":[{\"column_name\":\"src_ip\","
+ "\"op\":\"regexp\",\"value\":\"10\\\\..*\"}]}]}";
+ "\"filters_params\":\"{\\\"expressions\\\":[]}\","
+ "\"complete_columns\":\"[{\\\"column_name\\\":\\\"country\\\","
+ "\\\"value\\\":\\\"CN\\\",\\\"filters_relation\\\":\\\"and\\\","
+ "\\\"conditions\\\":[{\\\"column_name\\\":\\\"src_ip\\\","
+ "\\\"op\\\":\\\"regexp\\\",\\\"value\\\":\\\"10\\\\\\\\..*\\\"}]}]\"}";
return new RuleSource(id, name, name, "json", content, true, "0");
}
@@ -110,7 +110,7 @@ class CompiledRuleCompilerTest {
}
@Test
void decodesStringMappersButIgnoresStringFiltersAndCompletionsLikeLegacy() {
void decodesStringMappersFiltersAndCompletionsLikeLegacyFastjsonAccessors() {
RuleSource legacyShape = source(1996235894300086273L, "embedded-json", "json", "{"
+ "\"mappers\":\"[{\\\"origin_field\\\":\\\"src\\\","
+ "\\\"dest_field\\\":\\\"dest\\\","
@@ -124,8 +124,18 @@ class CompiledRuleCompilerTest {
CompiledRule compiled = compiler.compile(legacyShape);
assertEquals(1, compiled.getMapperByOriginField().get("src").size());
assertTrue(compiled.getFilters().isEmpty());
assertTrue(compiled.getCompletions().isEmpty());
assertEquals(1, compiled.getFilters().size());
assertEquals(1, compiled.getCompletions().size());
Map<String, Object> allowed = new java.util.LinkedHashMap<>();
allowed.put("dest", "allowed");
assertFalse(compiled.isFiltered(allowed));
compiled.complete(allowed);
assertEquals("CN", allowed.get("country"));
Map<String, Object> blocked = new java.util.LinkedHashMap<>();
blocked.put("dest", "blocked");
assertTrue(compiled.isFiltered(blocked));
}
@Test