feat(api-pull): 完善全悉设备接入并修复 NDR 边界重复采集
- 增加全悉设备 API 日志拉取及消费处理 - 完善全悉日志解析、字段映射和标准化逻辑 - NDR 采集窗口统一采用左闭右开规则 - 对设备秒级查询范围向外取整，并在本地精确过滤 - 区分接口原始返回数和实际接收数，保证分页偏移正确推进 - 补充时间窗口、边界过滤和分页推进测试 - 更新设备 API 拉取及对接说明文档
This commit is contained in:
王绍宏
+5
@@ -944,6 +944,11 @@ public class LogNormalProcessor {
|
||||
(HashMap<String, Object>) ((HashMap<String, Object>) map.get("action")).get("param");
|
||||
normalColumMap.put("dest_field_value",
|
||||
getMappingValue(action_param, entry.getValue().toString()));
|
||||
} else if (((HashMap<String, Object>) map.get("action")).get("type").equals("bounded_text")) {
|
||||
Map<String, Object> params = (Map<String, Object>)
|
||||
((Map<String, Object>) map.get("action")).get("param");
|
||||
int bytes = com.common.util.BoundedText.limit(params.get("max_utf8_bytes"));
|
||||
normalColumMap.put("dest_field_value", com.common.util.BoundedText.map(entry.getValue(), bytes));
|
||||
} else if (((HashMap<String, Object>) map.get("action")).get("type").equals("time")) {
|
||||
normalColumMap.put("action_param",
|
||||
((HashMap<String, Object>) map.get("action")).get("param"));
|
||||
|
||||
+4
-1
@@ -8,7 +8,7 @@ import java.util.Map;
|
||||
|
||||
/** 单个字段映射的不可变执行计划。 */
|
||||
public final class CompiledMapper {
|
||||
public enum Action { EQUAL, MAPPING, TIME, LEGACY_NULL }
|
||||
public enum Action { EQUAL, MAPPING, TIME, BOUNDED_TEXT, LEGACY_NULL }
|
||||
|
||||
private final String originField;
|
||||
private final String destinationField;
|
||||
@@ -33,6 +33,9 @@ public final class CompiledMapper {
|
||||
case MAPPING:
|
||||
// 保持旧执行器语义：源值为 null 时规则执行失败，而不是静默映射成字符串 "null"。
|
||||
return sanitize(mapping.get(sourceValue.toString()));
|
||||
case BOUNDED_TEXT:
|
||||
return sanitize(com.common.util.BoundedText.map(sourceValue,
|
||||
com.common.util.BoundedText.limit(mapping.get("max_utf8_bytes"))));
|
||||
case TIME:
|
||||
try {
|
||||
return TimeConversionUtils.convertToMillis(sourceValue.toString(), timezone);
|
||||
|
||||
+7
@@ -144,6 +144,13 @@ public class CompiledRuleCompiler {
|
||||
mapping.put(entry.getKey(), freeze(objectMapper.convertValue(entry.getValue(), Object.class)));
|
||||
}
|
||||
compiled = new CompiledMapper(origin, destination, CompiledMapper.Action.MAPPING, mapping, null);
|
||||
} else if ("bounded_text".equals(type)) {
|
||||
JsonNode limit = param == null ? null : param.get("max_utf8_bytes");
|
||||
if (limit == null || !limit.isIntegralNumber() || !limit.canConvertToInt())
|
||||
throw invalid(source, "bounded_text requires integer max_utf8_bytes");
|
||||
int bytes = com.common.util.BoundedText.limit(limit.intValue());
|
||||
compiled = new CompiledMapper(origin, destination, CompiledMapper.Action.BOUNDED_TEXT,
|
||||
Collections.singletonMap("max_utf8_bytes", bytes), null);
|
||||
} else if ("time".equals(type)) {
|
||||
String timezone = param == null ? null : text(param.get("timezone"));
|
||||
compiled = new CompiledMapper(origin, destination, CompiledMapper.Action.TIME,
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.common.util;
|
||||
|
||||
/** Explicit opt-in rule action: keep text only within its UTF-8 budget, never truncate or decode. */
|
||||
public final class BoundedText {
|
||||
private BoundedText() { }
|
||||
|
||||
public static int limit(Object value) {
|
||||
if (!(value instanceof Number)) throw new IllegalArgumentException("max_utf8_bytes must be an integer");
|
||||
Number n = (Number) value;
|
||||
int limit = n.intValue();
|
||||
if (n.doubleValue() != limit || limit < 1 || limit > 1048576)
|
||||
throw new IllegalArgumentException("max_utf8_bytes out of range");
|
||||
return limit;
|
||||
}
|
||||
|
||||
public static String map(Object value, int maxBytes) {
|
||||
if (!(value instanceof String)) return null;
|
||||
String text = (String) value;
|
||||
int bytes = 0;
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
char c = text.charAt(i);
|
||||
if (c <= 0x7f) bytes++;
|
||||
else if (c <= 0x7ff) bytes += 2;
|
||||
else if (Character.isHighSurrogate(c) && i + 1 < text.length()
|
||||
&& Character.isLowSurrogate(text.charAt(i + 1))) { bytes += 4; i++; }
|
||||
else if (Character.isSurrogate(c)) return null;
|
||||
else bytes += 3;
|
||||
if (bytes > maxBytes) return null;
|
||||
}
|
||||
return text;
|
||||
}
|
||||
}
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
package com.Modules.NormalData;
|
||||
|
||||
import com.Modules.NormalData.compiled.*;
|
||||
import com.common.mapper.*;
|
||||
import com.common.service.*;
|
||||
import com.config.*;
|
||||
import com.fasterxml.jackson.databind.*;
|
||||
import com.fasterxml.jackson.databind.node.*;
|
||||
import com.common.util.BoundedText;
|
||||
import java.nio.file.*;
|
||||
import java.time.*;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import static org.mockito.ArgumentMatchers.anyMap;
|
||||
|
||||
/** Shared sanitized vendor response + adapter golden identities -> real normalization -> real routing. */
|
||||
class QuanxiNormalizationTest {
|
||||
static final ObjectMapper JSON = new ObjectMapper();
|
||||
static final LocalDateTime RECEIVED = LocalDateTime.of(2026, 9, 6, 12, 0);
|
||||
static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS");
|
||||
|
||||
@Test void allTwentyDocumentsNormalizeAndUseAlarmInsertInBothEngines() throws Exception {
|
||||
JsonNode records = JSON.readTree(file("fixtures/api-response.json").toFile()).path("result").path("data");
|
||||
JsonNode expected = JSON.readTree(file("fixtures/expected-events.json").toFile());
|
||||
for (boolean compiled : new boolean[]{false, true}) {
|
||||
Fixture f = new Fixture(compiled);
|
||||
for (int i = 0; i < records.size(); i++) {
|
||||
JsonNode r = records.get(i);
|
||||
Map<String, Object> row = f.prepare(r, expected.get(i).path("event_id").asText());
|
||||
assertEquals(r.path("src_ip").asText(), row.get("src_ip"));
|
||||
assertEquals(r.path("dest_ip").asText(), row.get("dest_ip"));
|
||||
assertEquals(r.path("src_port").asInt(), row.get("src_port"));
|
||||
assertEquals(r.path("dest_port").asInt(), row.get("dest_port"));
|
||||
assertEquals(r.path("proto").asText(), row.get("proto"));
|
||||
assertEquals(r.path("app_proto").asText(), row.get("protocol"));
|
||||
assertEquals(r.path("attacker").asText(), row.get("attacker_ip"));
|
||||
assertEquals(r.path("victim").asText(), row.get("victim_ip"));
|
||||
assertEquals(r.path("direction").asText(), row.get("direction"));
|
||||
assertEquals(r.path("name").asText(), row.get("origin_event_name"));
|
||||
assertEquals(r.path("type").asText(), row.get("origin_event_type"));
|
||||
assertEquals(r.path("severity").asInt(), row.get("origin_event_level"));
|
||||
assertEquals(r.path("result").asText(), row.get("origin_attack_result"));
|
||||
assertEquals(r.path("doc_id").asText(), row.get("origin_event_id"));
|
||||
assertEquals(r.path("flow_id").asText(), row.get("tc_flow_id"));
|
||||
assertEquals(-1, row.get("event_level")); // Explicit unknown, not guessed vendor severity.
|
||||
assertEquals(-1, row.get("attack_result"));
|
||||
assertEquals(LocalDateTime.ofInstant(Instant.ofEpochMilli(r.path("timestamp").asLong()),
|
||||
ZoneId.of("Asia/Shanghai")), row.get("log_time"));
|
||||
assertEquals(RECEIVED, row.get("created_at"));
|
||||
assertEquals(row.get("id"), f.prepare(r, expected.get(i).path("event_id").asText()).get("id"));
|
||||
JsonNode h = r.path("appbrief").path("http");
|
||||
for (String[] field : HTTP_FIELDS) {
|
||||
assertEquals(h.has(field[0]) ? h.get(field[0]).asText().trim() : null, row.get(field[1]));
|
||||
}
|
||||
assertTrue(f.store.isAlarmData(row));
|
||||
assertEquals(1, f.store.insertDynamic(row));
|
||||
}
|
||||
verify(f.alarm, times(20)).insertDynamic(anyMap());
|
||||
verifyNoInteractions(f.normal);
|
||||
}
|
||||
}
|
||||
|
||||
@Test void missingWrongTypesOversizeAndExistingTopLevelFieldsAreHandledWithoutChangingRaw() throws Exception {
|
||||
for (boolean compiled : new boolean[]{false, true}) {
|
||||
Fixture f = new Fixture(compiled);
|
||||
ObjectNode original = (ObjectNode) JSON.readTree(file("fixtures/api-response.json").toFile())
|
||||
.path("result").path("data").get(6);
|
||||
for (String[] field : HTTP_FIELDS) {
|
||||
for (JsonNode value : Arrays.asList(NullNode.instance, IntNode.valueOf(42),
|
||||
JSON.createArrayNode().add("bad"), JSON.createObjectNode().put("bad", true),
|
||||
TextNode.valueOf(repeat("x", 10001)))) {
|
||||
ObjectNode record = original.deepCopy();
|
||||
record.with("appbrief").with("http").set(field[0], value);
|
||||
record.put(field[1], "unrelated-existing-value");
|
||||
String before = record.toString();
|
||||
assertNull(f.prepare(record, "synthetic-id").get(field[1]));
|
||||
assertEquals(before, record.toString());
|
||||
}
|
||||
ObjectNode record = original.deepCopy();
|
||||
record.with("appbrief").with("http").put(field[0], repeat("x", 10000));
|
||||
assertEquals(10000, ((String) f.prepare(record, "synthetic-id").get(field[1])).length());
|
||||
record.with("appbrief").with("http").remove(field[0]);
|
||||
assertNull(f.prepare(record, "synthetic-id").get(field[1]));
|
||||
}
|
||||
original.put("payload", "R0VUIC8gSFRUUC8xLjE=");
|
||||
original.remove("appbrief");
|
||||
for (String[] field : HTTP_FIELDS) assertNull(f.prepare(original, "synthetic-id").get(field[1]));
|
||||
}
|
||||
}
|
||||
|
||||
@Test void utf8BoundaryUsesBytesAndRejectsMalformedSurrogates() {
|
||||
assertNotNull(BoundedText.map(repeat("中", 3333), 10000));
|
||||
assertNull(BoundedText.map(repeat("中", 3334), 10000));
|
||||
assertNotNull(BoundedText.map(repeat("😀", 2500), 10000));
|
||||
assertNull(BoundedText.map(repeat("😀", 2501), 10000));
|
||||
assertNull(BoundedText.map("\uD800", 10000));
|
||||
assertNull(BoundedText.map(new Object(), 10000));
|
||||
assertEquals("", BoundedText.map("", 10000));
|
||||
assertThrows(IllegalArgumentException.class, () -> BoundedText.limit(0));
|
||||
assertThrows(IllegalArgumentException.class, () -> BoundedText.limit(1.5));
|
||||
}
|
||||
|
||||
@Test void ruleDestinationsAreActualAlarmSqlColumnsAndHttpPathsAreExact() throws Exception {
|
||||
JsonNode rule = JSON.readTree(file("normalization-rule.json").toFile());
|
||||
String sql = new String(Files.readAllBytes(file("../../../syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml")), "UTF-8");
|
||||
for (JsonNode m : rule.path("mappers")) assertTrue(sql.contains("dataMap." + m.path("dest_field").asText() + " "));
|
||||
for (String[] field : HTTP_FIELDS) {
|
||||
boolean found = false;
|
||||
for (JsonNode m : rule.path("mappers")) if (field[1].equals(m.path("dest_field").asText())) {
|
||||
assertEquals("appbrief.http." + field[0], m.path("origin_field").asText()); found = true;
|
||||
}
|
||||
assertTrue(found);
|
||||
}
|
||||
}
|
||||
|
||||
static final String[][] HTTP_FIELDS = {{"req_header_raw", "http_req_header"}, {"req_body", "http_req_body"},
|
||||
{"resp_header_raw", "http_resp_header"}, {"resp_body", "http_resp_body"}};
|
||||
static String repeat(String s, int n) { StringBuilder b = new StringBuilder(); for (int i=0;i<n;i++) b.append(s); return b.toString(); }
|
||||
static Path file(String name) {
|
||||
Path root = Paths.get("").toAbsolutePath();
|
||||
while (!Files.isDirectory(root.resolve("design/device-integration-docs/quanxi"))) root = root.getParent();
|
||||
return root.resolve("design/device-integration-docs/quanxi/" + name).normalize();
|
||||
}
|
||||
static class Fixture {
|
||||
final DeviceCollectTaskService tasks = mock(DeviceCollectTaskService.class);
|
||||
final DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
final SyslogNormalAlarmMapper alarm = mock(SyslogNormalAlarmMapper.class);
|
||||
final SyslogNormalDataMapper normal = mock(SyslogNormalDataMapper.class);
|
||||
final SyslogNormalDataService store = new SyslogNormalDataService();
|
||||
final LogNormalProcessor processor;
|
||||
final BatchRuleSnapshot snapshot;
|
||||
Fixture(boolean compiled) throws Exception {
|
||||
ReflectionTestUtils.setField(store, "syslogNormalAlarmMapper", alarm);
|
||||
ReflectionTestUtils.setField(store, "syslogNormalDataMapper", normal);
|
||||
ReflectionTestUtils.setField(store, "statsChangeCaptureService", mock(StatsChangeCaptureService.class));
|
||||
ReflectionTestUtils.setField(store, "directStatsProjectionWriter", mock(DirectStatsProjectionWriter.class));
|
||||
when(alarm.insertDynamic(anyMap())).thenReturn(1);
|
||||
NormalizationOptimizationProperties p = new NormalizationOptimizationProperties();
|
||||
p.getCompiledRule().setEnabled(compiled); p.getCompiledRule().setShadowCompareEnabled(false);
|
||||
p.getRuleSelection().setEnabled(true); p.getRuleSelection().setShadowEnabled(false);
|
||||
CompiledRuleCompiler compiler = new CompiledRuleCompiler(JSON);
|
||||
processor = new LogNormalProcessor(mock(DmNormalizeRuleService.class), mock(DmColumnService.class), store,
|
||||
mock(SyslogNonNormalMessageService.class), new LogDataFilterService(), new LogDataCompleteService(),
|
||||
tasks, devices, mock(DmColumnMapper.class), mock(DmNormalizeRuleMapper.class),
|
||||
mock(KafkaConsumerProperties.class), mock(AppProperties.class), new CompiledRuleCache(compiler, p), p);
|
||||
String content = new String(Files.readAllBytes(file("normalization-rule.json")), "UTF-8");
|
||||
RuleSource source = new RuleSource(700L, "quanxi", "quanxi", "json", content, true, "0");
|
||||
snapshot = new BatchRuleSnapshot(Collections.singletonMap(7L, Collections.singletonList(source)), 1L);
|
||||
}
|
||||
Map<String, Object> prepare(JsonNode doc, String id) {
|
||||
LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(doc.path("timestamp").asLong()), ZoneId.of("Asia/Shanghai"));
|
||||
String message = "[receive_time=" + TIME.format(RECEIVED) + " event_time=" + TIME.format(time)
|
||||
+ " event_id=" + id + " device_id=7 device_name=fixture vendor=quanxi data_type=json device_collect_id=3]" + doc;
|
||||
LogNormalProcessor.NormalizationResult result = processor.prepare(message, id, "raw", RECEIVED,
|
||||
snapshot, new BatchEnrichmentContext(false, tasks, devices), new RuleExecutionMetrics());
|
||||
assertEquals(LogNormalProcessor.ProcessingOutcome.SUCCESS, result.getOutcome());
|
||||
assertEquals(1, result.getRows().size());
|
||||
return result.getRows().get(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user