feat(api-pull): 完善全悉设备接入并修复 NDR 边界重复采集
- 增加全悉设备 API 日志拉取及消费处理 - 完善全悉日志解析、字段映射和标准化逻辑 - NDR 采集窗口统一采用左闭右开规则 - 对设备秒级查询范围向外取整，并在本地精确过滤 - 区分接口原始返回数和实际接收数，保证分页偏移正确推进 - 补充时间窗口、边界过滤和分页推进测试 - 更新设备 API 拉取及对接说明文档
This commit is contained in:
王绍宏
+165
@@ -0,0 +1,165 @@
|
||||
package com.Modules.NormalData;
|
||||
|
||||
import com.Modules.NormalData.compiled.*;
|
||||
import com.common.mapper.*;
|
||||
import com.common.service.*;
|
||||
import com.config.*;
|
||||
import com.fasterxml.jackson.databind.*;
|
||||
import com.fasterxml.jackson.databind.node.*;
|
||||
import com.common.util.BoundedText;
|
||||
import java.nio.file.*;
|
||||
import java.time.*;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import static org.mockito.ArgumentMatchers.anyMap;
|
||||
|
||||
/** Shared sanitized vendor response + adapter golden identities -> real normalization -> real routing. */
|
||||
class QuanxiNormalizationTest {
|
||||
static final ObjectMapper JSON = new ObjectMapper();
|
||||
static final LocalDateTime RECEIVED = LocalDateTime.of(2026, 9, 6, 12, 0);
|
||||
static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS");
|
||||
|
||||
@Test void allTwentyDocumentsNormalizeAndUseAlarmInsertInBothEngines() throws Exception {
|
||||
JsonNode records = JSON.readTree(file("fixtures/api-response.json").toFile()).path("result").path("data");
|
||||
JsonNode expected = JSON.readTree(file("fixtures/expected-events.json").toFile());
|
||||
for (boolean compiled : new boolean[]{false, true}) {
|
||||
Fixture f = new Fixture(compiled);
|
||||
for (int i = 0; i < records.size(); i++) {
|
||||
JsonNode r = records.get(i);
|
||||
Map<String, Object> row = f.prepare(r, expected.get(i).path("event_id").asText());
|
||||
assertEquals(r.path("src_ip").asText(), row.get("src_ip"));
|
||||
assertEquals(r.path("dest_ip").asText(), row.get("dest_ip"));
|
||||
assertEquals(r.path("src_port").asInt(), row.get("src_port"));
|
||||
assertEquals(r.path("dest_port").asInt(), row.get("dest_port"));
|
||||
assertEquals(r.path("proto").asText(), row.get("proto"));
|
||||
assertEquals(r.path("app_proto").asText(), row.get("protocol"));
|
||||
assertEquals(r.path("attacker").asText(), row.get("attacker_ip"));
|
||||
assertEquals(r.path("victim").asText(), row.get("victim_ip"));
|
||||
assertEquals(r.path("direction").asText(), row.get("direction"));
|
||||
assertEquals(r.path("name").asText(), row.get("origin_event_name"));
|
||||
assertEquals(r.path("type").asText(), row.get("origin_event_type"));
|
||||
assertEquals(r.path("severity").asInt(), row.get("origin_event_level"));
|
||||
assertEquals(r.path("result").asText(), row.get("origin_attack_result"));
|
||||
assertEquals(r.path("doc_id").asText(), row.get("origin_event_id"));
|
||||
assertEquals(r.path("flow_id").asText(), row.get("tc_flow_id"));
|
||||
assertEquals(-1, row.get("event_level")); // Explicit unknown, not guessed vendor severity.
|
||||
assertEquals(-1, row.get("attack_result"));
|
||||
assertEquals(LocalDateTime.ofInstant(Instant.ofEpochMilli(r.path("timestamp").asLong()),
|
||||
ZoneId.of("Asia/Shanghai")), row.get("log_time"));
|
||||
assertEquals(RECEIVED, row.get("created_at"));
|
||||
assertEquals(row.get("id"), f.prepare(r, expected.get(i).path("event_id").asText()).get("id"));
|
||||
JsonNode h = r.path("appbrief").path("http");
|
||||
for (String[] field : HTTP_FIELDS) {
|
||||
assertEquals(h.has(field[0]) ? h.get(field[0]).asText().trim() : null, row.get(field[1]));
|
||||
}
|
||||
assertTrue(f.store.isAlarmData(row));
|
||||
assertEquals(1, f.store.insertDynamic(row));
|
||||
}
|
||||
verify(f.alarm, times(20)).insertDynamic(anyMap());
|
||||
verifyNoInteractions(f.normal);
|
||||
}
|
||||
}
|
||||
|
||||
@Test void missingWrongTypesOversizeAndExistingTopLevelFieldsAreHandledWithoutChangingRaw() throws Exception {
|
||||
for (boolean compiled : new boolean[]{false, true}) {
|
||||
Fixture f = new Fixture(compiled);
|
||||
ObjectNode original = (ObjectNode) JSON.readTree(file("fixtures/api-response.json").toFile())
|
||||
.path("result").path("data").get(6);
|
||||
for (String[] field : HTTP_FIELDS) {
|
||||
for (JsonNode value : Arrays.asList(NullNode.instance, IntNode.valueOf(42),
|
||||
JSON.createArrayNode().add("bad"), JSON.createObjectNode().put("bad", true),
|
||||
TextNode.valueOf(repeat("x", 10001)))) {
|
||||
ObjectNode record = original.deepCopy();
|
||||
record.with("appbrief").with("http").set(field[0], value);
|
||||
record.put(field[1], "unrelated-existing-value");
|
||||
String before = record.toString();
|
||||
assertNull(f.prepare(record, "synthetic-id").get(field[1]));
|
||||
assertEquals(before, record.toString());
|
||||
}
|
||||
ObjectNode record = original.deepCopy();
|
||||
record.with("appbrief").with("http").put(field[0], repeat("x", 10000));
|
||||
assertEquals(10000, ((String) f.prepare(record, "synthetic-id").get(field[1])).length());
|
||||
record.with("appbrief").with("http").remove(field[0]);
|
||||
assertNull(f.prepare(record, "synthetic-id").get(field[1]));
|
||||
}
|
||||
original.put("payload", "R0VUIC8gSFRUUC8xLjE=");
|
||||
original.remove("appbrief");
|
||||
for (String[] field : HTTP_FIELDS) assertNull(f.prepare(original, "synthetic-id").get(field[1]));
|
||||
}
|
||||
}
|
||||
|
||||
@Test void utf8BoundaryUsesBytesAndRejectsMalformedSurrogates() {
|
||||
assertNotNull(BoundedText.map(repeat("中", 3333), 10000));
|
||||
assertNull(BoundedText.map(repeat("中", 3334), 10000));
|
||||
assertNotNull(BoundedText.map(repeat("😀", 2500), 10000));
|
||||
assertNull(BoundedText.map(repeat("😀", 2501), 10000));
|
||||
assertNull(BoundedText.map("\uD800", 10000));
|
||||
assertNull(BoundedText.map(new Object(), 10000));
|
||||
assertEquals("", BoundedText.map("", 10000));
|
||||
assertThrows(IllegalArgumentException.class, () -> BoundedText.limit(0));
|
||||
assertThrows(IllegalArgumentException.class, () -> BoundedText.limit(1.5));
|
||||
}
|
||||
|
||||
@Test void ruleDestinationsAreActualAlarmSqlColumnsAndHttpPathsAreExact() throws Exception {
|
||||
JsonNode rule = JSON.readTree(file("normalization-rule.json").toFile());
|
||||
String sql = new String(Files.readAllBytes(file("../../../syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml")), "UTF-8");
|
||||
for (JsonNode m : rule.path("mappers")) assertTrue(sql.contains("dataMap." + m.path("dest_field").asText() + " "));
|
||||
for (String[] field : HTTP_FIELDS) {
|
||||
boolean found = false;
|
||||
for (JsonNode m : rule.path("mappers")) if (field[1].equals(m.path("dest_field").asText())) {
|
||||
assertEquals("appbrief.http." + field[0], m.path("origin_field").asText()); found = true;
|
||||
}
|
||||
assertTrue(found);
|
||||
}
|
||||
}
|
||||
|
||||
static final String[][] HTTP_FIELDS = {{"req_header_raw", "http_req_header"}, {"req_body", "http_req_body"},
|
||||
{"resp_header_raw", "http_resp_header"}, {"resp_body", "http_resp_body"}};
|
||||
static String repeat(String s, int n) { StringBuilder b = new StringBuilder(); for (int i=0;i<n;i++) b.append(s); return b.toString(); }
|
||||
static Path file(String name) {
|
||||
Path root = Paths.get("").toAbsolutePath();
|
||||
while (!Files.isDirectory(root.resolve("design/device-integration-docs/quanxi"))) root = root.getParent();
|
||||
return root.resolve("design/device-integration-docs/quanxi/" + name).normalize();
|
||||
}
|
||||
static class Fixture {
|
||||
final DeviceCollectTaskService tasks = mock(DeviceCollectTaskService.class);
|
||||
final DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
final SyslogNormalAlarmMapper alarm = mock(SyslogNormalAlarmMapper.class);
|
||||
final SyslogNormalDataMapper normal = mock(SyslogNormalDataMapper.class);
|
||||
final SyslogNormalDataService store = new SyslogNormalDataService();
|
||||
final LogNormalProcessor processor;
|
||||
final BatchRuleSnapshot snapshot;
|
||||
Fixture(boolean compiled) throws Exception {
|
||||
ReflectionTestUtils.setField(store, "syslogNormalAlarmMapper", alarm);
|
||||
ReflectionTestUtils.setField(store, "syslogNormalDataMapper", normal);
|
||||
ReflectionTestUtils.setField(store, "statsChangeCaptureService", mock(StatsChangeCaptureService.class));
|
||||
ReflectionTestUtils.setField(store, "directStatsProjectionWriter", mock(DirectStatsProjectionWriter.class));
|
||||
when(alarm.insertDynamic(anyMap())).thenReturn(1);
|
||||
NormalizationOptimizationProperties p = new NormalizationOptimizationProperties();
|
||||
p.getCompiledRule().setEnabled(compiled); p.getCompiledRule().setShadowCompareEnabled(false);
|
||||
p.getRuleSelection().setEnabled(true); p.getRuleSelection().setShadowEnabled(false);
|
||||
CompiledRuleCompiler compiler = new CompiledRuleCompiler(JSON);
|
||||
processor = new LogNormalProcessor(mock(DmNormalizeRuleService.class), mock(DmColumnService.class), store,
|
||||
mock(SyslogNonNormalMessageService.class), new LogDataFilterService(), new LogDataCompleteService(),
|
||||
tasks, devices, mock(DmColumnMapper.class), mock(DmNormalizeRuleMapper.class),
|
||||
mock(KafkaConsumerProperties.class), mock(AppProperties.class), new CompiledRuleCache(compiler, p), p);
|
||||
String content = new String(Files.readAllBytes(file("normalization-rule.json")), "UTF-8");
|
||||
RuleSource source = new RuleSource(700L, "quanxi", "quanxi", "json", content, true, "0");
|
||||
snapshot = new BatchRuleSnapshot(Collections.singletonMap(7L, Collections.singletonList(source)), 1L);
|
||||
}
|
||||
Map<String, Object> prepare(JsonNode doc, String id) {
|
||||
LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(doc.path("timestamp").asLong()), ZoneId.of("Asia/Shanghai"));
|
||||
String message = "[receive_time=" + TIME.format(RECEIVED) + " event_time=" + TIME.format(time)
|
||||
+ " event_id=" + id + " device_id=7 device_name=fixture vendor=quanxi data_type=json device_collect_id=3]" + doc;
|
||||
LogNormalProcessor.NormalizationResult result = processor.prepare(message, id, "raw", RECEIVED,
|
||||
snapshot, new BatchEnrichmentContext(false, tasks, devices), new RuleExecutionMetrics());
|
||||
assertEquals(LogNormalProcessor.ProcessingOutcome.SUCCESS, result.getOutcome());
|
||||
assertEquals(1, result.getRows().size());
|
||||
return result.getRows().get(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user