feat(syslog): 完成高吞吐可靠链路与统计存储优化

- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
2026-08-20 00:18:33 +08:00
parent a3852fc0f9
commit 29445c6825
303 changed files with 20419 additions and 4481 deletions
@@ -0,0 +1,145 @@
+package com.haobang.dispatch;
+
+import com.common.entity.DeviceDevice;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.haobang.config.AppProperties;
+import com.haobang.wal.SyslogIngressEvent;
+import com.haobang.wal.SyslogWalService;
+import com.kafka.kafkaProducer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Path;
+import java.nio.file.Files;
+import java.time.LocalDateTime;
+import java.util.concurrent.CompletableFuture;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class SyslogWalDispatcherTest {
+    @TempDir Path tempDirectory;
+    private SyslogWalService wal;
+    private SyslogWalDispatcher dispatcher;
+
+    @AfterEach
+    void tearDown() {
+        if (dispatcher != null) dispatcher.shutdown();
+        if (wal != null) wal.shutdown();
+    }
+
+    @Test
+    void outOfOrderKafkaAckAdvancesOnlyContiguousCheckpoint() throws Exception {
+        AppProperties properties = properties(tempDirectory);
+        properties.setSyslogPointTimeWriteEnabled(true);
+        wal = new SyslogWalService(properties, new ObjectMapper());
+        wal.initialize();
+        SyslogIngressEvent firstEvent = wal.prepareForWal(event("20260811-event-1", "first"));
+        SyslogIngressEvent secondEvent = wal.prepareForWal(event("20260811-event-2", "second"));
+        wal.tryAppend(firstEvent);
+        wal.tryAppend(secondEvent);
+        waitUntil(() -> wal.getAppendedCount() == 2L);
+
+        DeviceDevice device = new DeviceDevice();
+        device.setId(126);
+        device.setName("device");
+        device.setVendor("vendor");
+        DeviceContextResolver resolver = mock(DeviceContextResolver.class);
+        when(resolver.resolve(anyString(), anyString()))
+                .thenReturn(DeviceContextResolver.Resolution.registered(device));
+
+        CompletableFuture<kafkaProducer.DeliveryResult> first = new CompletableFuture<>();
+        CompletableFuture<kafkaProducer.DeliveryResult> second = new CompletableFuture<>();
+        kafkaProducer producer = mock(kafkaProducer.class);
+        when(producer.messagePush(any(DeviceDevice.class), anyString(), anyString(), anyString(), any()))
+                .thenAnswer(invocation -> "20260811-event-1".equals(invocation.getArgument(3)) ? first : second);
+
+        dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
+        dispatcher.start();
+        waitUntil(() -> dispatcher.getInflight() == 2L);
+
+        second.complete(null);
+        Thread.sleep(100L);
+        assertEquals(0L, wal.getCheckpointSequence());
+
+        first.complete(null);
+        waitUntil(() -> wal.getCheckpointSequence() == 2L);
+        assertEquals(2L, dispatcher.getKafkaAcked());
+        assertEquals(0L, dispatcher.getRetryCount());
+        verify(producer).messagePush(any(DeviceDevice.class), eq("first"),
+                eq("20260811200000000"), eq("20260811-event-1"), eq(firstEvent.getPointTimeNs()));
+        verify(producer).messagePush(any(DeviceDevice.class), eq("second"),
+                eq("20260811200000000"), eq("20260811-event-2"), eq(secondEvent.getPointTimeNs()));
+    }
+
+    @Test
+    void terminalDeviceStateIsPersistedBeforeCheckpointAdvances() throws Exception {
+        AppProperties properties = properties(tempDirectory);
+        wal = new SyslogWalService(properties, new ObjectMapper());
+        wal.initialize();
+        wal.tryAppend(event("20260811-event-1", "first"));
+        waitUntil(() -> wal.getAppendedCount() == 1L);
+        DeviceContextResolver resolver = mock(DeviceContextResolver.class);
+        when(resolver.resolve(anyString(), anyString())).thenReturn(DeviceContextResolver.Resolution.terminal(
+                DeviceContextResolver.Type.UNKNOWN_DEVICE, null, "not registered"));
+        kafkaProducer producer = mock(kafkaProducer.class);
+
+        dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
+        dispatcher.start();
+
+        waitUntil(() -> wal.getCheckpointSequence() == 1L);
+        assertEquals(1L, dispatcher.getTerminalFiltered());
+        assertTrue(Files.list(tempDirectory.resolve("terminal")).findAny().isPresent());
+        verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
+    }
+
+    private SyslogIngressEvent event(String id, String raw) {
+        return new SyslogIngressEvent(id, LocalDateTime.of(2026, 8, 11, 20, 0),
+                "192.168.4.19", 514, "TCP", raw);
+    }
+
+    private AppProperties properties(Path directory) {
+        AppProperties value = new AppProperties();
+        value.setSyslogWalEnabled(true);
+        value.setSyslogWalDirectory(directory.toString());
+        value.setSyslogWalQueueCapacity(16);
+        value.setSyslogWalHighWatermarkPercent(70);
+        value.setSyslogWalLowWatermarkPercent(40);
+        value.setSyslogWalRetryIntervalMs(5);
+        value.setSyslogWalDiskRetryIntervalMs(50);
+        value.setSyslogWalFlushRecords(1);
+        value.setSyslogWalFlushIntervalMs(5);
+        value.setSyslogWalSegmentMaxBytes(1024 * 1024);
+        value.setSyslogWalRetentionDays(0);
+        value.setSyslogWalShutdownTimeoutMs(5000);
+        value.setSyslogWalCheckpointFlushRecords(1);
+        value.setSyslogWalCheckpointFlushIntervalMs(5);
+        value.setSyslogWalDiskHighWatermarkPercent(98);
+        value.setSyslogWalDiskCriticalWatermarkPercent(99);
+        value.setSyslogWalMinFreeBytes(0);
+        value.setSyslogCharset("UTF-8");
+        value.setSyslogDispatchWorkerThreads(2);
+        value.setSyslogDispatchMaxInflight(8);
+        value.setSyslogDispatchRetryBackoffMs(10);
+        value.setSyslogDispatchShutdownTimeoutMs(2000);
+        value.setSyslogDeviceContextCacheTtlMs(60000);
+        value.setSyslogStatusLogIntervalMs(60000);
+        return value;
+    }
+
+    private void waitUntil(Check condition) throws Exception {
+        long deadline = System.currentTimeMillis() + 5000L;
+        while (!condition.done() && System.currentTimeMillis() < deadline) Thread.sleep(10L);
+        assertTrue(condition.done(), "condition not reached before timeout");
+    }
+
+    private interface Check { boolean done() throws Exception; }
+}