feat(syslog): 完成高吞吐可靠链路与统计存储优化
- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
王绍宏
+58
@@ -0,0 +1,58 @@
|
||||
package com.haobang.dispatch;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.common.service.DeviceUnknownService;
|
||||
import com.haobang.config.AppProperties;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Collections;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class DeviceContextResolverTest {
|
||||
@Test
|
||||
void repeatedSourceIpQueriesDatabaseOnlyOnceWithinTtl() {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
device.setIp("192.168.4.19");
|
||||
device.setDeviceCollectId(1);
|
||||
when(service.getByIpSafely("192.168.4.19")).thenReturn(Collections.singletonList(device));
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setAppServiceDeviceCollectId(1);
|
||||
properties.setSyslogDeviceContextCacheTtlMs(60000L);
|
||||
DeviceContextResolver resolver = new DeviceContextResolver(
|
||||
service, mock(DeviceUnknownService.class), properties);
|
||||
|
||||
assertTrue(resolver.resolve("192.168.4.19", "TCP").isRegistered());
|
||||
assertTrue(resolver.resolve("192.168.4.19", "TCP").isRegistered());
|
||||
|
||||
verify(service, times(1)).getByIpSafely("192.168.4.19");
|
||||
assertEquals(1L, resolver.getMissCount());
|
||||
assertEquals(1L, resolver.getHitCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deviceAssignedToOtherCollectorIsTerminal() {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
device.setDeviceCollectId(2);
|
||||
when(service.getByIpSafely("192.168.4.19")).thenReturn(Collections.singletonList(device));
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setAppServiceDeviceCollectId(1);
|
||||
properties.setSyslogDeviceContextCacheTtlMs(60000L);
|
||||
|
||||
DeviceContextResolver.Resolution result =
|
||||
new DeviceContextResolver(service, mock(DeviceUnknownService.class), properties)
|
||||
.resolve("192.168.4.19", "TCP");
|
||||
|
||||
assertEquals(DeviceContextResolver.Type.NOT_ASSIGNED, result.getType());
|
||||
}
|
||||
}
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
package com.haobang.dispatch;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.wal.SyslogIngressEvent;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import com.kafka.kafkaProducer;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Files;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class SyslogWalDispatcherTest {
|
||||
@TempDir Path tempDirectory;
|
||||
private SyslogWalService wal;
|
||||
private SyslogWalDispatcher dispatcher;
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
if (dispatcher != null) dispatcher.shutdown();
|
||||
if (wal != null) wal.shutdown();
|
||||
}
|
||||
|
||||
@Test
|
||||
void outOfOrderKafkaAckAdvancesOnlyContiguousCheckpoint() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setSyslogPointTimeWriteEnabled(true);
|
||||
wal = new SyslogWalService(properties, new ObjectMapper());
|
||||
wal.initialize();
|
||||
SyslogIngressEvent firstEvent = wal.prepareForWal(event("20260811-event-1", "first"));
|
||||
SyslogIngressEvent secondEvent = wal.prepareForWal(event("20260811-event-2", "second"));
|
||||
wal.tryAppend(firstEvent);
|
||||
wal.tryAppend(secondEvent);
|
||||
waitUntil(() -> wal.getAppendedCount() == 2L);
|
||||
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
device.setName("device");
|
||||
device.setVendor("vendor");
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
when(resolver.resolve(anyString(), anyString()))
|
||||
.thenReturn(DeviceContextResolver.Resolution.registered(device));
|
||||
|
||||
CompletableFuture<kafkaProducer.DeliveryResult> first = new CompletableFuture<>();
|
||||
CompletableFuture<kafkaProducer.DeliveryResult> second = new CompletableFuture<>();
|
||||
kafkaProducer producer = mock(kafkaProducer.class);
|
||||
when(producer.messagePush(any(DeviceDevice.class), anyString(), anyString(), anyString(), any()))
|
||||
.thenAnswer(invocation -> "20260811-event-1".equals(invocation.getArgument(3)) ? first : second);
|
||||
|
||||
dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
|
||||
dispatcher.start();
|
||||
waitUntil(() -> dispatcher.getInflight() == 2L);
|
||||
|
||||
second.complete(null);
|
||||
Thread.sleep(100L);
|
||||
assertEquals(0L, wal.getCheckpointSequence());
|
||||
|
||||
first.complete(null);
|
||||
waitUntil(() -> wal.getCheckpointSequence() == 2L);
|
||||
assertEquals(2L, dispatcher.getKafkaAcked());
|
||||
assertEquals(0L, dispatcher.getRetryCount());
|
||||
verify(producer).messagePush(any(DeviceDevice.class), eq("first"),
|
||||
eq("20260811200000000"), eq("20260811-event-1"), eq(firstEvent.getPointTimeNs()));
|
||||
verify(producer).messagePush(any(DeviceDevice.class), eq("second"),
|
||||
eq("20260811200000000"), eq("20260811-event-2"), eq(secondEvent.getPointTimeNs()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void terminalDeviceStateIsPersistedBeforeCheckpointAdvances() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
wal = new SyslogWalService(properties, new ObjectMapper());
|
||||
wal.initialize();
|
||||
wal.tryAppend(event("20260811-event-1", "first"));
|
||||
waitUntil(() -> wal.getAppendedCount() == 1L);
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
when(resolver.resolve(anyString(), anyString())).thenReturn(DeviceContextResolver.Resolution.terminal(
|
||||
DeviceContextResolver.Type.UNKNOWN_DEVICE, null, "not registered"));
|
||||
kafkaProducer producer = mock(kafkaProducer.class);
|
||||
|
||||
dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
|
||||
dispatcher.start();
|
||||
|
||||
waitUntil(() -> wal.getCheckpointSequence() == 1L);
|
||||
assertEquals(1L, dispatcher.getTerminalFiltered());
|
||||
assertTrue(Files.list(tempDirectory.resolve("terminal")).findAny().isPresent());
|
||||
verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
private SyslogIngressEvent event(String id, String raw) {
|
||||
return new SyslogIngressEvent(id, LocalDateTime.of(2026, 8, 11, 20, 0),
|
||||
"192.168.4.19", 514, "TCP", raw);
|
||||
}
|
||||
|
||||
private AppProperties properties(Path directory) {
|
||||
AppProperties value = new AppProperties();
|
||||
value.setSyslogWalEnabled(true);
|
||||
value.setSyslogWalDirectory(directory.toString());
|
||||
value.setSyslogWalQueueCapacity(16);
|
||||
value.setSyslogWalHighWatermarkPercent(70);
|
||||
value.setSyslogWalLowWatermarkPercent(40);
|
||||
value.setSyslogWalRetryIntervalMs(5);
|
||||
value.setSyslogWalDiskRetryIntervalMs(50);
|
||||
value.setSyslogWalFlushRecords(1);
|
||||
value.setSyslogWalFlushIntervalMs(5);
|
||||
value.setSyslogWalSegmentMaxBytes(1024 * 1024);
|
||||
value.setSyslogWalRetentionDays(0);
|
||||
value.setSyslogWalShutdownTimeoutMs(5000);
|
||||
value.setSyslogWalCheckpointFlushRecords(1);
|
||||
value.setSyslogWalCheckpointFlushIntervalMs(5);
|
||||
value.setSyslogWalDiskHighWatermarkPercent(98);
|
||||
value.setSyslogWalDiskCriticalWatermarkPercent(99);
|
||||
value.setSyslogWalMinFreeBytes(0);
|
||||
value.setSyslogCharset("UTF-8");
|
||||
value.setSyslogDispatchWorkerThreads(2);
|
||||
value.setSyslogDispatchMaxInflight(8);
|
||||
value.setSyslogDispatchRetryBackoffMs(10);
|
||||
value.setSyslogDispatchShutdownTimeoutMs(2000);
|
||||
value.setSyslogDeviceContextCacheTtlMs(60000);
|
||||
value.setSyslogStatusLogIntervalMs(60000);
|
||||
return value;
|
||||
}
|
||||
|
||||
private void waitUntil(Check condition) throws Exception {
|
||||
long deadline = System.currentTimeMillis() + 5000L;
|
||||
while (!condition.done() && System.currentTimeMillis() < deadline) Thread.sleep(10L);
|
||||
assertTrue(condition.done(), "condition not reached before timeout");
|
||||
}
|
||||
|
||||
private interface Check { boolean done() throws Exception; }
|
||||
}
|
||||
+7
-1
@@ -16,7 +16,13 @@ import java.util.List;
|
||||
/**
|
||||
* 达梦数据库连接测试 — 查询 device_device 表
|
||||
*/
|
||||
@SpringBootTest
|
||||
@SpringBootTest(properties = {
|
||||
"syslog.server.enabled=false",
|
||||
"syslog.wal.directory=${java.io.tmpdir}/syslog-serve-device-test-${random.uuid}",
|
||||
"syslog.wal.disk-high-watermark-percent=98",
|
||||
"syslog.wal.disk-critical-watermark-percent=99",
|
||||
"syslog.wal.min-free-bytes=0"
|
||||
})
|
||||
@ActiveProfiles("dev")
|
||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||
class DeviceDeviceTest {
|
||||
|
||||
+7
-1
@@ -3,7 +3,13 @@ package com.haobang.syslog;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
|
||||
@SpringBootTest
|
||||
@SpringBootTest(properties = {
|
||||
"syslog.server.enabled=false",
|
||||
"syslog.wal.directory=${java.io.tmpdir}/syslog-serve-context-${random.uuid}",
|
||||
"syslog.wal.disk-high-watermark-percent=98",
|
||||
"syslog.wal.disk-critical-watermark-percent=99",
|
||||
"syslog.wal.min-free-bytes=0"
|
||||
})
|
||||
class SysjavacollectApplicationTests {
|
||||
|
||||
@Test
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class PointTimeAllocatorTest {
|
||||
@Test void millionSuffixesAreUniqueAndCycleInsideReceiveMillisecond(){PointTimeAllocator a=new PointTimeAllocator();long ms=1_786_686_912_123L;Set<Long>s=new HashSet<>(1_333_334);for(int i=0;i<1_000_000;i++){long v=a.allocate(ms);assertEquals(ms,Math.floorDiv(v,1_000_000L));assertTrue(s.add(Math.floorMod(v,1_000_000L)));}assertEquals(0,Math.floorMod(a.allocate(ms),1_000_000L));}
|
||||
@Test void concurrentCallsAreAtomic()throws Exception{PointTimeAllocator a=new PointTimeAllocator();Set<Long>s=ConcurrentHashMap.newKeySet();ExecutorService p=Executors.newFixedThreadPool(8);for(int i=0;i<100_000;i++)p.submit(()->s.add(a.allocate(1_786_686_912_123L)));p.shutdown();assertTrue(p.awaitTermination(20,TimeUnit.SECONDS));assertEquals(100_000,s.size());}
|
||||
@Test void arithmeticOverflowFailsExplicitly(){assertThrows(ArithmeticException.class,()->new PointTimeAllocator().allocate(Long.MAX_VALUE));}
|
||||
}
|
||||
+315
@@ -0,0 +1,315 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.config.CacheConfig;
|
||||
import com.haobang.config.AppProperties;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class SyslogWalServiceTest {
|
||||
@TempDir Path tempDirectory;
|
||||
private SyslogWalService service;
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
if (service != null) service.shutdown();
|
||||
}
|
||||
|
||||
@Test
|
||||
void forceBeforeCursorAndPersistSchemaV2() throws Exception {
|
||||
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
|
||||
service.initialize();
|
||||
|
||||
SyslogIngressEvent event = event("20260811-event-1", "first");
|
||||
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED, service.tryAppend(event));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord record = cursor.next(1000L);
|
||||
assertNotNull(record);
|
||||
assertEquals(1L, record.getSequence());
|
||||
assertEquals(event.getEventId(), record.getEventId());
|
||||
assertEquals(event.getRawMessage(), record.getRawMessage());
|
||||
assertTrue(record.getEndOffset() > record.getStartOffset());
|
||||
}
|
||||
|
||||
Path segment = Files.list(tempDirectory.resolve("active"))
|
||||
.filter(path -> path.getFileName().toString().startsWith("syslog-wal-"))
|
||||
.findFirst().orElseThrow(AssertionError::new);
|
||||
List<String> lines = Files.readAllLines(segment, StandardCharsets.UTF_8);
|
||||
JsonNode json = new ObjectMapper().readTree(lines.get(0));
|
||||
assertEquals(2, json.path("schema_version").asInt());
|
||||
assertEquals("20260811-event-1", json.path("event_id").asText());
|
||||
assertEquals(64, json.path("raw_sha256").asText().length());
|
||||
}
|
||||
|
||||
@Test
|
||||
void enabledPointTimeWritesSchemaV3AndRestoresSameIdentity() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setSyslogPointTimeWriteEnabled(true);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
SyslogIngressEvent prepared = service.prepareForWal(event("20260811-v3", "v3"));
|
||||
assertNotNull(prepared.getPointTimeNs());
|
||||
service.tryAppend(prepared);
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord restored = cursor.next(1000L);
|
||||
assertEquals(3, restored.getSchemaVersion());
|
||||
assertEquals(prepared.getPointTimeNs(), restored.getPointTimeNs());
|
||||
assertEquals(SyslogWalRecordValidator.receiveTimeMillis(restored.getReceiveTime()),
|
||||
Math.floorDiv(restored.getPointTimeNs(), PointTimeAllocator.NANOS_PER_MILLI));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void mixedV2V3RestartScanAndRecoveryRemainCompatible() throws Exception {
|
||||
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
|
||||
SyslogWalRecord v2 = walRecord(1L, "20260811-v2", "old");
|
||||
SyslogWalRecord v3 = walRecord(2L, "20260811-v3", "new"); v3.setSchemaVersion(3);
|
||||
v3.setPointTimeNs(Math.multiplyExact(SyslogWalRecordValidator.receiveTimeMillis(v3.getReceiveTime()), 1_000_000L));
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"),
|
||||
(mapper.writeValueAsString(v2)+"\n"+mapper.writeValueAsString(v3)+"\n").getBytes(StandardCharsets.UTF_8));
|
||||
service = new SyslogWalService(properties(tempDirectory), mapper); service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
assertEquals(2, cursor.next(1000L).getSchemaVersion());
|
||||
SyslogWalRecord restoredV3 = cursor.next(1000L); assertEquals(3, restoredV3.getSchemaVersion()); assertEquals(v3.getPointTimeNs(), restoredV3.getPointTimeNs());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void damagedV3IsQuarantinedAndNeverFallsBackAsV2() throws Exception {
|
||||
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
|
||||
String invalid = "{\"schema_version\":3,\"sequence\":1,\"event_id\":\"20260811-bad\","
|
||||
+ "\"receive_time\":\"2026-08-11T20:00:00.123\",\"point_time_ns\":-1}\n";
|
||||
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"), invalid.getBytes(StandardCharsets.UTF_8));
|
||||
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper()); service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord tombstone = cursor.next(1000L); assertTrue(tombstone.isQuarantinedInvalid()); assertEquals(3,tombstone.getSchemaVersion());
|
||||
}
|
||||
assertTrue(Files.list(tempDirectory.resolve("quarantine")).findAny().isPresent());
|
||||
assertEquals(1L, service.getQuarantinedCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatorRejectsEveryIllegalV3PointTimeShape() throws Exception {
|
||||
ObjectMapper mapper=new ObjectMapper();
|
||||
String prefix="{\"schema_version\":3,\"receive_time\":\"2026-08-11T20:00:00.123\"";
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+"}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":\"x\"}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":-1}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":1786636800124000000}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree("{\"schema_version\":4}")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void productionRedisMapperIsSanitizedBeforeWalReadAndWrite() throws Exception {
|
||||
/*
|
||||
* 生产环境注入的是 CacheConfig 的 ObjectMapper，而不是测试常用的 new ObjectMapper()。
|
||||
* 这个回归测试必须使用真实配置，否则无法复现现场“写入 @class、读取又被白名单拒绝”的故障。
|
||||
*/
|
||||
ObjectMapper productionMapper = new CacheConfig().objectMapper();
|
||||
service = new SyslogWalService(properties(tempDirectory), productionMapper);
|
||||
service.initialize();
|
||||
service.tryAppend(event("20260811-production-mapper", "new-format"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
|
||||
Path segment = Files.list(tempDirectory.resolve("active"))
|
||||
.findFirst().orElseThrow(AssertionError::new);
|
||||
JsonNode persisted = new ObjectMapper().readTree(Files.readAllLines(
|
||||
segment, StandardCharsets.UTF_8).get(0));
|
||||
assertFalse(persisted.has("@class"), "WAL must never persist Java class metadata");
|
||||
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord restored = cursor.next(1000L);
|
||||
assertNotNull(restored);
|
||||
assertEquals("20260811-production-mapper", restored.getEventId());
|
||||
assertEquals("new-format", restored.getRawMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void repairedReaderAcceptsExistingSchemaV2RecordWithLegacyClassMetadata() throws Exception {
|
||||
Path active = tempDirectory.resolve("active");
|
||||
Files.createDirectories(active);
|
||||
Path segment = active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl");
|
||||
|
||||
// 使用发生现场故障的生产 Mapper 生成旧格式，确保样本确实包含 @class。
|
||||
ObjectMapper productionMapper = new CacheConfig().objectMapper();
|
||||
SyslogWalRecord legacy = walRecord(1L, "20260811-legacy-class", "existing-wal");
|
||||
String legacyJson = productionMapper.writeValueAsString(legacy);
|
||||
assertTrue(legacyJson.contains("\"@class\":\"com.haobang.wal.SyslogWalRecord\""));
|
||||
Files.write(segment, (legacyJson + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
service = new SyslogWalService(properties(tempDirectory), productionMapper);
|
||||
service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord restored = cursor.next(1000L);
|
||||
assertNotNull(restored);
|
||||
assertEquals(1L, restored.getSequence());
|
||||
assertEquals("20260811-legacy-class", restored.getEventId());
|
||||
assertEquals("existing-wal", restored.getRawMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void restartResumesAfterPersistedCheckpointWithoutChangingEventIdentity() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
service.tryAppend(event("20260811-event-1", "first"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord first = cursor.next(1000L);
|
||||
service.markCheckpoint(first);
|
||||
waitUntil(() -> service.getCheckpointSequence() == 1L);
|
||||
}
|
||||
service.shutdown();
|
||||
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
assertEquals(1L, service.getCheckpointSequence());
|
||||
service.tryAppend(event("20260811-event-2", "second"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord second = cursor.next(1000L);
|
||||
assertNotNull(second);
|
||||
assertEquals(2L, second.getSequence());
|
||||
assertEquals("20260811-event-2", second.getEventId());
|
||||
assertEquals("2026-08-11T20:00:00.123", second.getReceiveTime());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void restartTruncatesIncompleteTailAndKeepsCompleteRecord() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
service.tryAppend(event("20260811-event-1", "first"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
service.shutdown();
|
||||
|
||||
Path segment = Files.list(tempDirectory.resolve("active")).findFirst().orElseThrow(AssertionError::new);
|
||||
Files.write(segment, "{\"schema_version\":2".getBytes(StandardCharsets.UTF_8),
|
||||
java.nio.file.StandardOpenOption.APPEND);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord record = cursor.next(1000L);
|
||||
assertNotNull(record);
|
||||
assertEquals("20260811-event-1", record.getEventId());
|
||||
}
|
||||
byte[] bytes = Files.readAllBytes(segment);
|
||||
assertEquals((byte) '\n', bytes[bytes.length - 1]);
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacyImporterImportsOnlyRequestedEventIdAndPreservesIdentity() throws Exception {
|
||||
Path archive = tempDirectory.resolve("legacy");
|
||||
Files.createDirectories(archive);
|
||||
String requestedId = "20260811-a3d874a0-6f50-4f0c-9e7a-9c20303360f5";
|
||||
String ignoredId = "20260811-53df7faa-7f78-4ce1-8eac-475965d9eeed";
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
Files.write(archive.resolve("syslog-raw-20260811-test.jsonl"), (
|
||||
legacyLine(mapper, requestedId, "requested") + "\n"
|
||||
+ legacyLine(mapper, ignoredId, "ignored") + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
Path ids = tempDirectory.resolve("rejected.txt");
|
||||
Files.write(ids, ("error eventId=" + requestedId).getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
AppProperties properties = properties(tempDirectory.resolve("wal"));
|
||||
properties.setSyslogWalLegacyImportEnabled(true);
|
||||
properties.setSyslogWalLegacyImportArchiveDirectory(archive.toString());
|
||||
properties.setSyslogWalLegacyImportEventIdFile(ids.toString());
|
||||
service = new SyslogWalService(properties, mapper);
|
||||
service.initialize();
|
||||
new SyslogLegacyArchiveImporter(properties, mapper, service).importIfRequested();
|
||||
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord imported = cursor.next(1000L);
|
||||
assertNotNull(imported);
|
||||
assertEquals(requestedId, imported.getEventId());
|
||||
assertEquals("2026-08-11T03:55:57.109", imported.getReceiveTime());
|
||||
assertEquals("requested", imported.getRawMessage());
|
||||
assertEquals(null, cursor.next(50L));
|
||||
}
|
||||
}
|
||||
|
||||
private String legacyLine(ObjectMapper mapper, String eventId, String raw) throws Exception {
|
||||
java.util.Map<String, Object> value = new java.util.LinkedHashMap<>();
|
||||
value.put("schema_version", 1);
|
||||
value.put("event_id", eventId);
|
||||
value.put("receive_time", "2026-08-11T03:55:57.109");
|
||||
value.put("source_ip", "192.168.4.19");
|
||||
value.put("source_port", 49768);
|
||||
value.put("protocol", "TCP");
|
||||
value.put("raw_message", raw);
|
||||
return mapper.writeValueAsString(value);
|
||||
}
|
||||
|
||||
private SyslogIngressEvent event(String eventId, String raw) {
|
||||
return new SyslogIngressEvent(eventId,
|
||||
LocalDateTime.of(2026, 8, 11, 20, 0, 0, 123_000_000),
|
||||
"192.168.4.19", 514, "TCP", raw);
|
||||
}
|
||||
|
||||
private SyslogWalRecord walRecord(long sequence, String eventId, String raw) {
|
||||
SyslogWalRecord record = new SyslogWalRecord();
|
||||
record.setSequence(sequence);
|
||||
record.setEventId(eventId);
|
||||
record.setReceiveTime("2026-08-11T23:32:32.531");
|
||||
record.setSourceIp("192.168.4.19");
|
||||
record.setSourcePort(45324);
|
||||
record.setProtocol("TCP");
|
||||
record.setCharset("UTF-8");
|
||||
record.setRawLength(raw.getBytes(StandardCharsets.UTF_8).length);
|
||||
record.setRawSha256("684b334d5690440f210e5778d304f778b5eaff5b30477667029968a711604400");
|
||||
record.setRawMessage(raw);
|
||||
return record;
|
||||
}
|
||||
|
||||
private AppProperties properties(Path directory) {
|
||||
AppProperties value = new AppProperties();
|
||||
value.setSyslogWalEnabled(true);
|
||||
value.setSyslogWalDirectory(directory.toString());
|
||||
value.setSyslogWalQueueCapacity(16);
|
||||
value.setSyslogWalHighWatermarkPercent(70);
|
||||
value.setSyslogWalLowWatermarkPercent(40);
|
||||
value.setSyslogWalRetryIntervalMs(5);
|
||||
value.setSyslogWalDiskRetryIntervalMs(50);
|
||||
value.setSyslogWalFlushRecords(1);
|
||||
value.setSyslogWalFlushIntervalMs(5);
|
||||
value.setSyslogWalSegmentMaxBytes(1024 * 1024);
|
||||
value.setSyslogWalRetentionDays(0);
|
||||
value.setSyslogWalShutdownTimeoutMs(5000);
|
||||
value.setSyslogWalCheckpointFlushRecords(1);
|
||||
value.setSyslogWalCheckpointFlushIntervalMs(5);
|
||||
value.setSyslogWalDiskHighWatermarkPercent(98);
|
||||
value.setSyslogWalDiskCriticalWatermarkPercent(99);
|
||||
value.setSyslogWalMinFreeBytes(0);
|
||||
value.setSyslogCharset("UTF-8");
|
||||
return value;
|
||||
}
|
||||
|
||||
private void waitUntil(Check condition) throws Exception {
|
||||
long deadline = System.currentTimeMillis() + 5000L;
|
||||
while (!condition.done() && System.currentTimeMillis() < deadline) Thread.sleep(10L);
|
||||
assertTrue(condition.done(), "condition not reached before timeout");
|
||||
}
|
||||
|
||||
private interface Check { boolean done() throws Exception; }
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
package com.kafka;
|
||||
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.haobang.util.DeviceInfoUtil;
|
||||
import org.apache.kafka.clients.producer.Producer;
|
||||
import org.apache.kafka.clients.producer.ProducerRecord;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
class KafkaProducerEventIdTest {
|
||||
|
||||
@Test
|
||||
void eventIdDateUsesShanghaiTimeInsteadOfJvmDefaultTimeZone() {
|
||||
Instant shanghaiMidnight = Instant.parse("2026-08-10T16:00:00Z");
|
||||
|
||||
String eventId = kafkaProducer.newEventId(shanghaiMidnight);
|
||||
|
||||
assertTrue(eventId.matches("20260811-[0-9a-f-]{36}"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void eventIdStrategyUsesTheSameEventIdAsProducerRecordKey() {
|
||||
kafkaProducer producer = producer("event_id");
|
||||
String eventId = "20260811-a3d874a0-6f50-4f0c-9e7a-9c20303360f5";
|
||||
|
||||
ProducerRecord<String, String> record = producer.buildRecord(126, eventId, "ciphertext");
|
||||
|
||||
assertEquals("agent-01-syslog-topic", record.topic());
|
||||
assertEquals(eventId, record.key());
|
||||
assertEquals("ciphertext", record.value());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deviceIdStrategyRemainsAvailableForRollback() {
|
||||
ProducerRecord<String, String> record =
|
||||
producer("device_id").buildRecord(126, "20260811-event", "ciphertext");
|
||||
|
||||
assertEquals("126", record.key());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidPartitionKeyStrategyFailsFast() {
|
||||
assertThrows(IllegalArgumentException.class, () -> producer("random"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void kafkaBusinessHeaderAddsOnlyPointTimeAndKeepsEventIdName() {
|
||||
DeviceDevice device = new DeviceDevice(); device.setId(126); device.setName("device"); device.setVendor("vendor");
|
||||
AppProperties properties = new AppProperties(); properties.setAppServiceDataType("json"); properties.setAppServiceDeviceCollectId(1);
|
||||
String payload = DeviceInfoUtil.getFullLogString(device, "raw", "20260814000000000",
|
||||
"20260814-event", 1_786_636_800_000_000_123L, properties);
|
||||
assertTrue(payload.contains("event_id=20260814-event"));
|
||||
assertTrue(payload.contains("point_time_ns=1786636800000000123"));
|
||||
assertTrue(payload.contains("receive_time=20260814000000000"));
|
||||
assertFalse(payload.contains("schema_version"));
|
||||
assertFalse(payload.contains("event_id_ns"));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private kafkaProducer producer(String strategy) {
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setKafkaProducerTopic("agent-01-syslog-topic");
|
||||
return new kafkaProducer(properties, mock(Producer.class), strategy);
|
||||
}
|
||||
}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.wal.SyslogIngressEvent;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.embedded.EmbeddedChannel;
|
||||
import io.netty.channel.socket.DatagramPacket;
|
||||
import io.netty.handler.codec.DelimiterBasedFrameDecoder;
|
||||
import io.netty.handler.codec.Delimiters;
|
||||
import io.netty.handler.codec.string.StringDecoder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class SyslogCharsetDecodingTest {
|
||||
private static final String CHINESE_MESSAGE =
|
||||
"<134>发现TCP传输层一句话小马行为 alarm_type=文件上传";
|
||||
|
||||
@Test
|
||||
void tcpUsesConfiguredUtf8CharsetBeforeAppendingToWal() {
|
||||
SyslogWalService wal = acceptingWal();
|
||||
EmbeddedChannel channel = new EmbeddedChannel(
|
||||
new DelimiterBasedFrameDecoder(4096, Delimiters.lineDelimiter()),
|
||||
new StringDecoder(StandardCharsets.UTF_8),
|
||||
new SyslogMessageHandler(StandardCharsets.UTF_8, wal));
|
||||
channel.connect(new InetSocketAddress("127.0.0.1", 5514));
|
||||
|
||||
channel.writeInbound(Unpooled.wrappedBuffer(
|
||||
(CHINESE_MESSAGE + "\n").getBytes(StandardCharsets.UTF_8)));
|
||||
|
||||
assertWalReceivedOriginalChinese(wal);
|
||||
channel.finishAndReleaseAll();
|
||||
}
|
||||
|
||||
@Test
|
||||
void udpUsesConfiguredUtf8CharsetBeforeAppendingToWal() {
|
||||
SyslogWalService wal = acceptingWal();
|
||||
EmbeddedChannel channel = new EmbeddedChannel(
|
||||
new SyslogMessageHandler(StandardCharsets.UTF_8, wal));
|
||||
DatagramPacket packet = new DatagramPacket(
|
||||
Unpooled.wrappedBuffer(CHINESE_MESSAGE.getBytes(StandardCharsets.UTF_8)),
|
||||
new InetSocketAddress("127.0.0.1", 514),
|
||||
new InetSocketAddress("192.168.4.19", 5514));
|
||||
|
||||
channel.writeInbound(packet);
|
||||
|
||||
assertWalReceivedOriginalChinese(wal);
|
||||
channel.finishAndReleaseAll();
|
||||
}
|
||||
|
||||
private SyslogWalService acceptingWal() {
|
||||
SyslogWalService wal = mock(SyslogWalService.class);
|
||||
when(wal.prepareForWal(any())).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
when(wal.tryAppend(any())).thenReturn(SyslogWalService.AdmissionResult.ACCEPTED);
|
||||
return wal;
|
||||
}
|
||||
|
||||
private void assertWalReceivedOriginalChinese(SyslogWalService wal) {
|
||||
ArgumentCaptor<SyslogIngressEvent> event = ArgumentCaptor.forClass(SyslogIngressEvent.class);
|
||||
verify(wal).tryAppend(event.capture());
|
||||
assertEquals(CHINESE_MESSAGE, event.getValue().getRawMessage());
|
||||
}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import io.netty.channel.embedded.EmbeddedChannel;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class SyslogMessageHandlerBackpressureTest {
|
||||
@Test
|
||||
void tcpWalBackpressureDisablesAutoReadInsteadOfDroppingCurrentFrame() {
|
||||
SyslogWalService wal = mock(SyslogWalService.class);
|
||||
when(wal.prepareForWal(any())).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
when(wal.tryAppend(any())).thenReturn(SyslogWalService.AdmissionResult.BACKPRESSURE);
|
||||
when(wal.getRetryIntervalMs()).thenReturn(60_000L);
|
||||
when(wal.getQueueCapacity()).thenReturn(10_000);
|
||||
EmbeddedChannel channel = new EmbeddedChannel(new SyslogMessageHandler(StandardCharsets.UTF_8, wal));
|
||||
channel.connect(new InetSocketAddress("127.0.0.1", 5514));
|
||||
|
||||
channel.writeInbound("<134>Aug 11 20:00:00 device event");
|
||||
|
||||
assertFalse(channel.config().isAutoRead());
|
||||
channel.close();
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.config.AppProperties;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class SyslogProfileCharsetConfigurationTest {
|
||||
@Test
|
||||
void prodProfileResolvesSyslogCharsetThroughSpringEnvironment() {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(TestConfiguration.class)
|
||||
.web(WebApplicationType.NONE)
|
||||
.profiles("prod")
|
||||
.properties("spring.main.banner-mode=off", "logging.level.root=OFF")
|
||||
.run()) {
|
||||
assertEquals("UTF-8", context.getBean(AppProperties.class).getSyslogCharset());
|
||||
}
|
||||
}
|
||||
|
||||
@Configuration
|
||||
@Import(AppProperties.class)
|
||||
static class TestConfiguration {
|
||||
}
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
class SyslogServerLifecycleCharsetTest {
|
||||
@Test
|
||||
void invalidCharsetStopsWalConstructionBeforeOpeningSyslogPorts() {
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setSyslogCharset("NOT-A-REAL-CHARSET");
|
||||
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> new SyslogWalService(properties, new ObjectMapper()));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user