feat(syslog): 完成高吞吐可靠链路与统计存储优化
- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
王绍宏
+16
@@ -0,0 +1,16 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class PointTimeAllocatorTest {
|
||||
@Test void millionSuffixesAreUniqueAndCycleInsideReceiveMillisecond(){PointTimeAllocator a=new PointTimeAllocator();long ms=1_786_686_912_123L;Set<Long>s=new HashSet<>(1_333_334);for(int i=0;i<1_000_000;i++){long v=a.allocate(ms);assertEquals(ms,Math.floorDiv(v,1_000_000L));assertTrue(s.add(Math.floorMod(v,1_000_000L)));}assertEquals(0,Math.floorMod(a.allocate(ms),1_000_000L));}
|
||||
@Test void concurrentCallsAreAtomic()throws Exception{PointTimeAllocator a=new PointTimeAllocator();Set<Long>s=ConcurrentHashMap.newKeySet();ExecutorService p=Executors.newFixedThreadPool(8);for(int i=0;i<100_000;i++)p.submit(()->s.add(a.allocate(1_786_686_912_123L)));p.shutdown();assertTrue(p.awaitTermination(20,TimeUnit.SECONDS));assertEquals(100_000,s.size());}
|
||||
@Test void arithmeticOverflowFailsExplicitly(){assertThrows(ArithmeticException.class,()->new PointTimeAllocator().allocate(Long.MAX_VALUE));}
|
||||
}
|
||||
+315
@@ -0,0 +1,315 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.config.CacheConfig;
|
||||
import com.haobang.config.AppProperties;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class SyslogWalServiceTest {
|
||||
@TempDir Path tempDirectory;
|
||||
private SyslogWalService service;
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
if (service != null) service.shutdown();
|
||||
}
|
||||
|
||||
@Test
|
||||
void forceBeforeCursorAndPersistSchemaV2() throws Exception {
|
||||
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
|
||||
service.initialize();
|
||||
|
||||
SyslogIngressEvent event = event("20260811-event-1", "first");
|
||||
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED, service.tryAppend(event));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord record = cursor.next(1000L);
|
||||
assertNotNull(record);
|
||||
assertEquals(1L, record.getSequence());
|
||||
assertEquals(event.getEventId(), record.getEventId());
|
||||
assertEquals(event.getRawMessage(), record.getRawMessage());
|
||||
assertTrue(record.getEndOffset() > record.getStartOffset());
|
||||
}
|
||||
|
||||
Path segment = Files.list(tempDirectory.resolve("active"))
|
||||
.filter(path -> path.getFileName().toString().startsWith("syslog-wal-"))
|
||||
.findFirst().orElseThrow(AssertionError::new);
|
||||
List<String> lines = Files.readAllLines(segment, StandardCharsets.UTF_8);
|
||||
JsonNode json = new ObjectMapper().readTree(lines.get(0));
|
||||
assertEquals(2, json.path("schema_version").asInt());
|
||||
assertEquals("20260811-event-1", json.path("event_id").asText());
|
||||
assertEquals(64, json.path("raw_sha256").asText().length());
|
||||
}
|
||||
|
||||
@Test
|
||||
void enabledPointTimeWritesSchemaV3AndRestoresSameIdentity() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setSyslogPointTimeWriteEnabled(true);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
SyslogIngressEvent prepared = service.prepareForWal(event("20260811-v3", "v3"));
|
||||
assertNotNull(prepared.getPointTimeNs());
|
||||
service.tryAppend(prepared);
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord restored = cursor.next(1000L);
|
||||
assertEquals(3, restored.getSchemaVersion());
|
||||
assertEquals(prepared.getPointTimeNs(), restored.getPointTimeNs());
|
||||
assertEquals(SyslogWalRecordValidator.receiveTimeMillis(restored.getReceiveTime()),
|
||||
Math.floorDiv(restored.getPointTimeNs(), PointTimeAllocator.NANOS_PER_MILLI));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void mixedV2V3RestartScanAndRecoveryRemainCompatible() throws Exception {
|
||||
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
|
||||
SyslogWalRecord v2 = walRecord(1L, "20260811-v2", "old");
|
||||
SyslogWalRecord v3 = walRecord(2L, "20260811-v3", "new"); v3.setSchemaVersion(3);
|
||||
v3.setPointTimeNs(Math.multiplyExact(SyslogWalRecordValidator.receiveTimeMillis(v3.getReceiveTime()), 1_000_000L));
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"),
|
||||
(mapper.writeValueAsString(v2)+"\n"+mapper.writeValueAsString(v3)+"\n").getBytes(StandardCharsets.UTF_8));
|
||||
service = new SyslogWalService(properties(tempDirectory), mapper); service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
assertEquals(2, cursor.next(1000L).getSchemaVersion());
|
||||
SyslogWalRecord restoredV3 = cursor.next(1000L); assertEquals(3, restoredV3.getSchemaVersion()); assertEquals(v3.getPointTimeNs(), restoredV3.getPointTimeNs());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void damagedV3IsQuarantinedAndNeverFallsBackAsV2() throws Exception {
|
||||
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
|
||||
String invalid = "{\"schema_version\":3,\"sequence\":1,\"event_id\":\"20260811-bad\","
|
||||
+ "\"receive_time\":\"2026-08-11T20:00:00.123\",\"point_time_ns\":-1}\n";
|
||||
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"), invalid.getBytes(StandardCharsets.UTF_8));
|
||||
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper()); service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord tombstone = cursor.next(1000L); assertTrue(tombstone.isQuarantinedInvalid()); assertEquals(3,tombstone.getSchemaVersion());
|
||||
}
|
||||
assertTrue(Files.list(tempDirectory.resolve("quarantine")).findAny().isPresent());
|
||||
assertEquals(1L, service.getQuarantinedCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatorRejectsEveryIllegalV3PointTimeShape() throws Exception {
|
||||
ObjectMapper mapper=new ObjectMapper();
|
||||
String prefix="{\"schema_version\":3,\"receive_time\":\"2026-08-11T20:00:00.123\"";
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+"}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":\"x\"}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":-1}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":1786636800124000000}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree("{\"schema_version\":4}")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void productionRedisMapperIsSanitizedBeforeWalReadAndWrite() throws Exception {
|
||||
/*
|
||||
* 生产环境注入的是 CacheConfig 的 ObjectMapper，而不是测试常用的 new ObjectMapper()。
|
||||
* 这个回归测试必须使用真实配置，否则无法复现现场“写入 @class、读取又被白名单拒绝”的故障。
|
||||
*/
|
||||
ObjectMapper productionMapper = new CacheConfig().objectMapper();
|
||||
service = new SyslogWalService(properties(tempDirectory), productionMapper);
|
||||
service.initialize();
|
||||
service.tryAppend(event("20260811-production-mapper", "new-format"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
|
||||
Path segment = Files.list(tempDirectory.resolve("active"))
|
||||
.findFirst().orElseThrow(AssertionError::new);
|
||||
JsonNode persisted = new ObjectMapper().readTree(Files.readAllLines(
|
||||
segment, StandardCharsets.UTF_8).get(0));
|
||||
assertFalse(persisted.has("@class"), "WAL must never persist Java class metadata");
|
||||
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord restored = cursor.next(1000L);
|
||||
assertNotNull(restored);
|
||||
assertEquals("20260811-production-mapper", restored.getEventId());
|
||||
assertEquals("new-format", restored.getRawMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void repairedReaderAcceptsExistingSchemaV2RecordWithLegacyClassMetadata() throws Exception {
|
||||
Path active = tempDirectory.resolve("active");
|
||||
Files.createDirectories(active);
|
||||
Path segment = active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl");
|
||||
|
||||
// 使用发生现场故障的生产 Mapper 生成旧格式，确保样本确实包含 @class。
|
||||
ObjectMapper productionMapper = new CacheConfig().objectMapper();
|
||||
SyslogWalRecord legacy = walRecord(1L, "20260811-legacy-class", "existing-wal");
|
||||
String legacyJson = productionMapper.writeValueAsString(legacy);
|
||||
assertTrue(legacyJson.contains("\"@class\":\"com.haobang.wal.SyslogWalRecord\""));
|
||||
Files.write(segment, (legacyJson + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
service = new SyslogWalService(properties(tempDirectory), productionMapper);
|
||||
service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord restored = cursor.next(1000L);
|
||||
assertNotNull(restored);
|
||||
assertEquals(1L, restored.getSequence());
|
||||
assertEquals("20260811-legacy-class", restored.getEventId());
|
||||
assertEquals("existing-wal", restored.getRawMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void restartResumesAfterPersistedCheckpointWithoutChangingEventIdentity() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
service.tryAppend(event("20260811-event-1", "first"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord first = cursor.next(1000L);
|
||||
service.markCheckpoint(first);
|
||||
waitUntil(() -> service.getCheckpointSequence() == 1L);
|
||||
}
|
||||
service.shutdown();
|
||||
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
assertEquals(1L, service.getCheckpointSequence());
|
||||
service.tryAppend(event("20260811-event-2", "second"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord second = cursor.next(1000L);
|
||||
assertNotNull(second);
|
||||
assertEquals(2L, second.getSequence());
|
||||
assertEquals("20260811-event-2", second.getEventId());
|
||||
assertEquals("2026-08-11T20:00:00.123", second.getReceiveTime());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void restartTruncatesIncompleteTailAndKeepsCompleteRecord() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
service.tryAppend(event("20260811-event-1", "first"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
service.shutdown();
|
||||
|
||||
Path segment = Files.list(tempDirectory.resolve("active")).findFirst().orElseThrow(AssertionError::new);
|
||||
Files.write(segment, "{\"schema_version\":2".getBytes(StandardCharsets.UTF_8),
|
||||
java.nio.file.StandardOpenOption.APPEND);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord record = cursor.next(1000L);
|
||||
assertNotNull(record);
|
||||
assertEquals("20260811-event-1", record.getEventId());
|
||||
}
|
||||
byte[] bytes = Files.readAllBytes(segment);
|
||||
assertEquals((byte) '\n', bytes[bytes.length - 1]);
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacyImporterImportsOnlyRequestedEventIdAndPreservesIdentity() throws Exception {
|
||||
Path archive = tempDirectory.resolve("legacy");
|
||||
Files.createDirectories(archive);
|
||||
String requestedId = "20260811-a3d874a0-6f50-4f0c-9e7a-9c20303360f5";
|
||||
String ignoredId = "20260811-53df7faa-7f78-4ce1-8eac-475965d9eeed";
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
Files.write(archive.resolve("syslog-raw-20260811-test.jsonl"), (
|
||||
legacyLine(mapper, requestedId, "requested") + "\n"
|
||||
+ legacyLine(mapper, ignoredId, "ignored") + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
Path ids = tempDirectory.resolve("rejected.txt");
|
||||
Files.write(ids, ("error eventId=" + requestedId).getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
AppProperties properties = properties(tempDirectory.resolve("wal"));
|
||||
properties.setSyslogWalLegacyImportEnabled(true);
|
||||
properties.setSyslogWalLegacyImportArchiveDirectory(archive.toString());
|
||||
properties.setSyslogWalLegacyImportEventIdFile(ids.toString());
|
||||
service = new SyslogWalService(properties, mapper);
|
||||
service.initialize();
|
||||
new SyslogLegacyArchiveImporter(properties, mapper, service).importIfRequested();
|
||||
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord imported = cursor.next(1000L);
|
||||
assertNotNull(imported);
|
||||
assertEquals(requestedId, imported.getEventId());
|
||||
assertEquals("2026-08-11T03:55:57.109", imported.getReceiveTime());
|
||||
assertEquals("requested", imported.getRawMessage());
|
||||
assertEquals(null, cursor.next(50L));
|
||||
}
|
||||
}
|
||||
|
||||
private String legacyLine(ObjectMapper mapper, String eventId, String raw) throws Exception {
|
||||
java.util.Map<String, Object> value = new java.util.LinkedHashMap<>();
|
||||
value.put("schema_version", 1);
|
||||
value.put("event_id", eventId);
|
||||
value.put("receive_time", "2026-08-11T03:55:57.109");
|
||||
value.put("source_ip", "192.168.4.19");
|
||||
value.put("source_port", 49768);
|
||||
value.put("protocol", "TCP");
|
||||
value.put("raw_message", raw);
|
||||
return mapper.writeValueAsString(value);
|
||||
}
|
||||
|
||||
private SyslogIngressEvent event(String eventId, String raw) {
|
||||
return new SyslogIngressEvent(eventId,
|
||||
LocalDateTime.of(2026, 8, 11, 20, 0, 0, 123_000_000),
|
||||
"192.168.4.19", 514, "TCP", raw);
|
||||
}
|
||||
|
||||
private SyslogWalRecord walRecord(long sequence, String eventId, String raw) {
|
||||
SyslogWalRecord record = new SyslogWalRecord();
|
||||
record.setSequence(sequence);
|
||||
record.setEventId(eventId);
|
||||
record.setReceiveTime("2026-08-11T23:32:32.531");
|
||||
record.setSourceIp("192.168.4.19");
|
||||
record.setSourcePort(45324);
|
||||
record.setProtocol("TCP");
|
||||
record.setCharset("UTF-8");
|
||||
record.setRawLength(raw.getBytes(StandardCharsets.UTF_8).length);
|
||||
record.setRawSha256("684b334d5690440f210e5778d304f778b5eaff5b30477667029968a711604400");
|
||||
record.setRawMessage(raw);
|
||||
return record;
|
||||
}
|
||||
|
||||
private AppProperties properties(Path directory) {
|
||||
AppProperties value = new AppProperties();
|
||||
value.setSyslogWalEnabled(true);
|
||||
value.setSyslogWalDirectory(directory.toString());
|
||||
value.setSyslogWalQueueCapacity(16);
|
||||
value.setSyslogWalHighWatermarkPercent(70);
|
||||
value.setSyslogWalLowWatermarkPercent(40);
|
||||
value.setSyslogWalRetryIntervalMs(5);
|
||||
value.setSyslogWalDiskRetryIntervalMs(50);
|
||||
value.setSyslogWalFlushRecords(1);
|
||||
value.setSyslogWalFlushIntervalMs(5);
|
||||
value.setSyslogWalSegmentMaxBytes(1024 * 1024);
|
||||
value.setSyslogWalRetentionDays(0);
|
||||
value.setSyslogWalShutdownTimeoutMs(5000);
|
||||
value.setSyslogWalCheckpointFlushRecords(1);
|
||||
value.setSyslogWalCheckpointFlushIntervalMs(5);
|
||||
value.setSyslogWalDiskHighWatermarkPercent(98);
|
||||
value.setSyslogWalDiskCriticalWatermarkPercent(99);
|
||||
value.setSyslogWalMinFreeBytes(0);
|
||||
value.setSyslogCharset("UTF-8");
|
||||
return value;
|
||||
}
|
||||
|
||||
private void waitUntil(Check condition) throws Exception {
|
||||
long deadline = System.currentTimeMillis() + 5000L;
|
||||
while (!condition.done() && System.currentTimeMillis() < deadline) Thread.sleep(10L);
|
||||
assertTrue(condition.done(), "condition not reached before timeout");
|
||||
}
|
||||
|
||||
private interface Check { boolean done() throws Exception; }
|
||||
}
|
||||
Reference in New Issue
Block a user