316 lines
16 KiB
Java
316 lines
16 KiB
Java
|
package com.haobang.wal;
|
||
|
|
|
||
|
|
import com.fasterxml.jackson.databind.JsonNode;
|
||
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||
|
|
import com.config.CacheConfig;
|
||
|
|
import com.haobang.config.AppProperties;
|
||
|
|
import org.junit.jupiter.api.AfterEach;
|
||
|
|
import org.junit.jupiter.api.Test;
|
||
|
|
import org.junit.jupiter.api.io.TempDir;
|
||
|
|
|
||
|
|
import java.nio.charset.StandardCharsets;
|
||
|
|
import java.nio.file.Files;
|
||
|
|
import java.nio.file.Path;
|
||
|
|
import java.time.LocalDateTime;
|
||
|
|
import java.util.List;
|
||
|
|
|
||
|
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||
|
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||
|
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||
|
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||
|
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||
|
|
|
||
|
|
class SyslogWalServiceTest {
|
||
|
|
@TempDir Path tempDirectory;
|
||
|
|
private SyslogWalService service;
|
||
|
|
|
||
|
|
@AfterEach
|
||
|
|
void tearDown() {
|
||
|
|
if (service != null) service.shutdown();
|
||
|
|
}
|
||
|
|
|
||
|
|
@Test
|
||
|
|
void forceBeforeCursorAndPersistSchemaV2() throws Exception {
|
||
|
|
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
|
||
|
|
service.initialize();
|
||
|
|
|
||
|
|
SyslogIngressEvent event = event("20260811-event-1", "first");
|
||
|
|
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED, service.tryAppend(event));
|
||
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
||
|
|
|
||
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||
|
|
SyslogWalRecord record = cursor.next(1000L);
|
||
|
|
assertNotNull(record);
|
||
|
|
assertEquals(1L, record.getSequence());
|
||
|
|
assertEquals(event.getEventId(), record.getEventId());
|
||
|
|
assertEquals(event.getRawMessage(), record.getRawMessage());
|
||
|
|
assertTrue(record.getEndOffset() > record.getStartOffset());
|
||
|
|
}
|
||
|
|
|
||
|
|
Path segment = Files.list(tempDirectory.resolve("active"))
|
||
|
|
.filter(path -> path.getFileName().toString().startsWith("syslog-wal-"))
|
||
|
|
.findFirst().orElseThrow(AssertionError::new);
|
||
|
|
List<String> lines = Files.readAllLines(segment, StandardCharsets.UTF_8);
|
||
|
|
JsonNode json = new ObjectMapper().readTree(lines.get(0));
|
||
|
|
assertEquals(2, json.path("schema_version").asInt());
|
||
|
|
assertEquals("20260811-event-1", json.path("event_id").asText());
|
||
|
|
assertEquals(64, json.path("raw_sha256").asText().length());
|
||
|
|
}
|
||
|
|
|
||
|
|
@Test
|
||
|
|
void enabledPointTimeWritesSchemaV3AndRestoresSameIdentity() throws Exception {
|
||
|
|
AppProperties properties = properties(tempDirectory);
|
||
|
|
properties.setSyslogPointTimeWriteEnabled(true);
|
||
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
||
|
|
service.initialize();
|
||
|
|
SyslogIngressEvent prepared = service.prepareForWal(event("20260811-v3", "v3"));
|
||
|
|
assertNotNull(prepared.getPointTimeNs());
|
||
|
|
service.tryAppend(prepared);
|
||
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
||
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||
|
|
SyslogWalRecord restored = cursor.next(1000L);
|
||
|
|
assertEquals(3, restored.getSchemaVersion());
|
||
|
|
assertEquals(prepared.getPointTimeNs(), restored.getPointTimeNs());
|
||
|
|
assertEquals(SyslogWalRecordValidator.receiveTimeMillis(restored.getReceiveTime()),
|
||
|
|
Math.floorDiv(restored.getPointTimeNs(), PointTimeAllocator.NANOS_PER_MILLI));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
@Test
|
||
|
|
void mixedV2V3RestartScanAndRecoveryRemainCompatible() throws Exception {
|
||
|
|
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
|
||
|
|
SyslogWalRecord v2 = walRecord(1L, "20260811-v2", "old");
|
||
|
|
SyslogWalRecord v3 = walRecord(2L, "20260811-v3", "new"); v3.setSchemaVersion(3);
|
||
|
|
v3.setPointTimeNs(Math.multiplyExact(SyslogWalRecordValidator.receiveTimeMillis(v3.getReceiveTime()), 1_000_000L));
|
||
|
|
ObjectMapper mapper = new ObjectMapper();
|
||
|
|
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"),
|
||
|
|
(mapper.writeValueAsString(v2)+"\n"+mapper.writeValueAsString(v3)+"\n").getBytes(StandardCharsets.UTF_8));
|
||
|
|
service = new SyslogWalService(properties(tempDirectory), mapper); service.initialize();
|
||
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||
|
|
assertEquals(2, cursor.next(1000L).getSchemaVersion());
|
||
|
|
SyslogWalRecord restoredV3 = cursor.next(1000L); assertEquals(3, restoredV3.getSchemaVersion()); assertEquals(v3.getPointTimeNs(), restoredV3.getPointTimeNs());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
@Test
|
||
|
|
void damagedV3IsQuarantinedAndNeverFallsBackAsV2() throws Exception {
|
||
|
|
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
|
||
|
|
String invalid = "{\"schema_version\":3,\"sequence\":1,\"event_id\":\"20260811-bad\","
|
||
|
|
+ "\"receive_time\":\"2026-08-11T20:00:00.123\",\"point_time_ns\":-1}\n";
|
||
|
|
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"), invalid.getBytes(StandardCharsets.UTF_8));
|
||
|
|
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper()); service.initialize();
|
||
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||
|
|
SyslogWalRecord tombstone = cursor.next(1000L); assertTrue(tombstone.isQuarantinedInvalid()); assertEquals(3,tombstone.getSchemaVersion());
|
||
|
|
}
|
||
|
|
assertTrue(Files.list(tempDirectory.resolve("quarantine")).findAny().isPresent());
|
||
|
|
assertEquals(1L, service.getQuarantinedCount());
|
||
|
|
}
|
||
|
|
|
||
|
|
@Test
|
||
|
|
void validatorRejectsEveryIllegalV3PointTimeShape() throws Exception {
|
||
|
|
ObjectMapper mapper=new ObjectMapper();
|
||
|
|
String prefix="{\"schema_version\":3,\"receive_time\":\"2026-08-11T20:00:00.123\"";
|
||
|
|
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+"}")));
|
||
|
|
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":\"x\"}")));
|
||
|
|
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":-1}")));
|
||
|
|
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":1786636800124000000}")));
|
||
|
|
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree("{\"schema_version\":4}")));
|
||
|
|
}
|
||
|
|
|
||
|
|
@Test
|
||
|
|
void productionRedisMapperIsSanitizedBeforeWalReadAndWrite() throws Exception {
|
||
|
|
/*
|
||
|
|
* 生产环境注入的是 CacheConfig 的 ObjectMapper，而不是测试常用的 new ObjectMapper()。
|
||
|
|
* 这个回归测试必须使用真实配置，否则无法复现现场“写入 @class、读取又被白名单拒绝”的故障。
|
||
|
|
*/
|
||
|
|
ObjectMapper productionMapper = new CacheConfig().objectMapper();
|
||
|
|
service = new SyslogWalService(properties(tempDirectory), productionMapper);
|
||
|
|
service.initialize();
|
||
|
|
service.tryAppend(event("20260811-production-mapper", "new-format"));
|
||
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
||
|
|
|
||
|
|
Path segment = Files.list(tempDirectory.resolve("active"))
|
||
|
|
.findFirst().orElseThrow(AssertionError::new);
|
||
|
|
JsonNode persisted = new ObjectMapper().readTree(Files.readAllLines(
|
||
|
|
segment, StandardCharsets.UTF_8).get(0));
|
||
|
|
assertFalse(persisted.has("@class"), "WAL must never persist Java class metadata");
|
||
|
|
|
||
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||
|
|
SyslogWalRecord restored = cursor.next(1000L);
|
||
|
|
assertNotNull(restored);
|
||
|
|
assertEquals("20260811-production-mapper", restored.getEventId());
|
||
|
|
assertEquals("new-format", restored.getRawMessage());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
@Test
|
||
|
|
void repairedReaderAcceptsExistingSchemaV2RecordWithLegacyClassMetadata() throws Exception {
|
||
|
|
Path active = tempDirectory.resolve("active");
|
||
|
|
Files.createDirectories(active);
|
||
|
|
Path segment = active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl");
|
||
|
|
|
||
|
|
// 使用发生现场故障的生产 Mapper 生成旧格式，确保样本确实包含 @class。
|
||
|
|
ObjectMapper productionMapper = new CacheConfig().objectMapper();
|
||
|
|
SyslogWalRecord legacy = walRecord(1L, "20260811-legacy-class", "existing-wal");
|
||
|
|
String legacyJson = productionMapper.writeValueAsString(legacy);
|
||
|
|
assertTrue(legacyJson.contains("\"@class\":\"com.haobang.wal.SyslogWalRecord\""));
|
||
|
|
Files.write(segment, (legacyJson + "\n").getBytes(StandardCharsets.UTF_8));
|
||
|
|
|
||
|
|
service = new SyslogWalService(properties(tempDirectory), productionMapper);
|
||
|
|
service.initialize();
|
||
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||
|
|
SyslogWalRecord restored = cursor.next(1000L);
|
||
|
|
assertNotNull(restored);
|
||
|
|
assertEquals(1L, restored.getSequence());
|
||
|
|
assertEquals("20260811-legacy-class", restored.getEventId());
|
||
|
|
assertEquals("existing-wal", restored.getRawMessage());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
@Test
|
||
|
|
void restartResumesAfterPersistedCheckpointWithoutChangingEventIdentity() throws Exception {
|
||
|
|
AppProperties properties = properties(tempDirectory);
|
||
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
||
|
|
service.initialize();
|
||
|
|
service.tryAppend(event("20260811-event-1", "first"));
|
||
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
||
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||
|
|
SyslogWalRecord first = cursor.next(1000L);
|
||
|
|
service.markCheckpoint(first);
|
||
|
|
waitUntil(() -> service.getCheckpointSequence() == 1L);
|
||
|
|
}
|
||
|
|
service.shutdown();
|
||
|
|
|
||
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
||
|
|
service.initialize();
|
||
|
|
assertEquals(1L, service.getCheckpointSequence());
|
||
|
|
service.tryAppend(event("20260811-event-2", "second"));
|
||
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
||
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||
|
|
SyslogWalRecord second = cursor.next(1000L);
|
||
|
|
assertNotNull(second);
|
||
|
|
assertEquals(2L, second.getSequence());
|
||
|
|
assertEquals("20260811-event-2", second.getEventId());
|
||
|
|
assertEquals("2026-08-11T20:00:00.123", second.getReceiveTime());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
@Test
|
||
|
|
void restartTruncatesIncompleteTailAndKeepsCompleteRecord() throws Exception {
|
||
|
|
AppProperties properties = properties(tempDirectory);
|
||
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
||
|
|
service.initialize();
|
||
|
|
service.tryAppend(event("20260811-event-1", "first"));
|
||
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
||
|
|
service.shutdown();
|
||
|
|
|
||
|
|
Path segment = Files.list(tempDirectory.resolve("active")).findFirst().orElseThrow(AssertionError::new);
|
||
|
|
Files.write(segment, "{\"schema_version\":2".getBytes(StandardCharsets.UTF_8),
|
||
|
|
java.nio.file.StandardOpenOption.APPEND);
|
||
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
||
|
|
service.initialize();
|
||
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||
|
|
SyslogWalRecord record = cursor.next(1000L);
|
||
|
|
assertNotNull(record);
|
||
|
|
assertEquals("20260811-event-1", record.getEventId());
|
||
|
|
}
|
||
|
|
byte[] bytes = Files.readAllBytes(segment);
|
||
|
|
assertEquals((byte) '\n', bytes[bytes.length - 1]);
|
||
|
|
}
|
||
|
|
|
||
|
|
@Test
|
||
|
|
void legacyImporterImportsOnlyRequestedEventIdAndPreservesIdentity() throws Exception {
|
||
|
|
Path archive = tempDirectory.resolve("legacy");
|
||
|
|
Files.createDirectories(archive);
|
||
|
|
String requestedId = "20260811-a3d874a0-6f50-4f0c-9e7a-9c20303360f5";
|
||
|
|
String ignoredId = "20260811-53df7faa-7f78-4ce1-8eac-475965d9eeed";
|
||
|
|
ObjectMapper mapper = new ObjectMapper();
|
||
|
|
Files.write(archive.resolve("syslog-raw-20260811-test.jsonl"), (
|
||
|
|
legacyLine(mapper, requestedId, "requested") + "\n"
|
||
|
|
+ legacyLine(mapper, ignoredId, "ignored") + "\n").getBytes(StandardCharsets.UTF_8));
|
||
|
|
Path ids = tempDirectory.resolve("rejected.txt");
|
||
|
|
Files.write(ids, ("error eventId=" + requestedId).getBytes(StandardCharsets.UTF_8));
|
||
|
|
|
||
|
|
AppProperties properties = properties(tempDirectory.resolve("wal"));
|
||
|
|
properties.setSyslogWalLegacyImportEnabled(true);
|
||
|
|
properties.setSyslogWalLegacyImportArchiveDirectory(archive.toString());
|
||
|
|
properties.setSyslogWalLegacyImportEventIdFile(ids.toString());
|
||
|
|
service = new SyslogWalService(properties, mapper);
|
||
|
|
service.initialize();
|
||
|
|
new SyslogLegacyArchiveImporter(properties, mapper, service).importIfRequested();
|
||
|
|
|
||
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||
|
|
SyslogWalRecord imported = cursor.next(1000L);
|
||
|
|
assertNotNull(imported);
|
||
|
|
assertEquals(requestedId, imported.getEventId());
|
||
|
|
assertEquals("2026-08-11T03:55:57.109", imported.getReceiveTime());
|
||
|
|
assertEquals("requested", imported.getRawMessage());
|
||
|
|
assertEquals(null, cursor.next(50L));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private String legacyLine(ObjectMapper mapper, String eventId, String raw) throws Exception {
|
||
|
|
java.util.Map<String, Object> value = new java.util.LinkedHashMap<>();
|
||
|
|
value.put("schema_version", 1);
|
||
|
|
value.put("event_id", eventId);
|
||
|
|
value.put("receive_time", "2026-08-11T03:55:57.109");
|
||
|
|
value.put("source_ip", "192.168.4.19");
|
||
|
|
value.put("source_port", 49768);
|
||
|
|
value.put("protocol", "TCP");
|
||
|
|
value.put("raw_message", raw);
|
||
|
|
return mapper.writeValueAsString(value);
|
||
|
|
}
|
||
|
|
|
||
|
|
private SyslogIngressEvent event(String eventId, String raw) {
|
||
|
|
return new SyslogIngressEvent(eventId,
|
||
|
|
LocalDateTime.of(2026, 8, 11, 20, 0, 0, 123_000_000),
|
||
|
|
"192.168.4.19", 514, "TCP", raw);
|
||
|
|
}
|
||
|
|
|
||
|
|
private SyslogWalRecord walRecord(long sequence, String eventId, String raw) {
|
||
|
|
SyslogWalRecord record = new SyslogWalRecord();
|
||
|
|
record.setSequence(sequence);
|
||
|
|
record.setEventId(eventId);
|
||
|
|
record.setReceiveTime("2026-08-11T23:32:32.531");
|
||
|
|
record.setSourceIp("192.168.4.19");
|
||
|
|
record.setSourcePort(45324);
|
||
|
|
record.setProtocol("TCP");
|
||
|
|
record.setCharset("UTF-8");
|
||
|
|
record.setRawLength(raw.getBytes(StandardCharsets.UTF_8).length);
|
||
|
|
record.setRawSha256("684b334d5690440f210e5778d304f778b5eaff5b30477667029968a711604400");
|
||
|
|
record.setRawMessage(raw);
|
||
|
|
return record;
|
||
|
|
}
|
||
|
|
|
||
|
|
private AppProperties properties(Path directory) {
|
||
|
|
AppProperties value = new AppProperties();
|
||
|
|
value.setSyslogWalEnabled(true);
|
||
|
|
value.setSyslogWalDirectory(directory.toString());
|
||
|
|
value.setSyslogWalQueueCapacity(16);
|
||
|
|
value.setSyslogWalHighWatermarkPercent(70);
|
||
|
|
value.setSyslogWalLowWatermarkPercent(40);
|
||
|
|
value.setSyslogWalRetryIntervalMs(5);
|
||
|
|
value.setSyslogWalDiskRetryIntervalMs(50);
|
||
|
|
value.setSyslogWalFlushRecords(1);
|
||
|
|
value.setSyslogWalFlushIntervalMs(5);
|
||
|
|
value.setSyslogWalSegmentMaxBytes(1024 * 1024);
|
||
|
|
value.setSyslogWalRetentionDays(0);
|
||
|
|
value.setSyslogWalShutdownTimeoutMs(5000);
|
||
|
|
value.setSyslogWalCheckpointFlushRecords(1);
|
||
|
|
value.setSyslogWalCheckpointFlushIntervalMs(5);
|
||
|
|
value.setSyslogWalDiskHighWatermarkPercent(98);
|
||
|
|
value.setSyslogWalDiskCriticalWatermarkPercent(99);
|
||
|
|
value.setSyslogWalMinFreeBytes(0);
|
||
|
|
value.setSyslogCharset("UTF-8");
|
||
|
|
return value;
|
||
|
|
}
|
||
|
|
|
||
|
|
private void waitUntil(Check condition) throws Exception {
|
||
|
|
long deadline = System.currentTimeMillis() + 5000L;
|
||
|
|
while (!condition.done() && System.currentTimeMillis() < deadline) Thread.sleep(10L);
|
||
|
|
assertTrue(condition.done(), "condition not reached before timeout");
|
||
|
|
}
|
||
|
|
|
||
|
|
private interface Check { boolean done() throws Exception; }
|
||
|
|
}
|