2026-08-20 00:18:33 +08:00
|
|
|
package com.haobang.wal;
|
|
|
|
|
|
|
|
|
|
import com.fasterxml.jackson.databind.JsonNode;
|
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
|
|
import com.config.CacheConfig;
|
|
|
|
|
import com.haobang.config.AppProperties;
|
|
|
|
|
import org.junit.jupiter.api.AfterEach;
|
|
|
|
|
import org.junit.jupiter.api.Test;
|
|
|
|
|
import org.junit.jupiter.api.io.TempDir;
|
|
|
|
|
|
|
|
|
|
import java.nio.charset.StandardCharsets;
|
|
|
|
|
import java.nio.file.Files;
|
|
|
|
|
import java.nio.file.Path;
|
2026-09-06 16:46:21 +08:00
|
|
|
import java.nio.file.attribute.FileTime;
|
|
|
|
|
import java.time.Instant;
|
2026-08-20 00:18:33 +08:00
|
|
|
import java.time.LocalDateTime;
|
2026-09-06 16:46:21 +08:00
|
|
|
import java.time.ZoneId;
|
|
|
|
|
import java.time.temporal.ChronoUnit;
|
2026-08-20 00:18:33 +08:00
|
|
|
import java.util.List;
|
2026-09-06 16:46:21 +08:00
|
|
|
import java.util.concurrent.CompletableFuture;
|
|
|
|
|
import java.util.concurrent.TimeUnit;
|
2026-08-20 00:18:33 +08:00
|
|
|
|
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
|
|
|
|
|
|
|
|
|
class SyslogWalServiceTest {
|
2026-09-06 16:46:21 +08:00
|
|
|
private static final long GIB = 1024L * 1024L * 1024L;
|
|
|
|
|
|
2026-08-20 00:18:33 +08:00
|
|
|
@TempDir Path tempDirectory;
|
|
|
|
|
private SyslogWalService service;
|
|
|
|
|
|
|
|
|
|
@AfterEach
|
|
|
|
|
void tearDown() {
|
|
|
|
|
if (service != null) service.shutdown();
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-06 16:46:21 +08:00
|
|
|
@Test
|
|
|
|
|
void forceFailureCompletesAcceptedReceiptExceptionallyAndClosesAdmission() throws Exception {
|
|
|
|
|
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper(),
|
|
|
|
|
(channel, metadata) -> { throw new java.io.IOException("injected force failure"); });
|
|
|
|
|
service.initialize();
|
|
|
|
|
IngressEvent event = service.prepareIngressEvent(new IngressEvent("20260831-force-fail", Instant.now(),
|
|
|
|
|
Instant.parse("2026-08-31T02:00:58.127Z"), IngressType.API_PULL, "HTTPS", 1001, 2001,
|
|
|
|
|
"NDRSERVER_V1", "idx:failed", "ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{}", null));
|
|
|
|
|
AppendAttempt accepted = service.tryAppendUnified(event);
|
|
|
|
|
assertEquals(AdmissionStatus.ACCEPTED, accepted.getStatus());
|
|
|
|
|
assertThrows(java.util.concurrent.ExecutionException.class,
|
|
|
|
|
() -> accepted.getDurableStage().toCompletableFuture().get(3, TimeUnit.SECONDS));
|
|
|
|
|
assertNotNull(service.getFatalFailure());
|
|
|
|
|
assertEquals(AdmissionStatus.UNAVAILABLE, service.tryAppendUnified(event).getStatus());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void apiPullWritesV4AndReceiptCompletesOnlyAfterForce() throws Exception {
|
|
|
|
|
AppProperties properties = properties(tempDirectory);
|
|
|
|
|
properties.setSyslogWalFlushRecords(100);
|
|
|
|
|
properties.setSyslogWalFlushIntervalMs(300);
|
|
|
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
|
|
|
|
service.initialize();
|
|
|
|
|
Instant eventTime = Instant.parse("2026-08-31T02:00:58.127Z");
|
|
|
|
|
IngressEvent event = new IngressEvent("20260831-api-event", Instant.now(), eventTime,
|
|
|
|
|
IngressType.API_PULL, "HTTPS", 1001, 2001, "NDRSERVER_V1", "idx:42",
|
|
|
|
|
"ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{\"id\":42}", null);
|
|
|
|
|
event = service.prepareIngressEvent(event);
|
|
|
|
|
AppendAttempt attempt = service.tryAppendUnified(event);
|
|
|
|
|
assertEquals(AdmissionStatus.ACCEPTED, attempt.getStatus());
|
|
|
|
|
CompletableFuture<DurableReceipt> future = attempt.getDurableStage().toCompletableFuture();
|
|
|
|
|
assertFalse(future.isDone(), "入队或 write 不能提前完成 durable receipt");
|
|
|
|
|
|
|
|
|
|
DurableReceipt receipt = future.get(3, TimeUnit.SECONDS);
|
|
|
|
|
assertEquals(4, receipt.getWalSchemaVersion());
|
|
|
|
|
assertEquals("20260831-api-event", receipt.getEventId());
|
|
|
|
|
assertTrue(receipt.getWalSequence() > 0);
|
|
|
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
|
|
|
|
SyslogWalRecord restored = cursor.next(1000L);
|
|
|
|
|
assertEquals(4, restored.getSchemaVersion());
|
|
|
|
|
assertEquals("API_PULL", restored.getIngressType());
|
|
|
|
|
assertEquals(Integer.valueOf(1001), restored.getDeviceId());
|
|
|
|
|
assertEquals(Integer.valueOf(2001), restored.getCollectTaskId());
|
|
|
|
|
assertEquals("NDRSERVER_V1", restored.getAdapterType());
|
|
|
|
|
assertEquals("idx:42", restored.getExternalEventId());
|
|
|
|
|
assertEquals("2026-08-31T10:00:58.127", restored.getEventTime());
|
|
|
|
|
assertEquals(eventTime.toEpochMilli(), Math.floorDiv(restored.getPointTimeNs(), 1_000_000L));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void rejectedAppendNeverExposesDurableStage() {
|
|
|
|
|
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
|
|
|
|
|
IngressEvent event = new IngressEvent("20260831-api-event", Instant.now(), Instant.now(),
|
|
|
|
|
IngressType.API_PULL, "HTTPS", 1001, 2001, "NDRSERVER_V1", "42",
|
|
|
|
|
"ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{}", 1L);
|
|
|
|
|
AppendAttempt attempt = service.tryAppendUnified(event);
|
|
|
|
|
assertEquals(AdmissionStatus.UNAVAILABLE, attempt.getStatus());
|
|
|
|
|
assertEquals(null, attempt.getDurableStage());
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-12 19:22:52 +08:00
|
|
|
@Test
|
|
|
|
|
void oversizedMessageIsRejectedBeforeEnteringWal() {
|
|
|
|
|
AppProperties properties = properties(tempDirectory);
|
|
|
|
|
properties.setSyslogMessageMaxBytes(4L);
|
|
|
|
|
properties.setSyslogWalMaxPendingBytes(16L);
|
|
|
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
|
|
|
|
service.initialize();
|
|
|
|
|
|
|
|
|
|
SyslogWalService.AdmissionResult result = service.tryAppend(
|
|
|
|
|
event("20260811-too-large", "12345"));
|
|
|
|
|
|
|
|
|
|
assertEquals(SyslogWalService.AdmissionResult.OVERSIZED, result);
|
|
|
|
|
assertEquals(1L, service.getOversizedCount());
|
|
|
|
|
assertEquals(0, service.getQueueSize());
|
|
|
|
|
assertEquals(0L, service.getPendingAppendBytes());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void pendingByteBudgetBackpressuresEvenWhenRecordQueueHasCapacity() {
|
|
|
|
|
AppProperties properties = properties(tempDirectory);
|
|
|
|
|
properties.setSyslogMessageMaxBytes(5L);
|
|
|
|
|
properties.setSyslogWalMaxPendingBytes(5L);
|
|
|
|
|
properties.setSyslogWalFlushRecords(100);
|
|
|
|
|
properties.setSyslogWalFlushIntervalMs(10000L);
|
|
|
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
|
|
|
|
service.initialize();
|
|
|
|
|
|
|
|
|
|
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED,
|
|
|
|
|
service.tryAppend(event("20260811-first", "12345")));
|
|
|
|
|
assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE,
|
|
|
|
|
service.tryAppend(event("20260811-second", "x")));
|
|
|
|
|
assertEquals(5L, service.getPendingAppendBytes());
|
|
|
|
|
assertEquals(1L, service.getBackpressureCount());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void disabledHardLimitLetsOneOverBudgetRecordMakeProgressExclusively() {
|
|
|
|
|
AppProperties properties = properties(tempDirectory);
|
|
|
|
|
properties.setSyslogMessageMaxBytes(0L);
|
|
|
|
|
properties.setSyslogWalMaxPendingBytes(4L);
|
|
|
|
|
properties.setSyslogWalFlushRecords(100);
|
|
|
|
|
properties.setSyslogWalFlushIntervalMs(10000L);
|
|
|
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
|
|
|
|
service.initialize();
|
|
|
|
|
|
|
|
|
|
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED,
|
|
|
|
|
service.tryAppend(event("20260811-over-budget", "12345")));
|
|
|
|
|
assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE,
|
|
|
|
|
service.tryAppend(event("20260811-waits", "x")));
|
|
|
|
|
assertEquals(5L, service.getPendingAppendBytes());
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-20 00:18:33 +08:00
|
|
|
@Test
|
|
|
|
|
void forceBeforeCursorAndPersistSchemaV2() throws Exception {
|
|
|
|
|
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
|
|
|
|
|
service.initialize();
|
|
|
|
|
|
|
|
|
|
SyslogIngressEvent event = event("20260811-event-1", "first");
|
|
|
|
|
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED, service.tryAppend(event));
|
|
|
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
|
|
|
|
|
|
|
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
|
|
|
|
SyslogWalRecord record = cursor.next(1000L);
|
|
|
|
|
assertNotNull(record);
|
|
|
|
|
assertEquals(1L, record.getSequence());
|
|
|
|
|
assertEquals(event.getEventId(), record.getEventId());
|
|
|
|
|
assertEquals(event.getRawMessage(), record.getRawMessage());
|
|
|
|
|
assertTrue(record.getEndOffset() > record.getStartOffset());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Path segment = Files.list(tempDirectory.resolve("active"))
|
|
|
|
|
.filter(path -> path.getFileName().toString().startsWith("syslog-wal-"))
|
|
|
|
|
.findFirst().orElseThrow(AssertionError::new);
|
|
|
|
|
List<String> lines = Files.readAllLines(segment, StandardCharsets.UTF_8);
|
|
|
|
|
JsonNode json = new ObjectMapper().readTree(lines.get(0));
|
|
|
|
|
assertEquals(2, json.path("schema_version").asInt());
|
|
|
|
|
assertEquals("20260811-event-1", json.path("event_id").asText());
|
|
|
|
|
assertEquals(64, json.path("raw_sha256").asText().length());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void enabledPointTimeWritesSchemaV3AndRestoresSameIdentity() throws Exception {
|
|
|
|
|
AppProperties properties = properties(tempDirectory);
|
|
|
|
|
properties.setSyslogPointTimeWriteEnabled(true);
|
|
|
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
|
|
|
|
service.initialize();
|
|
|
|
|
SyslogIngressEvent prepared = service.prepareForWal(event("20260811-v3", "v3"));
|
|
|
|
|
assertNotNull(prepared.getPointTimeNs());
|
|
|
|
|
service.tryAppend(prepared);
|
|
|
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
|
|
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
|
|
|
|
SyslogWalRecord restored = cursor.next(1000L);
|
|
|
|
|
assertEquals(3, restored.getSchemaVersion());
|
|
|
|
|
assertEquals(prepared.getPointTimeNs(), restored.getPointTimeNs());
|
|
|
|
|
assertEquals(SyslogWalRecordValidator.receiveTimeMillis(restored.getReceiveTime()),
|
|
|
|
|
Math.floorDiv(restored.getPointTimeNs(), PointTimeAllocator.NANOS_PER_MILLI));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void mixedV2V3RestartScanAndRecoveryRemainCompatible() throws Exception {
|
|
|
|
|
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
|
|
|
|
|
SyslogWalRecord v2 = walRecord(1L, "20260811-v2", "old");
|
|
|
|
|
SyslogWalRecord v3 = walRecord(2L, "20260811-v3", "new"); v3.setSchemaVersion(3);
|
|
|
|
|
v3.setPointTimeNs(Math.multiplyExact(SyslogWalRecordValidator.receiveTimeMillis(v3.getReceiveTime()), 1_000_000L));
|
|
|
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
|
|
|
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"),
|
|
|
|
|
(mapper.writeValueAsString(v2)+"\n"+mapper.writeValueAsString(v3)+"\n").getBytes(StandardCharsets.UTF_8));
|
|
|
|
|
service = new SyslogWalService(properties(tempDirectory), mapper); service.initialize();
|
|
|
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
|
|
|
|
assertEquals(2, cursor.next(1000L).getSchemaVersion());
|
|
|
|
|
SyslogWalRecord restoredV3 = cursor.next(1000L); assertEquals(3, restoredV3.getSchemaVersion()); assertEquals(v3.getPointTimeNs(), restoredV3.getPointTimeNs());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void damagedV3IsQuarantinedAndNeverFallsBackAsV2() throws Exception {
|
|
|
|
|
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
|
|
|
|
|
String invalid = "{\"schema_version\":3,\"sequence\":1,\"event_id\":\"20260811-bad\","
|
|
|
|
|
+ "\"receive_time\":\"2026-08-11T20:00:00.123\",\"point_time_ns\":-1}\n";
|
|
|
|
|
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"), invalid.getBytes(StandardCharsets.UTF_8));
|
|
|
|
|
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper()); service.initialize();
|
|
|
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
|
|
|
|
SyslogWalRecord tombstone = cursor.next(1000L); assertTrue(tombstone.isQuarantinedInvalid()); assertEquals(3,tombstone.getSchemaVersion());
|
|
|
|
|
}
|
|
|
|
|
assertTrue(Files.list(tempDirectory.resolve("quarantine")).findAny().isPresent());
|
|
|
|
|
assertEquals(1L, service.getQuarantinedCount());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void validatorRejectsEveryIllegalV3PointTimeShape() throws Exception {
|
|
|
|
|
ObjectMapper mapper=new ObjectMapper();
|
|
|
|
|
String prefix="{\"schema_version\":3,\"receive_time\":\"2026-08-11T20:00:00.123\"";
|
|
|
|
|
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+"}")));
|
|
|
|
|
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":\"x\"}")));
|
|
|
|
|
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":-1}")));
|
|
|
|
|
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":1786636800124000000}")));
|
|
|
|
|
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree("{\"schema_version\":4}")));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void productionRedisMapperIsSanitizedBeforeWalReadAndWrite() throws Exception {
|
|
|
|
|
/*
|
|
|
|
|
* 生产环境注入的是 CacheConfig 的 ObjectMapper,而不是测试常用的 new ObjectMapper()。
|
|
|
|
|
* 这个回归测试必须使用真实配置,否则无法复现现场“写入 @class、读取又被白名单拒绝”的故障。
|
|
|
|
|
*/
|
|
|
|
|
ObjectMapper productionMapper = new CacheConfig().objectMapper();
|
|
|
|
|
service = new SyslogWalService(properties(tempDirectory), productionMapper);
|
|
|
|
|
service.initialize();
|
|
|
|
|
service.tryAppend(event("20260811-production-mapper", "new-format"));
|
|
|
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
|
|
|
|
|
|
|
|
|
Path segment = Files.list(tempDirectory.resolve("active"))
|
|
|
|
|
.findFirst().orElseThrow(AssertionError::new);
|
|
|
|
|
JsonNode persisted = new ObjectMapper().readTree(Files.readAllLines(
|
|
|
|
|
segment, StandardCharsets.UTF_8).get(0));
|
|
|
|
|
assertFalse(persisted.has("@class"), "WAL must never persist Java class metadata");
|
|
|
|
|
|
|
|
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
|
|
|
|
SyslogWalRecord restored = cursor.next(1000L);
|
|
|
|
|
assertNotNull(restored);
|
|
|
|
|
assertEquals("20260811-production-mapper", restored.getEventId());
|
|
|
|
|
assertEquals("new-format", restored.getRawMessage());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void repairedReaderAcceptsExistingSchemaV2RecordWithLegacyClassMetadata() throws Exception {
|
|
|
|
|
Path active = tempDirectory.resolve("active");
|
|
|
|
|
Files.createDirectories(active);
|
|
|
|
|
Path segment = active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl");
|
|
|
|
|
|
|
|
|
|
// 使用发生现场故障的生产 Mapper 生成旧格式,确保样本确实包含 @class。
|
|
|
|
|
ObjectMapper productionMapper = new CacheConfig().objectMapper();
|
|
|
|
|
SyslogWalRecord legacy = walRecord(1L, "20260811-legacy-class", "existing-wal");
|
|
|
|
|
String legacyJson = productionMapper.writeValueAsString(legacy);
|
|
|
|
|
assertTrue(legacyJson.contains("\"@class\":\"com.haobang.wal.SyslogWalRecord\""));
|
|
|
|
|
Files.write(segment, (legacyJson + "\n").getBytes(StandardCharsets.UTF_8));
|
|
|
|
|
|
|
|
|
|
service = new SyslogWalService(properties(tempDirectory), productionMapper);
|
|
|
|
|
service.initialize();
|
|
|
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
|
|
|
|
SyslogWalRecord restored = cursor.next(1000L);
|
|
|
|
|
assertNotNull(restored);
|
|
|
|
|
assertEquals(1L, restored.getSequence());
|
|
|
|
|
assertEquals("20260811-legacy-class", restored.getEventId());
|
|
|
|
|
assertEquals("existing-wal", restored.getRawMessage());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-06 16:46:21 +08:00
|
|
|
@Test
|
|
|
|
|
void largeDiskUsesTenGiBAsSmallerRemainingCapacityLimit() {
|
|
|
|
|
long twoTiB = 2L * 1024L * GIB;
|
|
|
|
|
|
|
|
|
|
assertEquals(10L * GIB,
|
|
|
|
|
SyslogWalService.calculateDiskStopThresholdBytes(twoTiB, 90, 10L * GIB));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void smallDiskUsesRemainingTenPercentAsSmallerCapacityLimit() {
|
|
|
|
|
long fiftyGiB = 50L * GIB;
|
|
|
|
|
|
|
|
|
|
assertEquals(5L * GIB,
|
|
|
|
|
SyslogWalService.calculateDiskStopThresholdBytes(fiftyGiB, 90, 10L * GIB));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void diskStopsAtInclusiveEffectiveCapacityBoundary() {
|
|
|
|
|
long threshold = 10L * GIB;
|
|
|
|
|
|
|
|
|
|
assertFalse(SyslogWalService.isDiskStopRequired(threshold + 1L, threshold));
|
|
|
|
|
assertTrue(SyslogWalService.isDiskStopRequired(threshold, threshold));
|
|
|
|
|
assertTrue(SyslogWalService.isDiskStopRequired(threshold - 1L, threshold));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void completedCleanupIsRateLimitedToThirtyMinutes() {
|
|
|
|
|
long lastCleanupNanos = 1_000L;
|
|
|
|
|
|
|
|
|
|
assertFalse(SyslogWalService.isCompletedCleanupDue(lastCleanupNanos,
|
|
|
|
|
lastCleanupNanos + SyslogWalService.COMPLETED_CLEANUP_INTERVAL_NANOS - 1L));
|
|
|
|
|
assertTrue(SyslogWalService.isCompletedCleanupDue(lastCleanupNanos,
|
|
|
|
|
lastCleanupNanos + SyslogWalService.COMPLETED_CLEANUP_INTERVAL_NANOS));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void startupImmediatelyDeletesOnlyExpiredCompletedSegments() throws Exception {
|
|
|
|
|
Path completed = tempDirectory.resolve("completed");
|
|
|
|
|
Files.createDirectories(completed);
|
|
|
|
|
Path expired = completed.resolve("syslog-wal-00000000000000000001-20260820-000000000.jsonl");
|
|
|
|
|
Path retained = completed.resolve("syslog-wal-00000000000000000002-20260827-000000000.jsonl");
|
|
|
|
|
Files.write(expired, new byte[] { '\n' });
|
|
|
|
|
Files.write(retained, new byte[] { '\n' });
|
|
|
|
|
Files.setLastModifiedTime(expired, FileTime.from(Instant.now().minus(4L, ChronoUnit.DAYS)));
|
|
|
|
|
|
|
|
|
|
AppProperties properties = properties(tempDirectory);
|
|
|
|
|
properties.setSyslogWalRetentionDays(2);
|
|
|
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
|
|
|
|
service.initialize();
|
|
|
|
|
|
|
|
|
|
assertFalse(Files.exists(expired));
|
|
|
|
|
assertTrue(Files.exists(retained));
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-20 00:18:33 +08:00
|
|
|
@Test
|
|
|
|
|
void restartResumesAfterPersistedCheckpointWithoutChangingEventIdentity() throws Exception {
|
|
|
|
|
AppProperties properties = properties(tempDirectory);
|
|
|
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
|
|
|
|
service.initialize();
|
|
|
|
|
service.tryAppend(event("20260811-event-1", "first"));
|
|
|
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
|
|
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
|
|
|
|
SyslogWalRecord first = cursor.next(1000L);
|
|
|
|
|
service.markCheckpoint(first);
|
|
|
|
|
waitUntil(() -> service.getCheckpointSequence() == 1L);
|
|
|
|
|
}
|
|
|
|
|
service.shutdown();
|
|
|
|
|
|
|
|
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
|
|
|
|
service.initialize();
|
|
|
|
|
assertEquals(1L, service.getCheckpointSequence());
|
|
|
|
|
service.tryAppend(event("20260811-event-2", "second"));
|
|
|
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
|
|
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
|
|
|
|
SyslogWalRecord second = cursor.next(1000L);
|
|
|
|
|
assertNotNull(second);
|
|
|
|
|
assertEquals(2L, second.getSequence());
|
|
|
|
|
assertEquals("20260811-event-2", second.getEventId());
|
|
|
|
|
assertEquals("2026-08-11T20:00:00.123", second.getReceiveTime());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void restartTruncatesIncompleteTailAndKeepsCompleteRecord() throws Exception {
|
|
|
|
|
AppProperties properties = properties(tempDirectory);
|
|
|
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
|
|
|
|
service.initialize();
|
|
|
|
|
service.tryAppend(event("20260811-event-1", "first"));
|
|
|
|
|
waitUntil(() -> service.getAppendedCount() == 1L);
|
|
|
|
|
service.shutdown();
|
|
|
|
|
|
|
|
|
|
Path segment = Files.list(tempDirectory.resolve("active")).findFirst().orElseThrow(AssertionError::new);
|
|
|
|
|
Files.write(segment, "{\"schema_version\":2".getBytes(StandardCharsets.UTF_8),
|
|
|
|
|
java.nio.file.StandardOpenOption.APPEND);
|
|
|
|
|
service = new SyslogWalService(properties, new ObjectMapper());
|
|
|
|
|
service.initialize();
|
|
|
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
|
|
|
|
SyslogWalRecord record = cursor.next(1000L);
|
|
|
|
|
assertNotNull(record);
|
|
|
|
|
assertEquals("20260811-event-1", record.getEventId());
|
|
|
|
|
}
|
|
|
|
|
byte[] bytes = Files.readAllBytes(segment);
|
|
|
|
|
assertEquals((byte) '\n', bytes[bytes.length - 1]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
void legacyImporterImportsOnlyRequestedEventIdAndPreservesIdentity() throws Exception {
|
|
|
|
|
Path archive = tempDirectory.resolve("legacy");
|
|
|
|
|
Files.createDirectories(archive);
|
|
|
|
|
String requestedId = "20260811-a3d874a0-6f50-4f0c-9e7a-9c20303360f5";
|
|
|
|
|
String ignoredId = "20260811-53df7faa-7f78-4ce1-8eac-475965d9eeed";
|
|
|
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
|
|
|
Files.write(archive.resolve("syslog-raw-20260811-test.jsonl"), (
|
|
|
|
|
legacyLine(mapper, requestedId, "requested") + "\n"
|
|
|
|
|
+ legacyLine(mapper, ignoredId, "ignored") + "\n").getBytes(StandardCharsets.UTF_8));
|
|
|
|
|
Path ids = tempDirectory.resolve("rejected.txt");
|
|
|
|
|
Files.write(ids, ("error eventId=" + requestedId).getBytes(StandardCharsets.UTF_8));
|
|
|
|
|
|
|
|
|
|
AppProperties properties = properties(tempDirectory.resolve("wal"));
|
|
|
|
|
properties.setSyslogWalLegacyImportEnabled(true);
|
|
|
|
|
properties.setSyslogWalLegacyImportArchiveDirectory(archive.toString());
|
|
|
|
|
properties.setSyslogWalLegacyImportEventIdFile(ids.toString());
|
|
|
|
|
service = new SyslogWalService(properties, mapper);
|
|
|
|
|
service.initialize();
|
|
|
|
|
new SyslogLegacyArchiveImporter(properties, mapper, service).importIfRequested();
|
|
|
|
|
|
|
|
|
|
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
|
|
|
|
SyslogWalRecord imported = cursor.next(1000L);
|
|
|
|
|
assertNotNull(imported);
|
|
|
|
|
assertEquals(requestedId, imported.getEventId());
|
|
|
|
|
assertEquals("2026-08-11T03:55:57.109", imported.getReceiveTime());
|
|
|
|
|
assertEquals("requested", imported.getRawMessage());
|
|
|
|
|
assertEquals(null, cursor.next(50L));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private String legacyLine(ObjectMapper mapper, String eventId, String raw) throws Exception {
|
|
|
|
|
java.util.Map<String, Object> value = new java.util.LinkedHashMap<>();
|
|
|
|
|
value.put("schema_version", 1);
|
|
|
|
|
value.put("event_id", eventId);
|
|
|
|
|
value.put("receive_time", "2026-08-11T03:55:57.109");
|
|
|
|
|
value.put("source_ip", "192.168.4.19");
|
|
|
|
|
value.put("source_port", 49768);
|
|
|
|
|
value.put("protocol", "TCP");
|
|
|
|
|
value.put("raw_message", raw);
|
|
|
|
|
return mapper.writeValueAsString(value);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private SyslogIngressEvent event(String eventId, String raw) {
|
|
|
|
|
return new SyslogIngressEvent(eventId,
|
|
|
|
|
LocalDateTime.of(2026, 8, 11, 20, 0, 0, 123_000_000),
|
|
|
|
|
"192.168.4.19", 514, "TCP", raw);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private SyslogWalRecord walRecord(long sequence, String eventId, String raw) {
|
|
|
|
|
SyslogWalRecord record = new SyslogWalRecord();
|
|
|
|
|
record.setSequence(sequence);
|
|
|
|
|
record.setEventId(eventId);
|
|
|
|
|
record.setReceiveTime("2026-08-11T23:32:32.531");
|
|
|
|
|
record.setSourceIp("192.168.4.19");
|
|
|
|
|
record.setSourcePort(45324);
|
|
|
|
|
record.setProtocol("TCP");
|
|
|
|
|
record.setCharset("UTF-8");
|
|
|
|
|
record.setRawLength(raw.getBytes(StandardCharsets.UTF_8).length);
|
|
|
|
|
record.setRawSha256("684b334d5690440f210e5778d304f778b5eaff5b30477667029968a711604400");
|
|
|
|
|
record.setRawMessage(raw);
|
|
|
|
|
return record;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private AppProperties properties(Path directory) {
|
|
|
|
|
AppProperties value = new AppProperties();
|
|
|
|
|
value.setSyslogWalEnabled(true);
|
|
|
|
|
value.setSyslogWalDirectory(directory.toString());
|
|
|
|
|
value.setSyslogWalQueueCapacity(16);
|
|
|
|
|
value.setSyslogWalHighWatermarkPercent(70);
|
|
|
|
|
value.setSyslogWalLowWatermarkPercent(40);
|
|
|
|
|
value.setSyslogWalRetryIntervalMs(5);
|
|
|
|
|
value.setSyslogWalDiskRetryIntervalMs(50);
|
|
|
|
|
value.setSyslogWalFlushRecords(1);
|
|
|
|
|
value.setSyslogWalFlushIntervalMs(5);
|
|
|
|
|
value.setSyslogWalSegmentMaxBytes(1024 * 1024);
|
|
|
|
|
value.setSyslogWalRetentionDays(0);
|
|
|
|
|
value.setSyslogWalShutdownTimeoutMs(5000);
|
|
|
|
|
value.setSyslogWalCheckpointFlushRecords(1);
|
|
|
|
|
value.setSyslogWalCheckpointFlushIntervalMs(5);
|
|
|
|
|
value.setSyslogWalDiskHighWatermarkPercent(98);
|
|
|
|
|
value.setSyslogWalMinFreeBytes(0);
|
|
|
|
|
value.setSyslogCharset("UTF-8");
|
|
|
|
|
return value;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void waitUntil(Check condition) throws Exception {
|
|
|
|
|
long deadline = System.currentTimeMillis() + 5000L;
|
|
|
|
|
while (!condition.done() && System.currentTimeMillis() < deadline) Thread.sleep(10L);
|
|
|
|
|
assertTrue(condition.done(), "condition not reached before timeout");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private interface Check { boolean done() throws Exception; }
|
|
|
|
|
}
|