package com.haobang.wal; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.config.CacheConfig; import com.haobang.config.AppProperties; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.FileTime; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertThrows; class SyslogWalServiceTest { private static final long GIB = 1024L * 1024L * 1024L; @TempDir Path tempDirectory; private SyslogWalService service; @AfterEach void tearDown() { if (service != null) service.shutdown(); } @Test void forceFailureCompletesAcceptedReceiptExceptionallyAndClosesAdmission() throws Exception { service = new SyslogWalService(properties(tempDirectory), new ObjectMapper(), (channel, metadata) -> { throw new java.io.IOException("injected force failure"); }); service.initialize(); IngressEvent event = service.prepareIngressEvent(new IngressEvent("20260831-force-fail", Instant.now(), Instant.parse("2026-08-31T02:00:58.127Z"), IngressType.API_PULL, "HTTPS", 1001, 2001, "NDRSERVER_V1", "idx:failed", "ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{}", null)); AppendAttempt accepted = service.tryAppendUnified(event); assertEquals(AdmissionStatus.ACCEPTED, accepted.getStatus()); assertThrows(java.util.concurrent.ExecutionException.class, () -> accepted.getDurableStage().toCompletableFuture().get(3, TimeUnit.SECONDS)); assertNotNull(service.getFatalFailure()); assertEquals(AdmissionStatus.UNAVAILABLE, service.tryAppendUnified(event).getStatus()); } @Test void apiPullWritesV4AndReceiptCompletesOnlyAfterForce() throws Exception { AppProperties properties = properties(tempDirectory); properties.setSyslogWalFlushRecords(100); properties.setSyslogWalFlushIntervalMs(300); service = new SyslogWalService(properties, new ObjectMapper()); service.initialize(); Instant eventTime = Instant.parse("2026-08-31T02:00:58.127Z"); IngressEvent event = new IngressEvent("20260831-api-event", Instant.now(), eventTime, IngressType.API_PULL, "HTTPS", 1001, 2001, "NDRSERVER_V1", "idx:42", "ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{\"id\":42}", null); event = service.prepareIngressEvent(event); AppendAttempt attempt = service.tryAppendUnified(event); assertEquals(AdmissionStatus.ACCEPTED, attempt.getStatus()); CompletableFuture future = attempt.getDurableStage().toCompletableFuture(); assertFalse(future.isDone(), "入队或 write 不能提前完成 durable receipt"); DurableReceipt receipt = future.get(3, TimeUnit.SECONDS); assertEquals(4, receipt.getWalSchemaVersion()); assertEquals("20260831-api-event", receipt.getEventId()); assertTrue(receipt.getWalSequence() > 0); try (SyslogWalService.WalCursor cursor = service.openCursor()) { SyslogWalRecord restored = cursor.next(1000L); assertEquals(4, restored.getSchemaVersion()); assertEquals("API_PULL", restored.getIngressType()); assertEquals(Integer.valueOf(1001), restored.getDeviceId()); assertEquals(Integer.valueOf(2001), restored.getCollectTaskId()); assertEquals("NDRSERVER_V1", restored.getAdapterType()); assertEquals("idx:42", restored.getExternalEventId()); assertEquals("2026-08-31T10:00:58.127", restored.getEventTime()); assertEquals(eventTime.toEpochMilli(), Math.floorDiv(restored.getPointTimeNs(), 1_000_000L)); } } @Test void rejectedAppendNeverExposesDurableStage() { service = new SyslogWalService(properties(tempDirectory), new ObjectMapper()); IngressEvent event = new IngressEvent("20260831-api-event", Instant.now(), Instant.now(), IngressType.API_PULL, "HTTPS", 1001, 2001, "NDRSERVER_V1", "42", "ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{}", 1L); AppendAttempt attempt = service.tryAppendUnified(event); assertEquals(AdmissionStatus.UNAVAILABLE, attempt.getStatus()); assertEquals(null, attempt.getDurableStage()); } @Test void oversizedMessageIsRejectedBeforeEnteringWal() { AppProperties properties = properties(tempDirectory); properties.setSyslogMessageMaxBytes(4L); properties.setSyslogWalMaxPendingBytes(16L); service = new SyslogWalService(properties, new ObjectMapper()); service.initialize(); SyslogWalService.AdmissionResult result = service.tryAppend( event("20260811-too-large", "12345")); assertEquals(SyslogWalService.AdmissionResult.OVERSIZED, result); assertEquals(1L, service.getOversizedCount()); assertEquals(0, service.getQueueSize()); assertEquals(0L, service.getPendingAppendBytes()); } @Test void pendingByteBudgetBackpressuresEvenWhenRecordQueueHasCapacity() { AppProperties properties = properties(tempDirectory); properties.setSyslogMessageMaxBytes(5L); properties.setSyslogWalMaxPendingBytes(5L); properties.setSyslogWalFlushRecords(100); properties.setSyslogWalFlushIntervalMs(10000L); service = new SyslogWalService(properties, new ObjectMapper()); service.initialize(); assertEquals(SyslogWalService.AdmissionResult.ACCEPTED, service.tryAppend(event("20260811-first", "12345"))); assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE, service.tryAppend(event("20260811-second", "x"))); assertEquals(5L, service.getPendingAppendBytes()); assertEquals(1L, service.getBackpressureCount()); } @Test void disabledHardLimitLetsOneOverBudgetRecordMakeProgressExclusively() { AppProperties properties = properties(tempDirectory); properties.setSyslogMessageMaxBytes(0L); properties.setSyslogWalMaxPendingBytes(4L); properties.setSyslogWalFlushRecords(100); properties.setSyslogWalFlushIntervalMs(10000L); service = new SyslogWalService(properties, new ObjectMapper()); service.initialize(); assertEquals(SyslogWalService.AdmissionResult.ACCEPTED, service.tryAppend(event("20260811-over-budget", "12345"))); assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE, service.tryAppend(event("20260811-waits", "x"))); assertEquals(5L, service.getPendingAppendBytes()); } @Test void forceBeforeCursorAndPersistSchemaV2() throws Exception { service = new SyslogWalService(properties(tempDirectory), new ObjectMapper()); service.initialize(); SyslogIngressEvent event = event("20260811-event-1", "first"); assertEquals(SyslogWalService.AdmissionResult.ACCEPTED, service.tryAppend(event)); waitUntil(() -> service.getAppendedCount() == 1L); try (SyslogWalService.WalCursor cursor = service.openCursor()) { SyslogWalRecord record = cursor.next(1000L); assertNotNull(record); assertEquals(1L, record.getSequence()); assertEquals(event.getEventId(), record.getEventId()); assertEquals(event.getRawMessage(), record.getRawMessage()); assertTrue(record.getEndOffset() > record.getStartOffset()); } Path segment = Files.list(tempDirectory.resolve("active")) .filter(path -> path.getFileName().toString().startsWith("syslog-wal-")) .findFirst().orElseThrow(AssertionError::new); List lines = Files.readAllLines(segment, StandardCharsets.UTF_8); JsonNode json = new ObjectMapper().readTree(lines.get(0)); assertEquals(2, json.path("schema_version").asInt()); assertEquals("20260811-event-1", json.path("event_id").asText()); assertEquals(64, json.path("raw_sha256").asText().length()); } @Test void enabledPointTimeWritesSchemaV3AndRestoresSameIdentity() throws Exception { AppProperties properties = properties(tempDirectory); properties.setSyslogPointTimeWriteEnabled(true); service = new SyslogWalService(properties, new ObjectMapper()); service.initialize(); SyslogIngressEvent prepared = service.prepareForWal(event("20260811-v3", "v3")); assertNotNull(prepared.getPointTimeNs()); service.tryAppend(prepared); waitUntil(() -> service.getAppendedCount() == 1L); try (SyslogWalService.WalCursor cursor = service.openCursor()) { SyslogWalRecord restored = cursor.next(1000L); assertEquals(3, restored.getSchemaVersion()); assertEquals(prepared.getPointTimeNs(), restored.getPointTimeNs()); assertEquals(SyslogWalRecordValidator.receiveTimeMillis(restored.getReceiveTime()), Math.floorDiv(restored.getPointTimeNs(), PointTimeAllocator.NANOS_PER_MILLI)); } } @Test void mixedV2V3RestartScanAndRecoveryRemainCompatible() throws Exception { Path active = tempDirectory.resolve("active"); Files.createDirectories(active); SyslogWalRecord v2 = walRecord(1L, "20260811-v2", "old"); SyslogWalRecord v3 = walRecord(2L, "20260811-v3", "new"); v3.setSchemaVersion(3); v3.setPointTimeNs(Math.multiplyExact(SyslogWalRecordValidator.receiveTimeMillis(v3.getReceiveTime()), 1_000_000L)); ObjectMapper mapper = new ObjectMapper(); Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"), (mapper.writeValueAsString(v2)+"\n"+mapper.writeValueAsString(v3)+"\n").getBytes(StandardCharsets.UTF_8)); service = new SyslogWalService(properties(tempDirectory), mapper); service.initialize(); try (SyslogWalService.WalCursor cursor = service.openCursor()) { assertEquals(2, cursor.next(1000L).getSchemaVersion()); SyslogWalRecord restoredV3 = cursor.next(1000L); assertEquals(3, restoredV3.getSchemaVersion()); assertEquals(v3.getPointTimeNs(), restoredV3.getPointTimeNs()); } } @Test void damagedV3IsQuarantinedAndNeverFallsBackAsV2() throws Exception { Path active = tempDirectory.resolve("active"); Files.createDirectories(active); String invalid = "{\"schema_version\":3,\"sequence\":1,\"event_id\":\"20260811-bad\"," + "\"receive_time\":\"2026-08-11T20:00:00.123\",\"point_time_ns\":-1}\n"; Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"), invalid.getBytes(StandardCharsets.UTF_8)); service = new SyslogWalService(properties(tempDirectory), new ObjectMapper()); service.initialize(); try (SyslogWalService.WalCursor cursor = service.openCursor()) { SyslogWalRecord tombstone = cursor.next(1000L); assertTrue(tombstone.isQuarantinedInvalid()); assertEquals(3,tombstone.getSchemaVersion()); } assertTrue(Files.list(tempDirectory.resolve("quarantine")).findAny().isPresent()); assertEquals(1L, service.getQuarantinedCount()); } @Test void validatorRejectsEveryIllegalV3PointTimeShape() throws Exception { ObjectMapper mapper=new ObjectMapper(); String prefix="{\"schema_version\":3,\"receive_time\":\"2026-08-11T20:00:00.123\""; assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+"}"))); assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":\"x\"}"))); assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":-1}"))); assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":1786636800124000000}"))); assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree("{\"schema_version\":4}"))); } @Test void productionRedisMapperIsSanitizedBeforeWalReadAndWrite() throws Exception { /* * 生产环境注入的是 CacheConfig 的 ObjectMapper,而不是测试常用的 new ObjectMapper()。 * 这个回归测试必须使用真实配置,否则无法复现现场“写入 @class、读取又被白名单拒绝”的故障。 */ ObjectMapper productionMapper = new CacheConfig().objectMapper(); service = new SyslogWalService(properties(tempDirectory), productionMapper); service.initialize(); service.tryAppend(event("20260811-production-mapper", "new-format")); waitUntil(() -> service.getAppendedCount() == 1L); Path segment = Files.list(tempDirectory.resolve("active")) .findFirst().orElseThrow(AssertionError::new); JsonNode persisted = new ObjectMapper().readTree(Files.readAllLines( segment, StandardCharsets.UTF_8).get(0)); assertFalse(persisted.has("@class"), "WAL must never persist Java class metadata"); try (SyslogWalService.WalCursor cursor = service.openCursor()) { SyslogWalRecord restored = cursor.next(1000L); assertNotNull(restored); assertEquals("20260811-production-mapper", restored.getEventId()); assertEquals("new-format", restored.getRawMessage()); } } @Test void repairedReaderAcceptsExistingSchemaV2RecordWithLegacyClassMetadata() throws Exception { Path active = tempDirectory.resolve("active"); Files.createDirectories(active); Path segment = active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"); // 使用发生现场故障的生产 Mapper 生成旧格式,确保样本确实包含 @class。 ObjectMapper productionMapper = new CacheConfig().objectMapper(); SyslogWalRecord legacy = walRecord(1L, "20260811-legacy-class", "existing-wal"); String legacyJson = productionMapper.writeValueAsString(legacy); assertTrue(legacyJson.contains("\"@class\":\"com.haobang.wal.SyslogWalRecord\"")); Files.write(segment, (legacyJson + "\n").getBytes(StandardCharsets.UTF_8)); service = new SyslogWalService(properties(tempDirectory), productionMapper); service.initialize(); try (SyslogWalService.WalCursor cursor = service.openCursor()) { SyslogWalRecord restored = cursor.next(1000L); assertNotNull(restored); assertEquals(1L, restored.getSequence()); assertEquals("20260811-legacy-class", restored.getEventId()); assertEquals("existing-wal", restored.getRawMessage()); } } @Test void largeDiskUsesTenGiBAsSmallerRemainingCapacityLimit() { long twoTiB = 2L * 1024L * GIB; assertEquals(10L * GIB, SyslogWalService.calculateDiskStopThresholdBytes(twoTiB, 90, 10L * GIB)); } @Test void smallDiskUsesRemainingTenPercentAsSmallerCapacityLimit() { long fiftyGiB = 50L * GIB; assertEquals(5L * GIB, SyslogWalService.calculateDiskStopThresholdBytes(fiftyGiB, 90, 10L * GIB)); } @Test void diskStopsAtInclusiveEffectiveCapacityBoundary() { long threshold = 10L * GIB; assertFalse(SyslogWalService.isDiskStopRequired(threshold + 1L, threshold)); assertTrue(SyslogWalService.isDiskStopRequired(threshold, threshold)); assertTrue(SyslogWalService.isDiskStopRequired(threshold - 1L, threshold)); } @Test void completedCleanupIsRateLimitedToThirtyMinutes() { long lastCleanupNanos = 1_000L; assertFalse(SyslogWalService.isCompletedCleanupDue(lastCleanupNanos, lastCleanupNanos + SyslogWalService.COMPLETED_CLEANUP_INTERVAL_NANOS - 1L)); assertTrue(SyslogWalService.isCompletedCleanupDue(lastCleanupNanos, lastCleanupNanos + SyslogWalService.COMPLETED_CLEANUP_INTERVAL_NANOS)); } @Test void startupImmediatelyDeletesOnlyExpiredCompletedSegments() throws Exception { Path completed = tempDirectory.resolve("completed"); Files.createDirectories(completed); Path expired = completed.resolve("syslog-wal-00000000000000000001-20260820-000000000.jsonl"); Path retained = completed.resolve("syslog-wal-00000000000000000002-20260827-000000000.jsonl"); Files.write(expired, new byte[] { '\n' }); Files.write(retained, new byte[] { '\n' }); Files.setLastModifiedTime(expired, FileTime.from(Instant.now().minus(4L, ChronoUnit.DAYS))); AppProperties properties = properties(tempDirectory); properties.setSyslogWalRetentionDays(2); service = new SyslogWalService(properties, new ObjectMapper()); service.initialize(); assertFalse(Files.exists(expired)); assertTrue(Files.exists(retained)); } @Test void restartResumesAfterPersistedCheckpointWithoutChangingEventIdentity() throws Exception { AppProperties properties = properties(tempDirectory); service = new SyslogWalService(properties, new ObjectMapper()); service.initialize(); service.tryAppend(event("20260811-event-1", "first")); waitUntil(() -> service.getAppendedCount() == 1L); try (SyslogWalService.WalCursor cursor = service.openCursor()) { SyslogWalRecord first = cursor.next(1000L); service.markCheckpoint(first); waitUntil(() -> service.getCheckpointSequence() == 1L); } service.shutdown(); service = new SyslogWalService(properties, new ObjectMapper()); service.initialize(); assertEquals(1L, service.getCheckpointSequence()); service.tryAppend(event("20260811-event-2", "second")); waitUntil(() -> service.getAppendedCount() == 1L); try (SyslogWalService.WalCursor cursor = service.openCursor()) { SyslogWalRecord second = cursor.next(1000L); assertNotNull(second); assertEquals(2L, second.getSequence()); assertEquals("20260811-event-2", second.getEventId()); assertEquals("2026-08-11T20:00:00.123", second.getReceiveTime()); } } @Test void restartTruncatesIncompleteTailAndKeepsCompleteRecord() throws Exception { AppProperties properties = properties(tempDirectory); service = new SyslogWalService(properties, new ObjectMapper()); service.initialize(); service.tryAppend(event("20260811-event-1", "first")); waitUntil(() -> service.getAppendedCount() == 1L); service.shutdown(); Path segment = Files.list(tempDirectory.resolve("active")).findFirst().orElseThrow(AssertionError::new); Files.write(segment, "{\"schema_version\":2".getBytes(StandardCharsets.UTF_8), java.nio.file.StandardOpenOption.APPEND); service = new SyslogWalService(properties, new ObjectMapper()); service.initialize(); try (SyslogWalService.WalCursor cursor = service.openCursor()) { SyslogWalRecord record = cursor.next(1000L); assertNotNull(record); assertEquals("20260811-event-1", record.getEventId()); } byte[] bytes = Files.readAllBytes(segment); assertEquals((byte) '\n', bytes[bytes.length - 1]); } @Test void legacyImporterImportsOnlyRequestedEventIdAndPreservesIdentity() throws Exception { Path archive = tempDirectory.resolve("legacy"); Files.createDirectories(archive); String requestedId = "20260811-a3d874a0-6f50-4f0c-9e7a-9c20303360f5"; String ignoredId = "20260811-53df7faa-7f78-4ce1-8eac-475965d9eeed"; ObjectMapper mapper = new ObjectMapper(); Files.write(archive.resolve("syslog-raw-20260811-test.jsonl"), ( legacyLine(mapper, requestedId, "requested") + "\n" + legacyLine(mapper, ignoredId, "ignored") + "\n").getBytes(StandardCharsets.UTF_8)); Path ids = tempDirectory.resolve("rejected.txt"); Files.write(ids, ("error eventId=" + requestedId).getBytes(StandardCharsets.UTF_8)); AppProperties properties = properties(tempDirectory.resolve("wal")); properties.setSyslogWalLegacyImportEnabled(true); properties.setSyslogWalLegacyImportArchiveDirectory(archive.toString()); properties.setSyslogWalLegacyImportEventIdFile(ids.toString()); service = new SyslogWalService(properties, mapper); service.initialize(); new SyslogLegacyArchiveImporter(properties, mapper, service).importIfRequested(); try (SyslogWalService.WalCursor cursor = service.openCursor()) { SyslogWalRecord imported = cursor.next(1000L); assertNotNull(imported); assertEquals(requestedId, imported.getEventId()); assertEquals("2026-08-11T03:55:57.109", imported.getReceiveTime()); assertEquals("requested", imported.getRawMessage()); assertEquals(null, cursor.next(50L)); } } private String legacyLine(ObjectMapper mapper, String eventId, String raw) throws Exception { java.util.Map value = new java.util.LinkedHashMap<>(); value.put("schema_version", 1); value.put("event_id", eventId); value.put("receive_time", "2026-08-11T03:55:57.109"); value.put("source_ip", "192.168.4.19"); value.put("source_port", 49768); value.put("protocol", "TCP"); value.put("raw_message", raw); return mapper.writeValueAsString(value); } private SyslogIngressEvent event(String eventId, String raw) { return new SyslogIngressEvent(eventId, LocalDateTime.of(2026, 8, 11, 20, 0, 0, 123_000_000), "192.168.4.19", 514, "TCP", raw); } private SyslogWalRecord walRecord(long sequence, String eventId, String raw) { SyslogWalRecord record = new SyslogWalRecord(); record.setSequence(sequence); record.setEventId(eventId); record.setReceiveTime("2026-08-11T23:32:32.531"); record.setSourceIp("192.168.4.19"); record.setSourcePort(45324); record.setProtocol("TCP"); record.setCharset("UTF-8"); record.setRawLength(raw.getBytes(StandardCharsets.UTF_8).length); record.setRawSha256("684b334d5690440f210e5778d304f778b5eaff5b30477667029968a711604400"); record.setRawMessage(raw); return record; } private AppProperties properties(Path directory) { AppProperties value = new AppProperties(); value.setSyslogWalEnabled(true); value.setSyslogWalDirectory(directory.toString()); value.setSyslogWalQueueCapacity(16); value.setSyslogWalHighWatermarkPercent(70); value.setSyslogWalLowWatermarkPercent(40); value.setSyslogWalRetryIntervalMs(5); value.setSyslogWalDiskRetryIntervalMs(50); value.setSyslogWalFlushRecords(1); value.setSyslogWalFlushIntervalMs(5); value.setSyslogWalSegmentMaxBytes(1024 * 1024); value.setSyslogWalRetentionDays(0); value.setSyslogWalShutdownTimeoutMs(5000); value.setSyslogWalCheckpointFlushRecords(1); value.setSyslogWalCheckpointFlushIntervalMs(5); value.setSyslogWalDiskHighWatermarkPercent(98); value.setSyslogWalMinFreeBytes(0); value.setSyslogCharset("UTF-8"); return value; } private void waitUntil(Check condition) throws Exception { long deadline = System.currentTimeMillis() + 5000L; while (!condition.done() && System.currentTimeMillis() < deadline) Thread.sleep(10L); assertTrue(condition.done(), "condition not reached before timeout"); } private interface Check { boolean done() throws Exception; } }