Files
ai-security-xdr/haobang-security-dm/syslog-serve/src/test/java/com/haobang/wal/SyslogWalServiceTest.java
T
王绍宏 0e9e6656db feat(syslog): 优化大日志通道并增加异常连续消费兜底
- 128 KiB 及以上日志默认进入大日志 Topic
- InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存
- 截断内容追加 HB_XDR_TRUNCATED 标记
- InfluxDB 写入失败时继续执行规则解析和达梦写入
- 未知或系统性异常优先写入普通/大日志 DLT
- DLT 写入失败时记录特殊告警并提交 offset,避免阻塞后续消息
- DLT 失败等待时间默认限制为 1 秒
- 增加连续性放行、DLT 成功及失败监控指标
- 更新 Docker Compose、v3 升级文档及相关测试
2026-09-12 19:22:52 +08:00

490 lines
24 KiB
Java

package com.haobang.wal;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.config.CacheConfig;
import com.haobang.config.AppProperties;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileTime;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertThrows;
class SyslogWalServiceTest {
private static final long GIB = 1024L * 1024L * 1024L;
@TempDir Path tempDirectory;
private SyslogWalService service;
@AfterEach
void tearDown() {
if (service != null) service.shutdown();
}
@Test
void forceFailureCompletesAcceptedReceiptExceptionallyAndClosesAdmission() throws Exception {
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper(),
(channel, metadata) -> { throw new java.io.IOException("injected force failure"); });
service.initialize();
IngressEvent event = service.prepareIngressEvent(new IngressEvent("20260831-force-fail", Instant.now(),
Instant.parse("2026-08-31T02:00:58.127Z"), IngressType.API_PULL, "HTTPS", 1001, 2001,
"NDRSERVER_V1", "idx:failed", "ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{}", null));
AppendAttempt accepted = service.tryAppendUnified(event);
assertEquals(AdmissionStatus.ACCEPTED, accepted.getStatus());
assertThrows(java.util.concurrent.ExecutionException.class,
() -> accepted.getDurableStage().toCompletableFuture().get(3, TimeUnit.SECONDS));
assertNotNull(service.getFatalFailure());
assertEquals(AdmissionStatus.UNAVAILABLE, service.tryAppendUnified(event).getStatus());
}
@Test
void apiPullWritesV4AndReceiptCompletesOnlyAfterForce() throws Exception {
AppProperties properties = properties(tempDirectory);
properties.setSyslogWalFlushRecords(100);
properties.setSyslogWalFlushIntervalMs(300);
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
Instant eventTime = Instant.parse("2026-08-31T02:00:58.127Z");
IngressEvent event = new IngressEvent("20260831-api-event", Instant.now(), eventTime,
IngressType.API_PULL, "HTTPS", 1001, 2001, "NDRSERVER_V1", "idx:42",
"ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{\"id\":42}", null);
event = service.prepareIngressEvent(event);
AppendAttempt attempt = service.tryAppendUnified(event);
assertEquals(AdmissionStatus.ACCEPTED, attempt.getStatus());
CompletableFuture<DurableReceipt> future = attempt.getDurableStage().toCompletableFuture();
assertFalse(future.isDone(), "入队或 write 不能提前完成 durable receipt");
DurableReceipt receipt = future.get(3, TimeUnit.SECONDS);
assertEquals(4, receipt.getWalSchemaVersion());
assertEquals("20260831-api-event", receipt.getEventId());
assertTrue(receipt.getWalSequence() > 0);
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
SyslogWalRecord restored = cursor.next(1000L);
assertEquals(4, restored.getSchemaVersion());
assertEquals("API_PULL", restored.getIngressType());
assertEquals(Integer.valueOf(1001), restored.getDeviceId());
assertEquals(Integer.valueOf(2001), restored.getCollectTaskId());
assertEquals("NDRSERVER_V1", restored.getAdapterType());
assertEquals("idx:42", restored.getExternalEventId());
assertEquals("2026-08-31T10:00:58.127", restored.getEventTime());
assertEquals(eventTime.toEpochMilli(), Math.floorDiv(restored.getPointTimeNs(), 1_000_000L));
}
}
@Test
void rejectedAppendNeverExposesDurableStage() {
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
IngressEvent event = new IngressEvent("20260831-api-event", Instant.now(), Instant.now(),
IngressType.API_PULL, "HTTPS", 1001, 2001, "NDRSERVER_V1", "42",
"ndr.example", 443, "JSON", StandardCharsets.UTF_8, "{}", 1L);
AppendAttempt attempt = service.tryAppendUnified(event);
assertEquals(AdmissionStatus.UNAVAILABLE, attempt.getStatus());
assertEquals(null, attempt.getDurableStage());
}
@Test
void oversizedMessageIsRejectedBeforeEnteringWal() {
AppProperties properties = properties(tempDirectory);
properties.setSyslogMessageMaxBytes(4L);
properties.setSyslogWalMaxPendingBytes(16L);
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
SyslogWalService.AdmissionResult result = service.tryAppend(
event("20260811-too-large", "12345"));
assertEquals(SyslogWalService.AdmissionResult.OVERSIZED, result);
assertEquals(1L, service.getOversizedCount());
assertEquals(0, service.getQueueSize());
assertEquals(0L, service.getPendingAppendBytes());
}
@Test
void pendingByteBudgetBackpressuresEvenWhenRecordQueueHasCapacity() {
AppProperties properties = properties(tempDirectory);
properties.setSyslogMessageMaxBytes(5L);
properties.setSyslogWalMaxPendingBytes(5L);
properties.setSyslogWalFlushRecords(100);
properties.setSyslogWalFlushIntervalMs(10000L);
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED,
service.tryAppend(event("20260811-first", "12345")));
assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE,
service.tryAppend(event("20260811-second", "x")));
assertEquals(5L, service.getPendingAppendBytes());
assertEquals(1L, service.getBackpressureCount());
}
@Test
void disabledHardLimitLetsOneOverBudgetRecordMakeProgressExclusively() {
AppProperties properties = properties(tempDirectory);
properties.setSyslogMessageMaxBytes(0L);
properties.setSyslogWalMaxPendingBytes(4L);
properties.setSyslogWalFlushRecords(100);
properties.setSyslogWalFlushIntervalMs(10000L);
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED,
service.tryAppend(event("20260811-over-budget", "12345")));
assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE,
service.tryAppend(event("20260811-waits", "x")));
assertEquals(5L, service.getPendingAppendBytes());
}
@Test
void forceBeforeCursorAndPersistSchemaV2() throws Exception {
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
service.initialize();
SyslogIngressEvent event = event("20260811-event-1", "first");
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED, service.tryAppend(event));
waitUntil(() -> service.getAppendedCount() == 1L);
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
SyslogWalRecord record = cursor.next(1000L);
assertNotNull(record);
assertEquals(1L, record.getSequence());
assertEquals(event.getEventId(), record.getEventId());
assertEquals(event.getRawMessage(), record.getRawMessage());
assertTrue(record.getEndOffset() > record.getStartOffset());
}
Path segment = Files.list(tempDirectory.resolve("active"))
.filter(path -> path.getFileName().toString().startsWith("syslog-wal-"))
.findFirst().orElseThrow(AssertionError::new);
List<String> lines = Files.readAllLines(segment, StandardCharsets.UTF_8);
JsonNode json = new ObjectMapper().readTree(lines.get(0));
assertEquals(2, json.path("schema_version").asInt());
assertEquals("20260811-event-1", json.path("event_id").asText());
assertEquals(64, json.path("raw_sha256").asText().length());
}
@Test
void enabledPointTimeWritesSchemaV3AndRestoresSameIdentity() throws Exception {
AppProperties properties = properties(tempDirectory);
properties.setSyslogPointTimeWriteEnabled(true);
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
SyslogIngressEvent prepared = service.prepareForWal(event("20260811-v3", "v3"));
assertNotNull(prepared.getPointTimeNs());
service.tryAppend(prepared);
waitUntil(() -> service.getAppendedCount() == 1L);
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
SyslogWalRecord restored = cursor.next(1000L);
assertEquals(3, restored.getSchemaVersion());
assertEquals(prepared.getPointTimeNs(), restored.getPointTimeNs());
assertEquals(SyslogWalRecordValidator.receiveTimeMillis(restored.getReceiveTime()),
Math.floorDiv(restored.getPointTimeNs(), PointTimeAllocator.NANOS_PER_MILLI));
}
}
@Test
void mixedV2V3RestartScanAndRecoveryRemainCompatible() throws Exception {
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
SyslogWalRecord v2 = walRecord(1L, "20260811-v2", "old");
SyslogWalRecord v3 = walRecord(2L, "20260811-v3", "new"); v3.setSchemaVersion(3);
v3.setPointTimeNs(Math.multiplyExact(SyslogWalRecordValidator.receiveTimeMillis(v3.getReceiveTime()), 1_000_000L));
ObjectMapper mapper = new ObjectMapper();
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"),
(mapper.writeValueAsString(v2)+"\n"+mapper.writeValueAsString(v3)+"\n").getBytes(StandardCharsets.UTF_8));
service = new SyslogWalService(properties(tempDirectory), mapper); service.initialize();
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
assertEquals(2, cursor.next(1000L).getSchemaVersion());
SyslogWalRecord restoredV3 = cursor.next(1000L); assertEquals(3, restoredV3.getSchemaVersion()); assertEquals(v3.getPointTimeNs(), restoredV3.getPointTimeNs());
}
}
@Test
void damagedV3IsQuarantinedAndNeverFallsBackAsV2() throws Exception {
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
String invalid = "{\"schema_version\":3,\"sequence\":1,\"event_id\":\"20260811-bad\","
+ "\"receive_time\":\"2026-08-11T20:00:00.123\",\"point_time_ns\":-1}\n";
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"), invalid.getBytes(StandardCharsets.UTF_8));
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper()); service.initialize();
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
SyslogWalRecord tombstone = cursor.next(1000L); assertTrue(tombstone.isQuarantinedInvalid()); assertEquals(3,tombstone.getSchemaVersion());
}
assertTrue(Files.list(tempDirectory.resolve("quarantine")).findAny().isPresent());
assertEquals(1L, service.getQuarantinedCount());
}
@Test
void validatorRejectsEveryIllegalV3PointTimeShape() throws Exception {
ObjectMapper mapper=new ObjectMapper();
String prefix="{\"schema_version\":3,\"receive_time\":\"2026-08-11T20:00:00.123\"";
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+"}")));
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":\"x\"}")));
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":-1}")));
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":1786636800124000000}")));
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree("{\"schema_version\":4}")));
}
@Test
void productionRedisMapperIsSanitizedBeforeWalReadAndWrite() throws Exception {
/*
* 生产环境注入的是 CacheConfig 的 ObjectMapper,而不是测试常用的 new ObjectMapper()。
* 这个回归测试必须使用真实配置,否则无法复现现场“写入 @class、读取又被白名单拒绝”的故障。
*/
ObjectMapper productionMapper = new CacheConfig().objectMapper();
service = new SyslogWalService(properties(tempDirectory), productionMapper);
service.initialize();
service.tryAppend(event("20260811-production-mapper", "new-format"));
waitUntil(() -> service.getAppendedCount() == 1L);
Path segment = Files.list(tempDirectory.resolve("active"))
.findFirst().orElseThrow(AssertionError::new);
JsonNode persisted = new ObjectMapper().readTree(Files.readAllLines(
segment, StandardCharsets.UTF_8).get(0));
assertFalse(persisted.has("@class"), "WAL must never persist Java class metadata");
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
SyslogWalRecord restored = cursor.next(1000L);
assertNotNull(restored);
assertEquals("20260811-production-mapper", restored.getEventId());
assertEquals("new-format", restored.getRawMessage());
}
}
@Test
void repairedReaderAcceptsExistingSchemaV2RecordWithLegacyClassMetadata() throws Exception {
Path active = tempDirectory.resolve("active");
Files.createDirectories(active);
Path segment = active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl");
// 使用发生现场故障的生产 Mapper 生成旧格式,确保样本确实包含 @class。
ObjectMapper productionMapper = new CacheConfig().objectMapper();
SyslogWalRecord legacy = walRecord(1L, "20260811-legacy-class", "existing-wal");
String legacyJson = productionMapper.writeValueAsString(legacy);
assertTrue(legacyJson.contains("\"@class\":\"com.haobang.wal.SyslogWalRecord\""));
Files.write(segment, (legacyJson + "\n").getBytes(StandardCharsets.UTF_8));
service = new SyslogWalService(properties(tempDirectory), productionMapper);
service.initialize();
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
SyslogWalRecord restored = cursor.next(1000L);
assertNotNull(restored);
assertEquals(1L, restored.getSequence());
assertEquals("20260811-legacy-class", restored.getEventId());
assertEquals("existing-wal", restored.getRawMessage());
}
}
@Test
void largeDiskUsesTenGiBAsSmallerRemainingCapacityLimit() {
long twoTiB = 2L * 1024L * GIB;
assertEquals(10L * GIB,
SyslogWalService.calculateDiskStopThresholdBytes(twoTiB, 90, 10L * GIB));
}
@Test
void smallDiskUsesRemainingTenPercentAsSmallerCapacityLimit() {
long fiftyGiB = 50L * GIB;
assertEquals(5L * GIB,
SyslogWalService.calculateDiskStopThresholdBytes(fiftyGiB, 90, 10L * GIB));
}
@Test
void diskStopsAtInclusiveEffectiveCapacityBoundary() {
long threshold = 10L * GIB;
assertFalse(SyslogWalService.isDiskStopRequired(threshold + 1L, threshold));
assertTrue(SyslogWalService.isDiskStopRequired(threshold, threshold));
assertTrue(SyslogWalService.isDiskStopRequired(threshold - 1L, threshold));
}
@Test
void completedCleanupIsRateLimitedToThirtyMinutes() {
long lastCleanupNanos = 1_000L;
assertFalse(SyslogWalService.isCompletedCleanupDue(lastCleanupNanos,
lastCleanupNanos + SyslogWalService.COMPLETED_CLEANUP_INTERVAL_NANOS - 1L));
assertTrue(SyslogWalService.isCompletedCleanupDue(lastCleanupNanos,
lastCleanupNanos + SyslogWalService.COMPLETED_CLEANUP_INTERVAL_NANOS));
}
@Test
void startupImmediatelyDeletesOnlyExpiredCompletedSegments() throws Exception {
Path completed = tempDirectory.resolve("completed");
Files.createDirectories(completed);
Path expired = completed.resolve("syslog-wal-00000000000000000001-20260820-000000000.jsonl");
Path retained = completed.resolve("syslog-wal-00000000000000000002-20260827-000000000.jsonl");
Files.write(expired, new byte[] { '\n' });
Files.write(retained, new byte[] { '\n' });
Files.setLastModifiedTime(expired, FileTime.from(Instant.now().minus(4L, ChronoUnit.DAYS)));
AppProperties properties = properties(tempDirectory);
properties.setSyslogWalRetentionDays(2);
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
assertFalse(Files.exists(expired));
assertTrue(Files.exists(retained));
}
@Test
void restartResumesAfterPersistedCheckpointWithoutChangingEventIdentity() throws Exception {
AppProperties properties = properties(tempDirectory);
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
service.tryAppend(event("20260811-event-1", "first"));
waitUntil(() -> service.getAppendedCount() == 1L);
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
SyslogWalRecord first = cursor.next(1000L);
service.markCheckpoint(first);
waitUntil(() -> service.getCheckpointSequence() == 1L);
}
service.shutdown();
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
assertEquals(1L, service.getCheckpointSequence());
service.tryAppend(event("20260811-event-2", "second"));
waitUntil(() -> service.getAppendedCount() == 1L);
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
SyslogWalRecord second = cursor.next(1000L);
assertNotNull(second);
assertEquals(2L, second.getSequence());
assertEquals("20260811-event-2", second.getEventId());
assertEquals("2026-08-11T20:00:00.123", second.getReceiveTime());
}
}
@Test
void restartTruncatesIncompleteTailAndKeepsCompleteRecord() throws Exception {
AppProperties properties = properties(tempDirectory);
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
service.tryAppend(event("20260811-event-1", "first"));
waitUntil(() -> service.getAppendedCount() == 1L);
service.shutdown();
Path segment = Files.list(tempDirectory.resolve("active")).findFirst().orElseThrow(AssertionError::new);
Files.write(segment, "{\"schema_version\":2".getBytes(StandardCharsets.UTF_8),
java.nio.file.StandardOpenOption.APPEND);
service = new SyslogWalService(properties, new ObjectMapper());
service.initialize();
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
SyslogWalRecord record = cursor.next(1000L);
assertNotNull(record);
assertEquals("20260811-event-1", record.getEventId());
}
byte[] bytes = Files.readAllBytes(segment);
assertEquals((byte) '\n', bytes[bytes.length - 1]);
}
@Test
void legacyImporterImportsOnlyRequestedEventIdAndPreservesIdentity() throws Exception {
Path archive = tempDirectory.resolve("legacy");
Files.createDirectories(archive);
String requestedId = "20260811-a3d874a0-6f50-4f0c-9e7a-9c20303360f5";
String ignoredId = "20260811-53df7faa-7f78-4ce1-8eac-475965d9eeed";
ObjectMapper mapper = new ObjectMapper();
Files.write(archive.resolve("syslog-raw-20260811-test.jsonl"), (
legacyLine(mapper, requestedId, "requested") + "\n"
+ legacyLine(mapper, ignoredId, "ignored") + "\n").getBytes(StandardCharsets.UTF_8));
Path ids = tempDirectory.resolve("rejected.txt");
Files.write(ids, ("error eventId=" + requestedId).getBytes(StandardCharsets.UTF_8));
AppProperties properties = properties(tempDirectory.resolve("wal"));
properties.setSyslogWalLegacyImportEnabled(true);
properties.setSyslogWalLegacyImportArchiveDirectory(archive.toString());
properties.setSyslogWalLegacyImportEventIdFile(ids.toString());
service = new SyslogWalService(properties, mapper);
service.initialize();
new SyslogLegacyArchiveImporter(properties, mapper, service).importIfRequested();
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
SyslogWalRecord imported = cursor.next(1000L);
assertNotNull(imported);
assertEquals(requestedId, imported.getEventId());
assertEquals("2026-08-11T03:55:57.109", imported.getReceiveTime());
assertEquals("requested", imported.getRawMessage());
assertEquals(null, cursor.next(50L));
}
}
private String legacyLine(ObjectMapper mapper, String eventId, String raw) throws Exception {
java.util.Map<String, Object> value = new java.util.LinkedHashMap<>();
value.put("schema_version", 1);
value.put("event_id", eventId);
value.put("receive_time", "2026-08-11T03:55:57.109");
value.put("source_ip", "192.168.4.19");
value.put("source_port", 49768);
value.put("protocol", "TCP");
value.put("raw_message", raw);
return mapper.writeValueAsString(value);
}
private SyslogIngressEvent event(String eventId, String raw) {
return new SyslogIngressEvent(eventId,
LocalDateTime.of(2026, 8, 11, 20, 0, 0, 123_000_000),
"192.168.4.19", 514, "TCP", raw);
}
private SyslogWalRecord walRecord(long sequence, String eventId, String raw) {
SyslogWalRecord record = new SyslogWalRecord();
record.setSequence(sequence);
record.setEventId(eventId);
record.setReceiveTime("2026-08-11T23:32:32.531");
record.setSourceIp("192.168.4.19");
record.setSourcePort(45324);
record.setProtocol("TCP");
record.setCharset("UTF-8");
record.setRawLength(raw.getBytes(StandardCharsets.UTF_8).length);
record.setRawSha256("684b334d5690440f210e5778d304f778b5eaff5b30477667029968a711604400");
record.setRawMessage(raw);
return record;
}
private AppProperties properties(Path directory) {
AppProperties value = new AppProperties();
value.setSyslogWalEnabled(true);
value.setSyslogWalDirectory(directory.toString());
value.setSyslogWalQueueCapacity(16);
value.setSyslogWalHighWatermarkPercent(70);
value.setSyslogWalLowWatermarkPercent(40);
value.setSyslogWalRetryIntervalMs(5);
value.setSyslogWalDiskRetryIntervalMs(50);
value.setSyslogWalFlushRecords(1);
value.setSyslogWalFlushIntervalMs(5);
value.setSyslogWalSegmentMaxBytes(1024 * 1024);
value.setSyslogWalRetentionDays(0);
value.setSyslogWalShutdownTimeoutMs(5000);
value.setSyslogWalCheckpointFlushRecords(1);
value.setSyslogWalCheckpointFlushIntervalMs(5);
value.setSyslogWalDiskHighWatermarkPercent(98);
value.setSyslogWalMinFreeBytes(0);
value.setSyslogCharset("UTF-8");
return value;
}
private void waitUntil(Check condition) throws Exception {
long deadline = System.currentTimeMillis() + 5000L;
while (!condition.done() && System.currentTimeMillis() < deadline) Thread.sleep(10L);
assertTrue(condition.done(), "condition not reached before timeout");
}
private interface Check { boolean done() throws Exception; }
}