8f1a8b6283
- 每分钟保存 WAL 全局状态和分 IP 接收统计,历史保留 7 天 - 记录接收、WAL 落盘、Kafka ACK、过滤、背压及丢弃数量 - 增加 wal_max_pending_bytes 和 wal_admission_unavailable 指标 - 增加 TCP pending 当前值及进程内峰值统计 - 网络异常后同一 IP 恢复接收时打印恢复日志 - 提供当前状态及按日期查询诊断历史的接口 - 连接关闭时清理 TCP pending 计数并停止无效重试 - 增加相关单元测试和现场排查文档
91 lines
4.0 KiB
Java
91 lines
4.0 KiB
Java
package com.haobang.diagnostics;
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.haobang.config.AppProperties;
|
|
import com.haobang.wal.SyslogWalStatusProvider;
|
|
import io.netty.handler.codec.TooLongFrameException;
|
|
import org.junit.jupiter.api.AfterEach;
|
|
import org.junit.jupiter.api.Test;
|
|
import org.junit.jupiter.api.io.TempDir;
|
|
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.nio.file.Files;
|
|
import java.nio.file.Path;
|
|
import java.time.LocalDate;
|
|
import java.time.ZoneId;
|
|
import java.util.Collections;
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
import static org.mockito.Mockito.mock;
|
|
import static org.mockito.Mockito.when;
|
|
|
|
class SyslogDiagnosticsPersistenceTest {
|
|
@TempDir Path tempDirectory;
|
|
private SyslogDiagnosticsPersistence persistence;
|
|
|
|
@AfterEach
|
|
void tearDown() {
|
|
if (persistence != null) persistence.shutdown();
|
|
}
|
|
|
|
@Test
|
|
void writesSnapshotAndEventsAndKeepsSevenCalendarDays() throws Exception {
|
|
Path snapshots = Files.createDirectories(tempDirectory.resolve("snapshots"));
|
|
Path events = Files.createDirectories(tempDirectory.resolve("events"));
|
|
LocalDate today = LocalDate.now(ZoneId.of("Asia/Shanghai"));
|
|
Path expired = snapshots.resolve("syslog-status-" + today.minusDays(7) + ".jsonl");
|
|
Path retained = events.resolve("syslog-events-" + today.minusDays(6) + ".jsonl");
|
|
Files.write(expired, "old\n".getBytes(StandardCharsets.UTF_8));
|
|
Files.write(retained, "keep\n".getBytes(StandardCharsets.UTF_8));
|
|
|
|
AppProperties properties = properties();
|
|
SyslogIpMetricsRegistry registry = new SyslogIpMetricsRegistry(properties);
|
|
SyslogWalStatusProvider provider = mock(SyslogWalStatusProvider.class);
|
|
when(provider.status()).thenReturn(Collections.singletonMap("wal_pending_approx", 0L));
|
|
persistence = new SyslogDiagnosticsPersistence(properties, registry, provider,
|
|
new ObjectMapper());
|
|
persistence.start();
|
|
|
|
registry.recordNetworkReceived("10.56.170.107", 35064, "TCP");
|
|
registry.recordNetworkException("10.56.170.107", 35064, "TCP",
|
|
new TooLongFrameException("too long"));
|
|
persistence.writeSnapshotSafely();
|
|
waitUntil(() -> Files.isRegularFile(persistence.eventFile(today)));
|
|
|
|
String snapshot = new String(Files.readAllBytes(persistence.snapshotFile(today)),
|
|
StandardCharsets.UTF_8);
|
|
String ipSnapshot = new String(Files.readAllBytes(persistence.ipSnapshotFile(today)),
|
|
StandardCharsets.UTF_8);
|
|
String event = new String(Files.readAllBytes(persistence.eventFile(today)),
|
|
StandardCharsets.UTF_8);
|
|
assertTrue(snapshot.contains("\"ip_stats_count\":1"));
|
|
assertTrue(ipSnapshot.contains("\"source_ip\":\"10.56.170.107\""));
|
|
assertTrue(ipSnapshot.contains("\"delta\""));
|
|
assertTrue(event.contains("\"event_type\":\"TOO_LONG_FRAME\""));
|
|
assertFalse(Files.exists(expired));
|
|
assertTrue(Files.exists(retained));
|
|
}
|
|
|
|
private AppProperties properties() {
|
|
AppProperties value = new AppProperties();
|
|
value.setSyslogDiagnosticsEnabled(true);
|
|
value.setSyslogDiagnosticsDirectory(tempDirectory.toString());
|
|
value.setSyslogDiagnosticsSnapshotIntervalMs(60_000L);
|
|
value.setSyslogDiagnosticsRetentionDays(7);
|
|
value.setSyslogDiagnosticsMaxSourceIps(100);
|
|
value.setSyslogDiagnosticsEventQueueCapacity(100);
|
|
value.setAppServiceDeviceCollectId(7);
|
|
value.setAppServiceDeviceCollectName("probe-7");
|
|
return value;
|
|
}
|
|
|
|
private void waitUntil(Check condition) throws Exception {
|
|
long deadline = System.currentTimeMillis() + 3000L;
|
|
while (!condition.done() && System.currentTimeMillis() < deadline) Thread.sleep(10L);
|
|
assertTrue(condition.done(), "condition not reached before timeout");
|
|
}
|
|
|
|
private interface Check { boolean done() throws Exception; }
|
|
}
|