feat(syslog): 优化大日志通道并增加异常连续消费兜底
- 128 KiB 及以上日志默认进入大日志 Topic - InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存 - 截断内容追加 HB_XDR_TRUNCATED 标记 - InfluxDB 写入失败时继续执行规则解析和达梦写入 - 未知或系统性异常优先写入普通/大日志 DLT - DLT 写入失败时记录特殊告警并提交 offset，避免阻塞后续消息 - DLT 失败等待时间默认限制为 1 秒 - 增加连续性放行、DLT 成功及失败监控指标 - 更新 Docker Compose、v3 升级文档及相关测试
This commit is contained in:
王绍宏
+73
-4
@@ -64,7 +64,7 @@ public class SyslogWalService {
|
||||
private static final String SEGMENT_GLOB = "syslog-wal-*.jsonl";
|
||||
static final long COMPLETED_CLEANUP_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(30L);
|
||||
|
||||
public enum AdmissionResult { ACCEPTED, BACKPRESSURE, UNAVAILABLE }
|
||||
public enum AdmissionResult { ACCEPTED, BACKPRESSURE, OVERSIZED, UNAVAILABLE }
|
||||
|
||||
private final AppProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
@@ -76,6 +76,8 @@ public class SyslogWalService {
|
||||
private final AtomicLong durableSequence = new AtomicLong();
|
||||
private final LongAdder appended = new LongAdder();
|
||||
private final LongAdder admissionBackpressure = new LongAdder();
|
||||
private final LongAdder admissionOversized = new LongAdder();
|
||||
private final AtomicLong pendingAppendBytes = new AtomicLong();
|
||||
private final LongAdder quarantined = new LongAdder();
|
||||
private final LongAdder terminalRecorded = new LongAdder();
|
||||
private final Object durableMonitor = new Object();
|
||||
@@ -211,6 +213,7 @@ public class SyslogWalService {
|
||||
AppendAttempt attempt = tryAppendUnified(toUnified(event));
|
||||
if (attempt.getStatus() == AdmissionStatus.ACCEPTED) return AdmissionResult.ACCEPTED;
|
||||
if (attempt.getStatus() == AdmissionStatus.BACKPRESSURE) return AdmissionResult.BACKPRESSURE;
|
||||
if (attempt.getStatus() == AdmissionStatus.OVERSIZED) return AdmissionResult.OVERSIZED;
|
||||
return AdmissionResult.UNAVAILABLE;
|
||||
}
|
||||
|
||||
@@ -231,8 +234,19 @@ public class SyslogWalService {
|
||||
if (!running || fatalFailure != null || diskState == DiskState.CRITICAL) {
|
||||
return AppendAttempt.rejected(AdmissionStatus.UNAVAILABLE);
|
||||
}
|
||||
PendingWalAppend pending = new PendingWalAppend(event);
|
||||
int rawBytes = event.getRawMessage().getBytes(event.getCharset()).length;
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& rawBytes > properties.getSyslogMessageMaxBytes()) {
|
||||
admissionOversized.increment();
|
||||
return AppendAttempt.rejected(AdmissionStatus.OVERSIZED);
|
||||
}
|
||||
if (!reservePendingBytes(rawBytes)) {
|
||||
admissionBackpressure.increment();
|
||||
return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE);
|
||||
}
|
||||
PendingWalAppend pending = new PendingWalAppend(event, rawBytes);
|
||||
if (!appendQueue.offer(pending)) {
|
||||
releasePendingBytes(rawBytes);
|
||||
admissionBackpressure.increment();
|
||||
return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE);
|
||||
}
|
||||
@@ -243,14 +257,18 @@ public class SyslogWalService {
|
||||
refreshDiskState(false);
|
||||
if (fatalFailure != null || diskState != DiskState.NORMAL) return true;
|
||||
int high = percentageCount(properties.getSyslogWalHighWatermarkPercent());
|
||||
return appendQueue != null && appendQueue.size() >= high;
|
||||
long byteHigh = percentageBytes(properties.getSyslogWalHighWatermarkPercent());
|
||||
return appendQueue != null && (appendQueue.size() >= high
|
||||
|| (byteHigh > 0L && pendingAppendBytes.get() >= byteHigh));
|
||||
}
|
||||
|
||||
public boolean canResumeTcp() {
|
||||
refreshDiskState(false);
|
||||
if (fatalFailure != null || diskState != DiskState.NORMAL) return false;
|
||||
int low = percentageCount(properties.getSyslogWalLowWatermarkPercent());
|
||||
return appendQueue != null && appendQueue.size() <= low;
|
||||
long byteLow = percentageBytes(properties.getSyslogWalLowWatermarkPercent());
|
||||
return appendQueue != null && appendQueue.size() <= low
|
||||
&& (byteLow == 0L || pendingAppendBytes.get() <= byteLow);
|
||||
}
|
||||
|
||||
public long getRetryIntervalMs() {
|
||||
@@ -263,6 +281,8 @@ public class SyslogWalService {
|
||||
public int getQueueCapacity() { return properties.getSyslogWalQueueCapacity(); }
|
||||
public long getAppendedCount() { return appended.sum(); }
|
||||
public long getBackpressureCount() { return admissionBackpressure.sum(); }
|
||||
public long getOversizedCount() { return admissionOversized.sum(); }
|
||||
public long getPendingAppendBytes() { return pendingAppendBytes.get(); }
|
||||
public long getQuarantinedCount() { return quarantined.sum(); }
|
||||
public long getTerminalRecordedCount() { return terminalRecorded.sum(); }
|
||||
public long getIngressPointTimeAssignedCount() { return pointTimeAllocator.getAssignedCount(); }
|
||||
@@ -495,6 +515,7 @@ public class SyslogWalService {
|
||||
pending.getSequence(),
|
||||
pending.getSchemaVersion(),
|
||||
durableAt);
|
||||
releasePendingBytes(pending.getRawBytes());
|
||||
pending.getDurableFuture().complete(receipt);
|
||||
}
|
||||
pendingForce.clear();
|
||||
@@ -538,12 +559,14 @@ public class SyslogWalService {
|
||||
|
||||
private void completePendingExceptionally(Throwable failure) {
|
||||
for (PendingWalAppend pending : pendingForce) {
|
||||
releasePendingBytes(pending.getRawBytes());
|
||||
pending.getDurableFuture().completeExceptionally(failure);
|
||||
}
|
||||
pendingForce.clear();
|
||||
if (appendQueue == null) return;
|
||||
PendingWalAppend pending;
|
||||
while ((pending = appendQueue.poll()) != null) {
|
||||
releasePendingBytes(pending.getRawBytes());
|
||||
pending.getDurableFuture().completeExceptionally(failure);
|
||||
}
|
||||
}
|
||||
@@ -552,6 +575,38 @@ public class SyslogWalService {
|
||||
return Math.max(1, (int) Math.floor(properties.getSyslogWalQueueCapacity() * percent / 100.0));
|
||||
}
|
||||
|
||||
private long percentageBytes(int percent) {
|
||||
long capacity = properties.getSyslogWalMaxPendingBytes();
|
||||
return capacity <= 0L ? 0L : Math.max(1L, capacity * percent / 100L);
|
||||
}
|
||||
|
||||
private boolean reservePendingBytes(long bytes) {
|
||||
long capacity = properties.getSyslogWalMaxPendingBytes();
|
||||
if (capacity <= 0L) {
|
||||
pendingAppendBytes.addAndGet(bytes);
|
||||
return true;
|
||||
}
|
||||
while (true) {
|
||||
long current = pendingAppendBytes.get();
|
||||
// 单条硬上限显式关闭时，允许一个超过预算的事件独占通道，避免永久 BACKPRESSURE。
|
||||
if (bytes > capacity) {
|
||||
if (current != 0L) return false;
|
||||
if (pendingAppendBytes.compareAndSet(0L, bytes)) return true;
|
||||
continue;
|
||||
}
|
||||
if (bytes > capacity - current) return false;
|
||||
if (pendingAppendBytes.compareAndSet(current, current + bytes)) return true;
|
||||
}
|
||||
}
|
||||
|
||||
private void releasePendingBytes(long bytes) {
|
||||
long remaining = pendingAppendBytes.addAndGet(-bytes);
|
||||
if (remaining < 0L) {
|
||||
pendingAppendBytes.set(0L);
|
||||
throw new IllegalStateException("WAL pending 字节计数出现负值");
|
||||
}
|
||||
}
|
||||
|
||||
private void validateConfiguration() {
|
||||
if (!properties.isSyslogWalEnabled()) {
|
||||
throw new IllegalArgumentException("syslog.wal.enabled=false 会破坏可靠接入边界，正式链路禁止关闭 WAL");
|
||||
@@ -563,6 +618,20 @@ public class SyslogWalService {
|
||||
|| properties.getSyslogWalCheckpointFlushIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("WAL 队列、刷盘、分段、游标和停机参数必须大于 0");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() < 0L
|
||||
|| properties.getSyslogLargeMessageThresholdBytes() < 0L
|
||||
|| properties.getSyslogWalMaxPendingBytes() < 0L) {
|
||||
throw new IllegalArgumentException("消息上限、大日志阈值和 WAL 字节预算不能小于 0");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& properties.getSyslogWalMaxPendingBytes() > 0L
|
||||
&& properties.getSyslogMessageMaxBytes() > properties.getSyslogWalMaxPendingBytes()) {
|
||||
throw new IllegalArgumentException("单条消息上限不能大于 WAL pending 字节预算");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& properties.getSyslogLargeMessageThresholdBytes() > properties.getSyslogMessageMaxBytes()) {
|
||||
throw new IllegalArgumentException("大日志分流阈值不能大于单条消息硬上限");
|
||||
}
|
||||
if (properties.getSyslogWalRetryIntervalMs() <= 0
|
||||
|| properties.getSyslogWalDiskRetryIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("WAL 队列和磁盘重试间隔必须大于 0");
|
||||
|
||||
Reference in New Issue
Block a user