feat(syslog): 优化大日志通道并增加异常连续消费兜底
- 128 KiB 及以上日志默认进入大日志 Topic - InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存 - 截断内容追加 HB_XDR_TRUNCATED 标记 - InfluxDB 写入失败时继续执行规则解析和达梦写入 - 未知或系统性异常优先写入普通/大日志 DLT - DLT 写入失败时记录特殊告警并提交 offset，避免阻塞后续消息 - DLT 失败等待时间默认限制为 1 秒 - 增加连续性放行、DLT 成功及失败监控指标 - 更新 Docker Compose、v3 升级文档及相关测试
This commit is contained in:
王绍宏
@@ -415,6 +415,8 @@ public class ApiPullRunner {
|
||||
while (true) {
|
||||
AppendAttempt attempt = ingress.tryAppend(event);
|
||||
if (attempt.getStatus() == AdmissionStatus.ACCEPTED) return attempt.getDurableStage();
|
||||
if (attempt.getStatus() == AdmissionStatus.OVERSIZED)
|
||||
throw new AdapterException("MESSAGE_TOO_LARGE", "设备事件超过单条消息硬上限");
|
||||
if (attempt.getStatus() == AdmissionStatus.UNAVAILABLE)
|
||||
throw new AdapterException("WAL_UNAVAILABLE", "WAL 当前不可用");
|
||||
if (System.nanoTime() >= deadline)
|
||||
|
||||
+20
-1
@@ -19,9 +19,17 @@ public class AppProperties {
|
||||
@Value("${syslog.udp.port:515}")
|
||||
private int syslogUdpPort;
|
||||
|
||||
@Value("${syslog.max.frame.length:65536}")
|
||||
@Value("${syslog.max.frame.length:6291456}")
|
||||
private int syslogMaxFrameLength;
|
||||
|
||||
/** 所有接入方式共用的单条原文字节硬上限；0 表示不限制。 */
|
||||
@Value("${syslog.message.max-bytes:6291456}")
|
||||
private long syslogMessageMaxBytes;
|
||||
|
||||
/** 超过该值投递大日志 Topic；0 表示关闭分流。 */
|
||||
@Value("${syslog.large-message.threshold-bytes:262144}")
|
||||
private long syslogLargeMessageThresholdBytes;
|
||||
|
||||
@Value("${syslog.buffer.size:1024}")
|
||||
private int syslogBufferSize;
|
||||
|
||||
@@ -50,6 +58,10 @@ public class AppProperties {
|
||||
@Value("${syslog.wal.queue-capacity:10000}")
|
||||
private int syslogWalQueueCapacity;
|
||||
|
||||
/** 队列与待 force 记录共用的内存字节预算；0 表示不限制。 */
|
||||
@Value("${syslog.wal.max-pending-bytes:268435456}")
|
||||
private long syslogWalMaxPendingBytes;
|
||||
|
||||
@Value("${syslog.wal.high-watermark-percent:70}")
|
||||
private int syslogWalHighWatermarkPercent;
|
||||
|
||||
@@ -114,6 +126,10 @@ public class AppProperties {
|
||||
@Value("${syslog.dispatch.max-inflight:5000}")
|
||||
private int syslogDispatchMaxInflight;
|
||||
|
||||
/** Dispatcher 正文在途字节预算；0 表示不限制。 */
|
||||
@Value("${syslog.dispatch.max-inflight-bytes:268435456}")
|
||||
private long syslogDispatchMaxInflightBytes;
|
||||
|
||||
@Value("${syslog.dispatch.retry-backoff-ms:1000}")
|
||||
private long syslogDispatchRetryBackoffMs;
|
||||
|
||||
@@ -166,6 +182,9 @@ public class AppProperties {
|
||||
@Value("${spring.kafka.producer.topic:agent-syslog-topic}")
|
||||
private String kafkaProducerTopic;
|
||||
|
||||
@Value("${spring.kafka.producer.large-topic:agent-syslog-topic.large}")
|
||||
private String kafkaProducerLargeTopic;
|
||||
|
||||
@Value("${spring.kafka.producer.ack-timeout-ms:130000}")
|
||||
private long kafkaProducerAckTimeoutMs;
|
||||
|
||||
|
||||
+91
-8
@@ -48,9 +48,11 @@ public class SyslogWalDispatcher {
|
||||
private final LongAdder kafkaAcked = new LongAdder();
|
||||
private final LongAdder terminalFiltered = new LongAdder();
|
||||
private final LongAdder retryCount = new LongAdder();
|
||||
private final LongAdder largeDispatched = new LongAdder();
|
||||
private final AtomicLong inflight = new AtomicLong();
|
||||
|
||||
private Semaphore permits;
|
||||
private ByteBudget byteBudget;
|
||||
private ExecutorService workers;
|
||||
private ScheduledExecutorService scheduler;
|
||||
private Thread readerThread;
|
||||
@@ -72,6 +74,7 @@ public class SyslogWalDispatcher {
|
||||
public void start() throws Exception {
|
||||
validateConfiguration();
|
||||
permits = new Semaphore(properties.getSyslogDispatchMaxInflight());
|
||||
byteBudget = new ByteBudget(properties.getSyslogDispatchMaxInflightBytes());
|
||||
workers = Executors.newFixedThreadPool(properties.getSyslogDispatchWorkerThreads(),
|
||||
namedFactory("syslog-dispatch-worker"));
|
||||
scheduler = Executors.newScheduledThreadPool(2, namedFactory("syslog-dispatch-scheduler"));
|
||||
@@ -85,8 +88,9 @@ public class SyslogWalDispatcher {
|
||||
scheduler.scheduleWithFixedDelay(this::logStatus,
|
||||
properties.getSyslogStatusLogIntervalMs(), properties.getSyslogStatusLogIntervalMs(),
|
||||
TimeUnit.MILLISECONDS);
|
||||
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, checkpointSequence={}",
|
||||
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, maxInflightBytes={}, checkpointSequence={}",
|
||||
properties.getSyslogDispatchWorkerThreads(), properties.getSyslogDispatchMaxInflight(),
|
||||
properties.getSyslogDispatchMaxInflightBytes(),
|
||||
walService.getCheckpointSequence());
|
||||
}
|
||||
|
||||
@@ -99,7 +103,11 @@ public class SyslogWalDispatcher {
|
||||
permits.release();
|
||||
continue;
|
||||
}
|
||||
record.setDispatchPayloadBytes(Math.max(1L,
|
||||
Math.max(record.getRawLength(), utf8Length(record.getRawMessage()))));
|
||||
byteBudget.acquire(record.getDispatchPayloadBytes());
|
||||
if (record.getSequence() != nextReadSequence) {
|
||||
byteBudget.release(record.getDispatchPayloadBytes());
|
||||
permits.release();
|
||||
throw new IllegalStateException("WAL sequence 不连续，拒绝越过: expected="
|
||||
+ nextReadSequence + ", actual=" + record.getSequence());
|
||||
@@ -158,8 +166,13 @@ public class SyslogWalDispatcher {
|
||||
eventTime = LocalDateTime.parse(walEventTime, WAL_TIME).format(HEADER_TIME);
|
||||
|
||||
// v2/v3 继续调用旧重载，确保升级前已落盘记录产生的 Kafka 消息完全不变。
|
||||
java.util.concurrent.CompletableFuture<kafkaProducer.DeliveryResult> deliveryFuture =
|
||||
record.getSchemaVersion() >= 4
|
||||
boolean large = properties.getSyslogLargeMessageThresholdBytes() > 0L
|
||||
&& record.getDispatchPayloadBytes() > properties.getSyslogLargeMessageThresholdBytes();
|
||||
java.util.concurrent.CompletableFuture<kafkaProducer.DeliveryResult> deliveryFuture = large
|
||||
? producer.messagePushLarge(
|
||||
resolution.getDevice(), record.getRawMessage(), receiveTime, eventTime,
|
||||
record.getEventId(), record.getPointTimeNs())
|
||||
: record.getSchemaVersion() >= 4
|
||||
? producer.messagePush(
|
||||
resolution.getDevice(),
|
||||
record.getRawMessage(),
|
||||
@@ -177,6 +190,7 @@ public class SyslogWalDispatcher {
|
||||
.whenComplete((delivery, failure) -> {
|
||||
if (failure == null) {
|
||||
kafkaAcked.increment();
|
||||
if (large) largeDispatched.increment();
|
||||
markComplete(record);
|
||||
} else {
|
||||
scheduleRetry(record, attempt, "KAFKA_ACK", unwrap(failure));
|
||||
@@ -206,7 +220,9 @@ public class SyslogWalDispatcher {
|
||||
private void markComplete(SyslogWalRecord record) {
|
||||
synchronized (completionMonitor) {
|
||||
if (record.getSequence() < nextCommitSequence) return;
|
||||
completedOutOfOrder.put(record.getSequence(), record);
|
||||
if (completedOutOfOrder.putIfAbsent(record.getSequence(), checkpointOnly(record)) != null) return;
|
||||
// 已 ACK 的正文不再占字节预算；条数许可仍要等连续 checkpoint，限制乱序完成表大小。
|
||||
byteBudget.release(record.getDispatchPayloadBytes());
|
||||
SyslogWalRecord latest = null;
|
||||
int released = 0;
|
||||
while (true) {
|
||||
@@ -224,16 +240,26 @@ public class SyslogWalDispatcher {
|
||||
}
|
||||
}
|
||||
|
||||
private SyslogWalRecord checkpointOnly(SyslogWalRecord source) {
|
||||
SyslogWalRecord value = new SyslogWalRecord();
|
||||
value.setSequence(source.getSequence());
|
||||
value.setSegment(source.getSegment());
|
||||
value.setStartOffset(source.getStartOffset());
|
||||
value.setEndOffset(source.getEndOffset());
|
||||
return value;
|
||||
}
|
||||
|
||||
private void logStatus() {
|
||||
if (!running.get()) return;
|
||||
log.info("Syslog WAL状态: networkReceived={}, appended={}, walPendingApprox={}, walQueue={}/{}, durableSequence={}, "
|
||||
+ "checkpointSequence={}, kafkaInflight={}, kafkaAcked={}, terminalFiltered={}, "
|
||||
log.info("Syslog WAL状态: networkReceived={}, appended={}, admissionOversized={}, walPendingApprox={}, walPendingBytes={}, walQueue={}/{}, durableSequence={}, "
|
||||
+ "checkpointSequence={}, kafkaInflight={}, kafkaInflightBytes={}, kafkaAcked={}, largeDispatched={}, terminalFiltered={}, "
|
||||
+ "retry={}, tcpPauseTotal={}, udpDropped={}, deviceCacheSize={}, deviceCacheHits={}, "
|
||||
+ "deviceCacheMisses={}",
|
||||
SyslogMessageHandler.getNetworkReceivedCount(), walService.getAppendedCount(),
|
||||
walService.getPendingRecordsApprox(),
|
||||
walService.getOversizedCount(), walService.getPendingRecordsApprox(), walService.getPendingAppendBytes(),
|
||||
walService.getQueueSize(), walService.getQueueCapacity(), walService.getDurableSequence(),
|
||||
walService.getCheckpointSequence(), inflight.get(), kafkaAcked.sum(), terminalFiltered.sum(),
|
||||
walService.getCheckpointSequence(), inflight.get(), byteBudget == null ? 0L : byteBudget.used(),
|
||||
kafkaAcked.sum(), largeDispatched.sum(), terminalFiltered.sum(),
|
||||
retryCount.sum(), SyslogMessageHandler.getTcpPauseCount(),
|
||||
SyslogMessageHandler.getUdpDroppedCount(), deviceResolver.getCacheSize(),
|
||||
deviceResolver.getHitCount(), deviceResolver.getMissCount());
|
||||
@@ -246,6 +272,23 @@ public class SyslogWalDispatcher {
|
||||
return failure;
|
||||
}
|
||||
|
||||
/** 不分配临时 byte[] 的 UTF-8 长度估算；非法孤立代理项按 3 字节保守计算。 */
|
||||
static long utf8Length(String value) {
|
||||
if (value == null || value.isEmpty()) return 0L;
|
||||
long bytes = 0L;
|
||||
for (int index = 0; index < value.length(); index++) {
|
||||
char current = value.charAt(index);
|
||||
if (current <= 0x7f) bytes++;
|
||||
else if (current <= 0x7ff) bytes += 2L;
|
||||
else if (Character.isHighSurrogate(current) && index + 1 < value.length()
|
||||
&& Character.isLowSurrogate(value.charAt(index + 1))) {
|
||||
bytes += 4L;
|
||||
index++;
|
||||
} else bytes += 3L;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private ThreadFactory namedFactory(String prefix) {
|
||||
AtomicLong sequence = new AtomicLong();
|
||||
return runnable -> {
|
||||
@@ -258,18 +301,58 @@ public class SyslogWalDispatcher {
|
||||
private void validateConfiguration() {
|
||||
if (properties.getSyslogDispatchWorkerThreads() <= 0
|
||||
|| properties.getSyslogDispatchMaxInflight() <= 0
|
||||
|| properties.getSyslogDispatchMaxInflightBytes() < 0L
|
||||
|| properties.getSyslogDispatchRetryBackoffMs() <= 0
|
||||
|| properties.getSyslogDispatchShutdownTimeoutMs() <= 0
|
||||
|| properties.getSyslogDeviceContextCacheTtlMs() <= 0
|
||||
|| properties.getSyslogStatusLogIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("Dispatcher 线程、在途、重试、缓存和状态日志参数必须大于 0");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& properties.getSyslogDispatchMaxInflightBytes() > 0L
|
||||
&& properties.getSyslogMessageMaxBytes() > properties.getSyslogDispatchMaxInflightBytes()) {
|
||||
throw new IllegalArgumentException("单条消息上限不能大于 Dispatcher 在途字节预算");
|
||||
}
|
||||
}
|
||||
|
||||
public long getInflight() { return inflight.get(); }
|
||||
public long getKafkaAcked() { return kafkaAcked.sum(); }
|
||||
public long getTerminalFiltered() { return terminalFiltered.sum(); }
|
||||
public long getRetryCount() { return retryCount.sum(); }
|
||||
public long getLargeDispatched() { return largeDispatched.sum(); }
|
||||
public long getInflightBytes() { return byteBudget == null ? 0L : byteBudget.used(); }
|
||||
|
||||
/** 同时保留条数 Semaphore 和字节预算，避免少量大消息占满堆。 */
|
||||
static final class ByteBudget {
|
||||
private final long capacity;
|
||||
private long used;
|
||||
|
||||
ByteBudget(long capacity) {
|
||||
if (capacity < 0L) throw new IllegalArgumentException("capacity 不能小于 0");
|
||||
this.capacity = capacity;
|
||||
}
|
||||
|
||||
synchronized void acquire(long bytes) throws InterruptedException {
|
||||
if (bytes <= 0L) throw new IllegalArgumentException("bytes 必须大于 0");
|
||||
if (capacity == 0L) { used += bytes; return; }
|
||||
if (bytes > capacity) {
|
||||
// 硬上限关闭时仍保证进展：超预算单条只能独占，不允许与其他正文并存。
|
||||
while (used != 0L) wait();
|
||||
used = bytes;
|
||||
return;
|
||||
}
|
||||
while (used > capacity - bytes) wait();
|
||||
used += bytes;
|
||||
}
|
||||
|
||||
synchronized void release(long bytes) {
|
||||
used -= bytes;
|
||||
if (used < 0L) throw new IllegalStateException("Dispatcher 字节预算释放越界");
|
||||
notifyAll();
|
||||
}
|
||||
|
||||
synchronized long used() { return used; }
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
|
||||
@@ -4,5 +4,6 @@ package com.haobang.wal;
|
||||
public enum AdmissionStatus {
|
||||
ACCEPTED,
|
||||
BACKPRESSURE,
|
||||
OVERSIZED,
|
||||
UNAVAILABLE
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ public final class AppendAttempt {
|
||||
|
||||
public static AppendAttempt rejected(AdmissionStatus status) {
|
||||
if (status == null || status == AdmissionStatus.ACCEPTED) {
|
||||
throw new IllegalArgumentException("rejected 只能使用 BACKPRESSURE 或 UNAVAILABLE");
|
||||
throw new IllegalArgumentException("rejected 不能使用 ACCEPTED");
|
||||
}
|
||||
return new AppendAttempt(status, null);
|
||||
}
|
||||
|
||||
+5
-1
@@ -5,18 +5,22 @@ import java.util.concurrent.CompletableFuture;
|
||||
/** WAL 写线程队列中的事件及其待完成 durable future。 sequence/schemaVersion 由唯一写线程赋值，生产者线程不得提前推断 WAL 位置。 */
|
||||
final class PendingWalAppend {
|
||||
private final IngressEvent event;
|
||||
private final int rawBytes;
|
||||
private final CompletableFuture<DurableReceipt> durableFuture = new CompletableFuture<>();
|
||||
private long sequence;
|
||||
private int schemaVersion;
|
||||
|
||||
PendingWalAppend(IngressEvent event) {
|
||||
PendingWalAppend(IngressEvent event, int rawBytes) {
|
||||
this.event = event;
|
||||
this.rawBytes = rawBytes;
|
||||
}
|
||||
|
||||
IngressEvent getEvent() {
|
||||
return event;
|
||||
}
|
||||
|
||||
int getRawBytes() { return rawBytes; }
|
||||
|
||||
CompletableFuture<DurableReceipt> getDurableFuture() {
|
||||
return durableFuture;
|
||||
}
|
||||
|
||||
+4
@@ -112,6 +112,10 @@ public class SyslogLegacyArchiveImporter {
|
||||
while (true) {
|
||||
SyslogWalService.AdmissionResult result = walService.tryAppend(event);
|
||||
if (result == SyslogWalService.AdmissionResult.ACCEPTED) return;
|
||||
if (result == SyslogWalService.AdmissionResult.OVERSIZED) {
|
||||
throw new IllegalStateException("旧归档事件超过当前单条消息硬上限，拒绝永久重试: eventId="
|
||||
+ event.getEventId());
|
||||
}
|
||||
if (result == SyslogWalService.AdmissionResult.UNAVAILABLE) {
|
||||
throw new IllegalStateException("WAL 不可用，无法导入旧归档 eventId=" + event.getEventId(),
|
||||
walService.getFatalFailure());
|
||||
|
||||
@@ -87,6 +87,8 @@ public class SyslogWalRecord {
|
||||
@JsonIgnore private long endOffset;
|
||||
@JsonIgnore private boolean quarantinedInvalid;
|
||||
@JsonIgnore private String validationError;
|
||||
/** Dispatcher 对正文使用的 UTF-8/源编码较大值；只存在内存中，不改变 WAL 协议。 */
|
||||
@JsonIgnore private long dispatchPayloadBytes;
|
||||
|
||||
public int getSchemaVersion() {
|
||||
return schemaVersion;
|
||||
@@ -232,6 +234,14 @@ public class SyslogWalRecord {
|
||||
this.rawLength = rawLength;
|
||||
}
|
||||
|
||||
public long getDispatchPayloadBytes() {
|
||||
return dispatchPayloadBytes;
|
||||
}
|
||||
|
||||
public void setDispatchPayloadBytes(long dispatchPayloadBytes) {
|
||||
this.dispatchPayloadBytes = dispatchPayloadBytes;
|
||||
}
|
||||
|
||||
public String getRawSha256() {
|
||||
return rawSha256;
|
||||
}
|
||||
|
||||
+73
-4
@@ -64,7 +64,7 @@ public class SyslogWalService {
|
||||
private static final String SEGMENT_GLOB = "syslog-wal-*.jsonl";
|
||||
static final long COMPLETED_CLEANUP_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(30L);
|
||||
|
||||
public enum AdmissionResult { ACCEPTED, BACKPRESSURE, UNAVAILABLE }
|
||||
public enum AdmissionResult { ACCEPTED, BACKPRESSURE, OVERSIZED, UNAVAILABLE }
|
||||
|
||||
private final AppProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
@@ -76,6 +76,8 @@ public class SyslogWalService {
|
||||
private final AtomicLong durableSequence = new AtomicLong();
|
||||
private final LongAdder appended = new LongAdder();
|
||||
private final LongAdder admissionBackpressure = new LongAdder();
|
||||
private final LongAdder admissionOversized = new LongAdder();
|
||||
private final AtomicLong pendingAppendBytes = new AtomicLong();
|
||||
private final LongAdder quarantined = new LongAdder();
|
||||
private final LongAdder terminalRecorded = new LongAdder();
|
||||
private final Object durableMonitor = new Object();
|
||||
@@ -211,6 +213,7 @@ public class SyslogWalService {
|
||||
AppendAttempt attempt = tryAppendUnified(toUnified(event));
|
||||
if (attempt.getStatus() == AdmissionStatus.ACCEPTED) return AdmissionResult.ACCEPTED;
|
||||
if (attempt.getStatus() == AdmissionStatus.BACKPRESSURE) return AdmissionResult.BACKPRESSURE;
|
||||
if (attempt.getStatus() == AdmissionStatus.OVERSIZED) return AdmissionResult.OVERSIZED;
|
||||
return AdmissionResult.UNAVAILABLE;
|
||||
}
|
||||
|
||||
@@ -231,8 +234,19 @@ public class SyslogWalService {
|
||||
if (!running || fatalFailure != null || diskState == DiskState.CRITICAL) {
|
||||
return AppendAttempt.rejected(AdmissionStatus.UNAVAILABLE);
|
||||
}
|
||||
PendingWalAppend pending = new PendingWalAppend(event);
|
||||
int rawBytes = event.getRawMessage().getBytes(event.getCharset()).length;
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& rawBytes > properties.getSyslogMessageMaxBytes()) {
|
||||
admissionOversized.increment();
|
||||
return AppendAttempt.rejected(AdmissionStatus.OVERSIZED);
|
||||
}
|
||||
if (!reservePendingBytes(rawBytes)) {
|
||||
admissionBackpressure.increment();
|
||||
return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE);
|
||||
}
|
||||
PendingWalAppend pending = new PendingWalAppend(event, rawBytes);
|
||||
if (!appendQueue.offer(pending)) {
|
||||
releasePendingBytes(rawBytes);
|
||||
admissionBackpressure.increment();
|
||||
return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE);
|
||||
}
|
||||
@@ -243,14 +257,18 @@ public class SyslogWalService {
|
||||
refreshDiskState(false);
|
||||
if (fatalFailure != null || diskState != DiskState.NORMAL) return true;
|
||||
int high = percentageCount(properties.getSyslogWalHighWatermarkPercent());
|
||||
return appendQueue != null && appendQueue.size() >= high;
|
||||
long byteHigh = percentageBytes(properties.getSyslogWalHighWatermarkPercent());
|
||||
return appendQueue != null && (appendQueue.size() >= high
|
||||
|| (byteHigh > 0L && pendingAppendBytes.get() >= byteHigh));
|
||||
}
|
||||
|
||||
public boolean canResumeTcp() {
|
||||
refreshDiskState(false);
|
||||
if (fatalFailure != null || diskState != DiskState.NORMAL) return false;
|
||||
int low = percentageCount(properties.getSyslogWalLowWatermarkPercent());
|
||||
return appendQueue != null && appendQueue.size() <= low;
|
||||
long byteLow = percentageBytes(properties.getSyslogWalLowWatermarkPercent());
|
||||
return appendQueue != null && appendQueue.size() <= low
|
||||
&& (byteLow == 0L || pendingAppendBytes.get() <= byteLow);
|
||||
}
|
||||
|
||||
public long getRetryIntervalMs() {
|
||||
@@ -263,6 +281,8 @@ public class SyslogWalService {
|
||||
public int getQueueCapacity() { return properties.getSyslogWalQueueCapacity(); }
|
||||
public long getAppendedCount() { return appended.sum(); }
|
||||
public long getBackpressureCount() { return admissionBackpressure.sum(); }
|
||||
public long getOversizedCount() { return admissionOversized.sum(); }
|
||||
public long getPendingAppendBytes() { return pendingAppendBytes.get(); }
|
||||
public long getQuarantinedCount() { return quarantined.sum(); }
|
||||
public long getTerminalRecordedCount() { return terminalRecorded.sum(); }
|
||||
public long getIngressPointTimeAssignedCount() { return pointTimeAllocator.getAssignedCount(); }
|
||||
@@ -495,6 +515,7 @@ public class SyslogWalService {
|
||||
pending.getSequence(),
|
||||
pending.getSchemaVersion(),
|
||||
durableAt);
|
||||
releasePendingBytes(pending.getRawBytes());
|
||||
pending.getDurableFuture().complete(receipt);
|
||||
}
|
||||
pendingForce.clear();
|
||||
@@ -538,12 +559,14 @@ public class SyslogWalService {
|
||||
|
||||
private void completePendingExceptionally(Throwable failure) {
|
||||
for (PendingWalAppend pending : pendingForce) {
|
||||
releasePendingBytes(pending.getRawBytes());
|
||||
pending.getDurableFuture().completeExceptionally(failure);
|
||||
}
|
||||
pendingForce.clear();
|
||||
if (appendQueue == null) return;
|
||||
PendingWalAppend pending;
|
||||
while ((pending = appendQueue.poll()) != null) {
|
||||
releasePendingBytes(pending.getRawBytes());
|
||||
pending.getDurableFuture().completeExceptionally(failure);
|
||||
}
|
||||
}
|
||||
@@ -552,6 +575,38 @@ public class SyslogWalService {
|
||||
return Math.max(1, (int) Math.floor(properties.getSyslogWalQueueCapacity() * percent / 100.0));
|
||||
}
|
||||
|
||||
private long percentageBytes(int percent) {
|
||||
long capacity = properties.getSyslogWalMaxPendingBytes();
|
||||
return capacity <= 0L ? 0L : Math.max(1L, capacity * percent / 100L);
|
||||
}
|
||||
|
||||
private boolean reservePendingBytes(long bytes) {
|
||||
long capacity = properties.getSyslogWalMaxPendingBytes();
|
||||
if (capacity <= 0L) {
|
||||
pendingAppendBytes.addAndGet(bytes);
|
||||
return true;
|
||||
}
|
||||
while (true) {
|
||||
long current = pendingAppendBytes.get();
|
||||
// 单条硬上限显式关闭时，允许一个超过预算的事件独占通道，避免永久 BACKPRESSURE。
|
||||
if (bytes > capacity) {
|
||||
if (current != 0L) return false;
|
||||
if (pendingAppendBytes.compareAndSet(0L, bytes)) return true;
|
||||
continue;
|
||||
}
|
||||
if (bytes > capacity - current) return false;
|
||||
if (pendingAppendBytes.compareAndSet(current, current + bytes)) return true;
|
||||
}
|
||||
}
|
||||
|
||||
private void releasePendingBytes(long bytes) {
|
||||
long remaining = pendingAppendBytes.addAndGet(-bytes);
|
||||
if (remaining < 0L) {
|
||||
pendingAppendBytes.set(0L);
|
||||
throw new IllegalStateException("WAL pending 字节计数出现负值");
|
||||
}
|
||||
}
|
||||
|
||||
private void validateConfiguration() {
|
||||
if (!properties.isSyslogWalEnabled()) {
|
||||
throw new IllegalArgumentException("syslog.wal.enabled=false 会破坏可靠接入边界，正式链路禁止关闭 WAL");
|
||||
@@ -563,6 +618,20 @@ public class SyslogWalService {
|
||||
|| properties.getSyslogWalCheckpointFlushIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("WAL 队列、刷盘、分段、游标和停机参数必须大于 0");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() < 0L
|
||||
|| properties.getSyslogLargeMessageThresholdBytes() < 0L
|
||||
|| properties.getSyslogWalMaxPendingBytes() < 0L) {
|
||||
throw new IllegalArgumentException("消息上限、大日志阈值和 WAL 字节预算不能小于 0");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& properties.getSyslogWalMaxPendingBytes() > 0L
|
||||
&& properties.getSyslogMessageMaxBytes() > properties.getSyslogWalMaxPendingBytes()) {
|
||||
throw new IllegalArgumentException("单条消息上限不能大于 WAL pending 字节预算");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& properties.getSyslogLargeMessageThresholdBytes() > properties.getSyslogMessageMaxBytes()) {
|
||||
throw new IllegalArgumentException("大日志分流阈值不能大于单条消息硬上限");
|
||||
}
|
||||
if (properties.getSyslogWalRetryIntervalMs() <= 0
|
||||
|| properties.getSyslogWalDiskRetryIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("WAL 队列和磁盘重试间隔必须大于 0");
|
||||
|
||||
+5
@@ -29,13 +29,16 @@ public class SyslogWalStatusController {
|
||||
public Map<String, Object> status() {
|
||||
Map<String, Object> value = new LinkedHashMap<>();
|
||||
value.put("network_received", SyslogMessageHandler.getNetworkReceivedCount());
|
||||
value.put("network_oversized_dropped", SyslogMessageHandler.getOversizedDroppedCount());
|
||||
value.put("wal_appended", wal.getAppendedCount());
|
||||
value.put("wal_queue_size", wal.getQueueSize());
|
||||
value.put("wal_queue_capacity", wal.getQueueCapacity());
|
||||
value.put("wal_pending_bytes", wal.getPendingAppendBytes());
|
||||
value.put("wal_durable_sequence", wal.getDurableSequence());
|
||||
value.put("wal_checkpoint_sequence", wal.getCheckpointSequence());
|
||||
value.put("wal_pending_approx", wal.getPendingRecordsApprox());
|
||||
value.put("wal_admission_backpressure", wal.getBackpressureCount());
|
||||
value.put("wal_admission_oversized", wal.getOversizedCount());
|
||||
value.put("wal_terminal_recorded", wal.getTerminalRecordedCount());
|
||||
value.put("wal_quarantined", wal.getQuarantinedCount());
|
||||
Map<String, Long> pointTimeAssigned = new LinkedHashMap<>();
|
||||
@@ -49,7 +52,9 @@ public class SyslogWalStatusController {
|
||||
value.put("wal_fatal_error", wal.getFatalFailure() == null
|
||||
? null : wal.getFatalFailure().toString());
|
||||
value.put("dispatch_inflight", dispatcher.getInflight());
|
||||
value.put("dispatch_inflight_bytes", dispatcher.getInflightBytes());
|
||||
value.put("kafka_acked", dispatcher.getKafkaAcked());
|
||||
value.put("kafka_large_dispatched", dispatcher.getLargeDispatched());
|
||||
value.put("dispatch_terminal_filtered", dispatcher.getTerminalFiltered());
|
||||
value.put("dispatch_retry", dispatcher.getRetryCount());
|
||||
value.put("tcp_pause_total", SyslogMessageHandler.getTcpPauseCount());
|
||||
|
||||
Reference in New Issue
Block a user