feat(syslog): 优化大日志通道并增加异常连续消费兜底
- 128 KiB 及以上日志默认进入大日志 Topic - InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存 - 截断内容追加 HB_XDR_TRUNCATED 标记 - InfluxDB 写入失败时继续执行规则解析和达梦写入 - 未知或系统性异常优先写入普通/大日志 DLT - DLT 写入失败时记录特殊告警并提交 offset，避免阻塞后续消息 - DLT 失败等待时间默认限制为 1 秒 - 增加连续性放行、DLT 成功及失败监控指标 - 更新 Docker Compose、v3 升级文档及相关测试
This commit is contained in:
王绍宏
@@ -415,6 +415,8 @@ public class ApiPullRunner {
|
||||
while (true) {
|
||||
AppendAttempt attempt = ingress.tryAppend(event);
|
||||
if (attempt.getStatus() == AdmissionStatus.ACCEPTED) return attempt.getDurableStage();
|
||||
if (attempt.getStatus() == AdmissionStatus.OVERSIZED)
|
||||
throw new AdapterException("MESSAGE_TOO_LARGE", "设备事件超过单条消息硬上限");
|
||||
if (attempt.getStatus() == AdmissionStatus.UNAVAILABLE)
|
||||
throw new AdapterException("WAL_UNAVAILABLE", "WAL 当前不可用");
|
||||
if (System.nanoTime() >= deadline)
|
||||
|
||||
+20
-1
@@ -19,9 +19,17 @@ public class AppProperties {
|
||||
@Value("${syslog.udp.port:515}")
|
||||
private int syslogUdpPort;
|
||||
|
||||
@Value("${syslog.max.frame.length:65536}")
|
||||
@Value("${syslog.max.frame.length:6291456}")
|
||||
private int syslogMaxFrameLength;
|
||||
|
||||
/** 所有接入方式共用的单条原文字节硬上限；0 表示不限制。 */
|
||||
@Value("${syslog.message.max-bytes:6291456}")
|
||||
private long syslogMessageMaxBytes;
|
||||
|
||||
/** 超过该值投递大日志 Topic；0 表示关闭分流。 */
|
||||
@Value("${syslog.large-message.threshold-bytes:262144}")
|
||||
private long syslogLargeMessageThresholdBytes;
|
||||
|
||||
@Value("${syslog.buffer.size:1024}")
|
||||
private int syslogBufferSize;
|
||||
|
||||
@@ -50,6 +58,10 @@ public class AppProperties {
|
||||
@Value("${syslog.wal.queue-capacity:10000}")
|
||||
private int syslogWalQueueCapacity;
|
||||
|
||||
/** 队列与待 force 记录共用的内存字节预算；0 表示不限制。 */
|
||||
@Value("${syslog.wal.max-pending-bytes:268435456}")
|
||||
private long syslogWalMaxPendingBytes;
|
||||
|
||||
@Value("${syslog.wal.high-watermark-percent:70}")
|
||||
private int syslogWalHighWatermarkPercent;
|
||||
|
||||
@@ -114,6 +126,10 @@ public class AppProperties {
|
||||
@Value("${syslog.dispatch.max-inflight:5000}")
|
||||
private int syslogDispatchMaxInflight;
|
||||
|
||||
/** Dispatcher 正文在途字节预算；0 表示不限制。 */
|
||||
@Value("${syslog.dispatch.max-inflight-bytes:268435456}")
|
||||
private long syslogDispatchMaxInflightBytes;
|
||||
|
||||
@Value("${syslog.dispatch.retry-backoff-ms:1000}")
|
||||
private long syslogDispatchRetryBackoffMs;
|
||||
|
||||
@@ -166,6 +182,9 @@ public class AppProperties {
|
||||
@Value("${spring.kafka.producer.topic:agent-syslog-topic}")
|
||||
private String kafkaProducerTopic;
|
||||
|
||||
@Value("${spring.kafka.producer.large-topic:agent-syslog-topic.large}")
|
||||
private String kafkaProducerLargeTopic;
|
||||
|
||||
@Value("${spring.kafka.producer.ack-timeout-ms:130000}")
|
||||
private long kafkaProducerAckTimeoutMs;
|
||||
|
||||
|
||||
+91
-8
@@ -48,9 +48,11 @@ public class SyslogWalDispatcher {
|
||||
private final LongAdder kafkaAcked = new LongAdder();
|
||||
private final LongAdder terminalFiltered = new LongAdder();
|
||||
private final LongAdder retryCount = new LongAdder();
|
||||
private final LongAdder largeDispatched = new LongAdder();
|
||||
private final AtomicLong inflight = new AtomicLong();
|
||||
|
||||
private Semaphore permits;
|
||||
private ByteBudget byteBudget;
|
||||
private ExecutorService workers;
|
||||
private ScheduledExecutorService scheduler;
|
||||
private Thread readerThread;
|
||||
@@ -72,6 +74,7 @@ public class SyslogWalDispatcher {
|
||||
public void start() throws Exception {
|
||||
validateConfiguration();
|
||||
permits = new Semaphore(properties.getSyslogDispatchMaxInflight());
|
||||
byteBudget = new ByteBudget(properties.getSyslogDispatchMaxInflightBytes());
|
||||
workers = Executors.newFixedThreadPool(properties.getSyslogDispatchWorkerThreads(),
|
||||
namedFactory("syslog-dispatch-worker"));
|
||||
scheduler = Executors.newScheduledThreadPool(2, namedFactory("syslog-dispatch-scheduler"));
|
||||
@@ -85,8 +88,9 @@ public class SyslogWalDispatcher {
|
||||
scheduler.scheduleWithFixedDelay(this::logStatus,
|
||||
properties.getSyslogStatusLogIntervalMs(), properties.getSyslogStatusLogIntervalMs(),
|
||||
TimeUnit.MILLISECONDS);
|
||||
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, checkpointSequence={}",
|
||||
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, maxInflightBytes={}, checkpointSequence={}",
|
||||
properties.getSyslogDispatchWorkerThreads(), properties.getSyslogDispatchMaxInflight(),
|
||||
properties.getSyslogDispatchMaxInflightBytes(),
|
||||
walService.getCheckpointSequence());
|
||||
}
|
||||
|
||||
@@ -99,7 +103,11 @@ public class SyslogWalDispatcher {
|
||||
permits.release();
|
||||
continue;
|
||||
}
|
||||
record.setDispatchPayloadBytes(Math.max(1L,
|
||||
Math.max(record.getRawLength(), utf8Length(record.getRawMessage()))));
|
||||
byteBudget.acquire(record.getDispatchPayloadBytes());
|
||||
if (record.getSequence() != nextReadSequence) {
|
||||
byteBudget.release(record.getDispatchPayloadBytes());
|
||||
permits.release();
|
||||
throw new IllegalStateException("WAL sequence 不连续，拒绝越过: expected="
|
||||
+ nextReadSequence + ", actual=" + record.getSequence());
|
||||
@@ -158,8 +166,13 @@ public class SyslogWalDispatcher {
|
||||
eventTime = LocalDateTime.parse(walEventTime, WAL_TIME).format(HEADER_TIME);
|
||||
|
||||
// v2/v3 继续调用旧重载，确保升级前已落盘记录产生的 Kafka 消息完全不变。
|
||||
java.util.concurrent.CompletableFuture<kafkaProducer.DeliveryResult> deliveryFuture =
|
||||
record.getSchemaVersion() >= 4
|
||||
boolean large = properties.getSyslogLargeMessageThresholdBytes() > 0L
|
||||
&& record.getDispatchPayloadBytes() > properties.getSyslogLargeMessageThresholdBytes();
|
||||
java.util.concurrent.CompletableFuture<kafkaProducer.DeliveryResult> deliveryFuture = large
|
||||
? producer.messagePushLarge(
|
||||
resolution.getDevice(), record.getRawMessage(), receiveTime, eventTime,
|
||||
record.getEventId(), record.getPointTimeNs())
|
||||
: record.getSchemaVersion() >= 4
|
||||
? producer.messagePush(
|
||||
resolution.getDevice(),
|
||||
record.getRawMessage(),
|
||||
@@ -177,6 +190,7 @@ public class SyslogWalDispatcher {
|
||||
.whenComplete((delivery, failure) -> {
|
||||
if (failure == null) {
|
||||
kafkaAcked.increment();
|
||||
if (large) largeDispatched.increment();
|
||||
markComplete(record);
|
||||
} else {
|
||||
scheduleRetry(record, attempt, "KAFKA_ACK", unwrap(failure));
|
||||
@@ -206,7 +220,9 @@ public class SyslogWalDispatcher {
|
||||
private void markComplete(SyslogWalRecord record) {
|
||||
synchronized (completionMonitor) {
|
||||
if (record.getSequence() < nextCommitSequence) return;
|
||||
completedOutOfOrder.put(record.getSequence(), record);
|
||||
if (completedOutOfOrder.putIfAbsent(record.getSequence(), checkpointOnly(record)) != null) return;
|
||||
// 已 ACK 的正文不再占字节预算；条数许可仍要等连续 checkpoint，限制乱序完成表大小。
|
||||
byteBudget.release(record.getDispatchPayloadBytes());
|
||||
SyslogWalRecord latest = null;
|
||||
int released = 0;
|
||||
while (true) {
|
||||
@@ -224,16 +240,26 @@ public class SyslogWalDispatcher {
|
||||
}
|
||||
}
|
||||
|
||||
private SyslogWalRecord checkpointOnly(SyslogWalRecord source) {
|
||||
SyslogWalRecord value = new SyslogWalRecord();
|
||||
value.setSequence(source.getSequence());
|
||||
value.setSegment(source.getSegment());
|
||||
value.setStartOffset(source.getStartOffset());
|
||||
value.setEndOffset(source.getEndOffset());
|
||||
return value;
|
||||
}
|
||||
|
||||
private void logStatus() {
|
||||
if (!running.get()) return;
|
||||
log.info("Syslog WAL状态: networkReceived={}, appended={}, walPendingApprox={}, walQueue={}/{}, durableSequence={}, "
|
||||
+ "checkpointSequence={}, kafkaInflight={}, kafkaAcked={}, terminalFiltered={}, "
|
||||
log.info("Syslog WAL状态: networkReceived={}, appended={}, admissionOversized={}, walPendingApprox={}, walPendingBytes={}, walQueue={}/{}, durableSequence={}, "
|
||||
+ "checkpointSequence={}, kafkaInflight={}, kafkaInflightBytes={}, kafkaAcked={}, largeDispatched={}, terminalFiltered={}, "
|
||||
+ "retry={}, tcpPauseTotal={}, udpDropped={}, deviceCacheSize={}, deviceCacheHits={}, "
|
||||
+ "deviceCacheMisses={}",
|
||||
SyslogMessageHandler.getNetworkReceivedCount(), walService.getAppendedCount(),
|
||||
walService.getPendingRecordsApprox(),
|
||||
walService.getOversizedCount(), walService.getPendingRecordsApprox(), walService.getPendingAppendBytes(),
|
||||
walService.getQueueSize(), walService.getQueueCapacity(), walService.getDurableSequence(),
|
||||
walService.getCheckpointSequence(), inflight.get(), kafkaAcked.sum(), terminalFiltered.sum(),
|
||||
walService.getCheckpointSequence(), inflight.get(), byteBudget == null ? 0L : byteBudget.used(),
|
||||
kafkaAcked.sum(), largeDispatched.sum(), terminalFiltered.sum(),
|
||||
retryCount.sum(), SyslogMessageHandler.getTcpPauseCount(),
|
||||
SyslogMessageHandler.getUdpDroppedCount(), deviceResolver.getCacheSize(),
|
||||
deviceResolver.getHitCount(), deviceResolver.getMissCount());
|
||||
@@ -246,6 +272,23 @@ public class SyslogWalDispatcher {
|
||||
return failure;
|
||||
}
|
||||
|
||||
/** 不分配临时 byte[] 的 UTF-8 长度估算；非法孤立代理项按 3 字节保守计算。 */
|
||||
static long utf8Length(String value) {
|
||||
if (value == null || value.isEmpty()) return 0L;
|
||||
long bytes = 0L;
|
||||
for (int index = 0; index < value.length(); index++) {
|
||||
char current = value.charAt(index);
|
||||
if (current <= 0x7f) bytes++;
|
||||
else if (current <= 0x7ff) bytes += 2L;
|
||||
else if (Character.isHighSurrogate(current) && index + 1 < value.length()
|
||||
&& Character.isLowSurrogate(value.charAt(index + 1))) {
|
||||
bytes += 4L;
|
||||
index++;
|
||||
} else bytes += 3L;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private ThreadFactory namedFactory(String prefix) {
|
||||
AtomicLong sequence = new AtomicLong();
|
||||
return runnable -> {
|
||||
@@ -258,18 +301,58 @@ public class SyslogWalDispatcher {
|
||||
private void validateConfiguration() {
|
||||
if (properties.getSyslogDispatchWorkerThreads() <= 0
|
||||
|| properties.getSyslogDispatchMaxInflight() <= 0
|
||||
|| properties.getSyslogDispatchMaxInflightBytes() < 0L
|
||||
|| properties.getSyslogDispatchRetryBackoffMs() <= 0
|
||||
|| properties.getSyslogDispatchShutdownTimeoutMs() <= 0
|
||||
|| properties.getSyslogDeviceContextCacheTtlMs() <= 0
|
||||
|| properties.getSyslogStatusLogIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("Dispatcher 线程、在途、重试、缓存和状态日志参数必须大于 0");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& properties.getSyslogDispatchMaxInflightBytes() > 0L
|
||||
&& properties.getSyslogMessageMaxBytes() > properties.getSyslogDispatchMaxInflightBytes()) {
|
||||
throw new IllegalArgumentException("单条消息上限不能大于 Dispatcher 在途字节预算");
|
||||
}
|
||||
}
|
||||
|
||||
public long getInflight() { return inflight.get(); }
|
||||
public long getKafkaAcked() { return kafkaAcked.sum(); }
|
||||
public long getTerminalFiltered() { return terminalFiltered.sum(); }
|
||||
public long getRetryCount() { return retryCount.sum(); }
|
||||
public long getLargeDispatched() { return largeDispatched.sum(); }
|
||||
public long getInflightBytes() { return byteBudget == null ? 0L : byteBudget.used(); }
|
||||
|
||||
/** 同时保留条数 Semaphore 和字节预算，避免少量大消息占满堆。 */
|
||||
static final class ByteBudget {
|
||||
private final long capacity;
|
||||
private long used;
|
||||
|
||||
ByteBudget(long capacity) {
|
||||
if (capacity < 0L) throw new IllegalArgumentException("capacity 不能小于 0");
|
||||
this.capacity = capacity;
|
||||
}
|
||||
|
||||
synchronized void acquire(long bytes) throws InterruptedException {
|
||||
if (bytes <= 0L) throw new IllegalArgumentException("bytes 必须大于 0");
|
||||
if (capacity == 0L) { used += bytes; return; }
|
||||
if (bytes > capacity) {
|
||||
// 硬上限关闭时仍保证进展：超预算单条只能独占，不允许与其他正文并存。
|
||||
while (used != 0L) wait();
|
||||
used = bytes;
|
||||
return;
|
||||
}
|
||||
while (used > capacity - bytes) wait();
|
||||
used += bytes;
|
||||
}
|
||||
|
||||
synchronized void release(long bytes) {
|
||||
used -= bytes;
|
||||
if (used < 0L) throw new IllegalStateException("Dispatcher 字节预算释放越界");
|
||||
notifyAll();
|
||||
}
|
||||
|
||||
synchronized long used() { return used; }
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
|
||||
@@ -4,5 +4,6 @@ package com.haobang.wal;
|
||||
public enum AdmissionStatus {
|
||||
ACCEPTED,
|
||||
BACKPRESSURE,
|
||||
OVERSIZED,
|
||||
UNAVAILABLE
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ public final class AppendAttempt {
|
||||
|
||||
public static AppendAttempt rejected(AdmissionStatus status) {
|
||||
if (status == null || status == AdmissionStatus.ACCEPTED) {
|
||||
throw new IllegalArgumentException("rejected 只能使用 BACKPRESSURE 或 UNAVAILABLE");
|
||||
throw new IllegalArgumentException("rejected 不能使用 ACCEPTED");
|
||||
}
|
||||
return new AppendAttempt(status, null);
|
||||
}
|
||||
|
||||
+5
-1
@@ -5,18 +5,22 @@ import java.util.concurrent.CompletableFuture;
|
||||
/** WAL 写线程队列中的事件及其待完成 durable future。 sequence/schemaVersion 由唯一写线程赋值，生产者线程不得提前推断 WAL 位置。 */
|
||||
final class PendingWalAppend {
|
||||
private final IngressEvent event;
|
||||
private final int rawBytes;
|
||||
private final CompletableFuture<DurableReceipt> durableFuture = new CompletableFuture<>();
|
||||
private long sequence;
|
||||
private int schemaVersion;
|
||||
|
||||
PendingWalAppend(IngressEvent event) {
|
||||
PendingWalAppend(IngressEvent event, int rawBytes) {
|
||||
this.event = event;
|
||||
this.rawBytes = rawBytes;
|
||||
}
|
||||
|
||||
IngressEvent getEvent() {
|
||||
return event;
|
||||
}
|
||||
|
||||
int getRawBytes() { return rawBytes; }
|
||||
|
||||
CompletableFuture<DurableReceipt> getDurableFuture() {
|
||||
return durableFuture;
|
||||
}
|
||||
|
||||
+4
@@ -112,6 +112,10 @@ public class SyslogLegacyArchiveImporter {
|
||||
while (true) {
|
||||
SyslogWalService.AdmissionResult result = walService.tryAppend(event);
|
||||
if (result == SyslogWalService.AdmissionResult.ACCEPTED) return;
|
||||
if (result == SyslogWalService.AdmissionResult.OVERSIZED) {
|
||||
throw new IllegalStateException("旧归档事件超过当前单条消息硬上限，拒绝永久重试: eventId="
|
||||
+ event.getEventId());
|
||||
}
|
||||
if (result == SyslogWalService.AdmissionResult.UNAVAILABLE) {
|
||||
throw new IllegalStateException("WAL 不可用，无法导入旧归档 eventId=" + event.getEventId(),
|
||||
walService.getFatalFailure());
|
||||
|
||||
@@ -87,6 +87,8 @@ public class SyslogWalRecord {
|
||||
@JsonIgnore private long endOffset;
|
||||
@JsonIgnore private boolean quarantinedInvalid;
|
||||
@JsonIgnore private String validationError;
|
||||
/** Dispatcher 对正文使用的 UTF-8/源编码较大值；只存在内存中，不改变 WAL 协议。 */
|
||||
@JsonIgnore private long dispatchPayloadBytes;
|
||||
|
||||
public int getSchemaVersion() {
|
||||
return schemaVersion;
|
||||
@@ -232,6 +234,14 @@ public class SyslogWalRecord {
|
||||
this.rawLength = rawLength;
|
||||
}
|
||||
|
||||
public long getDispatchPayloadBytes() {
|
||||
return dispatchPayloadBytes;
|
||||
}
|
||||
|
||||
public void setDispatchPayloadBytes(long dispatchPayloadBytes) {
|
||||
this.dispatchPayloadBytes = dispatchPayloadBytes;
|
||||
}
|
||||
|
||||
public String getRawSha256() {
|
||||
return rawSha256;
|
||||
}
|
||||
|
||||
+73
-4
@@ -64,7 +64,7 @@ public class SyslogWalService {
|
||||
private static final String SEGMENT_GLOB = "syslog-wal-*.jsonl";
|
||||
static final long COMPLETED_CLEANUP_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(30L);
|
||||
|
||||
public enum AdmissionResult { ACCEPTED, BACKPRESSURE, UNAVAILABLE }
|
||||
public enum AdmissionResult { ACCEPTED, BACKPRESSURE, OVERSIZED, UNAVAILABLE }
|
||||
|
||||
private final AppProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
@@ -76,6 +76,8 @@ public class SyslogWalService {
|
||||
private final AtomicLong durableSequence = new AtomicLong();
|
||||
private final LongAdder appended = new LongAdder();
|
||||
private final LongAdder admissionBackpressure = new LongAdder();
|
||||
private final LongAdder admissionOversized = new LongAdder();
|
||||
private final AtomicLong pendingAppendBytes = new AtomicLong();
|
||||
private final LongAdder quarantined = new LongAdder();
|
||||
private final LongAdder terminalRecorded = new LongAdder();
|
||||
private final Object durableMonitor = new Object();
|
||||
@@ -211,6 +213,7 @@ public class SyslogWalService {
|
||||
AppendAttempt attempt = tryAppendUnified(toUnified(event));
|
||||
if (attempt.getStatus() == AdmissionStatus.ACCEPTED) return AdmissionResult.ACCEPTED;
|
||||
if (attempt.getStatus() == AdmissionStatus.BACKPRESSURE) return AdmissionResult.BACKPRESSURE;
|
||||
if (attempt.getStatus() == AdmissionStatus.OVERSIZED) return AdmissionResult.OVERSIZED;
|
||||
return AdmissionResult.UNAVAILABLE;
|
||||
}
|
||||
|
||||
@@ -231,8 +234,19 @@ public class SyslogWalService {
|
||||
if (!running || fatalFailure != null || diskState == DiskState.CRITICAL) {
|
||||
return AppendAttempt.rejected(AdmissionStatus.UNAVAILABLE);
|
||||
}
|
||||
PendingWalAppend pending = new PendingWalAppend(event);
|
||||
int rawBytes = event.getRawMessage().getBytes(event.getCharset()).length;
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& rawBytes > properties.getSyslogMessageMaxBytes()) {
|
||||
admissionOversized.increment();
|
||||
return AppendAttempt.rejected(AdmissionStatus.OVERSIZED);
|
||||
}
|
||||
if (!reservePendingBytes(rawBytes)) {
|
||||
admissionBackpressure.increment();
|
||||
return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE);
|
||||
}
|
||||
PendingWalAppend pending = new PendingWalAppend(event, rawBytes);
|
||||
if (!appendQueue.offer(pending)) {
|
||||
releasePendingBytes(rawBytes);
|
||||
admissionBackpressure.increment();
|
||||
return AppendAttempt.rejected(AdmissionStatus.BACKPRESSURE);
|
||||
}
|
||||
@@ -243,14 +257,18 @@ public class SyslogWalService {
|
||||
refreshDiskState(false);
|
||||
if (fatalFailure != null || diskState != DiskState.NORMAL) return true;
|
||||
int high = percentageCount(properties.getSyslogWalHighWatermarkPercent());
|
||||
return appendQueue != null && appendQueue.size() >= high;
|
||||
long byteHigh = percentageBytes(properties.getSyslogWalHighWatermarkPercent());
|
||||
return appendQueue != null && (appendQueue.size() >= high
|
||||
|| (byteHigh > 0L && pendingAppendBytes.get() >= byteHigh));
|
||||
}
|
||||
|
||||
public boolean canResumeTcp() {
|
||||
refreshDiskState(false);
|
||||
if (fatalFailure != null || diskState != DiskState.NORMAL) return false;
|
||||
int low = percentageCount(properties.getSyslogWalLowWatermarkPercent());
|
||||
return appendQueue != null && appendQueue.size() <= low;
|
||||
long byteLow = percentageBytes(properties.getSyslogWalLowWatermarkPercent());
|
||||
return appendQueue != null && appendQueue.size() <= low
|
||||
&& (byteLow == 0L || pendingAppendBytes.get() <= byteLow);
|
||||
}
|
||||
|
||||
public long getRetryIntervalMs() {
|
||||
@@ -263,6 +281,8 @@ public class SyslogWalService {
|
||||
public int getQueueCapacity() { return properties.getSyslogWalQueueCapacity(); }
|
||||
public long getAppendedCount() { return appended.sum(); }
|
||||
public long getBackpressureCount() { return admissionBackpressure.sum(); }
|
||||
public long getOversizedCount() { return admissionOversized.sum(); }
|
||||
public long getPendingAppendBytes() { return pendingAppendBytes.get(); }
|
||||
public long getQuarantinedCount() { return quarantined.sum(); }
|
||||
public long getTerminalRecordedCount() { return terminalRecorded.sum(); }
|
||||
public long getIngressPointTimeAssignedCount() { return pointTimeAllocator.getAssignedCount(); }
|
||||
@@ -495,6 +515,7 @@ public class SyslogWalService {
|
||||
pending.getSequence(),
|
||||
pending.getSchemaVersion(),
|
||||
durableAt);
|
||||
releasePendingBytes(pending.getRawBytes());
|
||||
pending.getDurableFuture().complete(receipt);
|
||||
}
|
||||
pendingForce.clear();
|
||||
@@ -538,12 +559,14 @@ public class SyslogWalService {
|
||||
|
||||
private void completePendingExceptionally(Throwable failure) {
|
||||
for (PendingWalAppend pending : pendingForce) {
|
||||
releasePendingBytes(pending.getRawBytes());
|
||||
pending.getDurableFuture().completeExceptionally(failure);
|
||||
}
|
||||
pendingForce.clear();
|
||||
if (appendQueue == null) return;
|
||||
PendingWalAppend pending;
|
||||
while ((pending = appendQueue.poll()) != null) {
|
||||
releasePendingBytes(pending.getRawBytes());
|
||||
pending.getDurableFuture().completeExceptionally(failure);
|
||||
}
|
||||
}
|
||||
@@ -552,6 +575,38 @@ public class SyslogWalService {
|
||||
return Math.max(1, (int) Math.floor(properties.getSyslogWalQueueCapacity() * percent / 100.0));
|
||||
}
|
||||
|
||||
private long percentageBytes(int percent) {
|
||||
long capacity = properties.getSyslogWalMaxPendingBytes();
|
||||
return capacity <= 0L ? 0L : Math.max(1L, capacity * percent / 100L);
|
||||
}
|
||||
|
||||
private boolean reservePendingBytes(long bytes) {
|
||||
long capacity = properties.getSyslogWalMaxPendingBytes();
|
||||
if (capacity <= 0L) {
|
||||
pendingAppendBytes.addAndGet(bytes);
|
||||
return true;
|
||||
}
|
||||
while (true) {
|
||||
long current = pendingAppendBytes.get();
|
||||
// 单条硬上限显式关闭时，允许一个超过预算的事件独占通道，避免永久 BACKPRESSURE。
|
||||
if (bytes > capacity) {
|
||||
if (current != 0L) return false;
|
||||
if (pendingAppendBytes.compareAndSet(0L, bytes)) return true;
|
||||
continue;
|
||||
}
|
||||
if (bytes > capacity - current) return false;
|
||||
if (pendingAppendBytes.compareAndSet(current, current + bytes)) return true;
|
||||
}
|
||||
}
|
||||
|
||||
private void releasePendingBytes(long bytes) {
|
||||
long remaining = pendingAppendBytes.addAndGet(-bytes);
|
||||
if (remaining < 0L) {
|
||||
pendingAppendBytes.set(0L);
|
||||
throw new IllegalStateException("WAL pending 字节计数出现负值");
|
||||
}
|
||||
}
|
||||
|
||||
private void validateConfiguration() {
|
||||
if (!properties.isSyslogWalEnabled()) {
|
||||
throw new IllegalArgumentException("syslog.wal.enabled=false 会破坏可靠接入边界，正式链路禁止关闭 WAL");
|
||||
@@ -563,6 +618,20 @@ public class SyslogWalService {
|
||||
|| properties.getSyslogWalCheckpointFlushIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("WAL 队列、刷盘、分段、游标和停机参数必须大于 0");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() < 0L
|
||||
|| properties.getSyslogLargeMessageThresholdBytes() < 0L
|
||||
|| properties.getSyslogWalMaxPendingBytes() < 0L) {
|
||||
throw new IllegalArgumentException("消息上限、大日志阈值和 WAL 字节预算不能小于 0");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& properties.getSyslogWalMaxPendingBytes() > 0L
|
||||
&& properties.getSyslogMessageMaxBytes() > properties.getSyslogWalMaxPendingBytes()) {
|
||||
throw new IllegalArgumentException("单条消息上限不能大于 WAL pending 字节预算");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& properties.getSyslogLargeMessageThresholdBytes() > properties.getSyslogMessageMaxBytes()) {
|
||||
throw new IllegalArgumentException("大日志分流阈值不能大于单条消息硬上限");
|
||||
}
|
||||
if (properties.getSyslogWalRetryIntervalMs() <= 0
|
||||
|| properties.getSyslogWalDiskRetryIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("WAL 队列和磁盘重试间隔必须大于 0");
|
||||
|
||||
+5
@@ -29,13 +29,16 @@ public class SyslogWalStatusController {
|
||||
public Map<String, Object> status() {
|
||||
Map<String, Object> value = new LinkedHashMap<>();
|
||||
value.put("network_received", SyslogMessageHandler.getNetworkReceivedCount());
|
||||
value.put("network_oversized_dropped", SyslogMessageHandler.getOversizedDroppedCount());
|
||||
value.put("wal_appended", wal.getAppendedCount());
|
||||
value.put("wal_queue_size", wal.getQueueSize());
|
||||
value.put("wal_queue_capacity", wal.getQueueCapacity());
|
||||
value.put("wal_pending_bytes", wal.getPendingAppendBytes());
|
||||
value.put("wal_durable_sequence", wal.getDurableSequence());
|
||||
value.put("wal_checkpoint_sequence", wal.getCheckpointSequence());
|
||||
value.put("wal_pending_approx", wal.getPendingRecordsApprox());
|
||||
value.put("wal_admission_backpressure", wal.getBackpressureCount());
|
||||
value.put("wal_admission_oversized", wal.getOversizedCount());
|
||||
value.put("wal_terminal_recorded", wal.getTerminalRecordedCount());
|
||||
value.put("wal_quarantined", wal.getQuarantinedCount());
|
||||
Map<String, Long> pointTimeAssigned = new LinkedHashMap<>();
|
||||
@@ -49,7 +52,9 @@ public class SyslogWalStatusController {
|
||||
value.put("wal_fatal_error", wal.getFatalFailure() == null
|
||||
? null : wal.getFatalFailure().toString());
|
||||
value.put("dispatch_inflight", dispatcher.getInflight());
|
||||
value.put("dispatch_inflight_bytes", dispatcher.getInflightBytes());
|
||||
value.put("kafka_acked", dispatcher.getKafkaAcked());
|
||||
value.put("kafka_large_dispatched", dispatcher.getLargeDispatched());
|
||||
value.put("dispatch_terminal_filtered", dispatcher.getTerminalFiltered());
|
||||
value.put("dispatch_retry", dispatcher.getRetryCount());
|
||||
value.put("tcp_pause_total", SyslogMessageHandler.getTcpPauseCount());
|
||||
|
||||
@@ -53,7 +53,7 @@ public class kafkaProducer {
|
||||
@Value("${spring.kafka.producer.compression-type:lz4}") String compressionType,
|
||||
@Value("${spring.kafka.producer.properties.max.block.ms:10000}") int maxBlockMs,
|
||||
@Value("${spring.kafka.producer.properties.buffer.memory:134217728}") long bufferMemory,
|
||||
@Value("${spring.kafka.producer.properties.max.request.size:8388608}") int maxRequestSize,
|
||||
@Value("${spring.kafka.producer.properties.max.request.size:33554432}") int maxRequestSize,
|
||||
@Value("${syslog.kafka.partition-key-strategy:event_id}") String partitionKeyStrategy) {
|
||||
this.appProperties = appProperties;
|
||||
this.partitionKeyStrategy = PartitionKeyStrategy.parse(partitionKeyStrategy);
|
||||
@@ -130,12 +130,34 @@ public class kafkaProducer {
|
||||
String eventTime,
|
||||
String eventId,
|
||||
Long pointTimeNs) {
|
||||
return messagePushToTopic(appProperties.getKafkaProducerTopic(), deviceDevice, syslog,
|
||||
receiveTime, eventTime, eventId, pointTimeNs);
|
||||
}
|
||||
|
||||
/** 大日志只改变目标 Topic，载荷协议、event_id 和 ACK 语义与普通通道完全一致。 */
|
||||
public CompletableFuture<DeliveryResult> messagePushLarge(DeviceDevice deviceDevice,
|
||||
String syslog,
|
||||
String receiveTime,
|
||||
String eventTime,
|
||||
String eventId,
|
||||
Long pointTimeNs) {
|
||||
return messagePushToTopic(appProperties.getKafkaProducerLargeTopic(), deviceDevice, syslog,
|
||||
receiveTime, eventTime, eventId, pointTimeNs);
|
||||
}
|
||||
|
||||
private CompletableFuture<DeliveryResult> messagePushToTopic(String topic,
|
||||
DeviceDevice deviceDevice,
|
||||
String syslog,
|
||||
String receiveTime,
|
||||
String eventTime,
|
||||
String eventId,
|
||||
Long pointTimeNs) {
|
||||
// [流程节点3.1] event_id 在接入边界生成并写入加密载荷，Retry/Replay 不再重新生成。
|
||||
String fullMessage = DeviceInfoUtil.getFullLogString(
|
||||
deviceDevice, syslog, receiveTime, eventTime, eventId, pointTimeNs, appProperties);
|
||||
// [流程节点3.2] Kafka 中保存携带 event_id 的完整密文，原始正文不写普通应用日志。
|
||||
String encrypted = Sm4Util.encryptCbc(fullMessage, appProperties.getSm4Key());
|
||||
ProducerRecord<String, String> record = buildRecord(deviceDevice.getId(), eventId, encrypted);
|
||||
ProducerRecord<String, String> record = buildRecord(topic, deviceDevice.getId(), eventId, encrypted);
|
||||
|
||||
CompletableFuture<DeliveryResult> result = new CompletableFuture<>();
|
||||
// [SYSLOG-ALARM][KPROD][已修复] 回调成功才表示 Broker 接管；send() 返回本身不再记录 PRODUCED。
|
||||
@@ -150,8 +172,13 @@ public class kafkaProducer {
|
||||
}
|
||||
|
||||
ProducerRecord<String, String> buildRecord(Integer deviceId, String eventId, String encrypted) {
|
||||
return buildRecord(appProperties.getKafkaProducerTopic(), deviceId, eventId, encrypted);
|
||||
}
|
||||
|
||||
ProducerRecord<String, String> buildRecord(String topic, Integer deviceId, String eventId, String encrypted) {
|
||||
if (topic == null || topic.trim().isEmpty()) throw new IllegalArgumentException("Kafka Topic 不能为空");
|
||||
String partitionKey = partitionKeyStrategy.partitionKey(deviceId, eventId);
|
||||
return new ProducerRecord<>(appProperties.getKafkaProducerTopic(), partitionKey, encrypted);
|
||||
return new ProducerRecord<>(topic, partitionKey, encrypted);
|
||||
}
|
||||
|
||||
public static String newEventId() {
|
||||
|
||||
@@ -33,6 +33,7 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
|
||||
private static final LongAdder NETWORK_RECEIVED = new LongAdder();
|
||||
private static final LongAdder TCP_PAUSE_COUNT = new LongAdder();
|
||||
private static final LongAdder UDP_DROPPED = new LongAdder();
|
||||
private static final LongAdder OVERSIZED_DROPPED = new LongAdder();
|
||||
|
||||
/** 每个 TCP Channel 独立 Handler，pending 只保存关闭 AUTO_READ 前已进入 Pipeline 的帧。 */
|
||||
private final Deque<SyslogIngressEvent> pendingTcp = new ArrayDeque<>();
|
||||
@@ -93,6 +94,10 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
|
||||
private void acceptUdp(SyslogWalService wal, SyslogIngressEvent event) {
|
||||
SyslogWalService.AdmissionResult result = wal.tryAppend(event);
|
||||
if (result == SyslogWalService.AdmissionResult.ACCEPTED) return;
|
||||
if (result == SyslogWalService.AdmissionResult.OVERSIZED) {
|
||||
recordOversized(event, "UDP");
|
||||
return;
|
||||
}
|
||||
UDP_DROPPED.increment();
|
||||
long dropped = UDP_DROPPED.sum();
|
||||
if (dropped == 1 || dropped % 100 == 0) {
|
||||
@@ -110,6 +115,11 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
|
||||
return;
|
||||
}
|
||||
SyslogWalService.AdmissionResult result = wal.tryAppend(event);
|
||||
if (result == SyslogWalService.AdmissionResult.OVERSIZED) {
|
||||
recordOversized(event, "TCP");
|
||||
ctx.close();
|
||||
return;
|
||||
}
|
||||
if (result != SyslogWalService.AdmissionResult.ACCEPTED) {
|
||||
pendingTcp.addLast(event);
|
||||
pauseAndSchedule(ctx, wal);
|
||||
@@ -144,6 +154,11 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
|
||||
while (!pendingTcp.isEmpty()) {
|
||||
SyslogIngressEvent event = pendingTcp.peekFirst();
|
||||
SyslogWalService.AdmissionResult result = wal.tryAppend(event);
|
||||
if (result == SyslogWalService.AdmissionResult.OVERSIZED) {
|
||||
pendingTcp.removeFirst();
|
||||
recordOversized(event, "TCP");
|
||||
continue;
|
||||
}
|
||||
if (result != SyslogWalService.AdmissionResult.ACCEPTED) {
|
||||
scheduleDrain(ctx, wal);
|
||||
return;
|
||||
@@ -172,4 +187,12 @@ public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
|
||||
public static long getNetworkReceivedCount() { return NETWORK_RECEIVED.sum(); }
|
||||
public static long getTcpPauseCount() { return TCP_PAUSE_COUNT.sum(); }
|
||||
public static long getUdpDroppedCount() { return UDP_DROPPED.sum(); }
|
||||
public static long getOversizedDroppedCount() { return OVERSIZED_DROPPED.sum(); }
|
||||
|
||||
private void recordOversized(SyslogIngressEvent event, String protocol) {
|
||||
OVERSIZED_DROPPED.increment();
|
||||
log.error("Syslog 超过单条消息硬上限，已拒绝且不记录原文: eventId={}, protocol={}, sourceIp={}, chars={}, rejectedTotal={}",
|
||||
event.getEventId(), protocol, event.getSourceIp(), event.getRawMessage().length(),
|
||||
OVERSIZED_DROPPED.sum());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,6 +47,10 @@ public class SyslogServerBoth {
|
||||
this.syslogCharset = java.util.Objects.requireNonNull(syslogCharset, "syslogCharset");
|
||||
this.tcpPort = properties.getSyslogTcpPort();
|
||||
this.udpPort = properties.getSyslogUdpPort();
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& properties.getSyslogMaxFrameLength() < properties.getSyslogMessageMaxBytes()) {
|
||||
throw new IllegalArgumentException("syslog.max.frame.length 不能小于 syslog.message.max-bytes");
|
||||
}
|
||||
}
|
||||
|
||||
public void start() throws InterruptedException {
|
||||
|
||||
@@ -9,7 +9,7 @@ server.error.include-binding-errors=always
|
||||
# Syslog Server Configuration
|
||||
syslog.tcp.port=514
|
||||
syslog.udp.port=514
|
||||
syslog.max.frame.length=262144
|
||||
syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456}
|
||||
syslog.buffer.size=1000
|
||||
# syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8
|
||||
syslog.charset=GBK
|
||||
|
||||
@@ -9,7 +9,7 @@ server.error.include-binding-errors=always
|
||||
# Syslog Server Configuration
|
||||
syslog.tcp.port=514
|
||||
syslog.udp.port=514
|
||||
syslog.max.frame.length=262144
|
||||
syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456}
|
||||
syslog.buffer.size=1000
|
||||
# syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8
|
||||
syslog.charset=GBK
|
||||
@@ -56,4 +56,4 @@ spring.redis.lettuce.pool.max-idle=10
|
||||
spring.redis.lettuce.pool.min-idle=5
|
||||
|
||||
# 生产环境缓存时间较长
|
||||
spring.cache.redis.time-to-live=3600000
|
||||
spring.cache.redis.time-to-live=3600000
|
||||
|
||||
@@ -9,7 +9,7 @@ server.error.include-binding-errors=always
|
||||
# Syslog Server Configuration
|
||||
syslog.tcp.port=514
|
||||
syslog.udp.port=514
|
||||
syslog.max.frame.length=2621440
|
||||
syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456}
|
||||
syslog.buffer.size=10000
|
||||
# syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8
|
||||
syslog.charset=UTF-8
|
||||
|
||||
@@ -9,7 +9,7 @@ server.error.include-binding-errors=always
|
||||
# Syslog Server Configuration
|
||||
syslog.tcp.port=514
|
||||
syslog.udp.port=514
|
||||
syslog.max.frame.length=262144
|
||||
syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456}
|
||||
syslog.buffer.size=1000
|
||||
# syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8
|
||||
syslog.charset=GBK
|
||||
|
||||
@@ -20,7 +20,11 @@ spring.kafka.producer.properties.linger.ms=5
|
||||
spring.kafka.producer.properties.batch.size=65536
|
||||
spring.kafka.producer.compression-type=lz4
|
||||
spring.kafka.producer.ack-timeout-ms=130000
|
||||
syslog.max.frame.length=262144
|
||||
syslog.max.frame.length=${SYSLOG_MAX_FRAME_LENGTH:6291456}
|
||||
# 单条原文硬上限；0 表示不限制。超过阈值的消息在进入 WAL 前被明确拒绝。
|
||||
syslog.message.max-bytes=${SYSLOG_MESSAGE_MAX_BYTES:6291456}
|
||||
# 超过 256KiB 的已接纳消息投递独立大日志 Topic；0 表示关闭分流。
|
||||
syslog.large-message.threshold-bytes=${SYSLOG_LARGE_MESSAGE_THRESHOLD_BYTES:262144}
|
||||
syslog.buffer.size=1000
|
||||
# syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8
|
||||
syslog.charset=GBK
|
||||
@@ -31,6 +35,8 @@ syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a
|
||||
syslog.wal.enabled=${SYSLOG_WAL_ENABLED:true}
|
||||
syslog.wal.directory=${SYSLOG_WAL_DIRECTORY:${SYSLOG_RAW_ARCHIVE_DIRECTORY:/data/syslog-wal}}
|
||||
syslog.wal.queue-capacity=${SYSLOG_WAL_QUEUE_CAPACITY:10000}
|
||||
# 队列和待 force 记录共享的内存字节预算，防止大日志把“有界条数”放大成无界堆占用。
|
||||
syslog.wal.max-pending-bytes=${SYSLOG_WAL_MAX_PENDING_BYTES:268435456}
|
||||
syslog.wal.high-watermark-percent=${SYSLOG_WAL_HIGH_WATERMARK_PERCENT:70}
|
||||
syslog.wal.low-watermark-percent=${SYSLOG_WAL_LOW_WATERMARK_PERCENT:40}
|
||||
syslog.wal.retry-interval-ms=${SYSLOG_WAL_RETRY_INTERVAL_MS:10}
|
||||
@@ -52,6 +58,7 @@ syslog.point-time.write-enabled=${SYSLOG_POINT_TIME_WRITE_ENABLED:true}
|
||||
# WAL 后台投递 Kafka：内存只保留有限在途消息，失败记录留在 WAL 自动重试。
|
||||
syslog.dispatch.worker-threads=${SYSLOG_DISPATCH_WORKER_THREADS:4}
|
||||
syslog.dispatch.max-inflight=${SYSLOG_DISPATCH_MAX_INFLIGHT:5000}
|
||||
syslog.dispatch.max-inflight-bytes=${SYSLOG_DISPATCH_MAX_INFLIGHT_BYTES:268435456}
|
||||
syslog.dispatch.retry-backoff-ms=${SYSLOG_DISPATCH_RETRY_BACKOFF_MS:1000}
|
||||
syslog.dispatch.shutdown-timeout-ms=${SYSLOG_DISPATCH_SHUTDOWN_TIMEOUT_MS:30000}
|
||||
syslog.device-context.cache-ttl-ms=${SYSLOG_DEVICE_CONTEXT_CACHE_TTL_MS:300000}
|
||||
@@ -77,6 +84,7 @@ app.service.data_type=json
|
||||
# kafka Configuration
|
||||
spring.kafka.producer.bootstrap-servers=192.168.4.99:9092
|
||||
spring.kafka.producer.topic =${KAFKA_PRODUCER_TOPIC:test-topic}
|
||||
spring.kafka.producer.large-topic=${KAFKA_PRODUCER_LARGE_TOPIC:${spring.kafka.producer.topic}.large}
|
||||
# kafka Configuration 新增优化配置
|
||||
spring.kafka.producer.properties.retries=10
|
||||
spring.kafka.producer.properties.retry.backoff.ms=500
|
||||
@@ -87,7 +95,8 @@ spring.kafka.producer.properties.delivery.timeout.ms=120000
|
||||
spring.kafka.producer.properties.max.block.ms=10000
|
||||
spring.kafka.producer.properties.buffer.memory=134217728
|
||||
# 允许投递加密后超过 Kafka 默认 1 MiB 的 Syslog；Broker/Topic 上限必须不小于此值。
|
||||
spring.kafka.producer.properties.max.request.size=${KAFKA_PRODUCER_MAX_REQUEST_SIZE:8388608}
|
||||
# 6MiB GBK 原文转 UTF-8 后再做 Base64 加密可能明显膨胀，按 32MiB 留出确定余量。
|
||||
spring.kafka.producer.properties.max.request.size=${KAFKA_PRODUCER_MAX_REQUEST_SIZE:33554432}
|
||||
# event_id 使高流量设备的独立事件均匀分布到多个分区；device_id 仅用于兼容回滚。
|
||||
syslog.kafka.partition-key-strategy=${SYSLOG_KAFKA_PARTITION_KEY_STRATEGY:event_id}
|
||||
|
||||
|
||||
+51
@@ -121,6 +121,56 @@ class SyslogWalDispatcherTest {
|
||||
verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void largeRecordUsesDedicatedKafkaLaneAndReleasesByteBudgetAfterAck() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setSyslogLargeMessageThresholdBytes(4L);
|
||||
properties.setSyslogMessageMaxBytes(32L);
|
||||
properties.setSyslogWalMaxPendingBytes(64L);
|
||||
properties.setSyslogDispatchMaxInflightBytes(64L);
|
||||
wal = new SyslogWalService(properties, new ObjectMapper());
|
||||
wal.initialize();
|
||||
wal.tryAppend(event("20260811-large", "12345"));
|
||||
waitUntil(() -> wal.getAppendedCount() == 1L);
|
||||
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
when(resolver.resolve(anyString(), anyString()))
|
||||
.thenReturn(DeviceContextResolver.Resolution.registered(device));
|
||||
kafkaProducer producer = mock(kafkaProducer.class);
|
||||
when(producer.messagePushLarge(any(DeviceDevice.class), anyString(), anyString(),
|
||||
anyString(), anyString(), any()))
|
||||
.thenReturn(CompletableFuture.completedFuture(null));
|
||||
|
||||
dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
|
||||
dispatcher.start();
|
||||
waitUntil(() -> wal.getCheckpointSequence() == 1L);
|
||||
|
||||
verify(producer).messagePushLarge(eq(device), eq("12345"),
|
||||
eq("20260811200000000"), eq("20260811200000000"),
|
||||
eq("20260811-large"), eq(null));
|
||||
verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
|
||||
assertEquals(1L, dispatcher.getLargeDispatched());
|
||||
assertEquals(0L, dispatcher.getInflightBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void singleRecordLargerThanByteBudgetCanRunExclusivelyWhenHardLimitIsDisabled() throws Exception {
|
||||
SyslogWalDispatcher.ByteBudget budget = new SyslogWalDispatcher.ByteBudget(4L);
|
||||
|
||||
budget.acquire(5L);
|
||||
|
||||
assertEquals(5L, budget.used());
|
||||
budget.release(5L);
|
||||
assertEquals(0L, budget.used());
|
||||
}
|
||||
|
||||
@Test
|
||||
void utf8BudgetCountsChineseAndEmojiWithoutAllocatingEncodedPayload() {
|
||||
assertEquals(11L, SyslogWalDispatcher.utf8Length("中文🙂x"));
|
||||
}
|
||||
|
||||
private SyslogIngressEvent event(String id, String raw) {
|
||||
return new SyslogIngressEvent(id, LocalDateTime.of(2026, 8, 11, 20, 0),
|
||||
"192.168.4.19", 514, "TCP", raw);
|
||||
@@ -147,6 +197,7 @@ class SyslogWalDispatcherTest {
|
||||
value.setSyslogCharset("UTF-8");
|
||||
value.setSyslogDispatchWorkerThreads(2);
|
||||
value.setSyslogDispatchMaxInflight(8);
|
||||
value.setSyslogDispatchMaxInflightBytes(0L);
|
||||
value.setSyslogDispatchRetryBackoffMs(10);
|
||||
value.setSyslogDispatchShutdownTimeoutMs(2000);
|
||||
value.setSyslogDeviceContextCacheTtlMs(60000);
|
||||
|
||||
+52
@@ -98,6 +98,58 @@ class SyslogWalServiceTest {
|
||||
assertEquals(null, attempt.getDurableStage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void oversizedMessageIsRejectedBeforeEnteringWal() {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setSyslogMessageMaxBytes(4L);
|
||||
properties.setSyslogWalMaxPendingBytes(16L);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
|
||||
SyslogWalService.AdmissionResult result = service.tryAppend(
|
||||
event("20260811-too-large", "12345"));
|
||||
|
||||
assertEquals(SyslogWalService.AdmissionResult.OVERSIZED, result);
|
||||
assertEquals(1L, service.getOversizedCount());
|
||||
assertEquals(0, service.getQueueSize());
|
||||
assertEquals(0L, service.getPendingAppendBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void pendingByteBudgetBackpressuresEvenWhenRecordQueueHasCapacity() {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setSyslogMessageMaxBytes(5L);
|
||||
properties.setSyslogWalMaxPendingBytes(5L);
|
||||
properties.setSyslogWalFlushRecords(100);
|
||||
properties.setSyslogWalFlushIntervalMs(10000L);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
|
||||
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED,
|
||||
service.tryAppend(event("20260811-first", "12345")));
|
||||
assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE,
|
||||
service.tryAppend(event("20260811-second", "x")));
|
||||
assertEquals(5L, service.getPendingAppendBytes());
|
||||
assertEquals(1L, service.getBackpressureCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void disabledHardLimitLetsOneOverBudgetRecordMakeProgressExclusively() {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setSyslogMessageMaxBytes(0L);
|
||||
properties.setSyslogWalMaxPendingBytes(4L);
|
||||
properties.setSyslogWalFlushRecords(100);
|
||||
properties.setSyslogWalFlushIntervalMs(10000L);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
|
||||
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED,
|
||||
service.tryAppend(event("20260811-over-budget", "12345")));
|
||||
assertEquals(SyslogWalService.AdmissionResult.BACKPRESSURE,
|
||||
service.tryAppend(event("20260811-waits", "x")));
|
||||
assertEquals(5L, service.getPendingAppendBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void forceBeforeCursorAndPersistSchemaV2() throws Exception {
|
||||
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
|
||||
|
||||
+14
@@ -63,6 +63,20 @@ class KafkaProducerEventIdTest {
|
||||
assertEquals("126", record.key());
|
||||
}
|
||||
|
||||
@Test
|
||||
void largeLaneUsesDedicatedTopicWithoutChangingPartitionKey() {
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setKafkaProducerTopic("raw");
|
||||
properties.setKafkaProducerLargeTopic("raw.large");
|
||||
kafkaProducer producer = new kafkaProducer(properties, mock(Producer.class), "event_id");
|
||||
|
||||
ProducerRecord<String, String> record = producer.buildRecord(
|
||||
properties.getKafkaProducerLargeTopic(), 126, "20260811-event", "ciphertext");
|
||||
|
||||
assertEquals("raw.large", record.topic());
|
||||
assertEquals("20260811-event", record.key());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidPartitionKeyStrategyFailsFast() {
|
||||
assertThrows(IllegalArgumentException.class, () -> producer("random"));
|
||||
|
||||
Reference in New Issue
Block a user