feat(syslog): 优化大日志通道并增加异常连续消费兜底
- 128 KiB 及以上日志默认进入大日志 Topic - InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存 - 截断内容追加 HB_XDR_TRUNCATED 标记 - InfluxDB 写入失败时继续执行规则解析和达梦写入 - 未知或系统性异常优先写入普通/大日志 DLT - DLT 写入失败时记录特殊告警并提交 offset，避免阻塞后续消息 - DLT 失败等待时间默认限制为 1 秒 - 增加连续性放行、DLT 成功及失败监控指标 - 更新 Docker Compose、v3 升级文档及相关测试
This commit is contained in:
王绍宏
+91
-8
@@ -48,9 +48,11 @@ public class SyslogWalDispatcher {
|
||||
private final LongAdder kafkaAcked = new LongAdder();
|
||||
private final LongAdder terminalFiltered = new LongAdder();
|
||||
private final LongAdder retryCount = new LongAdder();
|
||||
private final LongAdder largeDispatched = new LongAdder();
|
||||
private final AtomicLong inflight = new AtomicLong();
|
||||
|
||||
private Semaphore permits;
|
||||
private ByteBudget byteBudget;
|
||||
private ExecutorService workers;
|
||||
private ScheduledExecutorService scheduler;
|
||||
private Thread readerThread;
|
||||
@@ -72,6 +74,7 @@ public class SyslogWalDispatcher {
|
||||
public void start() throws Exception {
|
||||
validateConfiguration();
|
||||
permits = new Semaphore(properties.getSyslogDispatchMaxInflight());
|
||||
byteBudget = new ByteBudget(properties.getSyslogDispatchMaxInflightBytes());
|
||||
workers = Executors.newFixedThreadPool(properties.getSyslogDispatchWorkerThreads(),
|
||||
namedFactory("syslog-dispatch-worker"));
|
||||
scheduler = Executors.newScheduledThreadPool(2, namedFactory("syslog-dispatch-scheduler"));
|
||||
@@ -85,8 +88,9 @@ public class SyslogWalDispatcher {
|
||||
scheduler.scheduleWithFixedDelay(this::logStatus,
|
||||
properties.getSyslogStatusLogIntervalMs(), properties.getSyslogStatusLogIntervalMs(),
|
||||
TimeUnit.MILLISECONDS);
|
||||
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, checkpointSequence={}",
|
||||
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, maxInflightBytes={}, checkpointSequence={}",
|
||||
properties.getSyslogDispatchWorkerThreads(), properties.getSyslogDispatchMaxInflight(),
|
||||
properties.getSyslogDispatchMaxInflightBytes(),
|
||||
walService.getCheckpointSequence());
|
||||
}
|
||||
|
||||
@@ -99,7 +103,11 @@ public class SyslogWalDispatcher {
|
||||
permits.release();
|
||||
continue;
|
||||
}
|
||||
record.setDispatchPayloadBytes(Math.max(1L,
|
||||
Math.max(record.getRawLength(), utf8Length(record.getRawMessage()))));
|
||||
byteBudget.acquire(record.getDispatchPayloadBytes());
|
||||
if (record.getSequence() != nextReadSequence) {
|
||||
byteBudget.release(record.getDispatchPayloadBytes());
|
||||
permits.release();
|
||||
throw new IllegalStateException("WAL sequence 不连续，拒绝越过: expected="
|
||||
+ nextReadSequence + ", actual=" + record.getSequence());
|
||||
@@ -158,8 +166,13 @@ public class SyslogWalDispatcher {
|
||||
eventTime = LocalDateTime.parse(walEventTime, WAL_TIME).format(HEADER_TIME);
|
||||
|
||||
// v2/v3 继续调用旧重载，确保升级前已落盘记录产生的 Kafka 消息完全不变。
|
||||
java.util.concurrent.CompletableFuture<kafkaProducer.DeliveryResult> deliveryFuture =
|
||||
record.getSchemaVersion() >= 4
|
||||
boolean large = properties.getSyslogLargeMessageThresholdBytes() > 0L
|
||||
&& record.getDispatchPayloadBytes() > properties.getSyslogLargeMessageThresholdBytes();
|
||||
java.util.concurrent.CompletableFuture<kafkaProducer.DeliveryResult> deliveryFuture = large
|
||||
? producer.messagePushLarge(
|
||||
resolution.getDevice(), record.getRawMessage(), receiveTime, eventTime,
|
||||
record.getEventId(), record.getPointTimeNs())
|
||||
: record.getSchemaVersion() >= 4
|
||||
? producer.messagePush(
|
||||
resolution.getDevice(),
|
||||
record.getRawMessage(),
|
||||
@@ -177,6 +190,7 @@ public class SyslogWalDispatcher {
|
||||
.whenComplete((delivery, failure) -> {
|
||||
if (failure == null) {
|
||||
kafkaAcked.increment();
|
||||
if (large) largeDispatched.increment();
|
||||
markComplete(record);
|
||||
} else {
|
||||
scheduleRetry(record, attempt, "KAFKA_ACK", unwrap(failure));
|
||||
@@ -206,7 +220,9 @@ public class SyslogWalDispatcher {
|
||||
private void markComplete(SyslogWalRecord record) {
|
||||
synchronized (completionMonitor) {
|
||||
if (record.getSequence() < nextCommitSequence) return;
|
||||
completedOutOfOrder.put(record.getSequence(), record);
|
||||
if (completedOutOfOrder.putIfAbsent(record.getSequence(), checkpointOnly(record)) != null) return;
|
||||
// 已 ACK 的正文不再占字节预算；条数许可仍要等连续 checkpoint，限制乱序完成表大小。
|
||||
byteBudget.release(record.getDispatchPayloadBytes());
|
||||
SyslogWalRecord latest = null;
|
||||
int released = 0;
|
||||
while (true) {
|
||||
@@ -224,16 +240,26 @@ public class SyslogWalDispatcher {
|
||||
}
|
||||
}
|
||||
|
||||
private SyslogWalRecord checkpointOnly(SyslogWalRecord source) {
|
||||
SyslogWalRecord value = new SyslogWalRecord();
|
||||
value.setSequence(source.getSequence());
|
||||
value.setSegment(source.getSegment());
|
||||
value.setStartOffset(source.getStartOffset());
|
||||
value.setEndOffset(source.getEndOffset());
|
||||
return value;
|
||||
}
|
||||
|
||||
private void logStatus() {
|
||||
if (!running.get()) return;
|
||||
log.info("Syslog WAL状态: networkReceived={}, appended={}, walPendingApprox={}, walQueue={}/{}, durableSequence={}, "
|
||||
+ "checkpointSequence={}, kafkaInflight={}, kafkaAcked={}, terminalFiltered={}, "
|
||||
log.info("Syslog WAL状态: networkReceived={}, appended={}, admissionOversized={}, walPendingApprox={}, walPendingBytes={}, walQueue={}/{}, durableSequence={}, "
|
||||
+ "checkpointSequence={}, kafkaInflight={}, kafkaInflightBytes={}, kafkaAcked={}, largeDispatched={}, terminalFiltered={}, "
|
||||
+ "retry={}, tcpPauseTotal={}, udpDropped={}, deviceCacheSize={}, deviceCacheHits={}, "
|
||||
+ "deviceCacheMisses={}",
|
||||
SyslogMessageHandler.getNetworkReceivedCount(), walService.getAppendedCount(),
|
||||
walService.getPendingRecordsApprox(),
|
||||
walService.getOversizedCount(), walService.getPendingRecordsApprox(), walService.getPendingAppendBytes(),
|
||||
walService.getQueueSize(), walService.getQueueCapacity(), walService.getDurableSequence(),
|
||||
walService.getCheckpointSequence(), inflight.get(), kafkaAcked.sum(), terminalFiltered.sum(),
|
||||
walService.getCheckpointSequence(), inflight.get(), byteBudget == null ? 0L : byteBudget.used(),
|
||||
kafkaAcked.sum(), largeDispatched.sum(), terminalFiltered.sum(),
|
||||
retryCount.sum(), SyslogMessageHandler.getTcpPauseCount(),
|
||||
SyslogMessageHandler.getUdpDroppedCount(), deviceResolver.getCacheSize(),
|
||||
deviceResolver.getHitCount(), deviceResolver.getMissCount());
|
||||
@@ -246,6 +272,23 @@ public class SyslogWalDispatcher {
|
||||
return failure;
|
||||
}
|
||||
|
||||
/** 不分配临时 byte[] 的 UTF-8 长度估算；非法孤立代理项按 3 字节保守计算。 */
|
||||
static long utf8Length(String value) {
|
||||
if (value == null || value.isEmpty()) return 0L;
|
||||
long bytes = 0L;
|
||||
for (int index = 0; index < value.length(); index++) {
|
||||
char current = value.charAt(index);
|
||||
if (current <= 0x7f) bytes++;
|
||||
else if (current <= 0x7ff) bytes += 2L;
|
||||
else if (Character.isHighSurrogate(current) && index + 1 < value.length()
|
||||
&& Character.isLowSurrogate(value.charAt(index + 1))) {
|
||||
bytes += 4L;
|
||||
index++;
|
||||
} else bytes += 3L;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private ThreadFactory namedFactory(String prefix) {
|
||||
AtomicLong sequence = new AtomicLong();
|
||||
return runnable -> {
|
||||
@@ -258,18 +301,58 @@ public class SyslogWalDispatcher {
|
||||
private void validateConfiguration() {
|
||||
if (properties.getSyslogDispatchWorkerThreads() <= 0
|
||||
|| properties.getSyslogDispatchMaxInflight() <= 0
|
||||
|| properties.getSyslogDispatchMaxInflightBytes() < 0L
|
||||
|| properties.getSyslogDispatchRetryBackoffMs() <= 0
|
||||
|| properties.getSyslogDispatchShutdownTimeoutMs() <= 0
|
||||
|| properties.getSyslogDeviceContextCacheTtlMs() <= 0
|
||||
|| properties.getSyslogStatusLogIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("Dispatcher 线程、在途、重试、缓存和状态日志参数必须大于 0");
|
||||
}
|
||||
if (properties.getSyslogMessageMaxBytes() > 0L
|
||||
&& properties.getSyslogDispatchMaxInflightBytes() > 0L
|
||||
&& properties.getSyslogMessageMaxBytes() > properties.getSyslogDispatchMaxInflightBytes()) {
|
||||
throw new IllegalArgumentException("单条消息上限不能大于 Dispatcher 在途字节预算");
|
||||
}
|
||||
}
|
||||
|
||||
public long getInflight() { return inflight.get(); }
|
||||
public long getKafkaAcked() { return kafkaAcked.sum(); }
|
||||
public long getTerminalFiltered() { return terminalFiltered.sum(); }
|
||||
public long getRetryCount() { return retryCount.sum(); }
|
||||
public long getLargeDispatched() { return largeDispatched.sum(); }
|
||||
public long getInflightBytes() { return byteBudget == null ? 0L : byteBudget.used(); }
|
||||
|
||||
/** 同时保留条数 Semaphore 和字节预算，避免少量大消息占满堆。 */
|
||||
static final class ByteBudget {
|
||||
private final long capacity;
|
||||
private long used;
|
||||
|
||||
ByteBudget(long capacity) {
|
||||
if (capacity < 0L) throw new IllegalArgumentException("capacity 不能小于 0");
|
||||
this.capacity = capacity;
|
||||
}
|
||||
|
||||
synchronized void acquire(long bytes) throws InterruptedException {
|
||||
if (bytes <= 0L) throw new IllegalArgumentException("bytes 必须大于 0");
|
||||
if (capacity == 0L) { used += bytes; return; }
|
||||
if (bytes > capacity) {
|
||||
// 硬上限关闭时仍保证进展：超预算单条只能独占，不允许与其他正文并存。
|
||||
while (used != 0L) wait();
|
||||
used = bytes;
|
||||
return;
|
||||
}
|
||||
while (used > capacity - bytes) wait();
|
||||
used += bytes;
|
||||
}
|
||||
|
||||
synchronized void release(long bytes) {
|
||||
used -= bytes;
|
||||
if (used < 0L) throw new IllegalStateException("Dispatcher 字节预算释放越界");
|
||||
notifyAll();
|
||||
}
|
||||
|
||||
synchronized long used() { return used; }
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
|
||||
Reference in New Issue
Block a user