feat(syslog-consumer-rule): 补充慢批次消息长度统计
- 记录Kafka批次消费消息条数 - 记录批次日志正文总字节数 - 记录批次内单条日志最大字节数 - 优先使用Kafka序列化长度,避免重复编码带来的额外开销
This commit is contained in:
+22
-4
@@ -358,8 +358,19 @@ public class SysLogProcessor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void beginSlowBatchTrace(String lane, List<ConsumerRecord<String, String>> records) {
|
private void beginSlowBatchTrace(String lane, List<ConsumerRecord<String, String>> records) {
|
||||||
|
long polledMessageBytes = 0L;
|
||||||
|
int maxMessageBytes = 0;
|
||||||
|
for (ConsumerRecord<String, String> record : records) {
|
||||||
|
int messageBytes = record.serializedValueSize();
|
||||||
|
if (messageBytes < 0) {
|
||||||
|
String value = record.value();
|
||||||
|
messageBytes = value == null ? 0 : value.getBytes(StandardCharsets.UTF_8).length;
|
||||||
|
}
|
||||||
|
polledMessageBytes += messageBytes;
|
||||||
|
maxMessageBytes = Math.max(maxMessageBytes, messageBytes);
|
||||||
|
}
|
||||||
slowBatchTrace.set(new SlowBatchTrace(lane, topicPartitions(records), records.size(),
|
slowBatchTrace.set(new SlowBatchTrace(lane, topicPartitions(records), records.size(),
|
||||||
System.nanoTime()));
|
polledMessageBytes, maxMessageBytes, System.nanoTime()));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void recordSlowBatchStages(int records, int prepared, int persisted, int normalizedRows,
|
private void recordSlowBatchStages(int records, int prepared, int persisted, int normalizedRows,
|
||||||
@@ -402,13 +413,15 @@ public class SysLogProcessor {
|
|||||||
if (trace == null) return;
|
if (trace == null) return;
|
||||||
long endToEndMs = elapsedMillis(trace.startedAtNanos);
|
long endToEndMs = elapsedMillis(trace.startedAtNanos);
|
||||||
if (slowBatchThresholdMs <= 0L || endToEndMs < slowBatchThresholdMs) return;
|
if (slowBatchThresholdMs <= 0L || endToEndMs < slowBatchThresholdMs) return;
|
||||||
log.warn("[SLOW_SYSLOG_BATCH] lane={}, topicPartitions={}, polledRecords={}, processedRecords={}, "
|
log.warn("[SLOW_SYSLOG_BATCH] lane={}, topicPartitions={}, polledRecords={}, "
|
||||||
+ "chunks={}, prepared={}, persisted={}, normalizedRows={}, prepareMs={}, ingressMs={}, "
|
+ "polledMessageBytes={}, maxMessageBytes={}, processedRecords={}, chunks={}, "
|
||||||
|
+ "prepared={}, persisted={}, normalizedRows={}, prepareMs={}, ingressMs={}, "
|
||||||
+ "normalizeMs={}, ruleComputeWorkerMs={}, messageParseWorkerMs={}, "
|
+ "normalizeMs={}, ruleComputeWorkerMs={}, messageParseWorkerMs={}, "
|
||||||
+ "fieldMappingWorkerMs={}, persistencePlanMs={}, alarmDmWriteMs={}, influxMs={}, "
|
+ "fieldMappingWorkerMs={}, persistencePlanMs={}, alarmDmWriteMs={}, influxMs={}, "
|
||||||
+ "remainingDmWriteMs={}, businessProcessMs={}, kafkaCommitMs={}, "
|
+ "remainingDmWriteMs={}, businessProcessMs={}, kafkaCommitMs={}, "
|
||||||
+ "kafkaCommitStatus={}, kafkaCommitErrorType={}, endToEndMs={}",
|
+ "kafkaCommitStatus={}, kafkaCommitErrorType={}, endToEndMs={}",
|
||||||
trace.lane, trace.topicPartitions, trace.polledRecords, trace.recordsProcessed,
|
trace.lane, trace.topicPartitions, trace.polledRecords,
|
||||||
|
trace.polledMessageBytes, trace.maxMessageBytes, trace.recordsProcessed,
|
||||||
trace.chunks, trace.prepared, trace.persisted, trace.normalizedRows,
|
trace.chunks, trace.prepared, trace.persisted, trace.normalizedRows,
|
||||||
trace.prepareMs, trace.ingressMs, trace.normalizeMs, trace.ruleComputeMs,
|
trace.prepareMs, trace.ingressMs, trace.normalizeMs, trace.ruleComputeMs,
|
||||||
trace.messageParseMs, trace.fieldMappingMs, trace.persistencePlanMs,
|
trace.messageParseMs, trace.fieldMappingMs, trace.persistencePlanMs,
|
||||||
@@ -1284,6 +1297,8 @@ public class SysLogProcessor {
|
|||||||
private final String lane;
|
private final String lane;
|
||||||
private final String topicPartitions;
|
private final String topicPartitions;
|
||||||
private final int polledRecords;
|
private final int polledRecords;
|
||||||
|
private final long polledMessageBytes;
|
||||||
|
private final int maxMessageBytes;
|
||||||
private final long startedAtNanos;
|
private final long startedAtNanos;
|
||||||
private int chunks;
|
private int chunks;
|
||||||
private int recordsProcessed;
|
private int recordsProcessed;
|
||||||
@@ -1302,10 +1317,13 @@ public class SysLogProcessor {
|
|||||||
private long remainingDmMs;
|
private long remainingDmMs;
|
||||||
|
|
||||||
private SlowBatchTrace(String lane, String topicPartitions, int polledRecords,
|
private SlowBatchTrace(String lane, String topicPartitions, int polledRecords,
|
||||||
|
long polledMessageBytes, int maxMessageBytes,
|
||||||
long startedAtNanos) {
|
long startedAtNanos) {
|
||||||
this.lane = lane;
|
this.lane = lane;
|
||||||
this.topicPartitions = topicPartitions;
|
this.topicPartitions = topicPartitions;
|
||||||
this.polledRecords = polledRecords;
|
this.polledRecords = polledRecords;
|
||||||
|
this.polledMessageBytes = polledMessageBytes;
|
||||||
|
this.maxMessageBytes = maxMessageBytes;
|
||||||
this.startedAtNanos = startedAtNanos;
|
this.startedAtNanos = startedAtNanos;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user