feat(syslog): 提升RAW消费及达梦批量写入上限

- 将普通RAW基础处理批量调整为3000条
- 将RAW自适应拉取上限调整为5000条
- 将普通、告警及非标达梦批量写入上限调整为2000条
- 启动日志增加达梦实际批量配置输出
- 补充自适应分块和达梦批量写入测试
- 更新v4吞吐优化升级说明
This commit is contained in:
王绍宏
2026-09-13 11:19:35 +08:00
parent e393350c8b
commit 7e6b2fc4dc
17 changed files with 168 additions and 47 deletions
@@ -7,8 +7,8 @@ public final class AdaptiveRawBatchSizer {
private final int maxRecords;
public AdaptiveRawBatchSizer(
@org.springframework.beans.factory.annotation.Value("${syslog.raw-listener.max-poll-records:500}") int baseRecords,
@org.springframework.beans.factory.annotation.Value("${syslog.raw-listener.adaptive-max-records:3000}") int maxRecords) {
@org.springframework.beans.factory.annotation.Value("${syslog.raw-listener.max-poll-records:3000}") int baseRecords,
@org.springframework.beans.factory.annotation.Value("${syslog.raw-listener.adaptive-max-records:5000}") int maxRecords) {
if (baseRecords < 1 || maxRecords < baseRecords) {
throw new IllegalArgumentException("普通自适应批量要求 1 <= 基础条数 <= 上限条数");
}
@@ -79,7 +79,7 @@ public class SysLogProcessor {
private long rawProcessingBatchMaxBytes = 16777216L;
@Autowired
private AdaptiveRawBatchSizer adaptiveRawBatchSizer = new AdaptiveRawBatchSizer(500, 3000);
private AdaptiveRawBatchSizer adaptiveRawBatchSizer = new AdaptiveRawBatchSizer(3000, 5000);
// 每个普通listener线程独立学习上一个业务块的阶段,避免不同节点/通道互相放大批量。
private final ThreadLocal<RawAdaptiveState> rawAdaptiveState = ThreadLocal.withInitial(RawAdaptiveState::new);
@@ -66,7 +66,7 @@ public class DmStandardizedBatchWriter {
@Value("${dm.standardized-write.mode:BATCH}")
private String mode;
@Value("${dm.standardized-write.batch-size:500}")
@Value("${dm.standardized-write.batch-size:2000}")
private int batchSize;
@Value("${syslog.slow-batch.threshold-ms:1000}")
@@ -44,7 +44,7 @@ public class SyslogNonNormalMessageServiceImpl
@Autowired(required = false)
private DmLargeTextLimiter largeTextLimiter;
@Value("${dm.non-normal-write.batch-size:500}")
@Value("${dm.non-normal-write.batch-size:2000}")
private int nonNormalBatchSize;
@Override
@@ -53,8 +53,8 @@ public class KafkaReliabilityConfig {
ConsumerFactory<Object, Object> consumerFactory,
DefaultErrorHandler syslogDefaultErrorHandler,
@Value("${syslog.raw-listener.concurrency:3}") int concurrency,
@Value("${syslog.raw-listener.max-poll-records:500}") int baseRecords,
@Value("${syslog.raw-listener.adaptive-max-records:3000}") int maxPollRecords) {
@Value("${syslog.raw-listener.max-poll-records:3000}") int baseRecords,
@Value("${syslog.raw-listener.adaptive-max-records:5000}") int maxPollRecords) {
if (concurrency < 1) {
throw new IllegalArgumentException("syslog.raw-listener.concurrency 必须大于 0");
}
@@ -30,8 +30,10 @@ public class SyslogBuildStartupLogger {
+ ", profiles=" + String.join(",", environment.getActiveProfiles())
+ ", rawEnabled=" + value("syslog.raw-consumer.enabled", "true")
+ ", rawConcurrency=" + value("syslog.raw-listener.concurrency", "3")
+ ", baseProcessingRecords=" + value("syslog.raw-listener.max-poll-records", "500")
+ ", adaptiveMaxRecords=" + value("syslog.raw-listener.adaptive-max-records", "3000")
+ ", baseProcessingRecords=" + value("syslog.raw-listener.max-poll-records", "3000")
+ ", adaptiveMaxRecords=" + value("syslog.raw-listener.adaptive-max-records", "5000")
+ ", dmStandardizedBatchSize=" + value("dm.standardized-write.batch-size", "2000")
+ ", dmNonNormalBatchSize=" + value("dm.non-normal-write.batch-size", "2000")
+ ", rawFieldMaxUtf8Bytes=" + value("syslog.raw-listener.field-max-utf8-bytes", "2048")
+ ", samplingEnabled=" + value("syslog.backlog-sampling.enabled", "true")
+ ", statsWriteMode=" + value("stats.write-mode", "DIRECT")
@@ -63,7 +63,7 @@ spring.kafka.consumer.topic=agent-01-syslog-topic
syslog.raw-consumer.enabled=true
# 通用/Retry消费者上限。RAW底层上限由adaptive-max-records决定(3000),业务基础批量由max-poll-records决定(500)。
# 通用/Retry消费者上限。RAW底层上限由adaptive-max-records决定(5000),业务基础批量由max-poll-records决定(3000)。
spring.kafka.consumer.max-poll-records=500
spring.kafka.consumer.properties.max.poll.interval.ms=900000
spring.kafka.consumer.properties.session.timeout.ms=45000
@@ -85,12 +85,12 @@ syslog.raw-listener.concurrency=${SYSLOG_RAW_LISTENER_CONCURRENCY:3}
# 慢日志统一阈值,默认1000毫秒(1秒):端到端批次、达梦事务、写入排队及非标异步写入。
# 达到阈值时打印诊断;0关闭耗时触发的慢日志,但异常/失败仍记录。已有同名环境变量优先。
syslog.slow-batch.threshold-ms=${SYSLOG_SLOW_BATCH_THRESHOLD_MS:1000}
# 普通业务基础批量500(保留旧配置名);不再表示底层poll上限。已有同名环境变量仍覆盖基础值。
# 目标业务批量=ceil(基础条数*10000/max(普通保留万分比,非标保留万分比)),并受下面3000上限限制。
syslog.raw-listener.max-poll-records=${SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS:500}
# 底层RAW max.poll.records固定为此值;无积压时也可能poll到3000,再按500分块,全部处理后才ACK。
# 普通业务基础批量3000(保留旧配置名);不再表示底层poll上限。已有同名环境变量仍覆盖基础值。
# 目标业务批量=ceil(基础条数*10000/max(普通保留万分比,非标保留万分比)),并受下面5000上限限制。
syslog.raw-listener.max-poll-records=${SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS:3000}
# 底层RAW max.poll.records固定为此值;无积压时也可能poll到5000,再按3000分块,全部处理后才ACK。
# 基础值必须大于0且不超过上限;Retry/大日志/独立统计不采用此自适应策略。
syslog.raw-listener.adaptive-max-records=${SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS:3000}
syslog.raw-listener.adaptive-max-records=${SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS:5000}
# 大批次按 Kafka value 序列化字节拆成16MiB业务块,全部完成后提交本次poll;单条不会切断。
syslog.raw-listener.processing-batch-max-bytes=${SYSLOG_RAW_PROCESSING_BATCH_MAX_BYTES:16777216}
# 每个业务块只向共享规则线程池提交8个worker,避免2000条对应2000个排队任务。
@@ -217,7 +217,8 @@ app.processor.process-timeout-ms=120000
# 失败时只对数据类错误有限二分;
# 连接、超时或提交结果未知会立即停止并保留 Kafka offset,禁止放大为逐条请求风暴。
dm.standardized-write.mode=BATCH
dm.standardized-write.batch-size=${DM_STANDARDIZED_WRITE_BATCH_SIZE:500}
# 普通表/告警表每个分组的小批次上限2000行;不足2000直接写入,不等待凑满。
dm.standardized-write.batch-size=${DM_STANDARDIZED_WRITE_BATCH_SIZE:2000}
dm.standardized-write.writer-concurrency=2
dm.standardized-write.writer-queue-capacity=100
# 标准化大字段和非标原文均在达梦持久化边界截断;0 表示不限制。
@@ -236,7 +237,8 @@ dm.standardized-write.max-split-depth=10
dm.standardized-write.max-fallback-operations=100
dm.standardized-write.circuit-breaker-failure-threshold=3
dm.standardized-write.circuit-breaker-backoff-ms=30000
dm.non-normal-write.batch-size=${DM_NON_NORMAL_WRITE_BATCH_SIZE:500}
# 非标表每条多行INSERT上限2000行;同名环境变量会覆盖JAR默认值。
dm.non-normal-write.batch-size=${DM_NON_NORMAL_WRITE_BATCH_SIZE:2000}
dm.non-normal-write.queue-capacity=${DM_NON_NORMAL_WRITE_QUEUE_CAPACITY:500}
dm.non-normal-write.max-concurrency-retries=${DM_NON_NORMAL_WRITE_MAX_CONCURRENCY_RETRIES:3}
dm.non-normal-write.concurrency-retry-backoff-ms=${DM_NON_NORMAL_WRITE_CONCURRENCY_RETRY_BACKOFF_MS:50}
@@ -4,6 +4,22 @@ import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
class AdaptiveRawBatchSizerTest {
@Test void newBaseStartsAt3000AndDegradedRatesAreCappedAt5000() {
AdaptiveRawBatchSizer sizer = new AdaptiveRawBatchSizer(3000, 5000);
assertEquals(3000, sizer.targetRecords(10000, 10000));
assertEquals(3750, sizer.targetRecords(8000, 8000));
assertEquals(5000, sizer.targetRecords(6000, 6000));
assertEquals(5000, sizer.targetRecords(1000, 2000));
}
@Test void largerCapKeepsDefaultRetentionTargetAndAllowsFiveThousand() {
AdaptiveRawBatchSizer sizer = new AdaptiveRawBatchSizer(500, 5000);
assertEquals(500, sizer.targetRecords(10000, 10000));
assertEquals(2500, sizer.targetRecords(1000, 2000));
assertEquals(5000, sizer.targetRecords(1000, 1000));
assertEquals(5000, sizer.targetRecords(0, 0));
}
@Test void derivesMultiplierAndCapsWithoutPerStageConfiguration() {
AdaptiveRawBatchSizer sizer = new AdaptiveRawBatchSizer(500, 3000);
assertEquals(500, sizer.targetRecords(10000, 10000));
@@ -23,6 +23,8 @@ class SysLogAdaptiveRawBatchTest {
@BeforeEach void setup() {
processor = spy(new SysLogProcessor());
// 保留对旧基础值覆盖配置的回归场景;新默认值另行验证。
ReflectionTestUtils.setField(processor, "adaptiveRawBatchSizer", new AdaptiveRawBatchSizer(500, 5000));
ReflectionTestUtils.setField(processor, "taskExecutor", (Executor) Runnable::run);
ReflectionTestUtils.setField(processor, "largeTaskExecutor", (Executor) Runnable::run);
ReflectionTestUtils.setField(processor, "largeProcessingBatchMaxBytes", 16777216L);
@@ -30,6 +32,25 @@ class SysLogAdaptiveRawBatchTest {
ack = mock(Acknowledgment.class);
}
@Test void newBaseSplitsAt3000ThenBacklogCanUse5000WithoutEarlyAck() {
ReflectionTestUtils.setField(processor, "adaptiveRawBatchSizer", new AdaptiveRawBatchSizer(3000, 5000));
List<Integer> sizes = new ArrayList<>();
doAnswer(call -> {
sizes.add(((List<?>) call.getArgument(0)).size());
verifyNoInteractions(ack);
processor.observeRawRetention(2000);
return null;
}).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
processor.consumeAndParallelProcess(records(5000), ack);
assertEquals(Arrays.asList(3000, 2000), sizes);
verify(ack).acknowledge();
clearInvocations(ack);
sizes.clear();
processor.consumeAndParallelProcess(records(5000), ack);
assertEquals(Collections.singletonList(5000), sizes);
verify(ack).acknowledge();
}
@Test void livePollSplitsInto500WithoutEarlyAckOrRecordLoss() {
List<Integer> sizes = new ArrayList<>();
List<Long> offsets = new ArrayList<>();
@@ -41,9 +62,9 @@ class SysLogAdaptiveRawBatchTest {
processor.observeRawRetention(10000);
return null;
}).when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
processor.consumeAndParallelProcess(records(3000), ack);
assertEquals(Arrays.asList(500, 500, 500, 500, 500, 500), sizes);
for (int i = 0; i < 3000; i++) assertEquals((long) i, offsets.get(i).longValue());
processor.consumeAndParallelProcess(records(5000), ack);
assertEquals(Collections.nCopies(10, 500), sizes);
for (int i = 0; i < 5000; i++) assertEquals((long) i, offsets.get(i).longValue());
verify(ack).acknowledge();
}
@@ -70,14 +91,14 @@ class SysLogAdaptiveRawBatchTest {
verify(ack, times(3)).acknowledge();
}
@Test void tenPercentHits3000CapAndIncompleteTailIsNotHeldWaitingForMore() {
@Test void tenPercentHits5000CapAndIncompleteTailIsNotHeldWaitingForMore() {
processor.observeRawRetention(1000);
List<Integer> sizes = new ArrayList<>();
doAnswer(call -> { sizes.add(((List<?>) call.getArgument(0)).size()); return null; })
.when(processor).processBatch(anyList(), anyBoolean(), anyString(), any());
processor.consumeAndParallelProcess(records(3000), ack);
processor.consumeAndParallelProcess(records(5000), ack);
processor.consumeAndParallelProcess(records(17), ack);
assertEquals(Arrays.asList(3000, 17), sizes);
assertEquals(Arrays.asList(5000, 17), sizes);
}
@Test void bytesLimitStillWinsAndOversizedSingleRecordIsNotLost() {
@@ -56,7 +56,7 @@ class SysLogSlowBatchTraceTest {
String log = onlyTrace();
assertTrue(log.contains("PREPARE="));
assertTrue(log.contains("ADAPTIVE_CHUNK_SELECTION="));
assertTrue(log.contains("adaptiveTargetHistogram={500=1}"));
assertTrue(log.contains("adaptiveTargetHistogram={3000=1}"));
assertTrue(log.contains("actualChunkHistogram={2=1}"));
assertTrue(log.contains("ALARM_DM="));
assertTrue(log.contains("raw:0=12..14"));
@@ -54,6 +54,25 @@ class DmStandardizedBatchWriterTest {
when(normalDataService.isAlarmData(any())).thenReturn(false);
}
@Test
void normalAndAlarmBatchesUse2000LimitAndWriteTailImmediately() {
ReflectionTestUtils.setField(writer, "batchSize", 2000);
List<Integer> sizes = new ArrayList<>();
doAnswer(invocation -> {
sizes.add(((List<?>) invocation.getArgument(1)).size());
return null;
}).when(batchExecutor).execute(anyBoolean(), anyList(), anyInt());
for (boolean alarm : new boolean[]{false, true}) {
when(normalDataService.isAlarmData(any())).thenReturn(alarm);
sizes.clear();
assertEquals(3000, writer.write(requests(3000, -1)).getSuccessfulRows());
assertEquals(java.util.Arrays.asList(2000, 1000), sizes);
sizes.clear();
assertEquals(500, writer.write(requests(500, -1)).getSuccessfulRows());
assertEquals(java.util.Collections.singletonList(500), sizes);
}
}
@Test
void alarmsKeepLegacyTruncationInBothBatchAndSingleMode() {
DmLargeTextLimiter limiter = new DmLargeTextLimiter();
@@ -39,6 +39,24 @@ class SyslogNonNormalMessageServiceBatchTest {
.thenAnswer(invocation -> ((List<?>) invocation.getArgument(0)).size());
}
@Test
void writes2000RowsPerInsertAndDoesNotHoldIncompleteTail() {
ReflectionTestUtils.setField(service, "nonNormalBatchSize", 2000);
List<SyslogNonNormalMessage> messages = new ArrayList<>();
for (int i = 0; i < 3000; i++) {
SyslogNonNormalMessage message = new SyslogNonNormalMessage();
message.setId("non-normal-" + i);
messages.add(message);
}
assertEquals(3000, service.saveBatchMessages(messages));
@SuppressWarnings("unchecked")
ArgumentCaptor<List<SyslogNonNormalMessage>> captor = ArgumentCaptor.forClass(List.class);
verify(mapper, times(2)).batchInsert(captor.capture());
assertEquals(2000, captor.getAllValues().get(0).size());
assertEquals(1000, captor.getAllValues().get(1).size());
verify(directStatsProjectionWriter).writeNonNormalRows(messages);
}
@Test
void chunksLargePollInsideOneServiceTransactionAndAggregatesStatsOnce() {
List<SyslogNonNormalMessage> messages = new ArrayList<>();
@@ -0,0 +1,30 @@
package com.config;
import com.common.service.DmStandardizedBatchWriter;
import com.common.service.impl.SyslogNonNormalMessageServiceImpl;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Value;
import java.io.InputStream;
import java.util.Properties;
import static org.junit.jupiter.api.Assertions.assertEquals;
class DmBatchDefaultsTest {
@Test
void propertiesAndInjectionFallbacksBothDefaultTo2000() throws Exception {
Properties properties = new Properties();
try (InputStream input = getClass().getResourceAsStream("/application.properties")) {
properties.load(input);
}
assertEquals("${DM_STANDARDIZED_WRITE_BATCH_SIZE:2000}",
properties.getProperty("dm.standardized-write.batch-size"));
assertEquals("${DM_NON_NORMAL_WRITE_BATCH_SIZE:2000}",
properties.getProperty("dm.non-normal-write.batch-size"));
assertEquals("${dm.standardized-write.batch-size:2000}",
DmStandardizedBatchWriter.class.getDeclaredField("batchSize").getAnnotation(Value.class).value());
assertEquals("${dm.non-normal-write.batch-size:2000}",
SyslogNonNormalMessageServiceImpl.class.getDeclaredField("nonNormalBatchSize")
.getAnnotation(Value.class).value());
}
}
@@ -19,21 +19,21 @@ import static org.mockito.Mockito.when;
class KafkaReliabilityConfigTest {
@Test
void rawDefaultIs500BaseAnd3000NativePollCeiling() throws Exception {
void rawDefaultIs3000BaseAnd5000NativePollCeiling() throws Exception {
java.util.Properties properties = new java.util.Properties();
try (java.io.InputStream input = getClass().getResourceAsStream("/application.properties")) {
properties.load(input);
}
assertEquals("${SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS:500}",
assertEquals("${SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS:3000}",
properties.getProperty("syslog.raw-listener.max-poll-records"));
java.lang.reflect.Method method = java.util.Arrays.stream(KafkaReliabilityConfig.class.getDeclaredMethods())
.filter(candidate -> candidate.getName().equals("rawKafkaListenerContainerFactory")).findFirst().get();
org.springframework.beans.factory.annotation.Value fallback = method.getParameters()[4]
.getAnnotation(org.springframework.beans.factory.annotation.Value.class);
assertEquals("${syslog.raw-listener.max-poll-records:500}", fallback.value());
assertEquals("${SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS:3000}",
assertEquals("${syslog.raw-listener.max-poll-records:3000}", fallback.value());
assertEquals("${SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS:5000}",
properties.getProperty("syslog.raw-listener.adaptive-max-records"));
assertEquals("${syslog.raw-listener.adaptive-max-records:3000}", method.getParameters()[5]
assertEquals("${syslog.raw-listener.adaptive-max-records:5000}", method.getParameters()[5]
.getAnnotation(org.springframework.beans.factory.annotation.Value.class).value());
}
@@ -66,13 +66,13 @@ class KafkaReliabilityConfigTest {
ConcurrentKafkaListenerContainerFactory<Object, Object> factory =
config.rawKafkaListenerContainerFactory(
configurer, consumerFactory, errorHandler, 3, 500, 3000);
configurer, consumerFactory, errorHandler, 3, 3000, 5000);
assertEquals(ContainerProperties.AckMode.MANUAL_IMMEDIATE,
factory.getContainerProperties().getAckMode());
ArgumentCaptor<ConsumerFactory> actual = ArgumentCaptor.forClass(ConsumerFactory.class);
verify(configurer).configure(any(), actual.capture());
assertEquals(3000, actual.getValue().getConfigurationProperties().get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
assertEquals(5000, actual.getValue().getConfigurationProperties().get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
assertEquals(false, actual.getValue().getConfigurationProperties().get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG));
assertEquals(500, consumerFactory.getConfigurationProperties().get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
}
@@ -15,6 +15,8 @@ class SyslogBuildStartupLoggerTest {
MockEnvironment environment = new MockEnvironment()
.withProperty("syslog.raw-listener.max-poll-records", "700")
.withProperty("syslog.raw-listener.adaptive-max-records", "2500")
.withProperty("dm.standardized-write.batch-size", "800")
.withProperty("dm.non-normal-write.batch-size", "900")
.withProperty("syslog.slow-batch.threshold-ms", "1500")
.withProperty("stats.projection-async.enabled", "false");
environment.setActiveProfiles("prod");
@@ -32,6 +34,8 @@ class SyslogBuildStartupLoggerTest {
assertTrue(message.contains("profiles=prod"));
assertTrue(message.contains("baseProcessingRecords=700"));
assertTrue(message.contains("adaptiveMaxRecords=2500"));
assertTrue(message.contains("dmStandardizedBatchSize=800"));
assertTrue(message.contains("dmNonNormalBatchSize=900"));
assertTrue(message.contains("statsPublisherEnabled=false"));
assertTrue(message.contains("slowBatchThresholdMs=1500"));
} finally {
@@ -46,8 +50,10 @@ class SyslogBuildStartupLoggerTest {
.withProperty("codeRevision", "fake")).startupSummary();
assertTrue(summary.contains("codeRevision=" + SyslogBuildStartupLogger.CODE_REVISION));
assertFalse(summary.contains("buildTime="));
assertTrue(summary.contains("baseProcessingRecords=500"));
assertTrue(summary.contains("adaptiveMaxRecords=3000"));
assertTrue(summary.contains("baseProcessingRecords=3000"));
assertTrue(summary.contains("adaptiveMaxRecords=5000"));
assertTrue(summary.contains("dmStandardizedBatchSize=2000"));
assertTrue(summary.contains("dmNonNormalBatchSize=2000"));
assertTrue(summary.contains("slowBatchThresholdMs=1000"));
assertTrue(summary.contains("rawFieldMaxUtf8Bytes=2048"));
}