8173c2ff3f
- 入口统计改用独立 Kafka 消费组,移除主消费同步统计写库 - DIRECT 模式入库计数改为事务提交后异步投递,取消事实事务内统计 Outbox 写入 - 采样审计、告警分钟桶通知改为后台处理,异常不阻断主消费或回滚告警 - RAW 基础业务批量默认 500,按保留率动态扩大,上限 3000 - 限制业务块字节数及规则任务数量,保留告警全量处理和整次 poll 完成后提交 - 细化 SLOW_SYSLOG_BATCH,补充批次关联、线程排队、数据库执行及提交耗时 - 更新 v4 升级说明,明确异步统计损失风险及现场验证方法 验证:JDK 8 构建成功,232 项测试通过。 升级:复用现有表和 Kafka Topic,无新增建表或建 Topic 脚本要求。
108 lines
5.0 KiB
Java
108 lines
5.0 KiB
Java
package com.common.service;
|
|
|
|
import com.common.mapper.BacklogRetentionAuditMapper;
|
|
import com.config.BacklogSamplingProperties;
|
|
import org.junit.jupiter.api.Test;
|
|
import org.springframework.test.util.ReflectionTestUtils;
|
|
import org.springframework.transaction.PlatformTransactionManager;
|
|
import org.springframework.transaction.TransactionDefinition;
|
|
import org.springframework.transaction.TransactionStatus;
|
|
|
|
import java.time.LocalDateTime;
|
|
import java.util.Arrays;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
import static org.mockito.ArgumentMatchers.any;
|
|
import static org.mockito.Mockito.mock;
|
|
import static org.mockito.Mockito.verify;
|
|
import static org.mockito.Mockito.when;
|
|
|
|
class BacklogSamplingAuditServiceTest {
|
|
|
|
@Test
|
|
void aggregatesSameHourTopicPartitionAndPolicyWithoutMutatingInput() {
|
|
BacklogRetentionDelta first = delta(1, 1, 0, 1, 0);
|
|
BacklogRetentionDelta second = delta(1, 0, 1, 0, 1);
|
|
|
|
List<BacklogRetentionDelta> result = BacklogSamplingAuditService.aggregate(
|
|
Arrays.asList(first, second));
|
|
|
|
assertEquals(1, result.size());
|
|
assertEquals(2L, result.get(0).getMessageTotal());
|
|
assertEquals(1L, result.get(0).getNormalRowsSaved());
|
|
assertEquals(1L, result.get(0).getNormalRowsDropped());
|
|
assertEquals(1L, result.get(0).getRawSaved());
|
|
assertEquals(1L, result.get(0).getRawDropped());
|
|
assertEquals(1L, first.getMessageTotal(), "聚合不能修改调用方的逐消息审计对象");
|
|
}
|
|
|
|
@Test
|
|
void auditFailureIsBackgroundOnlyAndNeverTriggersFullRetentionFallback() {
|
|
BacklogRetentionAuditMapper mapper = mock(BacklogRetentionAuditMapper.class);
|
|
when(mapper.upsert(any())).thenThrow(new IllegalStateException("table unavailable"));
|
|
PlatformTransactionManager transactionManager = transactionManager();
|
|
BacklogSamplingMetrics metrics = new BacklogSamplingMetrics();
|
|
BacklogSamplingAuditService service = new BacklogSamplingAuditService(
|
|
mapper, properties(), metrics, transactionManager);
|
|
|
|
boolean audited = service.recordAsync(Arrays.asList(
|
|
delta(2, 1, 1, 1, 1)));
|
|
|
|
assertTrue(audited);
|
|
org.mockito.Mockito.verifyNoInteractions(mapper, transactionManager);
|
|
((BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(service, "queue")).drain();
|
|
Map<String, Object> snapshot = metrics.snapshot();
|
|
assertEquals(1L, snapshot.get("audit_write_failures_total"));
|
|
assertEquals(0L, snapshot.get("audit_fallback_messages_total"));
|
|
assertEquals(2L, snapshot.get("audit_lost_or_unconfirmed_messages_total"));
|
|
assertFalse(service.recordAsync(Arrays.asList(delta(1, 0, 1, 0, 1))));
|
|
assertEquals(3L, metrics.snapshot().get("degraded_messages_total"));
|
|
}
|
|
|
|
@Test
|
|
void successfulAuditCommitsAndRecordsSamplingMetrics() {
|
|
BacklogRetentionAuditMapper mapper = mock(BacklogRetentionAuditMapper.class);
|
|
when(mapper.upsert(any())).thenReturn(1);
|
|
PlatformTransactionManager transactionManager = transactionManager();
|
|
BacklogSamplingMetrics metrics = new BacklogSamplingMetrics();
|
|
BacklogSamplingAuditService service = new BacklogSamplingAuditService(
|
|
mapper, properties(), metrics, transactionManager);
|
|
|
|
assertTrue(service.recordAsync(Arrays.asList(
|
|
delta(1, 0, 1, 0, 1))));
|
|
|
|
org.mockito.Mockito.verifyNoInteractions(mapper, transactionManager);
|
|
((BestEffortAuxiliaryQueue) ReflectionTestUtils.getField(service, "queue")).drain();
|
|
verify(mapper).upsert(any());
|
|
verify(transactionManager).commit(any());
|
|
assertEquals(1L, metrics.snapshot().get("degraded_messages_total"));
|
|
assertEquals(1L, metrics.snapshot().get("normal_rows_dropped_total"));
|
|
}
|
|
|
|
private BacklogSamplingProperties properties() {
|
|
BacklogSamplingProperties properties = new BacklogSamplingProperties();
|
|
ReflectionTestUtils.setField(properties, "auditTimeoutSeconds", 5);
|
|
ReflectionTestUtils.setField(properties, "auditRetryBackoffMs", 60_000L);
|
|
return properties;
|
|
}
|
|
|
|
private PlatformTransactionManager transactionManager() {
|
|
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
|
|
TransactionStatus status = mock(TransactionStatus.class);
|
|
when(transactionManager.getTransaction(any(TransactionDefinition.class))).thenReturn(status);
|
|
return transactionManager;
|
|
}
|
|
|
|
private BacklogRetentionDelta delta(long messages, long normalSaved, long normalDropped,
|
|
long rawSaved, long rawDropped) {
|
|
return new BacklogRetentionDelta(LocalDateTime.of(2026, 9, 12, 18, 0),
|
|
"raw", 0, "v1", 1000, 10000,
|
|
messages, 0, 0, 1, normalSaved, normalDropped,
|
|
0, 0, 0, 1, rawSaved, rawDropped, 3_600_000L, 3_600_000L);
|
|
}
|
|
}
|