feat(syslog): 增加积压采样免采样窗口
- 新增 start-lag-ms 配置，默认15分钟内全量保留 - 在15分钟至1小时之间按10级逐步降低保留率 - 延迟达到1小时后使用普通10%、非标20%的最低保留率 - 校验起始延迟必须小于最低保留率阈值 - 状态接口新增 start_lag_ms 配置展示 - 将采样策略版本升级为 v2 - 更新 Docker Compose 和 v4 升级配置说明 - 补充采样时间边界及非法配置单元测试
This commit is contained in:
王绍宏
+9
@@ -0,0 +1,9 @@
|
||||
package com.common.mapper;
|
||||
|
||||
import com.common.service.BacklogRetentionDelta;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface BacklogRetentionAuditMapper {
|
||||
int upsert(BacklogRetentionDelta delta);
|
||||
}
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
package com.common.service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Objects;
|
||||
|
||||
/** 一批积压消息按来源小时、Topic、Partition 聚合后的采样审计增量。 */
|
||||
public final class BacklogRetentionDelta {
|
||||
private final LocalDateTime sourceHour;
|
||||
private final String sourceTopic;
|
||||
private final int kafkaPartition;
|
||||
private final String policyVersion;
|
||||
private final int normalRetainBps;
|
||||
private final int nonNormalRetainBps;
|
||||
private long messageTotal;
|
||||
private long alarmRowsTotal;
|
||||
private long alarmRowsSaved;
|
||||
private long normalRowsTotal;
|
||||
private long normalRowsSaved;
|
||||
private long normalRowsDropped;
|
||||
private long nonNormalRowsTotal;
|
||||
private long nonNormalRowsSaved;
|
||||
private long nonNormalRowsDropped;
|
||||
private long rawTotal;
|
||||
private long rawSaved;
|
||||
private long rawDropped;
|
||||
private long minLagMs;
|
||||
private long maxLagMs;
|
||||
|
||||
public BacklogRetentionDelta(LocalDateTime sourceHour, String sourceTopic, int kafkaPartition,
|
||||
String policyVersion, int normalRetainBps, int nonNormalRetainBps,
|
||||
long messageTotal, long alarmRowsTotal, long alarmRowsSaved,
|
||||
long normalRowsTotal, long normalRowsSaved, long normalRowsDropped,
|
||||
long nonNormalRowsTotal, long nonNormalRowsSaved,
|
||||
long nonNormalRowsDropped, long rawTotal, long rawSaved,
|
||||
long rawDropped, long minLagMs, long maxLagMs) {
|
||||
this.sourceHour = Objects.requireNonNull(sourceHour, "sourceHour");
|
||||
this.sourceTopic = Objects.requireNonNull(sourceTopic, "sourceTopic");
|
||||
this.kafkaPartition = kafkaPartition;
|
||||
this.policyVersion = Objects.requireNonNull(policyVersion, "policyVersion");
|
||||
this.normalRetainBps = normalRetainBps;
|
||||
this.nonNormalRetainBps = nonNormalRetainBps;
|
||||
this.messageTotal = messageTotal;
|
||||
this.alarmRowsTotal = alarmRowsTotal;
|
||||
this.alarmRowsSaved = alarmRowsSaved;
|
||||
this.normalRowsTotal = normalRowsTotal;
|
||||
this.normalRowsSaved = normalRowsSaved;
|
||||
this.normalRowsDropped = normalRowsDropped;
|
||||
this.nonNormalRowsTotal = nonNormalRowsTotal;
|
||||
this.nonNormalRowsSaved = nonNormalRowsSaved;
|
||||
this.nonNormalRowsDropped = nonNormalRowsDropped;
|
||||
this.rawTotal = rawTotal;
|
||||
this.rawSaved = rawSaved;
|
||||
this.rawDropped = rawDropped;
|
||||
this.minLagMs = minLagMs;
|
||||
this.maxLagMs = maxLagMs;
|
||||
}
|
||||
|
||||
public void merge(BacklogRetentionDelta other) {
|
||||
if (!sameKey(other)) throw new IllegalArgumentException("不能合并不同维度的采样审计增量");
|
||||
messageTotal += other.messageTotal;
|
||||
alarmRowsTotal += other.alarmRowsTotal;
|
||||
alarmRowsSaved += other.alarmRowsSaved;
|
||||
normalRowsTotal += other.normalRowsTotal;
|
||||
normalRowsSaved += other.normalRowsSaved;
|
||||
normalRowsDropped += other.normalRowsDropped;
|
||||
nonNormalRowsTotal += other.nonNormalRowsTotal;
|
||||
nonNormalRowsSaved += other.nonNormalRowsSaved;
|
||||
nonNormalRowsDropped += other.nonNormalRowsDropped;
|
||||
rawTotal += other.rawTotal;
|
||||
rawSaved += other.rawSaved;
|
||||
rawDropped += other.rawDropped;
|
||||
minLagMs = Math.min(minLagMs, other.minLagMs);
|
||||
maxLagMs = Math.max(maxLagMs, other.maxLagMs);
|
||||
}
|
||||
|
||||
public BacklogRetentionDelta copy() {
|
||||
return new BacklogRetentionDelta(sourceHour, sourceTopic, kafkaPartition, policyVersion,
|
||||
normalRetainBps, nonNormalRetainBps, messageTotal, alarmRowsTotal,
|
||||
alarmRowsSaved, normalRowsTotal, normalRowsSaved, normalRowsDropped,
|
||||
nonNormalRowsTotal, nonNormalRowsSaved, nonNormalRowsDropped,
|
||||
rawTotal, rawSaved, rawDropped, minLagMs, maxLagMs);
|
||||
}
|
||||
|
||||
private boolean sameKey(BacklogRetentionDelta other) {
|
||||
return other != null && kafkaPartition == other.kafkaPartition
|
||||
&& normalRetainBps == other.normalRetainBps
|
||||
&& nonNormalRetainBps == other.nonNormalRetainBps
|
||||
&& sourceHour.equals(other.sourceHour)
|
||||
&& sourceTopic.equals(other.sourceTopic)
|
||||
&& policyVersion.equals(other.policyVersion);
|
||||
}
|
||||
|
||||
public String aggregationKey() {
|
||||
return sourceHour + "\u0000" + sourceTopic + "\u0000" + kafkaPartition + "\u0000"
|
||||
+ policyVersion + "\u0000" + normalRetainBps + "\u0000" + nonNormalRetainBps;
|
||||
}
|
||||
|
||||
public LocalDateTime getSourceHour() { return sourceHour; }
|
||||
public String getSourceTopic() { return sourceTopic; }
|
||||
public int getKafkaPartition() { return kafkaPartition; }
|
||||
public String getPolicyVersion() { return policyVersion; }
|
||||
public int getNormalRetainBps() { return normalRetainBps; }
|
||||
public int getNonNormalRetainBps() { return nonNormalRetainBps; }
|
||||
public long getMessageTotal() { return messageTotal; }
|
||||
public long getAlarmRowsTotal() { return alarmRowsTotal; }
|
||||
public long getAlarmRowsSaved() { return alarmRowsSaved; }
|
||||
public long getNormalRowsTotal() { return normalRowsTotal; }
|
||||
public long getNormalRowsSaved() { return normalRowsSaved; }
|
||||
public long getNormalRowsDropped() { return normalRowsDropped; }
|
||||
public long getNonNormalRowsTotal() { return nonNormalRowsTotal; }
|
||||
public long getNonNormalRowsSaved() { return nonNormalRowsSaved; }
|
||||
public long getNonNormalRowsDropped() { return nonNormalRowsDropped; }
|
||||
public long getRawTotal() { return rawTotal; }
|
||||
public long getRawSaved() { return rawSaved; }
|
||||
public long getRawDropped() { return rawDropped; }
|
||||
public long getMinLagMs() { return minLagMs; }
|
||||
public long getMaxLagMs() { return maxLagMs; }
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
package com.common.service;
|
||||
|
||||
import com.common.mapper.BacklogRetentionAuditMapper;
|
||||
import com.config.BacklogSamplingProperties;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/** 先提交小时审计，再允许调用方执行有损采样；审计失败时调用方必须回退全量保留。 */
|
||||
@Slf4j
|
||||
@Service
|
||||
public class BacklogSamplingAuditService {
|
||||
private final BacklogRetentionAuditMapper mapper;
|
||||
private final BacklogSamplingProperties properties;
|
||||
private final BacklogSamplingMetrics metrics;
|
||||
private final PlatformTransactionManager transactionManager;
|
||||
private volatile long retryAfterMs;
|
||||
|
||||
public BacklogSamplingAuditService(BacklogRetentionAuditMapper mapper,
|
||||
BacklogSamplingProperties properties,
|
||||
BacklogSamplingMetrics metrics,
|
||||
PlatformTransactionManager transactionManager) {
|
||||
this.mapper = mapper;
|
||||
this.properties = properties;
|
||||
this.metrics = metrics;
|
||||
this.transactionManager = transactionManager;
|
||||
}
|
||||
|
||||
/** 返回 false 表示不能安全采样，调用方应保留当前批次全部数据。 */
|
||||
public boolean recordBeforeDropping(List<BacklogRetentionDelta> deltas) {
|
||||
if (deltas == null || deltas.isEmpty()) return true;
|
||||
long affectedMessages = messageCount(deltas);
|
||||
long now = System.currentTimeMillis();
|
||||
if (now < retryAfterMs) {
|
||||
metrics.auditFailure(affectedMessages);
|
||||
return false;
|
||||
}
|
||||
|
||||
List<BacklogRetentionDelta> aggregated = aggregate(deltas);
|
||||
try {
|
||||
TransactionTemplate transaction = new TransactionTemplate(transactionManager);
|
||||
transaction.setTimeout(Math.max(1, properties.getAuditTimeoutSeconds()));
|
||||
transaction.execute(status -> {
|
||||
for (BacklogRetentionDelta delta : aggregated) mapper.upsert(delta);
|
||||
return null;
|
||||
});
|
||||
retryAfterMs = 0L;
|
||||
metrics.record(aggregated);
|
||||
return true;
|
||||
} catch (RuntimeException failure) {
|
||||
retryAfterMs = System.currentTimeMillis() + Math.max(1L, properties.getAuditRetryBackoffMs());
|
||||
metrics.auditFailure(affectedMessages);
|
||||
log.error("[BACKLOG_SAMPLING_AUDIT_FAILURE] 小时采样审计写入失败，当前批次自动回退全量保留: "
|
||||
+ "messages={}, groups={}, retryBackoffMs={}",
|
||||
affectedMessages, aggregated.size(), properties.getAuditRetryBackoffMs(), failure);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static List<BacklogRetentionDelta> aggregate(List<BacklogRetentionDelta> deltas) {
|
||||
Map<String, BacklogRetentionDelta> grouped = new LinkedHashMap<>();
|
||||
for (BacklogRetentionDelta delta : deltas) {
|
||||
BacklogRetentionDelta existing = grouped.get(delta.aggregationKey());
|
||||
if (existing == null) grouped.put(delta.aggregationKey(), delta.copy());
|
||||
else existing.merge(delta);
|
||||
}
|
||||
return new ArrayList<>(grouped.values());
|
||||
}
|
||||
|
||||
private long messageCount(List<BacklogRetentionDelta> deltas) {
|
||||
long total = 0L;
|
||||
for (BacklogRetentionDelta delta : deltas) total += delta.getMessageTotal();
|
||||
return total;
|
||||
}
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
package com.common.service;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
|
||||
/** 积压采样热路径指标；表审计失败时仍可从接口和日志发现。 */
|
||||
@Component
|
||||
public class BacklogSamplingMetrics {
|
||||
private final LongAdder degradedMessages = new LongAdder();
|
||||
private final LongAdder alarmRowsSaved = new LongAdder();
|
||||
private final LongAdder normalRowsSaved = new LongAdder();
|
||||
private final LongAdder normalRowsDropped = new LongAdder();
|
||||
private final LongAdder nonNormalRowsSaved = new LongAdder();
|
||||
private final LongAdder nonNormalRowsDropped = new LongAdder();
|
||||
private final LongAdder rawSaved = new LongAdder();
|
||||
private final LongAdder rawDropped = new LongAdder();
|
||||
private final LongAdder auditWriteFailures = new LongAdder();
|
||||
private final LongAdder auditFallbackMessages = new LongAdder();
|
||||
|
||||
public void record(List<BacklogRetentionDelta> deltas) {
|
||||
for (BacklogRetentionDelta delta : deltas) {
|
||||
degradedMessages.add(delta.getMessageTotal());
|
||||
alarmRowsSaved.add(delta.getAlarmRowsSaved());
|
||||
normalRowsSaved.add(delta.getNormalRowsSaved());
|
||||
normalRowsDropped.add(delta.getNormalRowsDropped());
|
||||
nonNormalRowsSaved.add(delta.getNonNormalRowsSaved());
|
||||
nonNormalRowsDropped.add(delta.getNonNormalRowsDropped());
|
||||
rawSaved.add(delta.getRawSaved());
|
||||
rawDropped.add(delta.getRawDropped());
|
||||
}
|
||||
}
|
||||
|
||||
public void auditFailure(long affectedMessages) {
|
||||
auditWriteFailures.increment();
|
||||
auditFallbackMessages.add(Math.max(0L, affectedMessages));
|
||||
}
|
||||
|
||||
public Map<String, Object> snapshot() {
|
||||
Map<String, Object> values = new LinkedHashMap<>();
|
||||
values.put("degraded_messages_total", degradedMessages.sum());
|
||||
values.put("alarm_rows_saved_total", alarmRowsSaved.sum());
|
||||
values.put("normal_rows_saved_total", normalRowsSaved.sum());
|
||||
values.put("normal_rows_dropped_total", normalRowsDropped.sum());
|
||||
values.put("non_normal_rows_saved_total", nonNormalRowsSaved.sum());
|
||||
values.put("non_normal_rows_dropped_total", nonNormalRowsDropped.sum());
|
||||
values.put("raw_saved_total", rawSaved.sum());
|
||||
values.put("raw_dropped_total", rawDropped.sum());
|
||||
values.put("audit_write_failures_total", auditWriteFailures.sum());
|
||||
values.put("audit_fallback_messages_total", auditFallbackMessages.sum());
|
||||
return values;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user