nanChen/ai-security-xdr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
44d327139b170fabf82633330369eb49f2a0210c
ai-security-xdr/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DirectStatsProjectionWriter.java
T

596 lines
28 KiB
Java
Raw Normal View History

feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
package com.common.service;
import com.common.entity.SyslogNonNormalMessage;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.dao.CannotAcquireLockException;
import org.springframework.dao.DeadlockLoserDataAccessException;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.sql.Date;
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
import java.sql.SQLException;
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
/**
feat(syslog): 优化大日志通道并增加异常连续消费兜底
2026-09-12 19:22:52 +08:00
* 把当前事实事务的内存聚合结果写入同步投影或可靠异步投影入口。
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
*
feat(syslog): 优化大日志通道并增加异常连续消费兜底
2026-09-12 19:22:52 +08:00
* <p>DIRECT 默认只追加一条批次 Outbox；SHADOW/显式回退仍同步更新统计表。调用方必须
* 已经开启事实事务，本类不会吞掉任何数据库异常。</p>
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
*/
@Slf4j
@Service
public class DirectStatsProjectionWriter {
private static final String DEVICE_MERGE_TEMPLATE =
"MERGE INTO %s t USING (SELECT ? stat_date,? device_id,? parse_delta,? non_log_delta FROM DUAL) s " +
"ON (t.stat_date=s.stat_date AND t.device_id=s.device_id) " +
"WHEN MATCHED THEN UPDATE SET t.parse_count=t.parse_count+s.parse_delta," +
"t.non_log_count=t.non_log_count+s.non_log_delta,t.updated_at=SYSDATE " +
"WHEN NOT MATCHED THEN INSERT (stat_date,device_id,parse_count,non_log_count," +
"last_receive_time,projection_seq,updated_at) " +
"VALUES (s.stat_date,s.device_id,s.parse_delta,s.non_log_delta,NULL,0,SYSDATE)";
private static final String RULE_MERGE_TEMPLATE =
"MERGE INTO %s t USING (SELECT ? stat_date,? normalize_rule_id,? max_log_time FROM DUAL) s " +
"ON (t.stat_date=s.stat_date AND t.normalize_rule_id=s.normalize_rule_id) " +
"WHEN MATCHED THEN UPDATE SET t.max_log_time=CASE " +
"WHEN s.max_log_time IS NULL THEN t.max_log_time " +
"WHEN t.max_log_time IS NULL OR t.max_log_time<s.max_log_time THEN s.max_log_time " +
"ELSE t.max_log_time END,t.updated_at=SYSDATE " +
"WHEN NOT MATCHED THEN INSERT (stat_date,normalize_rule_id,max_log_time,projection_seq,updated_at) " +
"VALUES (s.stat_date,s.normalize_rule_id,s.max_log_time,0,SYSDATE)";
private static final String DEVICE_EXACT_MERGE_TEMPLATE =
"MERGE INTO %s t USING (SELECT ? stat_date,? device_id,? parse_count,? non_log_count FROM DUAL) s " +
"ON (t.stat_date=s.stat_date AND t.device_id=s.device_id) " +
"WHEN MATCHED THEN UPDATE SET t.parse_count=s.parse_count," +
"t.non_log_count=s.non_log_count,t.updated_at=SYSDATE " +
"WHEN NOT MATCHED THEN INSERT (stat_date,device_id,parse_count,non_log_count," +
"last_receive_time,projection_seq,updated_at) " +
"VALUES (s.stat_date,s.device_id,s.parse_count,s.non_log_count,NULL,0,SYSDATE)";
private final JdbcTemplate jdbc;
private final StatsWriteModeProvider modeProvider;
private final DirectStatsWriteMetrics metrics;
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
private final StatsMaterializationPublisher materializationPublisher;
feat(syslog): 优化大日志通道并增加异常连续消费兜底
2026-09-12 19:22:52 +08:00
private final StatsProjectionOutboxWriter projectionOutboxWriter;
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
@Value("${stats.incremental.read-enabled:false}")
private boolean readEnabled;
@Value("${stats.direct.device-table:device_daily_stats}")
private String directDeviceTable;
@Value("${stats.direct.rule-table:normalize_rule_daily_stats}")
private String directRuleTable;
@Value("${stats.direct.shadow-device-table:device_daily_stats_direct_shadow}")
private String shadowDeviceTable;
@Value("${stats.direct.shadow-rule-table:normalize_rule_daily_stats_direct_shadow}")
private String shadowRuleTable;
@Value("${stats.direct.control-table:stats_direct_control}")
private String controlTable;
@Value("${stats.direct.today-stats-date-enabled:true}")
private boolean todayStatsDateEnabled;
@Value("${stats.business-zone:Asia/Shanghai}")
private String businessZone;
public DirectStatsProjectionWriter(JdbcTemplate jdbc,
StatsWriteModeProvider modeProvider,
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
DirectStatsWriteMetrics metrics,
feat(syslog): 优化大日志通道并增加异常连续消费兜底
2026-09-12 19:22:52 +08:00
StatsMaterializationPublisher materializationPublisher,
StatsProjectionOutboxWriter projectionOutboxWriter) {
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
this.jdbc = jdbc;
this.modeProvider = modeProvider;
this.metrics = metrics;
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
this.materializationPublisher = materializationPublisher;
feat(syslog): 优化大日志通道并增加异常连续消费兜底
2026-09-12 19:22:52 +08:00
this.projectionOutboxWriter = projectionOutboxWriter;
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
}
public WriteResult writeStandardizedRows(List<Map<String, Object>> rows) {
if (!shouldBuildAggregation()) {
handleFactOnlyMode();
return WriteResult.inactive(rows == null ? 0 : rows.size());
}
long aggregateStarted = System.nanoTime();
DirectStatsAccumulator accumulator = accumulator().addStandardizedRows(rows);
return write(accumulator, elapsedMs(aggregateStarted));
}
public WriteResult writeStandardizedRow(Map<String, Object> row) {
if (!shouldBuildAggregation()) {
handleFactOnlyMode();
return WriteResult.inactive(row == null ? 0 : 1);
}
long aggregateStarted = System.nanoTime();
DirectStatsAccumulator accumulator = accumulator().addStandardizedRow(row);
return write(accumulator, elapsedMs(aggregateStarted));
}
public WriteResult writeNonNormalRows(List<SyslogNonNormalMessage> messages) {
if (!shouldBuildAggregation()) {
handleFactOnlyMode();
return WriteResult.inactive(messages == null ? 0 : messages.size());
}
long aggregateStarted = System.nanoTime();
DirectStatsAccumulator accumulator = accumulator().addNonNormalRows(messages);
return write(accumulator, elapsedMs(aggregateStarted));
}
public WriteResult writeNonNormalInsert(SyslogNonNormalMessage message) {
if (!shouldBuildAggregation()) {
handleFactOnlyMode();
return WriteResult.inactive(message == null ? 0 : 1);
}
long aggregateStarted = System.nanoTime();
DirectStatsAccumulator accumulator = accumulator().addNonNormalInsert(message);
return write(accumulator, elapsedMs(aggregateStarted));
}
public WriteResult writeNonNormalUpdate(SyslogNonNormalMessage oldValue, SyslogNonNormalMessage patch) {
if (!shouldBuildAggregation()) {
handleFactOnlyMode();
return WriteResult.inactive(oldValue == null || patch == null ? 0 : 1);
}
long aggregateStarted = System.nanoTime();
DirectStatsAccumulator accumulator = accumulator().addNonNormalUpdate(oldValue, patch);
return writeExactDeviceGroups(accumulator, elapsedMs(aggregateStarted));
}
public void recordTransactionRetry() {
metrics.recordRetry();
}
public Map<String, Object> status() {
Map<String, Object> status = new LinkedHashMap<>(metrics.snapshot(modeProvider.getMode()));
status.put("read_enabled", readEnabled);
status.put("device_table", selectedDeviceTable());
status.put("rule_table", selectedRuleTable());
status.put("today_stats_date_enabled", todayStatsDateEnabled);
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
status.put("business_materialization_mode", asynchronousMaterialization()
? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS");
feat(syslog): 优化大日志通道并增加异常连续消费兜底
2026-09-12 19:22:52 +08:00
status.put("daily_projection_write_mode", asynchronousProjection()
? "DURABLE_OUTBOX" : "SYNCHRONOUS_MERGE");
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
return status;
}
private boolean shouldBuildAggregation() {
return modeProvider.writesDirectProjection();
}
private DirectStatsAccumulator accumulator() {
return new DirectStatsAccumulator(ZoneId.of(businessZone));
}
private void handleFactOnlyMode() {
if (!modeProvider.requiresRebuild()) {
return;
}
requireTransaction();
metrics.markRebuildRequired();
String table = safeTable(controlTable);
jdbc.update("MERGE INTO " + table + " t USING " +
"(SELECT 'SYSLOG_STATS' control_name FROM DUAL) s " +
"ON (t.control_name=s.control_name) " +
"WHEN MATCHED THEN UPDATE SET t.rebuild_required=1,t.updated_at=SYSDATE " +
"WHEN NOT MATCHED THEN INSERT (control_name,rebuild_required,updated_at) " +
"VALUES (s.control_name,1,SYSDATE)");
}
private WriteResult write(DirectStatsAccumulator accumulator, long aggregateMs) {
if (accumulator.isEmpty()) {
metrics.recordSuccess(accumulator.getInputRows(), 0, 0, aggregateMs, 0L, 0L);
return WriteResult.direct(accumulator.getInputRows(), 0, 0,
aggregateMs, 0L, 0L);
}
requireTransaction();
feat(syslog): 优化大日志通道并增加异常连续消费兜底
2026-09-12 19:22:52 +08:00
if (asynchronousProjection()) {
try {
long outboxStarted = System.nanoTime();
projectionOutboxWriter.append(StatsProjectionOutboxEvent.fromAccumulator(accumulator));
long outboxMs = elapsedMs(outboxStarted);
int deviceGroups = effectiveDeviceEntries(accumulator).size();
int ruleGroups = effectiveRuleEntries(accumulator).size();
metrics.recordSuccess(accumulator.getInputRows(), deviceGroups, ruleGroups,
aggregateMs, outboxMs, 0L);
return WriteResult.direct(accumulator.getInputRows(), deviceGroups, ruleGroups,
aggregateMs, outboxMs, 0L);
} catch (RuntimeException failure) {
metrics.recordFailure();
throw new DirectStatsWriteException(
"统计投影 Outbox 写入失败，事实事务必须整体回滚", failure);
}
}
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> devices =
effectiveDeviceEntries(accumulator);
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> rules =
effectiveRuleEntries(accumulator);
long mergeMs;
long materializeMs = 0L;
try {
long mergeStarted = System.nanoTime();
mergeDevices(devices);
mergeRules(rules);
mergeMs = elapsedMs(mergeStarted);
if (modeProvider.materializesBusinessFields() && readEnabled) {
long materializeStarted = System.nanoTime();
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
if (asynchronousMaterialization()) {
materializationPublisher.enqueueAfterCommit(materializationCommands(devices, rules));
} else {
materializeDevices(devices);
materializeRules(rules);
}
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
materializeMs = elapsedMs(materializeStarted);
}
metrics.recordSuccess(accumulator.getInputRows(), devices.size(), rules.size(),
aggregateMs, mergeMs, materializeMs);
log.debug("直接统计事务块完成: factRows={}, deviceGroups={}, ruleGroups={}, " +
"aggregateMs={}, statsMergeMs={}, materializeMs={}",
accumulator.getInputRows(), devices.size(), rules.size(),
aggregateMs, mergeMs, materializeMs);
return WriteResult.direct(accumulator.getInputRows(), devices.size(), rules.size(),
aggregateMs, mergeMs, materializeMs);
} catch (RuntimeException failure) {
metrics.recordFailure();
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
if (isConcurrentFailure(failure)) {
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
throw new ConcurrentStatsWriteException("直接统计聚合键并发写入失败，可安全回滚后有限重试", failure);
}
throw new DirectStatsWriteException("直接统计写入失败，事实事务必须整体回滚", failure);
}
}
/**
* UPDATE/物理 DELETE 属于低频管理路径。负 delta 可能移除当前最大值或命中未建立历史基线的日期，
* 因此直接从事实表精确重算受影响的小分组，不能把负数插入一个不存在的聚合键。
*/
private WriteResult writeExactDeviceGroups(DirectStatsAccumulator accumulator, long aggregateMs) {
if (accumulator.getDevices().isEmpty()) {
metrics.recordSuccess(accumulator.getInputRows(), 0, 0, aggregateMs, 0L, 0L);
return WriteResult.direct(accumulator.getInputRows(), 0, 0,
aggregateMs, 0L, 0L);
}
requireTransaction();
feat(syslog): 优化大日志通道并增加异常连续消费兜底
2026-09-12 19:22:52 +08:00
// Additive positive/negative changes commute in the serialized projection consumer. This
// keeps non-normal UPDATE statistics outside the fact transaction as well.
if (asynchronousProjection()) {
return write(accumulator, aggregateMs);
}
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
List<DirectStatsAccumulator.DailyKey> keys = new ArrayList<>(accumulator.getDevices().keySet());
try {
long mergeStarted = System.nanoTime();
List<Object[]> exactValues = new ArrayList<>(keys.size());
for (DirectStatsAccumulator.DailyKey key : keys) {
LocalDateTime start = key.getDate().atStartOfDay();
LocalDateTime end = key.getDate().plusDays(1).atStartOfDay();
Map<String, Object> counts = jdbc.queryForMap(
"SELECT " +
"((SELECT COUNT(*) FROM syslog_normal_data " +
"WHERE created_at>=? AND created_at<? AND device_id=?) + " +
" (SELECT COUNT(*) FROM syslog_normal_alarm " +
"WHERE created_at>=? AND created_at<? AND device_id=?)) parse_count," +
"(SELECT COUNT(*) FROM syslog_non_normal_message " +
"WHERE created_at>=? AND created_at<? AND device_id=?) non_log_count FROM DUAL",
start, end, key.getDimensionId(),
start, end, key.getDimensionId(),
start, end, key.getDimensionId());
exactValues.add(new Object[]{Date.valueOf(key.getDate()), key.getDimensionId(),
number(counts, "parse_count"), number(counts, "non_log_count")});
}
jdbc.batchUpdate(String.format(DEVICE_EXACT_MERGE_TEMPLATE,
safeTable(selectedDeviceTable())), exactValues);
long mergeMs = elapsedMs(mergeStarted);
long materializeMs = 0L;
if (modeProvider.materializesBusinessFields() && readEnabled) {
long materializeStarted = System.nanoTime();
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
if (asynchronousMaterialization()) {
materializationPublisher.enqueueAfterCommit(deviceMaterializationCommands(keys));
} else {
materializeDeviceKeys(keys);
}
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
materializeMs = elapsedMs(materializeStarted);
}
metrics.recordSuccess(accumulator.getInputRows(), keys.size(), 0,
aggregateMs, mergeMs, materializeMs);
return WriteResult.direct(accumulator.getInputRows(), keys.size(), 0,
aggregateMs, mergeMs, materializeMs);
} catch (RuntimeException failure) {
metrics.recordFailure();
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
if (isConcurrentFailure(failure)) {
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
throw new ConcurrentStatsWriteException("直接统计局部分组并发写入失败，可安全回滚后有限重试", failure);
}
throw new DirectStatsWriteException("直接统计局部分组重算失败，事实事务必须整体回滚", failure);
}
}
private List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>>
effectiveDeviceEntries(DirectStatsAccumulator accumulator) {
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> result =
new ArrayList<>();
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry
: accumulator.getDevices().entrySet()) {
DirectStatsAccumulator.DeviceDelta delta = entry.getValue();
if (delta.getParseDelta() != 0L || delta.getNonLogDelta() != 0L) {
result.add(entry);
}
}
return result;
}
private List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>>
effectiveRuleEntries(DirectStatsAccumulator accumulator) {
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> result =
new ArrayList<>();
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry
: accumulator.getRules().entrySet()) {
if (entry.getValue().getMaxLogTime() != null) {
result.add(entry);
}
}
return result;
}
private void mergeDevices(
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> entries) {
if (entries.isEmpty()) return;
List<Object[]> parameters = new ArrayList<>(entries.size());
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry : entries) {
parameters.add(new Object[]{Date.valueOf(entry.getKey().getDate()),
entry.getKey().getDimensionId(), entry.getValue().getParseDelta(),
entry.getValue().getNonLogDelta()});
}
jdbc.batchUpdate(String.format(DEVICE_MERGE_TEMPLATE, safeTable(selectedDeviceTable())), parameters);
}
private void mergeRules(
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> entries) {
if (entries.isEmpty()) return;
List<Object[]> parameters = new ArrayList<>(entries.size());
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry : entries) {
parameters.add(new Object[]{Date.valueOf(entry.getKey().getDate()),
entry.getKey().getDimensionId(), entry.getValue().getMaxLogTime()});
}
jdbc.batchUpdate(String.format(RULE_MERGE_TEMPLATE, safeTable(selectedRuleTable())), parameters);
}
private void materializeDevices(
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> entries) {
List<DirectStatsAccumulator.DailyKey> keys = new ArrayList<>(entries.size());
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry : entries) {
keys.add(entry.getKey());
}
materializeDeviceKeys(keys);
}
private void materializeDeviceKeys(List<DirectStatsAccumulator.DailyKey> keys) {
LocalDate today = LocalDate.now(ZoneId.of(businessZone));
List<Object[]> parameters = new ArrayList<>();
for (DirectStatsAccumulator.DailyKey key : keys) {
if (today.equals(key.getDate())) {
long deviceId = key.getDimensionId();
parameters.add(new Object[]{Date.valueOf(today), deviceId,
Date.valueOf(today), deviceId, Date.valueOf(today), deviceId});
}
}
if (parameters.isEmpty()) return;
String deviceTable = safeTable(selectedDeviceTable());
String dateAssignment = todayStatsDateEnabled ? ",today_stats_date=?" : "";
String sql = "UPDATE device_device SET " +
"today_parse_count=(SELECT parse_count FROM " + deviceTable +
" WHERE stat_date=? AND device_id=?)," +
"today_non_log_count=(SELECT non_log_count FROM " + deviceTable +
" WHERE stat_date=? AND device_id=?)" + dateAssignment +
",updated_at=SYSDATE WHERE id=?";
if (!todayStatsDateEnabled) {
List<Object[]> withoutDateColumn = new ArrayList<>(parameters.size());
for (Object[] row : parameters) {
withoutDateColumn.add(new Object[]{row[0], row[1], row[2], row[3], row[5]});
}
jdbc.batchUpdate(sql, withoutDateColumn);
} else {
jdbc.batchUpdate(sql, parameters);
}
}
private void materializeRules(
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> entries) {
SortedMap<Long, LocalDateTime> maxima = new TreeMap<>();
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry : entries) {
long ruleId = entry.getKey().getDimensionId();
LocalDateTime candidate = entry.getValue().getMaxLogTime();
LocalDateTime current = maxima.get(ruleId);
if (current == null || candidate.isAfter(current)) {
maxima.put(ruleId, candidate);
}
}
if (maxima.isEmpty()) return;
List<Object[]> parameters = new ArrayList<>(maxima.size());
for (Map.Entry<Long, LocalDateTime> entry : maxima.entrySet()) {
parameters.add(new Object[]{entry.getValue(), entry.getValue(), entry.getValue(), entry.getKey()});
}
jdbc.batchUpdate("UPDATE dm_normalize_rule SET first_data_saved_at=CASE " +
"WHEN ? IS NULL THEN first_data_saved_at " +
"WHEN first_data_saved_at IS NULL OR first_data_saved_at<? THEN ? " +
"ELSE first_data_saved_at END,updated_at=SYSDATE " +
"WHERE id=? AND del_flag='0'", parameters);
}
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
private boolean asynchronousMaterialization() {
return modeProvider.materializesBusinessFields() && readEnabled
&& materializationPublisher.isEnabled();
}
feat(syslog): 优化大日志通道并增加异常连续消费兜底
2026-09-12 19:22:52 +08:00
private boolean asynchronousProjection() {
return modeProvider.getMode() == StatsWriteModeProvider.Mode.DIRECT
&& projectionOutboxWriter != null && projectionOutboxWriter.isEnabled();
}
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
private List<StatsMaterializationCommand> materializationCommands(
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> devices,
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> rules) {
List<StatsMaterializationCommand> commands = new ArrayList<>(devices.size() + rules.size());
LocalDate today = LocalDate.now(ZoneId.of(businessZone));
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry : devices) {
DirectStatsAccumulator.DailyKey key = entry.getKey();
if (today.equals(key.getDate())) {
commands.add(StatsMaterializationCommand.device(key.getDate(), key.getDimensionId()));
}
}
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry : rules) {
DirectStatsAccumulator.DailyKey key = entry.getKey();
commands.add(StatsMaterializationCommand.rule(key.getDate(), key.getDimensionId()));
}
return commands;
}
private List<StatsMaterializationCommand> deviceMaterializationCommands(
List<DirectStatsAccumulator.DailyKey> keys) {
List<StatsMaterializationCommand> commands = new ArrayList<>(keys.size());
LocalDate today = LocalDate.now(ZoneId.of(businessZone));
for (DirectStatsAccumulator.DailyKey key : keys) {
if (today.equals(key.getDate())) {
commands.add(StatsMaterializationCommand.device(key.getDate(), key.getDimensionId()));
}
}
return commands;
}
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
private String selectedDeviceTable() {
return modeProvider.writesShadowProjection() ? shadowDeviceTable : directDeviceTable;
}
private String selectedRuleTable() {
return modeProvider.writesShadowProjection() ? shadowRuleTable : directRuleTable;
}
private static String safeTable(String table) {
if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
throw new IllegalStateException("非法统计表名: " + table);
}
return table;
}
private static void requireTransaction() {
if (!TransactionSynchronizationManager.isActualTransactionActive()) {
throw new IllegalStateException("直接统计必须与事实写入位于同一个 Spring 达梦事务");
}
}
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
/**
* 识别 Spring 与不同 JDBC 驱动对并发冲突的各种包装形式。
* 达梦可能把死锁翻译成 DataIntegrityViolationException，并仅在内层
* BatchUpdateException 的 SQLState/消息中保留 Deadlock 信息。
*/
public static boolean isConcurrentFailure(Throwable failure) {
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
Throwable current = failure;
while (current != null) {
if (current instanceof DuplicateKeyException
|| current instanceof CannotAcquireLockException
|| current instanceof DeadlockLoserDataAccessException) {
return true;
}
fix(syslog-consumer-rule): 解耦统计物化并解决多实例达梦死锁
2026-09-11 20:42:22 +08:00
if (current instanceof SQLException) {
String sqlState = ((SQLException) current).getSQLState();
if (sqlState != null && sqlState.startsWith("40")) {
return true;
}
}
String message = String.valueOf(current.getMessage())
.toLowerCase(java.util.Locale.ROOT);
if (message.contains("deadlock") || message.contains("死锁")
|| message.contains("unique constraint") || message.contains("duplicate key")) {
return true;
}
feat(syslog): 完成高吞吐可靠链路与统计存储优化
2026-08-20 00:18:33 +08:00
current = current.getCause();
}
return false;
}
private static long elapsedMs(long startedNanos) {
return (System.nanoTime() - startedNanos) / 1_000_000L;
}
private static long number(Map<String, Object> row, String name) {
Object value = row.get(name);
if (value == null) value = row.get(name.toUpperCase(java.util.Locale.ROOT));
if (value == null) {
for (Map.Entry<String, Object> entry : row.entrySet()) {
if (entry.getKey().equalsIgnoreCase(name)) {
value = entry.getValue();
break;
}
}
}
return value == null ? 0L : ((Number) value).longValue();
}
public static class DirectStatsWriteException extends RuntimeException {
DirectStatsWriteException(String message, Throwable cause) {
super(message, cause);
}
}
public static final class ConcurrentStatsWriteException extends DirectStatsWriteException {
ConcurrentStatsWriteException(String message, Throwable cause) {
super(message, cause);
}
}
/** 单个事实事务块的统计写入结果，用于批次 INFO 日志和事务耗时观测。 */
public static final class WriteResult {
private final boolean direct;
private final int inputRows;
private final int deviceGroups;
private final int ruleGroups;
private final long aggregateMs;
private final long mergeMs;
private final long materializeMs;
private WriteResult(boolean direct, int inputRows, int deviceGroups, int ruleGroups,
long aggregateMs, long mergeMs, long materializeMs) {
this.direct = direct;
this.inputRows = inputRows;
this.deviceGroups = deviceGroups;
this.ruleGroups = ruleGroups;
this.aggregateMs = aggregateMs;
this.mergeMs = mergeMs;
this.materializeMs = materializeMs;
}
static WriteResult inactive(int inputRows) {
return new WriteResult(false, inputRows, 0, 0, 0L, 0L, 0L);
}
static WriteResult direct(int inputRows, int deviceGroups, int ruleGroups,
long aggregateMs, long mergeMs, long materializeMs) {
return new WriteResult(true, inputRows, deviceGroups, ruleGroups,
aggregateMs, mergeMs, materializeMs);
}
public boolean isDirect() { return direct; }
public int getInputRows() { return inputRows; }
public int getDeviceGroups() { return deviceGroups; }
public int getRuleGroups() { return ruleGroups; }
public long getAggregateMs() { return aggregateMs; }
public long getMergeMs() { return mergeMs; }
public long getMaterializeMs() { return materializeMs; }
}
}
Reference in New Issue Copy Permalink