package com.common.service; import com.common.entity.SyslogNonNormalMessage; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.dao.CannotAcquireLockException; import org.springframework.dao.DeadlockLoserDataAccessException; import org.springframework.dao.DuplicateKeyException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionSynchronizationManager; import java.sql.Date; import java.sql.SQLException; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; import java.util.function.Supplier; /** * 把当前事实事务的内存聚合结果交给提交后的有界异步统计队列。 * *

DIRECT 连续消费优先,不写 Outbox 或统计表,统计异常记录后放行。 * 历史 SHADOW 模式仍同步写影子投影,保留原有校验语义。

*/ @Slf4j @Service public class DirectStatsProjectionWriter { private static final String DEVICE_MERGE_TEMPLATE = "MERGE INTO %s t USING (SELECT ? stat_date,? device_id,? parse_delta,? non_log_delta FROM DUAL) s " + "ON (t.stat_date=s.stat_date AND t.device_id=s.device_id) " + "WHEN MATCHED THEN UPDATE SET t.parse_count=t.parse_count+s.parse_delta," + "t.non_log_count=t.non_log_count+s.non_log_delta,t.updated_at=SYSDATE " + "WHEN NOT MATCHED THEN INSERT (stat_date,device_id,parse_count,non_log_count," + "last_receive_time,projection_seq,updated_at) " + "VALUES (s.stat_date,s.device_id,s.parse_delta,s.non_log_delta,NULL,0,SYSDATE)"; private static final String RULE_MERGE_TEMPLATE = "MERGE INTO %s t USING (SELECT ? stat_date,? normalize_rule_id,? max_log_time FROM DUAL) s " + "ON (t.stat_date=s.stat_date AND t.normalize_rule_id=s.normalize_rule_id) " + "WHEN MATCHED THEN UPDATE SET t.max_log_time=CASE " + "WHEN s.max_log_time IS NULL THEN t.max_log_time " + "WHEN t.max_log_time IS NULL OR t.max_log_time> rows) { return project(rows == null ? 0 : rows.size(), () -> accumulator().addStandardizedRows(rows), false); } public WriteResult writeStandardizedRow(Map row) { return project(row == null ? 0 : 1, () -> accumulator().addStandardizedRow(row), false); } public WriteResult writeNonNormalRows(List messages) { return project(messages == null ? 0 : messages.size(), () -> accumulator().addNonNormalRows(messages), false); } public WriteResult writeNonNormalInsert(SyslogNonNormalMessage message) { return project(message == null ? 0 : 1, () -> accumulator().addNonNormalInsert(message), false); } public WriteResult writeNonNormalUpdate(SyslogNonNormalMessage oldValue, SyslogNonNormalMessage patch) { return project(oldValue == null || patch == null ? 0 : 1, () -> accumulator().addNonNormalUpdate(oldValue, patch), true); } private WriteResult project(int rows, Supplier build, boolean exact) { try { if (!shouldBuildAggregation()) { handleFactOnlyMode(); return WriteResult.inactive(rows); } long started = System.nanoTime(); DirectStatsAccumulator aggregate = build.get(); return exact ? writeExactDeviceGroups(aggregate, elapsedMs(started)) : write(aggregate, elapsedMs(started)); } catch (RuntimeException failure) { if (!asynchronousProjection()) throw failure; metrics.recordFailure(); asyncPublisher.recordLoss(rows, "AGGREGATION_OR_REGISTRATION_FAILED", failure); return WriteResult.inactive(rows); } } public void recordTransactionRetry() { metrics.recordRetry(); } public Map status() { Map status = new LinkedHashMap<>(metrics.snapshot(modeProvider.getMode())); status.put("read_enabled", readEnabled); status.put("device_table", selectedDeviceTable()); status.put("rule_table", selectedRuleTable()); status.put("today_stats_date_enabled", todayStatsDateEnabled); status.put("business_materialization_mode", asynchronousMaterialization() ? "KAFKA_DIRTY_KEY" : "SYNCHRONOUS"); status.put("daily_projection_write_mode", asynchronousProjection() ? "BEST_EFFORT_AFTER_COMMIT_KAFKA" : "SYNCHRONOUS_MERGE"); status.put("async_publisher", asyncPublisher.status()); return status; } private boolean shouldBuildAggregation() { return modeProvider.writesDirectProjection(); } private DirectStatsAccumulator accumulator() { return new DirectStatsAccumulator(ZoneId.of(businessZone)); } private void handleFactOnlyMode() { if (!modeProvider.requiresRebuild()) { return; } requireTransaction(); metrics.markRebuildRequired(); String table = safeTable(controlTable); jdbc.update("MERGE INTO " + table + " t USING " + "(SELECT 'SYSLOG_STATS' control_name FROM DUAL) s " + "ON (t.control_name=s.control_name) " + "WHEN MATCHED THEN UPDATE SET t.rebuild_required=1,t.updated_at=SYSDATE " + "WHEN NOT MATCHED THEN INSERT (control_name,rebuild_required,updated_at) " + "VALUES (s.control_name,1,SYSDATE)"); } private WriteResult write(DirectStatsAccumulator accumulator, long aggregateMs) { if (accumulator.isEmpty()) { metrics.recordSuccess(accumulator.getInputRows(), 0, 0, aggregateMs, 0L, 0L); return WriteResult.direct(accumulator.getInputRows(), 0, 0, aggregateMs, 0L, 0L); } requireTransaction(); if (asynchronousProjection()) { long enqueueStarted = System.nanoTime(); asyncPublisher.enqueueAfterCommit(StatsProjectionOutboxEvent.fromAccumulator(accumulator), accumulator.getInputRows()); long enqueueMs = elapsedMs(enqueueStarted); int deviceGroups = effectiveDeviceEntries(accumulator).size(); int ruleGroups = effectiveRuleEntries(accumulator).size(); metrics.recordSuccess(accumulator.getInputRows(), deviceGroups, ruleGroups, aggregateMs, enqueueMs, 0L); return WriteResult.direct(accumulator.getInputRows(), deviceGroups, ruleGroups, aggregateMs, enqueueMs, 0L); } List> devices = effectiveDeviceEntries(accumulator); List> rules = effectiveRuleEntries(accumulator); long mergeMs; long materializeMs = 0L; try { long mergeStarted = System.nanoTime(); mergeDevices(devices); mergeRules(rules); mergeMs = elapsedMs(mergeStarted); if (modeProvider.materializesBusinessFields() && readEnabled) { long materializeStarted = System.nanoTime(); if (asynchronousMaterialization()) { materializationPublisher.enqueueAfterCommit(materializationCommands(devices, rules)); } else { materializeDevices(devices); materializeRules(rules); } materializeMs = elapsedMs(materializeStarted); } metrics.recordSuccess(accumulator.getInputRows(), devices.size(), rules.size(), aggregateMs, mergeMs, materializeMs); log.debug("直接统计事务块完成: factRows={}, deviceGroups={}, ruleGroups={}, " + "aggregateMs={}, statsMergeMs={}, materializeMs={}", accumulator.getInputRows(), devices.size(), rules.size(), aggregateMs, mergeMs, materializeMs); return WriteResult.direct(accumulator.getInputRows(), devices.size(), rules.size(), aggregateMs, mergeMs, materializeMs); } catch (RuntimeException failure) { metrics.recordFailure(); if (isConcurrentFailure(failure)) { throw new ConcurrentStatsWriteException("直接统计聚合键并发写入失败,可安全回滚后有限重试", failure); } throw new DirectStatsWriteException("直接统计写入失败,事实事务必须整体回滚", failure); } } /** * UPDATE/物理 DELETE 属于低频管理路径。负 delta 可能移除当前最大值或命中未建立历史基线的日期, * 因此直接从事实表精确重算受影响的小分组,不能把负数插入一个不存在的聚合键。 */ private WriteResult writeExactDeviceGroups(DirectStatsAccumulator accumulator, long aggregateMs) { if (accumulator.getDevices().isEmpty()) { metrics.recordSuccess(accumulator.getInputRows(), 0, 0, aggregateMs, 0L, 0L); return WriteResult.direct(accumulator.getInputRows(), 0, 0, aggregateMs, 0L, 0L); } requireTransaction(); // Additive positive/negative changes commute in the serialized projection consumer. This // keeps non-normal UPDATE statistics outside the fact transaction as well. if (asynchronousProjection()) { return write(accumulator, aggregateMs); } List keys = new ArrayList<>(accumulator.getDevices().keySet()); try { long mergeStarted = System.nanoTime(); List exactValues = new ArrayList<>(keys.size()); for (DirectStatsAccumulator.DailyKey key : keys) { LocalDateTime start = key.getDate().atStartOfDay(); LocalDateTime end = key.getDate().plusDays(1).atStartOfDay(); Map counts = jdbc.queryForMap( "SELECT " + "((SELECT COUNT(*) FROM syslog_normal_data " + "WHERE created_at>=? AND created_at=? AND created_at=? AND created_at> effectiveDeviceEntries(DirectStatsAccumulator accumulator) { List> result = new ArrayList<>(); for (Map.Entry entry : accumulator.getDevices().entrySet()) { DirectStatsAccumulator.DeviceDelta delta = entry.getValue(); if (delta.getParseDelta() != 0L || delta.getNonLogDelta() != 0L) { result.add(entry); } } return result; } private List> effectiveRuleEntries(DirectStatsAccumulator accumulator) { List> result = new ArrayList<>(); for (Map.Entry entry : accumulator.getRules().entrySet()) { if (entry.getValue().getMaxLogTime() != null) { result.add(entry); } } return result; } private void mergeDevices( List> entries) { if (entries.isEmpty()) return; List parameters = new ArrayList<>(entries.size()); for (Map.Entry entry : entries) { parameters.add(new Object[]{Date.valueOf(entry.getKey().getDate()), entry.getKey().getDimensionId(), entry.getValue().getParseDelta(), entry.getValue().getNonLogDelta()}); } jdbc.batchUpdate(String.format(DEVICE_MERGE_TEMPLATE, safeTable(selectedDeviceTable())), parameters); } private void mergeRules( List> entries) { if (entries.isEmpty()) return; List parameters = new ArrayList<>(entries.size()); for (Map.Entry entry : entries) { parameters.add(new Object[]{Date.valueOf(entry.getKey().getDate()), entry.getKey().getDimensionId(), entry.getValue().getMaxLogTime()}); } jdbc.batchUpdate(String.format(RULE_MERGE_TEMPLATE, safeTable(selectedRuleTable())), parameters); } private void materializeDevices( List> entries) { List keys = new ArrayList<>(entries.size()); for (Map.Entry entry : entries) { keys.add(entry.getKey()); } materializeDeviceKeys(keys); } private void materializeDeviceKeys(List keys) { LocalDate today = LocalDate.now(ZoneId.of(businessZone)); List parameters = new ArrayList<>(); for (DirectStatsAccumulator.DailyKey key : keys) { if (today.equals(key.getDate())) { long deviceId = key.getDimensionId(); parameters.add(new Object[]{Date.valueOf(today), deviceId, Date.valueOf(today), deviceId, Date.valueOf(today), deviceId}); } } if (parameters.isEmpty()) return; String deviceTable = safeTable(selectedDeviceTable()); String dateAssignment = todayStatsDateEnabled ? ",today_stats_date=?" : ""; String sql = "UPDATE device_device SET " + "today_parse_count=(SELECT parse_count FROM " + deviceTable + " WHERE stat_date=? AND device_id=?)," + "today_non_log_count=(SELECT non_log_count FROM " + deviceTable + " WHERE stat_date=? AND device_id=?)" + dateAssignment + ",updated_at=SYSDATE WHERE id=?"; if (!todayStatsDateEnabled) { List withoutDateColumn = new ArrayList<>(parameters.size()); for (Object[] row : parameters) { withoutDateColumn.add(new Object[]{row[0], row[1], row[2], row[3], row[5]}); } jdbc.batchUpdate(sql, withoutDateColumn); } else { jdbc.batchUpdate(sql, parameters); } } private void materializeRules( List> entries) { SortedMap maxima = new TreeMap<>(); for (Map.Entry entry : entries) { long ruleId = entry.getKey().getDimensionId(); LocalDateTime candidate = entry.getValue().getMaxLogTime(); LocalDateTime current = maxima.get(ruleId); if (current == null || candidate.isAfter(current)) { maxima.put(ruleId, candidate); } } if (maxima.isEmpty()) return; List parameters = new ArrayList<>(maxima.size()); for (Map.Entry entry : maxima.entrySet()) { parameters.add(new Object[]{entry.getValue(), entry.getValue(), entry.getValue(), entry.getKey()}); } jdbc.batchUpdate("UPDATE dm_normalize_rule SET first_data_saved_at=CASE " + "WHEN ? IS NULL THEN first_data_saved_at " + "WHEN first_data_saved_at IS NULL OR first_data_saved_at materializationCommands( List> devices, List> rules) { List commands = new ArrayList<>(devices.size() + rules.size()); LocalDate today = LocalDate.now(ZoneId.of(businessZone)); for (Map.Entry entry : devices) { DirectStatsAccumulator.DailyKey key = entry.getKey(); if (today.equals(key.getDate())) { commands.add(StatsMaterializationCommand.device(key.getDate(), key.getDimensionId())); } } for (Map.Entry entry : rules) { DirectStatsAccumulator.DailyKey key = entry.getKey(); commands.add(StatsMaterializationCommand.rule(key.getDate(), key.getDimensionId())); } return commands; } private List deviceMaterializationCommands( List keys) { List commands = new ArrayList<>(keys.size()); LocalDate today = LocalDate.now(ZoneId.of(businessZone)); for (DirectStatsAccumulator.DailyKey key : keys) { if (today.equals(key.getDate())) { commands.add(StatsMaterializationCommand.device(key.getDate(), key.getDimensionId())); } } return commands; } private String selectedDeviceTable() { return modeProvider.writesShadowProjection() ? shadowDeviceTable : directDeviceTable; } private String selectedRuleTable() { return modeProvider.writesShadowProjection() ? shadowRuleTable : directRuleTable; } private static String safeTable(String table) { if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) { throw new IllegalStateException("非法统计表名: " + table); } return table; } private static void requireTransaction() { if (!TransactionSynchronizationManager.isActualTransactionActive()) { throw new IllegalStateException("直接统计必须与事实写入位于同一个 Spring 达梦事务"); } } /** * 识别 Spring 与不同 JDBC 驱动对并发冲突的各种包装形式。 * 达梦可能把死锁翻译成 DataIntegrityViolationException,并仅在内层 * BatchUpdateException 的 SQLState/消息中保留 Deadlock 信息。 */ public static boolean isConcurrentFailure(Throwable failure) { Throwable current = failure; while (current != null) { if (current instanceof DuplicateKeyException || current instanceof CannotAcquireLockException || current instanceof DeadlockLoserDataAccessException) { return true; } if (current instanceof SQLException) { String sqlState = ((SQLException) current).getSQLState(); if (sqlState != null && sqlState.startsWith("40")) { return true; } } String message = String.valueOf(current.getMessage()) .toLowerCase(java.util.Locale.ROOT); if (message.contains("deadlock") || message.contains("死锁") || message.contains("unique constraint") || message.contains("duplicate key")) { return true; } current = current.getCause(); } return false; } private static long elapsedMs(long startedNanos) { return (System.nanoTime() - startedNanos) / 1_000_000L; } private static long number(Map row, String name) { Object value = row.get(name); if (value == null) value = row.get(name.toUpperCase(java.util.Locale.ROOT)); if (value == null) { for (Map.Entry entry : row.entrySet()) { if (entry.getKey().equalsIgnoreCase(name)) { value = entry.getValue(); break; } } } return value == null ? 0L : ((Number) value).longValue(); } public static class DirectStatsWriteException extends RuntimeException { DirectStatsWriteException(String message, Throwable cause) { super(message, cause); } } public static final class ConcurrentStatsWriteException extends DirectStatsWriteException { ConcurrentStatsWriteException(String message, Throwable cause) { super(message, cause); } } /** 单个事实事务块的统计写入结果,用于批次 INFO 日志和事务耗时观测。 */ public static final class WriteResult { private final boolean direct; private final int inputRows; private final int deviceGroups; private final int ruleGroups; private final long aggregateMs; private final long mergeMs; private final long materializeMs; private WriteResult(boolean direct, int inputRows, int deviceGroups, int ruleGroups, long aggregateMs, long mergeMs, long materializeMs) { this.direct = direct; this.inputRows = inputRows; this.deviceGroups = deviceGroups; this.ruleGroups = ruleGroups; this.aggregateMs = aggregateMs; this.mergeMs = mergeMs; this.materializeMs = materializeMs; } static WriteResult inactive(int inputRows) { return new WriteResult(false, inputRows, 0, 0, 0L, 0L, 0L); } static WriteResult direct(int inputRows, int deviceGroups, int ruleGroups, long aggregateMs, long mergeMs, long materializeMs) { return new WriteResult(true, inputRows, deviceGroups, ruleGroups, aggregateMs, mergeMs, materializeMs); } public boolean isDirect() { return direct; } public int getInputRows() { return inputRows; } public int getDeviceGroups() { return deviceGroups; } public int getRuleGroups() { return ruleGroups; } public long getAggregateMs() { return aggregateMs; } public long getMergeMs() { return mergeMs; } public long getMaterializeMs() { return materializeMs; } } }