496 lines
23 KiB
Java
496 lines
23 KiB
Java
|
package com.common.service;
|
||
|
|
|
||
|
|
import com.common.entity.SyslogNonNormalMessage;
|
||
|
|
import lombok.extern.slf4j.Slf4j;
|
||
|
|
import org.springframework.beans.factory.annotation.Value;
|
||
|
|
import org.springframework.dao.CannotAcquireLockException;
|
||
|
|
import org.springframework.dao.DeadlockLoserDataAccessException;
|
||
|
|
import org.springframework.dao.DuplicateKeyException;
|
||
|
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||
|
|
import org.springframework.stereotype.Service;
|
||
|
|
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||
|
|
|
||
|
|
import java.sql.Date;
|
||
|
|
import java.time.LocalDate;
|
||
|
|
import java.time.LocalDateTime;
|
||
|
|
import java.time.ZoneId;
|
||
|
|
import java.util.ArrayList;
|
||
|
|
import java.util.LinkedHashMap;
|
||
|
|
import java.util.List;
|
||
|
|
import java.util.Map;
|
||
|
|
import java.util.SortedMap;
|
||
|
|
import java.util.TreeMap;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* 把当前事实事务的内存聚合结果直接写入小型统计表。
|
||
|
|
*
|
||
|
|
* <p>调用方必须已经开启事实事务；本类不会新开事务，也不会吞掉任何数据库异常。</p>
|
||
|
|
*/
|
||
|
|
@Slf4j
|
||
|
|
@Service
|
||
|
|
public class DirectStatsProjectionWriter {
|
||
|
|
|
||
|
|
private static final String DEVICE_MERGE_TEMPLATE =
|
||
|
|
"MERGE INTO %s t USING (SELECT ? stat_date,? device_id,? parse_delta,? non_log_delta FROM DUAL) s " +
|
||
|
|
"ON (t.stat_date=s.stat_date AND t.device_id=s.device_id) " +
|
||
|
|
"WHEN MATCHED THEN UPDATE SET t.parse_count=t.parse_count+s.parse_delta," +
|
||
|
|
"t.non_log_count=t.non_log_count+s.non_log_delta,t.updated_at=SYSDATE " +
|
||
|
|
"WHEN NOT MATCHED THEN INSERT (stat_date,device_id,parse_count,non_log_count," +
|
||
|
|
"last_receive_time,projection_seq,updated_at) " +
|
||
|
|
"VALUES (s.stat_date,s.device_id,s.parse_delta,s.non_log_delta,NULL,0,SYSDATE)";
|
||
|
|
|
||
|
|
private static final String RULE_MERGE_TEMPLATE =
|
||
|
|
"MERGE INTO %s t USING (SELECT ? stat_date,? normalize_rule_id,? max_log_time FROM DUAL) s " +
|
||
|
|
"ON (t.stat_date=s.stat_date AND t.normalize_rule_id=s.normalize_rule_id) " +
|
||
|
|
"WHEN MATCHED THEN UPDATE SET t.max_log_time=CASE " +
|
||
|
|
"WHEN s.max_log_time IS NULL THEN t.max_log_time " +
|
||
|
|
"WHEN t.max_log_time IS NULL OR t.max_log_time<s.max_log_time THEN s.max_log_time " +
|
||
|
|
"ELSE t.max_log_time END,t.updated_at=SYSDATE " +
|
||
|
|
"WHEN NOT MATCHED THEN INSERT (stat_date,normalize_rule_id,max_log_time,projection_seq,updated_at) " +
|
||
|
|
"VALUES (s.stat_date,s.normalize_rule_id,s.max_log_time,0,SYSDATE)";
|
||
|
|
|
||
|
|
private static final String DEVICE_EXACT_MERGE_TEMPLATE =
|
||
|
|
"MERGE INTO %s t USING (SELECT ? stat_date,? device_id,? parse_count,? non_log_count FROM DUAL) s " +
|
||
|
|
"ON (t.stat_date=s.stat_date AND t.device_id=s.device_id) " +
|
||
|
|
"WHEN MATCHED THEN UPDATE SET t.parse_count=s.parse_count," +
|
||
|
|
"t.non_log_count=s.non_log_count,t.updated_at=SYSDATE " +
|
||
|
|
"WHEN NOT MATCHED THEN INSERT (stat_date,device_id,parse_count,non_log_count," +
|
||
|
|
"last_receive_time,projection_seq,updated_at) " +
|
||
|
|
"VALUES (s.stat_date,s.device_id,s.parse_count,s.non_log_count,NULL,0,SYSDATE)";
|
||
|
|
|
||
|
|
private final JdbcTemplate jdbc;
|
||
|
|
private final StatsWriteModeProvider modeProvider;
|
||
|
|
private final DirectStatsWriteMetrics metrics;
|
||
|
|
|
||
|
|
@Value("${stats.incremental.read-enabled:false}")
|
||
|
|
private boolean readEnabled;
|
||
|
|
|
||
|
|
@Value("${stats.direct.device-table:device_daily_stats}")
|
||
|
|
private String directDeviceTable;
|
||
|
|
|
||
|
|
@Value("${stats.direct.rule-table:normalize_rule_daily_stats}")
|
||
|
|
private String directRuleTable;
|
||
|
|
|
||
|
|
@Value("${stats.direct.shadow-device-table:device_daily_stats_direct_shadow}")
|
||
|
|
private String shadowDeviceTable;
|
||
|
|
|
||
|
|
@Value("${stats.direct.shadow-rule-table:normalize_rule_daily_stats_direct_shadow}")
|
||
|
|
private String shadowRuleTable;
|
||
|
|
|
||
|
|
@Value("${stats.direct.control-table:stats_direct_control}")
|
||
|
|
private String controlTable;
|
||
|
|
|
||
|
|
@Value("${stats.direct.today-stats-date-enabled:true}")
|
||
|
|
private boolean todayStatsDateEnabled;
|
||
|
|
|
||
|
|
@Value("${stats.business-zone:Asia/Shanghai}")
|
||
|
|
private String businessZone;
|
||
|
|
|
||
|
|
public DirectStatsProjectionWriter(JdbcTemplate jdbc,
|
||
|
|
StatsWriteModeProvider modeProvider,
|
||
|
|
DirectStatsWriteMetrics metrics) {
|
||
|
|
this.jdbc = jdbc;
|
||
|
|
this.modeProvider = modeProvider;
|
||
|
|
this.metrics = metrics;
|
||
|
|
}
|
||
|
|
|
||
|
|
public WriteResult writeStandardizedRows(List<Map<String, Object>> rows) {
|
||
|
|
if (!shouldBuildAggregation()) {
|
||
|
|
handleFactOnlyMode();
|
||
|
|
return WriteResult.inactive(rows == null ? 0 : rows.size());
|
||
|
|
}
|
||
|
|
long aggregateStarted = System.nanoTime();
|
||
|
|
DirectStatsAccumulator accumulator = accumulator().addStandardizedRows(rows);
|
||
|
|
return write(accumulator, elapsedMs(aggregateStarted));
|
||
|
|
}
|
||
|
|
|
||
|
|
public WriteResult writeStandardizedRow(Map<String, Object> row) {
|
||
|
|
if (!shouldBuildAggregation()) {
|
||
|
|
handleFactOnlyMode();
|
||
|
|
return WriteResult.inactive(row == null ? 0 : 1);
|
||
|
|
}
|
||
|
|
long aggregateStarted = System.nanoTime();
|
||
|
|
DirectStatsAccumulator accumulator = accumulator().addStandardizedRow(row);
|
||
|
|
return write(accumulator, elapsedMs(aggregateStarted));
|
||
|
|
}
|
||
|
|
|
||
|
|
public WriteResult writeNonNormalRows(List<SyslogNonNormalMessage> messages) {
|
||
|
|
if (!shouldBuildAggregation()) {
|
||
|
|
handleFactOnlyMode();
|
||
|
|
return WriteResult.inactive(messages == null ? 0 : messages.size());
|
||
|
|
}
|
||
|
|
long aggregateStarted = System.nanoTime();
|
||
|
|
DirectStatsAccumulator accumulator = accumulator().addNonNormalRows(messages);
|
||
|
|
return write(accumulator, elapsedMs(aggregateStarted));
|
||
|
|
}
|
||
|
|
|
||
|
|
public WriteResult writeNonNormalInsert(SyslogNonNormalMessage message) {
|
||
|
|
if (!shouldBuildAggregation()) {
|
||
|
|
handleFactOnlyMode();
|
||
|
|
return WriteResult.inactive(message == null ? 0 : 1);
|
||
|
|
}
|
||
|
|
long aggregateStarted = System.nanoTime();
|
||
|
|
DirectStatsAccumulator accumulator = accumulator().addNonNormalInsert(message);
|
||
|
|
return write(accumulator, elapsedMs(aggregateStarted));
|
||
|
|
}
|
||
|
|
|
||
|
|
public WriteResult writeNonNormalUpdate(SyslogNonNormalMessage oldValue, SyslogNonNormalMessage patch) {
|
||
|
|
if (!shouldBuildAggregation()) {
|
||
|
|
handleFactOnlyMode();
|
||
|
|
return WriteResult.inactive(oldValue == null || patch == null ? 0 : 1);
|
||
|
|
}
|
||
|
|
long aggregateStarted = System.nanoTime();
|
||
|
|
DirectStatsAccumulator accumulator = accumulator().addNonNormalUpdate(oldValue, patch);
|
||
|
|
return writeExactDeviceGroups(accumulator, elapsedMs(aggregateStarted));
|
||
|
|
}
|
||
|
|
|
||
|
|
public void recordTransactionRetry() {
|
||
|
|
metrics.recordRetry();
|
||
|
|
}
|
||
|
|
|
||
|
|
public Map<String, Object> status() {
|
||
|
|
Map<String, Object> status = new LinkedHashMap<>(metrics.snapshot(modeProvider.getMode()));
|
||
|
|
status.put("read_enabled", readEnabled);
|
||
|
|
status.put("device_table", selectedDeviceTable());
|
||
|
|
status.put("rule_table", selectedRuleTable());
|
||
|
|
status.put("today_stats_date_enabled", todayStatsDateEnabled);
|
||
|
|
return status;
|
||
|
|
}
|
||
|
|
|
||
|
|
private boolean shouldBuildAggregation() {
|
||
|
|
return modeProvider.writesDirectProjection();
|
||
|
|
}
|
||
|
|
|
||
|
|
private DirectStatsAccumulator accumulator() {
|
||
|
|
return new DirectStatsAccumulator(ZoneId.of(businessZone));
|
||
|
|
}
|
||
|
|
|
||
|
|
private void handleFactOnlyMode() {
|
||
|
|
if (!modeProvider.requiresRebuild()) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
requireTransaction();
|
||
|
|
metrics.markRebuildRequired();
|
||
|
|
String table = safeTable(controlTable);
|
||
|
|
jdbc.update("MERGE INTO " + table + " t USING " +
|
||
|
|
"(SELECT 'SYSLOG_STATS' control_name FROM DUAL) s " +
|
||
|
|
"ON (t.control_name=s.control_name) " +
|
||
|
|
"WHEN MATCHED THEN UPDATE SET t.rebuild_required=1,t.updated_at=SYSDATE " +
|
||
|
|
"WHEN NOT MATCHED THEN INSERT (control_name,rebuild_required,updated_at) " +
|
||
|
|
"VALUES (s.control_name,1,SYSDATE)");
|
||
|
|
}
|
||
|
|
|
||
|
|
private WriteResult write(DirectStatsAccumulator accumulator, long aggregateMs) {
|
||
|
|
if (accumulator.isEmpty()) {
|
||
|
|
metrics.recordSuccess(accumulator.getInputRows(), 0, 0, aggregateMs, 0L, 0L);
|
||
|
|
return WriteResult.direct(accumulator.getInputRows(), 0, 0,
|
||
|
|
aggregateMs, 0L, 0L);
|
||
|
|
}
|
||
|
|
requireTransaction();
|
||
|
|
|
||
|
|
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> devices =
|
||
|
|
effectiveDeviceEntries(accumulator);
|
||
|
|
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> rules =
|
||
|
|
effectiveRuleEntries(accumulator);
|
||
|
|
long mergeMs;
|
||
|
|
long materializeMs = 0L;
|
||
|
|
try {
|
||
|
|
long mergeStarted = System.nanoTime();
|
||
|
|
mergeDevices(devices);
|
||
|
|
mergeRules(rules);
|
||
|
|
mergeMs = elapsedMs(mergeStarted);
|
||
|
|
|
||
|
|
if (modeProvider.materializesBusinessFields() && readEnabled) {
|
||
|
|
long materializeStarted = System.nanoTime();
|
||
|
|
materializeDevices(devices);
|
||
|
|
materializeRules(rules);
|
||
|
|
materializeMs = elapsedMs(materializeStarted);
|
||
|
|
}
|
||
|
|
metrics.recordSuccess(accumulator.getInputRows(), devices.size(), rules.size(),
|
||
|
|
aggregateMs, mergeMs, materializeMs);
|
||
|
|
log.debug("直接统计事务块完成: factRows={}, deviceGroups={}, ruleGroups={}, " +
|
||
|
|
"aggregateMs={}, statsMergeMs={}, materializeMs={}",
|
||
|
|
accumulator.getInputRows(), devices.size(), rules.size(),
|
||
|
|
aggregateMs, mergeMs, materializeMs);
|
||
|
|
return WriteResult.direct(accumulator.getInputRows(), devices.size(), rules.size(),
|
||
|
|
aggregateMs, mergeMs, materializeMs);
|
||
|
|
} catch (RuntimeException failure) {
|
||
|
|
metrics.recordFailure();
|
||
|
|
if (isConcurrentWrite(failure)) {
|
||
|
|
throw new ConcurrentStatsWriteException("直接统计聚合键并发写入失败，可安全回滚后有限重试", failure);
|
||
|
|
}
|
||
|
|
throw new DirectStatsWriteException("直接统计写入失败，事实事务必须整体回滚", failure);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* UPDATE/物理 DELETE 属于低频管理路径。负 delta 可能移除当前最大值或命中未建立历史基线的日期，
|
||
|
|
* 因此直接从事实表精确重算受影响的小分组，不能把负数插入一个不存在的聚合键。
|
||
|
|
*/
|
||
|
|
private WriteResult writeExactDeviceGroups(DirectStatsAccumulator accumulator, long aggregateMs) {
|
||
|
|
if (accumulator.getDevices().isEmpty()) {
|
||
|
|
metrics.recordSuccess(accumulator.getInputRows(), 0, 0, aggregateMs, 0L, 0L);
|
||
|
|
return WriteResult.direct(accumulator.getInputRows(), 0, 0,
|
||
|
|
aggregateMs, 0L, 0L);
|
||
|
|
}
|
||
|
|
requireTransaction();
|
||
|
|
List<DirectStatsAccumulator.DailyKey> keys = new ArrayList<>(accumulator.getDevices().keySet());
|
||
|
|
try {
|
||
|
|
long mergeStarted = System.nanoTime();
|
||
|
|
List<Object[]> exactValues = new ArrayList<>(keys.size());
|
||
|
|
for (DirectStatsAccumulator.DailyKey key : keys) {
|
||
|
|
LocalDateTime start = key.getDate().atStartOfDay();
|
||
|
|
LocalDateTime end = key.getDate().plusDays(1).atStartOfDay();
|
||
|
|
Map<String, Object> counts = jdbc.queryForMap(
|
||
|
|
"SELECT " +
|
||
|
|
"((SELECT COUNT(*) FROM syslog_normal_data " +
|
||
|
|
"WHERE created_at>=? AND created_at<? AND device_id=?) + " +
|
||
|
|
" (SELECT COUNT(*) FROM syslog_normal_alarm " +
|
||
|
|
"WHERE created_at>=? AND created_at<? AND device_id=?)) parse_count," +
|
||
|
|
"(SELECT COUNT(*) FROM syslog_non_normal_message " +
|
||
|
|
"WHERE created_at>=? AND created_at<? AND device_id=?) non_log_count FROM DUAL",
|
||
|
|
start, end, key.getDimensionId(),
|
||
|
|
start, end, key.getDimensionId(),
|
||
|
|
start, end, key.getDimensionId());
|
||
|
|
exactValues.add(new Object[]{Date.valueOf(key.getDate()), key.getDimensionId(),
|
||
|
|
number(counts, "parse_count"), number(counts, "non_log_count")});
|
||
|
|
}
|
||
|
|
jdbc.batchUpdate(String.format(DEVICE_EXACT_MERGE_TEMPLATE,
|
||
|
|
safeTable(selectedDeviceTable())), exactValues);
|
||
|
|
long mergeMs = elapsedMs(mergeStarted);
|
||
|
|
long materializeMs = 0L;
|
||
|
|
if (modeProvider.materializesBusinessFields() && readEnabled) {
|
||
|
|
long materializeStarted = System.nanoTime();
|
||
|
|
materializeDeviceKeys(keys);
|
||
|
|
materializeMs = elapsedMs(materializeStarted);
|
||
|
|
}
|
||
|
|
metrics.recordSuccess(accumulator.getInputRows(), keys.size(), 0,
|
||
|
|
aggregateMs, mergeMs, materializeMs);
|
||
|
|
return WriteResult.direct(accumulator.getInputRows(), keys.size(), 0,
|
||
|
|
aggregateMs, mergeMs, materializeMs);
|
||
|
|
} catch (RuntimeException failure) {
|
||
|
|
metrics.recordFailure();
|
||
|
|
if (isConcurrentWrite(failure)) {
|
||
|
|
throw new ConcurrentStatsWriteException("直接统计局部分组并发写入失败，可安全回滚后有限重试", failure);
|
||
|
|
}
|
||
|
|
throw new DirectStatsWriteException("直接统计局部分组重算失败，事实事务必须整体回滚", failure);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>>
|
||
|
|
effectiveDeviceEntries(DirectStatsAccumulator accumulator) {
|
||
|
|
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> result =
|
||
|
|
new ArrayList<>();
|
||
|
|
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry
|
||
|
|
: accumulator.getDevices().entrySet()) {
|
||
|
|
DirectStatsAccumulator.DeviceDelta delta = entry.getValue();
|
||
|
|
if (delta.getParseDelta() != 0L || delta.getNonLogDelta() != 0L) {
|
||
|
|
result.add(entry);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
private List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>>
|
||
|
|
effectiveRuleEntries(DirectStatsAccumulator accumulator) {
|
||
|
|
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> result =
|
||
|
|
new ArrayList<>();
|
||
|
|
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry
|
||
|
|
: accumulator.getRules().entrySet()) {
|
||
|
|
if (entry.getValue().getMaxLogTime() != null) {
|
||
|
|
result.add(entry);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
private void mergeDevices(
|
||
|
|
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> entries) {
|
||
|
|
if (entries.isEmpty()) return;
|
||
|
|
List<Object[]> parameters = new ArrayList<>(entries.size());
|
||
|
|
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry : entries) {
|
||
|
|
parameters.add(new Object[]{Date.valueOf(entry.getKey().getDate()),
|
||
|
|
entry.getKey().getDimensionId(), entry.getValue().getParseDelta(),
|
||
|
|
entry.getValue().getNonLogDelta()});
|
||
|
|
}
|
||
|
|
jdbc.batchUpdate(String.format(DEVICE_MERGE_TEMPLATE, safeTable(selectedDeviceTable())), parameters);
|
||
|
|
}
|
||
|
|
|
||
|
|
private void mergeRules(
|
||
|
|
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> entries) {
|
||
|
|
if (entries.isEmpty()) return;
|
||
|
|
List<Object[]> parameters = new ArrayList<>(entries.size());
|
||
|
|
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry : entries) {
|
||
|
|
parameters.add(new Object[]{Date.valueOf(entry.getKey().getDate()),
|
||
|
|
entry.getKey().getDimensionId(), entry.getValue().getMaxLogTime()});
|
||
|
|
}
|
||
|
|
jdbc.batchUpdate(String.format(RULE_MERGE_TEMPLATE, safeTable(selectedRuleTable())), parameters);
|
||
|
|
}
|
||
|
|
|
||
|
|
private void materializeDevices(
|
||
|
|
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta>> entries) {
|
||
|
|
List<DirectStatsAccumulator.DailyKey> keys = new ArrayList<>(entries.size());
|
||
|
|
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry : entries) {
|
||
|
|
keys.add(entry.getKey());
|
||
|
|
}
|
||
|
|
materializeDeviceKeys(keys);
|
||
|
|
}
|
||
|
|
|
||
|
|
private void materializeDeviceKeys(List<DirectStatsAccumulator.DailyKey> keys) {
|
||
|
|
LocalDate today = LocalDate.now(ZoneId.of(businessZone));
|
||
|
|
List<Object[]> parameters = new ArrayList<>();
|
||
|
|
for (DirectStatsAccumulator.DailyKey key : keys) {
|
||
|
|
if (today.equals(key.getDate())) {
|
||
|
|
long deviceId = key.getDimensionId();
|
||
|
|
parameters.add(new Object[]{Date.valueOf(today), deviceId,
|
||
|
|
Date.valueOf(today), deviceId, Date.valueOf(today), deviceId});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (parameters.isEmpty()) return;
|
||
|
|
String deviceTable = safeTable(selectedDeviceTable());
|
||
|
|
String dateAssignment = todayStatsDateEnabled ? ",today_stats_date=?" : "";
|
||
|
|
String sql = "UPDATE device_device SET " +
|
||
|
|
"today_parse_count=(SELECT parse_count FROM " + deviceTable +
|
||
|
|
" WHERE stat_date=? AND device_id=?)," +
|
||
|
|
"today_non_log_count=(SELECT non_log_count FROM " + deviceTable +
|
||
|
|
" WHERE stat_date=? AND device_id=?)" + dateAssignment +
|
||
|
|
",updated_at=SYSDATE WHERE id=?";
|
||
|
|
if (!todayStatsDateEnabled) {
|
||
|
|
List<Object[]> withoutDateColumn = new ArrayList<>(parameters.size());
|
||
|
|
for (Object[] row : parameters) {
|
||
|
|
withoutDateColumn.add(new Object[]{row[0], row[1], row[2], row[3], row[5]});
|
||
|
|
}
|
||
|
|
jdbc.batchUpdate(sql, withoutDateColumn);
|
||
|
|
} else {
|
||
|
|
jdbc.batchUpdate(sql, parameters);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private void materializeRules(
|
||
|
|
List<Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta>> entries) {
|
||
|
|
SortedMap<Long, LocalDateTime> maxima = new TreeMap<>();
|
||
|
|
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry : entries) {
|
||
|
|
long ruleId = entry.getKey().getDimensionId();
|
||
|
|
LocalDateTime candidate = entry.getValue().getMaxLogTime();
|
||
|
|
LocalDateTime current = maxima.get(ruleId);
|
||
|
|
if (current == null || candidate.isAfter(current)) {
|
||
|
|
maxima.put(ruleId, candidate);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (maxima.isEmpty()) return;
|
||
|
|
List<Object[]> parameters = new ArrayList<>(maxima.size());
|
||
|
|
for (Map.Entry<Long, LocalDateTime> entry : maxima.entrySet()) {
|
||
|
|
parameters.add(new Object[]{entry.getValue(), entry.getValue(), entry.getValue(), entry.getKey()});
|
||
|
|
}
|
||
|
|
jdbc.batchUpdate("UPDATE dm_normalize_rule SET first_data_saved_at=CASE " +
|
||
|
|
"WHEN ? IS NULL THEN first_data_saved_at " +
|
||
|
|
"WHEN first_data_saved_at IS NULL OR first_data_saved_at<? THEN ? " +
|
||
|
|
"ELSE first_data_saved_at END,updated_at=SYSDATE " +
|
||
|
|
"WHERE id=? AND del_flag='0'", parameters);
|
||
|
|
}
|
||
|
|
|
||
|
|
private String selectedDeviceTable() {
|
||
|
|
return modeProvider.writesShadowProjection() ? shadowDeviceTable : directDeviceTable;
|
||
|
|
}
|
||
|
|
|
||
|
|
private String selectedRuleTable() {
|
||
|
|
return modeProvider.writesShadowProjection() ? shadowRuleTable : directRuleTable;
|
||
|
|
}
|
||
|
|
|
||
|
|
private static String safeTable(String table) {
|
||
|
|
if (table == null || !table.matches("[A-Za-z][A-Za-z0-9_]*")) {
|
||
|
|
throw new IllegalStateException("非法统计表名: " + table);
|
||
|
|
}
|
||
|
|
return table;
|
||
|
|
}
|
||
|
|
|
||
|
|
private static void requireTransaction() {
|
||
|
|
if (!TransactionSynchronizationManager.isActualTransactionActive()) {
|
||
|
|
throw new IllegalStateException("直接统计必须与事实写入位于同一个 Spring 达梦事务");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private static boolean isConcurrentWrite(Throwable failure) {
|
||
|
|
Throwable current = failure;
|
||
|
|
while (current != null) {
|
||
|
|
if (current instanceof DuplicateKeyException
|
||
|
|
|| current instanceof CannotAcquireLockException
|
||
|
|
|| current instanceof DeadlockLoserDataAccessException) {
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
current = current.getCause();
|
||
|
|
}
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
private static long elapsedMs(long startedNanos) {
|
||
|
|
return (System.nanoTime() - startedNanos) / 1_000_000L;
|
||
|
|
}
|
||
|
|
|
||
|
|
private static long number(Map<String, Object> row, String name) {
|
||
|
|
Object value = row.get(name);
|
||
|
|
if (value == null) value = row.get(name.toUpperCase(java.util.Locale.ROOT));
|
||
|
|
if (value == null) {
|
||
|
|
for (Map.Entry<String, Object> entry : row.entrySet()) {
|
||
|
|
if (entry.getKey().equalsIgnoreCase(name)) {
|
||
|
|
value = entry.getValue();
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return value == null ? 0L : ((Number) value).longValue();
|
||
|
|
}
|
||
|
|
|
||
|
|
public static class DirectStatsWriteException extends RuntimeException {
|
||
|
|
DirectStatsWriteException(String message, Throwable cause) {
|
||
|
|
super(message, cause);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public static final class ConcurrentStatsWriteException extends DirectStatsWriteException {
|
||
|
|
ConcurrentStatsWriteException(String message, Throwable cause) {
|
||
|
|
super(message, cause);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/** 单个事实事务块的统计写入结果，用于批次 INFO 日志和事务耗时观测。 */
|
||
|
|
public static final class WriteResult {
|
||
|
|
private final boolean direct;
|
||
|
|
private final int inputRows;
|
||
|
|
private final int deviceGroups;
|
||
|
|
private final int ruleGroups;
|
||
|
|
private final long aggregateMs;
|
||
|
|
private final long mergeMs;
|
||
|
|
private final long materializeMs;
|
||
|
|
|
||
|
|
private WriteResult(boolean direct, int inputRows, int deviceGroups, int ruleGroups,
|
||
|
|
long aggregateMs, long mergeMs, long materializeMs) {
|
||
|
|
this.direct = direct;
|
||
|
|
this.inputRows = inputRows;
|
||
|
|
this.deviceGroups = deviceGroups;
|
||
|
|
this.ruleGroups = ruleGroups;
|
||
|
|
this.aggregateMs = aggregateMs;
|
||
|
|
this.mergeMs = mergeMs;
|
||
|
|
this.materializeMs = materializeMs;
|
||
|
|
}
|
||
|
|
|
||
|
|
static WriteResult inactive(int inputRows) {
|
||
|
|
return new WriteResult(false, inputRows, 0, 0, 0L, 0L, 0L);
|
||
|
|
}
|
||
|
|
|
||
|
|
static WriteResult direct(int inputRows, int deviceGroups, int ruleGroups,
|
||
|
|
long aggregateMs, long mergeMs, long materializeMs) {
|
||
|
|
return new WriteResult(true, inputRows, deviceGroups, ruleGroups,
|
||
|
|
aggregateMs, mergeMs, materializeMs);
|
||
|
|
}
|
||
|
|
|
||
|
|
public boolean isDirect() { return direct; }
|
||
|
|
public int getInputRows() { return inputRows; }
|
||
|
|
public int getDeviceGroups() { return deviceGroups; }
|
||
|
|
public int getRuleGroups() { return ruleGroups; }
|
||
|
|
public long getAggregateMs() { return aggregateMs; }
|
||
|
|
public long getMergeMs() { return mergeMs; }
|
||
|
|
public long getMaterializeMs() { return materializeMs; }
|
||
|
|
}
|
||
|
|
}
|