From ce473b2c7d1e2acab61b723f04a0ec17b2365594 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=8E=8B=E7=BB=8D=E5=AE=8F?= <994233303@qq.com>
Date: Fri, 11 Sep 2026 15:42:24 +0800
Subject: [PATCH] =?UTF-8?q?perf(syslog-consumer-rule):=20=E9=9D=9E?=
=?UTF-8?q?=E6=A0=87=E6=97=A5=E5=BF=97=E6=94=B9=E4=B8=BA=E5=8D=95=E7=BA=BF?=
=?UTF-8?q?=E7=A8=8B=E5=BC=82=E6=AD=A5=E6=89=B9=E9=87=8F=E5=86=99=E5=85=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
- 非标写库结果不再参与 Kafka ACK
- 使用独立单线程和有界队列处理非标日志
- 非标数据按固定大小分批写入达梦
- 增加异步成功、失败和队列拒绝指标
- 补充异步解耦及批量写入测试
perf(syslog-consumer-rule): 按批次字段并集优化达梦批量写入
- 普通日志和告警日志按非空字段并集生成批写模板
- 为同批记录补齐缺失字段并绑定 NULL
- 使用 MyBatis ExecutorType.BATCH 复用 PreparedStatement
- 避免固定写入全部 373 个候选字段
- 非标日志保留固定 24 列多行批量写入
- 批量大小和非标队列容量默认调整为 500
- 补充字段并集与动态 SQL 一致性测试
---
.../NormalData/LogNormalProcessor.java | 55 +-
.../Modules/NormalData/SysLogProcessor.java | 93 +-
.../mapper/SyslogNormalAlarmMapper.java | 11 +-
.../common/mapper/SyslogNormalDataMapper.java | 11 +-
.../service/DmStandardizedBatchExecutor.java | 49 +-
.../service/DmStandardizedBatchWriter.java | 32 +-
.../service/SyslogNormalDataService.java | 26 +-
.../SyslogNonNormalMessageServiceImpl.java | 21 +-
.../java/com/config/ThreadPoolConfig.java | 21 +
.../src/main/resources/application.properties | 10 +-
.../mapper/SyslogNormalAlarmMapper.xml | 1503 ++++++++---------
.../mapper/SyslogNormalDataMapper.xml | 1503 ++++++++---------
.../LogNormalProcessorRuleSelectionTest.java | 33 +
.../NormalData/QuanxiNormalizationTest.java | 8 +-
.../SysLogProcessorParallelTest.java | 83 +
.../mapper/FixedColumnBatchMapperTest.java | 127 ++
.../DmStandardizedBatchExecutorTest.java | 41 +
.../DmStandardizedBatchWriterTest.java | 27 +-
...yslogNonNormalMessageServiceBatchTest.java | 61 +
...yslogNormalDataServiceIdempotencyTest.java | 6 +-
.../com/kafka/KafkaProducerEventIdTest.java | 4 +-
.../v1/kafka/02-skip-raw-backlog.sh | 195 +++
.../v1/kafka/kafka-cluster-upgrade.sh | 88 +
.../v2/influxdb/01-ensure-v2-bucket.sh | 3 +-
24 files changed, 2419 insertions(+), 1592 deletions(-)
create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/mapper/FixedColumnBatchMapperTest.java
create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmStandardizedBatchExecutorTest.java
create mode 100644 haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNonNormalMessageServiceBatchTest.java
create mode 100755 haobang-security-dm/upgrade-doc/v1/kafka/02-skip-raw-backlog.sh
create mode 100755 haobang-security-dm/upgrade-doc/v1/kafka/kafka-cluster-upgrade.sh
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java
index d51e83d..7355ad6 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java
@@ -50,18 +50,31 @@ public class LogNormalProcessor {
public static final class NormalizationResult {
private final ProcessingOutcome outcome;
private final List
*/
public NormalizationResult prepare(String logMsg, String syslogUUID, String syslogTopic,
LocalDateTime fallbackFirstReceivedAt) {
@@ -326,6 +349,8 @@ public class LogNormalProcessor {
ctx.executionMetrics = executionMetrics == null ? new RuleExecutionMetrics() : executionMetrics;
ctx.messageParseContext = new MessageParseContext(
optimizationProperties.getMessageParseCache().isEnabled(), globalMetrics);
+ // 活动 Kafka 链路必须把非标日志带回 poll 边界集中批写,禁止计算线程逐条开启达梦事务。
+ ctx.deferNonNormalPersistence = true;
return init(ctx);
}
@@ -373,7 +398,7 @@ public class LogNormalProcessor {
// 原文已在进入标准化前写入 InfluxDB。设备没有绑定规则是可预期的业务终态,
// 继续 Retry/DLT 不会凭空产生规则,只会重复消耗 Kafka、线程和日志 I/O。
// 因此只跳过达梦标准化并允许 ACK;系统依赖和规则执行异常仍由下方异常链处理。
- return NormalizationResult.filtered();
+ return NormalizationResult.filtered(ctx.deferredNonNormalMessages);
}
NormalizationOptimizationProperties.RuleSelection selectionConfig =
optimizationProperties.getRuleSelection();
@@ -420,9 +445,9 @@ public class LogNormalProcessor {
if (!decision.isSelected()) {
saveSelectionFailure(ctx, deviceId, "未命中规则",
"同设备多规则均未形成可用归属候选");
- // 规则均未命中是确定性的业务终态;非标保存成功后直接 ACK,避免 Retry
- // 为同一个 syslog_uuid 重复写入随机 ID 的非标记录。保存失败仍会由上层异常链重试。
- return NormalizationResult.filtered();
+ // 规则均未命中是确定性的业务终态;非标记录随结果返回并交给独立线程批写。
+ // 按现场临时策略,非标写入结果不参与原 Kafka 消息的 ACK。
+ return NormalizationResult.filtered(ctx.deferredNonNormalMessages);
}
RuleSelectionEngine.Candidate selected = decision.getWinner();
@@ -442,7 +467,7 @@ public class LogNormalProcessor {
saveSelectionFailure(ctx, deviceId, "多规则归属歧义",
"候选评分相同但业务结果不同, candidateRuleIds="
+ candidateRuleIds(decision.getTiedCandidates()));
- return NormalizationResult.filtered();
+ return NormalizationResult.filtered(ctx.deferredNonNormalMessages);
}
globalMetrics.ruleSelection("equivalent_tie");
prepared = byRule.get(selected.getSource().getId());
@@ -716,8 +741,12 @@ public class LogNormalProcessor {
}
private NormalizationResult finish(ProcessContext ctx) {
- if (ctx.savedCount > 0) return NormalizationResult.success(ctx.normalizedRows);
- if (ctx.filteredCount > 0 && ctx.rejectedCount == 0) return NormalizationResult.filtered();
+ if (ctx.savedCount > 0) {
+ return NormalizationResult.success(ctx.normalizedRows, ctx.deferredNonNormalMessages);
+ }
+ if (ctx.filteredCount > 0 && ctx.rejectedCount == 0) {
+ return NormalizationResult.filtered(ctx.deferredNonNormalMessages);
+ }
throw new IllegalArgumentException("Syslog 未产生可用标准化结果: eventId=" + ctx.strSyslogUUID);
}
@@ -1133,7 +1162,9 @@ public class LogNormalProcessor {
normalMessage.setCollectTaskId(collect_id);
if (collectTask != null)
normalMessage.setCollectTaskName(collectTask.getTaskName());
- if (!this.messageService.saveMessage(normalMessage)) {
+ if (ctx.deferNonNormalPersistence) {
+ ctx.deferredNonNormalMessages.add(normalMessage);
+ } else if (!this.messageService.saveMessage(normalMessage)) {
throw new IllegalStateException("非标日志服务返回保存失败");
}
ctx.isSaveNonNormal = true;
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java
index 7bc34d4..f60c1c3 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java
@@ -9,7 +9,9 @@ import com.config.AppProperties;
import com.config.KafkaConsumerProperties;
import com.common.service.DmStandardizedBatchWriter;
import com.common.service.IngressStatsProjectionWriter;
+import com.common.service.SyslogNonNormalMessageService;
import com.common.entity.IngressStatsEvent;
+import com.common.entity.SyslogNonNormalMessage;
import com.influxdb.client.write.Point;
import com.influx.raw.InfluxRawPointFactory;
import com.influx.raw.PointTimeResolver;
@@ -71,6 +73,13 @@ public class SysLogProcessor {
@Autowired
private DmStandardizedBatchWriter standardizedBatchWriter;
+ @Autowired
+ private SyslogNonNormalMessageService nonNormalMessageService;
+
+ @Autowired
+ @Qualifier("nonNormalWriterExecutor")
+ private Executor nonNormalWriterExecutor;
+
@Autowired
private IngressStatsProjectionWriter ingressStatsProjectionWriter;
@@ -83,6 +92,9 @@ public class SysLogProcessor {
@Autowired private RawInfluxMetrics rawInfluxMetrics;
private final AtomicInteger totalProcessed = new AtomicInteger(0);
+ private final AtomicLong nonNormalAsyncCompletedTotal = new AtomicLong();
+ private final AtomicLong nonNormalAsyncFailedTotal = new AtomicLong();
+ private final AtomicLong nonNormalAsyncRejectedTotal = new AtomicLong();
// [SYSLOG-ALARM][KACK][已修复-主链路停用] 历史辅助方法仍保留此计数用于兼容,但唯一活动 listener 不再用全局计数决定 ACK。
private final AtomicInteger currentBatchCount = new AtomicInteger(0);
@@ -133,8 +145,9 @@ public class SysLogProcessor {
}
/**
- * 统一批处理边界:原文批量写入 InfluxDB 后,有界并行执行标准化;
- * 只有整批全部到达可靠终态才提交 offset。
+ * 统一批处理边界:原文批量写入 InfluxDB 后,有界并行执行标准化。
+ * 正常消息仍需到达可靠终态才提交 offset;非标消息按现场临时策略异步落库,
+ * 其写库结果不参与 Kafka ACK。
*/
private void consumeBatch(List> records, Acknowledgment ack,
boolean enforceRetryDue, String lane) {
@@ -180,7 +193,9 @@ public class SysLogProcessor {
+ "ruleComputeMs={}, ruleSnapshotMs={}, messageParseMs={}, fieldMappingMs={}, "
+ "filterCompleteMs={}, enrichmentMs={}, ruleAssembleMs={}, compiledRuleHit={}, "
+ "compiledRuleMiss={}, messageParseHit={}, messageParseMiss={}, deviceUnique={}, "
- + "collectTaskUnique={}, shadowCompared={}, shadowMismatch={}, dmWriteMs={}, totalMs={}",
+ + "collectTaskUnique={}, shadowCompared={}, shadowMismatch={}, dmWriteMs={}, "
+ + "nonNormalQueued={}, nonNormalRejected={}, nonNormalAsyncCompletedTotal={}, "
+ + "nonNormalAsyncFailedTotal={}, nonNormalAsyncRejectedTotal={}, totalMs={}",
lane, topicPartitions(records), records.size(), preparedMessages.size(), persistedMessages.size(),
metrics.success.get(), metrics.filtered.get(), metrics.retry.get(), metrics.dlt.get(),
metrics.normalizedRows.get(), metrics.dmSuccessfulRows.get(), metrics.dmFailedMessages.get(), ingressGroups,
@@ -190,6 +205,9 @@ public class SysLogProcessor {
metrics.compiledRuleMisses(), metrics.messageParseHits(), metrics.messageParseMisses(),
metrics.deviceUnique(), metrics.collectTaskUnique(), metrics.shadowCompared(),
metrics.shadowDifferences(), metrics.dmWriteMs(),
+ metrics.nonNormalQueued(), metrics.nonNormalRejected(),
+ nonNormalAsyncCompletedTotal.get(), nonNormalAsyncFailedTotal.get(),
+ nonNormalAsyncRejectedTotal.get(),
System.currentTimeMillis() - batchStartedAt);
}
@@ -492,15 +510,18 @@ public class SysLogProcessor {
metrics.recordRuleExecution(executionMetrics, enrichmentContext);
List> writeRequests = new ArrayList<>();
+ List nonNormalMessages = new ArrayList<>();
Set pendingDatabaseMessages = new HashSet<>();
+ int filteredMessages = 0;
for (CompletableFuture future : futures) {
PreparedNormalization prepared = future.join();
if (prepared.terminalState != null) {
metrics.record(prepared.terminalState);
continue;
}
+ nonNormalMessages.addAll(prepared.result.getNonNormalMessages());
if (prepared.result.getOutcome() == LogNormalProcessor.ProcessingOutcome.FILTERED) {
- metrics.record(SyslogFailureRouter.TerminalState.FILTERED);
+ filteredMessages++;
continue;
}
@@ -528,6 +549,14 @@ public class SysLogProcessor {
metrics.record(SyslogFailureRouter.TerminalState.SUCCESS);
}
}
+ for (int i = 0; i < filteredMessages; i++) {
+ metrics.record(SyslogFailureRouter.TerminalState.FILTERED);
+ }
+
+ // 正常链路先达到可靠终态,再把非标日志交给独立单线程,避免正常写入失败重试时
+ // 提前制造非标重复数据。这里只提交任务,不等待结果,非标写入不参与 Kafka ACK。
+ // 队列满、写库失败或进程异常退出都可能丢非标数据,这是现场选择的性能优先策略。
+ submitNonNormalMessages(nonNormalMessages, metrics);
} catch (TimeoutException timeout) {
cancelOutstanding(futures);
throw new IllegalStateException("标准化批次处理超时,禁止提交 offset", timeout);
@@ -544,6 +573,48 @@ public class SysLogProcessor {
}
}
+ /**
+ * 把非标批次交给独立线程后立即返回。任何提交或执行失败只记录,不传播到 Kafka ACK 边界。
+ */
+ private void submitNonNormalMessages(List messages, BatchMetrics metrics) {
+ if (messages.isEmpty()) {
+ return;
+ }
+ List batch = new ArrayList<>(messages);
+ try {
+ nonNormalWriterExecutor.execute(() -> {
+ long startedAt = System.nanoTime();
+ try {
+ int inserted = nonNormalMessageService.saveBatchMessages(batch);
+ if (inserted != batch.size()) {
+ nonNormalAsyncFailedTotal.addAndGet(batch.size());
+ log.error("非标日志异步批写结果异常,Kafka offset 不回退: rows={}, inserted={}, "
+ + "durationMs={}, failedTotal={}",
+ batch.size(), inserted, elapsedMillis(startedAt),
+ nonNormalAsyncFailedTotal.get());
+ return;
+ }
+ nonNormalAsyncCompletedTotal.addAndGet(inserted);
+ log.debug("非标日志异步批写完成: rows={}, durationMs={}, completedTotal={}",
+ inserted, elapsedMillis(startedAt), nonNormalAsyncCompletedTotal.get());
+ } catch (Exception failure) {
+ nonNormalAsyncFailedTotal.addAndGet(batch.size());
+ log.error("非标日志异步批写失败,Kafka offset 不回退,当前非标数据可能丢失: "
+ + "rows={}, durationMs={}, failedTotal={}",
+ batch.size(), elapsedMillis(startedAt),
+ nonNormalAsyncFailedTotal.get(), failure);
+ }
+ });
+ metrics.recordNonNormalSubmission(batch.size(), 0);
+ } catch (RuntimeException rejected) {
+ nonNormalAsyncRejectedTotal.addAndGet(batch.size());
+ metrics.recordNonNormalSubmission(0, batch.size());
+ log.error("非标日志异步队列提交失败,当前非标数据已丢弃且不阻塞 Kafka ACK: "
+ + "rows={}, rejectedTotal={}",
+ batch.size(), nonNormalAsyncRejectedTotal.get(), rejected);
+ }
+ }
+
private static long elapsedMillis(long startedAtNanos) {
return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAtNanos);
}
@@ -681,6 +752,8 @@ public class SysLogProcessor {
private final AtomicLong ruleComputeMs = new AtomicLong();
private final AtomicLong ruleSnapshotMs = new AtomicLong();
private final AtomicLong dmWriteMs = new AtomicLong();
+ private final AtomicInteger nonNormalQueued = new AtomicInteger();
+ private final AtomicInteger nonNormalRejected = new AtomicInteger();
private volatile RuleExecutionMetrics ruleExecutionMetrics = new RuleExecutionMetrics();
private volatile int deviceUnique;
private volatile int collectTaskUnique;
@@ -703,6 +776,10 @@ public class SysLogProcessor {
return success.get();
}
+ int filteredCount() {
+ return filtered.get();
+ }
+
long ruleComputeMs() {
return ruleComputeMs.get();
}
@@ -711,6 +788,9 @@ public class SysLogProcessor {
return dmWriteMs.get();
}
+ int nonNormalQueued() { return nonNormalQueued.get(); }
+ int nonNormalRejected() { return nonNormalRejected.get(); }
+
long ruleSnapshotMs() { return ruleSnapshotMs.get(); }
long messageParseMs() { return ruleExecutionMetrics.getMessageParseMs(); }
long fieldMappingMs() { return ruleExecutionMetrics.getFieldMappingMs(); }
@@ -747,6 +827,11 @@ public class SysLogProcessor {
dmWriteMs.set(Math.max(0L, elapsedMs));
}
+ void recordNonNormalSubmission(int queued, int rejected) {
+ nonNormalQueued.addAndGet(Math.max(0, queued));
+ nonNormalRejected.addAndGet(Math.max(0, rejected));
+ }
+
/** 仅用于批次汇总观测,不参与 ACK 判断;ACK 仍由每条消息的可靠终态决定。 */
void recordDmWrite(int rows, int successfulRows, int failedMessages) {
this.normalizedRows.addAndGet(rows);
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java
index 5e3bc32..1232e47 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java
@@ -15,21 +15,16 @@ import java.util.Map;
public interface SyslogNormalAlarmMapper {
/**
- * 动态插入数据 - 使用 Map
+ * 使用当前小批次的字段并集模板插入告警日志。调用方保证同批 Map 具有相同键集合,
+ * 缺失值绑定为 NULL,使 MyBatis ExecutorType.BATCH 能复用 PreparedStatement。
*/
- int insertDynamic(@Param("dataMap") Map dataMap);
+ int insertBatchPlanned(@Param("dataMap") Map dataMap);
/**
* 动态插入数据 - 使用实体类
*/
int insertByEntity(SyslogNormalAlarm entity);
- /**
- * 批量插入数据
- */
- int batchInsert(@Param("dataList") List> dataList);
-
-
/**
* 统计24小时内分组数据量
*/
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java
index 3a64ed5..d65c29d 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java
@@ -12,21 +12,16 @@ import java.util.List;
public interface SyslogNormalDataMapper {
/**
- * 动态插入数据 - 使用 Map
+ * 使用当前小批次的字段并集模板插入普通日志。调用方保证同批 Map 具有相同键集合,
+ * 缺失值绑定为 NULL,使 MyBatis ExecutorType.BATCH 能复用 PreparedStatement。
*/
- int insertDynamic(@Param("dataMap") Map dataMap);
+ int insertBatchPlanned(@Param("dataMap") Map dataMap);
/**
* 动态插入数据 - 使用实体类
*/
int insertByEntity(SyslogNormalData entity);
- /**
- * 批量插入数据
- */
- int batchInsert(@Param("dataList") java.util.List> dataList);
-
-
/**
* 统计24小时内分组数据量
*/
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java
index deacf5d..ea560a3 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java
@@ -13,13 +13,18 @@ import org.springframework.transaction.support.TransactionSynchronizationAdapter
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.util.List;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.Map;
+import java.util.Set;
/**
* 在一个达梦小事务中执行真正的 MyBatis JDBC batch。
*
* 这个类只负责数据库执行,不负责错误分类、拆分或 Kafka 状态。上层写入器保证传入的所有行
- * 属于同一张表和同一个动态列签名,因此 MyBatis BatchExecutor 可以复用同一个 PreparedStatement。
+ * 普通日志和告警日志分别使用当前小批次的非空字段并集模板,因此同一目标表、同一分区日期内
+ * MyBatis BatchExecutor 可以复用同一个 PreparedStatement。
* 每个 {@link #execute(boolean, List, int)} 调用都会创建独立事务;失败时整个小批次回滚,上层随后
* 决定二分定位还是按系统故障停止。
*/
@@ -63,7 +68,7 @@ public class DmStandardizedBatchExecutor {
* 执行一个同构小批次。
*
* @param alarmTable true 写 syslog_normal_alarm,false 写 syslog_normal_data
- * @param rows 已按相同动态列集合分组的行
+ * @param rows 已按相同目标表和分区日期分组的行
* @param transactionTimeoutSeconds 当前小事务硬超时
*/
public void execute(boolean alarmTable, List> rows,
@@ -72,10 +77,11 @@ public class DmStandardizedBatchExecutor {
return;
}
+ List> alignedRows = alignBatchColumns(rows);
int attempt = 0;
while (true) {
try {
- executeTransaction(alarmTable, rows, transactionTimeoutSeconds, attempt);
+ executeTransaction(alarmTable, alignedRows, transactionTimeoutSeconds, attempt);
return;
} catch (DirectStatsProjectionWriter.ConcurrentStatsWriteException concurrentFailure) {
if (attempt >= Math.max(0, maxConcurrencyRetries)) {
@@ -89,6 +95,39 @@ public class DmStandardizedBatchExecutor {
}
}
+ /**
+ * 计算当前小批次所有非空字段的并集,并为每行建立相同键集合的浅拷贝。
+ *
+ * Mapper XML 只允许声明过的列,额外业务键即使出现在 Map 中也不会进入 SQL。
+ * 同一键在某行不存在时,Map#get 返回 null,最终由 JDBC 按 SQL NULL 绑定。
+ */
+ static List> alignBatchColumns(List> rows) {
+ Set columnUnion = new LinkedHashSet<>();
+ for (Map row : rows) {
+ if (row == null) {
+ throw new IllegalArgumentException("达梦批写行不能为空");
+ }
+ for (Map.Entry entry : row.entrySet()) {
+ if (entry.getKey() != null && entry.getValue() != null) {
+ columnUnion.add(entry.getKey());
+ }
+ }
+ }
+ if (columnUnion.isEmpty()) {
+ throw new IllegalArgumentException("达梦批写没有可插入的非空字段");
+ }
+
+ List> alignedRows = new ArrayList<>(rows.size());
+ for (Map row : rows) {
+ Map aligned = new LinkedHashMap<>(columnUnion.size());
+ for (String column : columnUnion) {
+ aligned.put(column, row.get(column));
+ }
+ alignedRows.add(aligned);
+ }
+ return alignedRows;
+ }
+
private void executeTransaction(boolean alarmTable, List> rows,
int transactionTimeoutSeconds, int retries) {
@@ -112,12 +151,12 @@ public class DmStandardizedBatchExecutor {
SyslogNormalAlarmMapper mapper = batchSqlSession.getMapper(SyslogNormalAlarmMapper.class);
for (Map row : rows) {
// BatchExecutor 下这里不会逐条提交;返回值也不是最终影响行数,不能据此判断成功。
- mapper.insertDynamic(row);
+ mapper.insertBatchPlanned(row);
}
} else {
SyslogNormalDataMapper mapper = batchSqlSession.getMapper(SyslogNormalDataMapper.class);
for (Map row : rows) {
- mapper.insertDynamic(row);
+ mapper.insertBatchPlanned(row);
}
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java
index f568935..6b0a947 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java
@@ -32,8 +32,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
-import java.util.Set;
-import java.util.TreeSet;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
@@ -48,7 +46,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* 写入器遵守三条不能破坏的可靠性规则:
*
*
- * - 正常行按目标表、分区日期和动态列签名分组后批量提交;
+ * - 普通日志和告警日志分别按目标表、分区日期计算批次字段并集后批量提交;
* - 数据类错误回滚当前小批次并有限二分,最终只把坏行交给 Retry/DLT;
* - 连接、超时、提交未知等系统故障绝不降级为大量单条请求,而是立即失败并保留 Kafka offset。
*
@@ -67,7 +65,7 @@ public class DmStandardizedBatchWriter {
@Value("${dm.standardized-write.mode:BATCH}")
private String mode;
- @Value("${dm.standardized-write.batch-size:100}")
+ @Value("${dm.standardized-write.batch-size:500}")
private int batchSize;
@Value("${dm.standardized-write.transaction-timeout-seconds:45}")
@@ -161,7 +159,7 @@ public class DmStandardizedBatchWriter {
}
/**
- * 相同目标表、log_time 日期和非空列集合才会进入同一批次。
+ * 两张标准化表在每个小批次计算字段并集,只按目标表和 log_time 日期分组。
* LinkedHashMap 保持稳定遍历顺序,便于测试和现场日志复现。
*/
private void writeGroupedBatches(List> requests, WriteReport report,
@@ -173,7 +171,7 @@ public class DmStandardizedBatchWriter {
groups.computeIfAbsent(key, ignored -> new ArrayList<>()).add(request);
} catch (RuntimeException invalidGroupingKey) {
report.recordFailure(request.getSource(), new PermanentRowWriteException(
- "标准化结果缺少稳定分区时间或列签名无效", invalidGroupingKey));
+ "标准化结果缺少稳定分区时间", invalidGroupingKey));
}
}
@@ -294,18 +292,7 @@ public class DmStandardizedBatchWriter {
boolean alarmTable = normalDataService.isAlarmData(data);
LocalDate partitionDate = partitionDate(data.get("log_time"));
- // MyBatis XML 的动态列判断只包含非 null 值,所以签名也只记录非 null 列。
- // TreeSet 提供固定顺序,避免 HashMap 插入顺序不同却被错误拆成多个批次。
- Set columns = new TreeSet<>();
- for (Map.Entry entry : data.entrySet()) {
- if (entry.getValue() != null) {
- columns.add(entry.getKey());
- }
- }
- if (columns.isEmpty()) {
- throw new IllegalArgumentException("标准化结果没有可写入列");
- }
- return new BatchKey(alarmTable, partitionDate, String.join(",", columns));
+ return new BatchKey(alarmTable, partitionDate);
}
private LocalDate partitionDate(Object value) {
@@ -499,12 +486,10 @@ public class DmStandardizedBatchWriter {
private static final class BatchKey {
private final boolean alarmTable;
private final LocalDate partitionDate;
- private final String columnSignature;
- private BatchKey(boolean alarmTable, LocalDate partitionDate, String columnSignature) {
+ private BatchKey(boolean alarmTable, LocalDate partitionDate) {
this.alarmTable = alarmTable;
this.partitionDate = partitionDate;
- this.columnSignature = columnSignature;
}
@Override
@@ -513,13 +498,12 @@ public class DmStandardizedBatchWriter {
if (!(other instanceof BatchKey)) return false;
BatchKey that = (BatchKey) other;
return alarmTable == that.alarmTable
- && Objects.equals(partitionDate, that.partitionDate)
- && Objects.equals(columnSignature, that.columnSignature);
+ && Objects.equals(partitionDate, that.partitionDate);
}
@Override
public int hashCode() {
- return Objects.hash(alarmTable, partitionDate, columnSignature);
+ return Objects.hash(alarmTable, partitionDate);
}
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java
index 91f9921..a679909 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java
@@ -15,6 +15,7 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.List;
+import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.time.LocalDateTime;
@@ -35,21 +36,22 @@ public class SyslogNormalDataService {
/**
- * 动态插入单条数据 - Map方式
+ * 标准化单条兼容入口:按 event_level 路由后,两张目标表均使用受白名单约束的动态列模板。
*/
@Transactional
public int insertDynamic(Map dataMap) {
// [流程节点7] 根据标准化结果分类落库:安全事件写入 syslog_normal_alarm,访问事件写入 syslog_normal_data。
// [SYSLOG-ALARM][DBBATCH] 每条标准化结果单独开启事务并执行动态 INSERT,会产生大量数据库往返、SQL 解析和提交开销;
- // 高峰期 DB 变慢又会放大 Kafka 批次超时。建议按目标表和固定列集合聚合批次写入,并用 event_id + 规则版本唯一键保证重试幂等。
+ // 高峰期 DB 变慢又会放大 Kafka 批次超时。建议按目标表和批次字段并集聚合写入,并用 event_id + 规则版本唯一键保证重试幂等。
prepareForInsert(dataMap);
// [流程节点7.2] event_level 非空走安全事件表;为空走访问事件表,这是两条后续分析链路的分叉点。
boolean alarmData = isAlarmData(dataMap);
+ Map plannedData = nonNullColumnMap(dataMap);
try {
int inserted = alarmData
- ? syslogNormalAlarmMapper.insertDynamic(dataMap)
- : syslogNormalDataMapper.insertDynamic(dataMap);
+ ? syslogNormalAlarmMapper.insertBatchPlanned(plannedData)
+ : syslogNormalDataMapper.insertBatchPlanned(plannedData);
if (inserted > 0) {
if (alarmData && lateAlarmBucketWriter != null) {
lateAlarmBucketWriter.record(java.util.Collections.singletonList(dataMap));
@@ -72,6 +74,17 @@ public class SyslogNormalDataService {
}
}
+ /** 单条兼容路径也只把实际非空字段交给列模板,避免原 Map 中的空键误触发列选择。 */
+ private Map nonNullColumnMap(Map dataMap) {
+ Map result = new LinkedHashMap<>();
+ for (Map.Entry entry : dataMap.entrySet()) {
+ if (entry.getKey() != null && entry.getValue() != null) {
+ result.put(entry.getKey(), entry.getValue());
+ }
+ }
+ return result;
+ }
+
/**
* 在单条和批量写入之前执行完全相同的数据准备与关键字段校验。
*
@@ -220,9 +233,8 @@ public class SyslogNormalDataService {
data.put("log_time", new java.util.Date());
}
}
- // 旧 mapper 的 batchInsert 只写固定的少数字段,DIRECT 下会造成“统计有 device/rule,
- // 事实行却丢了 device/rule”的永久差异。主 Kafka 链路使用 DmStandardizedBatchExecutor;
- // 这个低频兼容 API 逐条复用完整动态 INSERT,但仍处于本方法的同一个 Spring 事务中。
+ // 主 Kafka 链路使用 DmStandardizedBatchExecutor 执行真正的 JDBC batch;这个低频兼容
+ // API 在同一个 Spring 事务内复用受白名单约束的列模板,不能再调用旧的五字段 batch SQL。
int inserted = 0;
for (Map data : dataList) {
inserted += insertDynamic(data);
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java
index 6bd33f3..c49fc72 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java
@@ -11,12 +11,14 @@ import com.common.service.StatsChangeCaptureService;
import com.common.service.DirectStatsProjectionWriter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
+import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@@ -38,6 +40,9 @@ public class SyslogNonNormalMessageServiceImpl
@Autowired
private DirectStatsProjectionWriter directStatsProjectionWriter;
+ @Value("${dm.non-normal-write.batch-size:500}")
+ private int nonNormalBatchSize;
+
@Override
@Transactional(rollbackFor = Exception.class)
public boolean saveMessage(SyslogNonNormalMessage message) {
@@ -125,8 +130,20 @@ public class SyslogNonNormalMessageServiceImpl
}
}
- // 批量插入
- int inserted = messageMapper.batchInsert(messages);
+ // 同一个服务事务内分块执行固定列多行 INSERT,避免单条提交,也避免一个 Kafka poll
+ // 直接生成过大的 SQL。所有分块成功后才统一写 DIRECT 统计并提交事务。
+ int inserted = 0;
+ int effectiveBatchSize = Math.max(1, nonNormalBatchSize);
+ for (int start = 0; start < messages.size(); start += effectiveBatchSize) {
+ int end = Math.min(start + effectiveBatchSize, messages.size());
+ int affected = messageMapper.batchInsert(
+ new ArrayList<>(messages.subList(start, end)));
+ if (affected <= 0) {
+ throw new IllegalStateException("非标日志批量 INSERT 未写入记录: start="
+ + start + ", end=" + end);
+ }
+ inserted += affected;
+ }
if (inserted > 0) {
if (statsChangeCaptureService.isEnabled()) {
for (SyslogNonNormalMessage message : messages) {
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java
index dd5fedd..a57e542 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java
@@ -31,6 +31,9 @@ public class ThreadPoolConfig {
@Value("${dm.standardized-write.writer-queue-capacity:100}")
private int dmWriterQueueCapacity;
+ @Value("${dm.non-normal-write.queue-capacity:500}")
+ private int nonNormalWriterQueueCapacity;
+
@Bean("logNormalProcessorExecutor")
@Primary
public Executor logNormalProcessorExecutor() {
@@ -71,4 +74,22 @@ public class ThreadPoolConfig {
executor.initialize();
return executor;
}
+
+ /**
+ * 非标日志使用单独的串行写线程,与正常事实批写解耦。
+ * listener 不等待该线程,队列溢出或异步写入失败只记录错误,不阻塞 Kafka ACK。
+ */
+ @Bean("nonNormalWriterExecutor")
+ public Executor nonNormalWriterExecutor() {
+ ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+ executor.setCorePoolSize(1);
+ executor.setMaxPoolSize(1);
+ executor.setQueueCapacity(Math.max(1, nonNormalWriterQueueCapacity));
+ executor.setThreadNamePrefix("dm-non-normal-writer-");
+ executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
+ executor.setWaitForTasksToCompleteOnShutdown(true);
+ executor.setAwaitTerminationSeconds(60);
+ executor.initialize();
+ return executor;
+ }
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties
index efd1c5a..1943d00 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties
@@ -56,8 +56,11 @@ mybatis.type-aliases-package=com.common.entity
mybatis.configuration.default-statement-timeout=30
mybatis.configuration.default-fetch-size=1000
mybatis.configuration.map-underscore-to-camel-case=true
+# 批次列并集写入会为部分行绑定空字段;达梦对未指定类型的 NULL 参数兼容性较差,统一使用 JDBC NULL。
+mybatis.configuration.jdbc-type-for-null=NULL
mybatis.type-handlers-package=com.Modules.etl.handler
mybatis-plus.configuration.map-underscore-to-camel-case=true
+mybatis-plus.configuration.jdbc-type-for-null=NULL
mybatis-plus.type-handlers-package=com.Modules.etl.handler
# kafka Configuration
@@ -142,10 +145,11 @@ app.processor.batch-size=100
app.processor.process-timeout-ms=120000
# 达梦标准化表默认启用批量写入;现场需要紧急回退时可显式切换为 SINGLE。
-# BATCH 模式按目标表、log_time 日期和动态列集合分组,失败时只对数据类错误有限二分;
+# BATCH 模式按目标表和 log_time 日期分组,按小批次实际非空字段并集执行 JDBC Batch;
+# 失败时只对数据类错误有限二分;
# 连接、超时或提交结果未知会立即停止并保留 Kafka offset,禁止放大为逐条请求风暴。
dm.standardized-write.mode=BATCH
-dm.standardized-write.batch-size=100
+dm.standardized-write.batch-size=${DM_STANDARDIZED_WRITE_BATCH_SIZE:500}
dm.standardized-write.writer-concurrency=2
dm.standardized-write.writer-queue-capacity=100
dm.standardized-write.transaction-timeout-seconds=45
@@ -154,6 +158,8 @@ dm.standardized-write.max-split-depth=10
dm.standardized-write.max-fallback-operations=100
dm.standardized-write.circuit-breaker-failure-threshold=3
dm.standardized-write.circuit-breaker-backoff-ms=30000
+dm.non-normal-write.batch-size=${DM_NON_NORMAL_WRITE_BATCH_SIZE:500}
+dm.non-normal-write.queue-capacity=${DM_NON_NORMAL_WRITE_QUEUE_CAPACITY:500}
# ETL配置
etl.batch.page-size=1000
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml
index 9e84f53..ee8f6cf 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml
@@ -4,756 +4,759 @@
-
+
+
INSERT INTO syslog_normal_alarm
- id,
- created_at,
- log_time,
- device_id,
- webshell_type,
- vuirs_type,
- vuirs_url,
- class_filename,
- class_path,
- parent_class,
- jar_path,
- class_md5,
- class_loader,
- class_hashcode,
- class_loader_hashcode,
- tc_nameip,
- perform_sql,
- tc_account,
- tc_appname,
- process_uname,
- p_process_uname,
- container_name,
- container_id,
- http_resp_server,
- srcip_id,
- cdnip,
- natip,
- mail_sender,
- mail_receiver,
- vpn_mac,
- vpn_os,
- vpn_user,
- vpn_groupname,
- vpn_access_ip,
- dest_ip_apt,
- origin_attack_result,
- description,
- solution,
- attack_cause,
- username,
- tc_flow_id,
- login_result,
- cmdline,
- origin_attack_action,
- victim_domain,
- vpn_deviceid,
- vpn_access_action,
- file_access_time,
- file_name,
- tc_class,
- tc_name2,
- login_lasttime,
- origin_permissions,
- begin_permissions,
- print_time,
- printer,
- printer_type,
- print_pages,
- print_copies,
- src_device,
- dst_device,
- src_file,
- src_file_type,
- src_file_path,
- dst_file,
- dst_file_type,
- dst_file_path,
- dlp_policy_name,
- dlp_policy_type,
- dst_upload_url,
- process_uuid,
- p_process_uuid,
- env,
- brute_force_service,
- vuirs_name,
- http_req_length,
- http_req_content_type,
- tc_scan_port,
- tc_labels,
- http_resp_content_type,
- dns_msg_type,
- dns_answer_length,
- dns_ioc,
- tx_bytes,
- rx_bytes,
- all_bytes,
- duration_time,
- mail_attach_name,
- mail_subject,
- mail_message,
- mail_send_server,
- mail_agent,
- tls_version,
- tls_server_cert,
- tls_server_suite,
- tls_client_suites_len,
- tls_ja3,
- tls_ja3s,
- vpn_access_port,
- log_topic,
- collect_time,
- src_is_intranetip,
- src_ip_ioc,
- src_ip_apt,
- srcip_name,
- tc_client,
- srcip_organization_id,
- dest_ip_intranetip,
- dest_ip_ioc,
- desip_id,
- desip_name,
- tc_hostip,
- desip_organization_id,
- origin_confidence,
- origin_malscore,
- attacker_icampaign,
- attacker_host_asset_id,
- attacker_organization_id,
- victim_host_asset_id,
- victim_organization_id,
- logout_time,
- http_req_line,
- desip_security_scope_id,
- srcip_security_scope_id,
- http_resp_length,
- tc_attack_type,
- tc_realip,
- attacker_ip_lists,
- login_password,
- detail,
- attacker_country_code,
- attacker_region_code,
- victim_region_code,
- payload,
- http_referer,
- http_user_agent,
- http_session,
- http_query_string,
- file_path,
- file_permission,
- login_abnormal_type,
- file_tag,
- file_platform,
- target_ip,
- collect_date,
- tc_client_ip,
- tc_server_ip,
- tc_externalip,
- http_status_code,
- device_domian,
- src_ip_str,
- src_port_str,
- dest_ip_str,
- dest_port_str,
- pcap,
- ioc,
- malicious_family,
- vuln_cve,
- aliyun_type,
- attacker_host_asset_name,
- attacker_organization_name,
- ct_id,
- cve_list,
- desip_organization_name,
- dest_ip_group,
- file_gid,
- file_owner,
- file_ownergroup,
- file_uid,
- http_resp_cookie,
- origin_rule_id,
- origin_rule_name,
- service_name,
- src_ip_asset_group,
- srcip_organization_name,
- victim_host_asset_name,
- http_resp_codes,
- victim_organization_name,
- tc_type,
- direction,
- http_req_cookie,
- http_req_protocol,
- http_req_header_raw,
- http_url,
- uname,
- origin_hostname,
- origin_os,
- origin_agent_mac,
- origin_host_id,
- origin_agent_version,
- origin_agent_id,
- origin_agent_name,
- origin_work_group,
- origin_asset_group,
- origin_local_port,
- origin_agent_ip,
- origin_internal_ip,
- origin_external_ip,
- origin_local_addr,
- agent_id,
- agent_name,
- tc_title,
- log_id,
- event_date,
- event_time_ts,
- event_level,
- src_ip ,
- src_port,
- dest_ip,
- dest_port,
- event_time,
- attacker_country,
- src_mac,
- dest_mac,
- proto,
- dev_id,
- created_time,
- src_country,
- src_country_code,
- src_region,
- src_region_code,
- src_city,
- src_lon,
- http_method,
- http_host,
- http_req_header,
- http_req_body,
- http_resp_header,
- http_resp_body,
- file_type,
- file_md5,
- file_size,
- process,
- start_time,
- action,
- attacker_region,
- end_time,
- file_created_time,
- file_modified_time,
- tc_miguan_scan_port,
- process_path,
- parent_process_path,
- gname,
- exe_name,
- exe_path,
- login_time,
- login_times,
- check_item,
- check_type,
- attacker_ip,
- attacker_port,
- victim_ip,
- victim_port,
- attacker_city,
- attacker_lon,
- attacker_lat,
- victim_country,
- victim_region,
- victim_city,
- victim_lon,
- victim_lat,
- origin_event_id,
- origin_event_name,
- origin_event_category,
- origin_event_level,
- origin_attack_chain,
- engine_type,
- evil_payload,
- http_resp_status,
- dns_query,
- dns_query_type,
- dns_ttl,
- dns_answer,
- dns_subdomains,
- file_sha256,
- file_ssdeep,
- victim_country_code,
- http_xff_ip,
- tc_miguan_class,
- pid,
- ppid,
- process_name,
- backdoor_type,
- tty,
- sudo_user,
- sudo_group,
- origin_event_type,
- dest_domain,
- shell_cmdline,
- parent_cmdline,
- attack_chain,
- process_tree,
- host_file_sha256,
- host_file_md5,
- host_file_size,
- host_file_type,
- dest_country,
- dest_country_code,
- log_origin,
- dest_region,
- src_lat,
- dest_region_code,
- dest_city,
- dest_lon,
- dest_lat,
- event_category,
- attack_result,
- probe_ip,
- device_ip,
- device_manufacturer,
- device_name,
- product_name,
- __id,
- __count,
- __count_reason,
- event_type,
- protocol,
- shell_cmd,
- parent_name,
- host_file_path,
- uid,
- fall,
- tc_miguan_server_ip,
- dev_type,
- collect_method,
- field_cate_id,
- device_type,
- tc_miguan_client_ip,
- tc_miguan_name,
- origin_total_packages,
- origin_total_bytes,
- origin_peak_packages_rate,
- origin_peak_bytes_rate,
- origin_peak_flows_rate,
- apt_orgname,
- apt_orgmsg,
- mail_message_id,
- mail_bcc,
- mail_size,
- mail_attach_hashcode,
- mail_url,
- mail_cc,
- algorithm,
- miningpool_ip,
- process_md5,
- pprocess_md5,
- source_servername,
- origin_source_servername,
- mail_filename,
- dst_upload_appname,
- target_port,
- gid,
- origin_uid,
- origin_gid,
- target_ports,
- tc_miguan_name1,
- tc_miguan_class1,
- etl_time,
- tc_miguan_scan_port2,
- desip_security_scope,
- srcip_security_scope,
- collect_time_ts,
- tc_miguan_scan_port1,
- src_dev_name,
- collect_protocol,
- destination_system_type,
- destination_system,
- etl_host,
- normalize_rule_id,
- normalize_rule_name,
- syslog_uuid,
- syslog_topic,
+ id,
+ created_at,
+ log_time,
+ device_id,
+ webshell_type,
+ vuirs_type,
+ vuirs_url,
+ class_filename,
+ class_path,
+ parent_class,
+ jar_path,
+ class_md5,
+ class_loader,
+ class_hashcode,
+ class_loader_hashcode,
+ tc_nameip,
+ perform_sql,
+ tc_account,
+ tc_appname,
+ process_uname,
+ p_process_uname,
+ container_name,
+ container_id,
+ http_resp_server,
+ srcip_id,
+ cdnip,
+ natip,
+ mail_sender,
+ mail_receiver,
+ vpn_mac,
+ vpn_os,
+ vpn_user,
+ vpn_groupname,
+ vpn_access_ip,
+ dest_ip_apt,
+ origin_attack_result,
+ description,
+ solution,
+ attack_cause,
+ username,
+ tc_flow_id,
+ login_result,
+ cmdline,
+ origin_attack_action,
+ victim_domain,
+ vpn_deviceid,
+ vpn_access_action,
+ file_access_time,
+ file_name,
+ tc_class,
+ tc_name2,
+ login_lasttime,
+ origin_permissions,
+ begin_permissions,
+ print_time,
+ printer,
+ printer_type,
+ print_pages,
+ print_copies,
+ src_device,
+ dst_device,
+ src_file,
+ src_file_type,
+ src_file_path,
+ dst_file,
+ dst_file_type,
+ dst_file_path,
+ dlp_policy_name,
+ dlp_policy_type,
+ dst_upload_url,
+ process_uuid,
+ p_process_uuid,
+ env,
+ brute_force_service,
+ vuirs_name,
+ http_req_length,
+ http_req_content_type,
+ tc_scan_port,
+ tc_labels,
+ http_resp_content_type,
+ dns_msg_type,
+ dns_answer_length,
+ dns_ioc,
+ tx_bytes,
+ rx_bytes,
+ all_bytes,
+ duration_time,
+ mail_attach_name,
+ mail_subject,
+ mail_message,
+ mail_send_server,
+ mail_agent,
+ tls_version,
+ tls_server_cert,
+ tls_server_suite,
+ tls_client_suites_len,
+ tls_ja3,
+ tls_ja3s,
+ vpn_access_port,
+ log_topic,
+ collect_time,
+ src_is_intranetip,
+ src_ip_ioc,
+ src_ip_apt,
+ srcip_name,
+ tc_client,
+ srcip_organization_id,
+ dest_ip_intranetip,
+ dest_ip_ioc,
+ desip_id,
+ desip_name,
+ tc_hostip,
+ desip_organization_id,
+ origin_confidence,
+ origin_malscore,
+ attacker_icampaign,
+ attacker_host_asset_id,
+ attacker_organization_id,
+ victim_host_asset_id,
+ victim_organization_id,
+ logout_time,
+ http_req_line,
+ desip_security_scope_id,
+ srcip_security_scope_id,
+ http_resp_length,
+ tc_attack_type,
+ tc_realip,
+ attacker_ip_lists,
+ login_password,
+ detail,
+ attacker_country_code,
+ attacker_region_code,
+ victim_region_code,
+ payload,
+ http_referer,
+ http_user_agent,
+ http_session,
+ http_query_string,
+ file_path,
+ file_permission,
+ login_abnormal_type,
+ file_tag,
+ file_platform,
+ target_ip,
+ collect_date,
+ tc_client_ip,
+ tc_server_ip,
+ tc_externalip,
+ http_status_code,
+ device_domian,
+ src_ip_str,
+ src_port_str,
+ dest_ip_str,
+ dest_port_str,
+ pcap,
+ ioc,
+ malicious_family,
+ vuln_cve,
+ aliyun_type,
+ attacker_host_asset_name,
+ attacker_organization_name,
+ ct_id,
+ cve_list,
+ desip_organization_name,
+ dest_ip_group,
+ file_gid,
+ file_owner,
+ file_ownergroup,
+ file_uid,
+ http_resp_cookie,
+ origin_rule_id,
+ origin_rule_name,
+ service_name,
+ src_ip_asset_group,
+ srcip_organization_name,
+ victim_host_asset_name,
+ http_resp_codes,
+ victim_organization_name,
+ tc_type,
+ direction,
+ http_req_cookie,
+ http_req_protocol,
+ http_req_header_raw,
+ http_url,
+ uname,
+ origin_hostname,
+ origin_os,
+ origin_agent_mac,
+ origin_host_id,
+ origin_agent_version,
+ origin_agent_id,
+ origin_agent_name,
+ origin_work_group,
+ origin_asset_group,
+ origin_local_port,
+ origin_agent_ip,
+ origin_internal_ip,
+ origin_external_ip,
+ origin_local_addr,
+ agent_id,
+ agent_name,
+ tc_title,
+ log_id,
+ event_date,
+ event_time_ts,
+ event_level,
+ src_ip ,
+ src_port,
+ dest_ip,
+ dest_port,
+ event_time,
+ attacker_country,
+ src_mac,
+ dest_mac,
+ proto,
+ dev_id,
+ created_time,
+ src_country,
+ src_country_code,
+ src_region,
+ src_region_code,
+ src_city,
+ src_lon,
+ http_method,
+ http_host,
+ http_req_header,
+ http_req_body,
+ http_resp_header,
+ http_resp_body,
+ file_type,
+ file_md5,
+ file_size,
+ process,
+ start_time,
+ action,
+ attacker_region,
+ end_time,
+ file_created_time,
+ file_modified_time,
+ tc_miguan_scan_port,
+ process_path,
+ parent_process_path,
+ gname,
+ exe_name,
+ exe_path,
+ login_time,
+ login_times,
+ check_item,
+ check_type,
+ attacker_ip,
+ attacker_port,
+ victim_ip,
+ victim_port,
+ attacker_city,
+ attacker_lon,
+ attacker_lat,
+ victim_country,
+ victim_region,
+ victim_city,
+ victim_lon,
+ victim_lat,
+ origin_event_id,
+ origin_event_name,
+ origin_event_category,
+ origin_event_level,
+ origin_attack_chain,
+ engine_type,
+ evil_payload,
+ http_resp_status,
+ dns_query,
+ dns_query_type,
+ dns_ttl,
+ dns_answer,
+ dns_subdomains,
+ file_sha256,
+ file_ssdeep,
+ victim_country_code,
+ http_xff_ip,
+ tc_miguan_class,
+ pid,
+ ppid,
+ process_name,
+ backdoor_type,
+ tty,
+ sudo_user,
+ sudo_group,
+ origin_event_type,
+ dest_domain,
+ shell_cmdline,
+ parent_cmdline,
+ attack_chain,
+ process_tree,
+ host_file_sha256,
+ host_file_md5,
+ host_file_size,
+ host_file_type,
+ dest_country,
+ dest_country_code,
+ log_origin,
+ dest_region,
+ src_lat,
+ dest_region_code,
+ dest_city,
+ dest_lon,
+ dest_lat,
+ event_category,
+ attack_result,
+ probe_ip,
+ device_ip,
+ device_manufacturer,
+ device_name,
+ product_name,
+ __id,
+ __count,
+ __count_reason,
+ event_type,
+ protocol,
+ shell_cmd,
+ parent_name,
+ host_file_path,
+ uid,
+ fall,
+ tc_miguan_server_ip,
+ dev_type,
+ collect_method,
+ field_cate_id,
+ device_type,
+ tc_miguan_client_ip,
+ tc_miguan_name,
+ origin_total_packages,
+ origin_total_bytes,
+ origin_peak_packages_rate,
+ origin_peak_bytes_rate,
+ origin_peak_flows_rate,
+ apt_orgname,
+ apt_orgmsg,
+ mail_message_id,
+ mail_bcc,
+ mail_size,
+ mail_attach_hashcode,
+ mail_url,
+ mail_cc,
+ algorithm,
+ miningpool_ip,
+ process_md5,
+ pprocess_md5,
+ source_servername,
+ origin_source_servername,
+ mail_filename,
+ dst_upload_appname,
+ target_port,
+ gid,
+ origin_uid,
+ origin_gid,
+ target_ports,
+ tc_miguan_name1,
+ tc_miguan_class1,
+ etl_time,
+ tc_miguan_scan_port2,
+ desip_security_scope,
+ srcip_security_scope,
+ collect_time_ts,
+ tc_miguan_scan_port1,
+ src_dev_name,
+ collect_protocol,
+ destination_system_type,
+ destination_system,
+ etl_host,
+ normalize_rule_id,
+ normalize_rule_name,
+ syslog_uuid,
+ syslog_topic,
+ rule_version,
VALUES
- #{dataMap.id},
- #{dataMap.created_at},
- #{dataMap.log_time},
- #{dataMap.device_id},
- #{dataMap.webshell_type},
- #{dataMap.vuirs_type},
- #{dataMap.vuirs_url},
- #{dataMap.class_filename},
- #{dataMap.class_path},
- #{dataMap.parent_class},
- #{dataMap.jar_path},
- #{dataMap.class_md5},
- #{dataMap.class_loader},
- #{dataMap.class_hashcode},
- #{dataMap.class_loader_hashcode},
- #{dataMap.tc_nameip},
- #{dataMap.perform_sql},
- #{dataMap.tc_account},
- #{dataMap.tc_appname},
- #{dataMap.process_uname},
- #{dataMap.p_process_uname},
- #{dataMap.container_name},
- #{dataMap.container_id},
- #{dataMap.http_resp_server},
- #{dataMap.srcip_id},
- #{dataMap.cdnip},
- #{dataMap.natip},
- #{dataMap.mail_sender},
- #{dataMap.mail_receiver},
- #{dataMap.vpn_mac},
- #{dataMap.vpn_os},
- #{dataMap.vpn_user},
- #{dataMap.vpn_groupname},
- #{dataMap.vpn_access_ip},
- #{dataMap.dest_ip_apt},
- #{dataMap.origin_attack_result},
- #{dataMap.description},
- #{dataMap.solution},
- #{dataMap.attack_cause},
- #{dataMap.username},
- #{dataMap.tc_flow_id},
- #{dataMap.login_result},
- #{dataMap.cmdline},
- #{dataMap.origin_attack_action},
- #{dataMap.victim_domain},
- #{dataMap.vpn_deviceid},
- #{dataMap.vpn_access_action},
- #{dataMap.file_access_time},
- #{dataMap.file_name},
- #{dataMap.tc_class},
- #{dataMap.tc_name2},
- #{dataMap.login_lasttime},
- #{dataMap.origin_permissions},
- #{dataMap.begin_permissions},
- #{dataMap.print_time},
- #{dataMap.printer},
- #{dataMap.printer_type},
- #{dataMap.print_pages},
- #{dataMap.print_copies},
- #{dataMap.src_device},
- #{dataMap.dst_device},
- #{dataMap.src_file},
- #{dataMap.src_file_type},
- #{dataMap.src_file_path},
- #{dataMap.dst_file},
- #{dataMap.dst_file_type},
- #{dataMap.dst_file_path},
- #{dataMap.dlp_policy_name},
- #{dataMap.dlp_policy_type},
- #{dataMap.dst_upload_url},
- #{dataMap.process_uuid},
- #{dataMap.p_process_uuid},
- #{dataMap.env},
- #{dataMap.brute_force_service},
- #{dataMap.vuirs_name},
- #{dataMap.http_req_length},
- #{dataMap.http_req_content_type},
- #{dataMap.tc_scan_port},
- #{dataMap.tc_labels},
- #{dataMap.http_resp_content_type},
- #{dataMap.dns_msg_type},
- #{dataMap.dns_answer_length},
- #{dataMap.dns_ioc},
- #{dataMap.tx_bytes},
- #{dataMap.rx_bytes},
- #{dataMap.all_bytes},
- #{dataMap.duration_time},
- #{dataMap.mail_attach_name},
- #{dataMap.mail_subject},
- #{dataMap.mail_message},
- #{dataMap.mail_send_server},
- #{dataMap.mail_agent},
- #{dataMap.tls_version},
- #{dataMap.tls_server_cert},
- #{dataMap.tls_server_suite},
- #{dataMap.tls_client_suites_len},
- #{dataMap.tls_ja3},
- #{dataMap.tls_ja3s},
- #{dataMap.vpn_access_port},
- #{dataMap.log_topic},
- #{dataMap.collect_time},
- #{dataMap.src_is_intranetip},
- #{dataMap.src_ip_ioc},
- #{dataMap.src_ip_apt},
- #{dataMap.srcip_name},
- #{dataMap.tc_client},
- #{dataMap.srcip_organization_id},
- #{dataMap.dest_ip_intranetip},
- #{dataMap.dest_ip_ioc},
- #{dataMap.desip_id},
- #{dataMap.desip_name},
- #{dataMap.tc_hostip},
- #{dataMap.desip_organization_id},
- #{dataMap.origin_confidence},
- #{dataMap.origin_malscore},
- #{dataMap.attacker_icampaign},
- #{dataMap.attacker_host_asset_id},
- #{dataMap.attacker_organization_id},
- #{dataMap.victim_host_asset_id},
- #{dataMap.victim_organization_id},
- #{dataMap.logout_time},
- #{dataMap.http_req_line},
- #{dataMap.desip_security_scope_id},
- #{dataMap.srcip_security_scope_id},
- #{dataMap.http_resp_length},
- #{dataMap.tc_attack_type},
- #{dataMap.tc_realip},
- #{dataMap.attacker_ip_lists},
- #{dataMap.login_password},
- #{dataMap.detail},
- #{dataMap.attacker_country_code},
- #{dataMap.attacker_region_code},
- #{dataMap.victim_region_code},
- #{dataMap.payload},
- #{dataMap.http_referer},
- #{dataMap.http_user_agent},
- #{dataMap.http_session},
- #{dataMap.http_query_string},
- #{dataMap.file_path},
- #{dataMap.file_permission},
- #{dataMap.login_abnormal_type},
- #{dataMap.file_tag},
- #{dataMap.file_platform},
- #{dataMap.target_ip},
- #{dataMap.collect_date},
- #{dataMap.tc_client_ip},
- #{dataMap.tc_server_ip},
- #{dataMap.tc_externalip},
- #{dataMap.http_status_code},
- #{dataMap.device_domian},
- #{dataMap.src_ip_str},
- #{dataMap.src_port_str},
- #{dataMap.dest_ip_str} ,
- CAST(#{dataMap.dest_port_str} AS text),
- #{dataMap.pcap},
- #{dataMap.ioc},
- #{dataMap.malicious_family},
- #{dataMap.vuln_cve},
- #{dataMap.aliyun_type},
- #{dataMap.attacker_host_asset_name},
- #{dataMap.attacker_organization_name},
- #{dataMap.ct_id},
- #{dataMap.cve_list},
- #{dataMap.desip_organization_name},
- #{dataMap.dest_ip_group},
- #{dataMap.file_gid},
- #{dataMap.file_owner},
- #{dataMap.file_ownergroup},
- #{dataMap.file_uid},
- #{dataMap.http_resp_cookie},
- #{dataMap.origin_rule_id},
- #{dataMap.origin_rule_name},
- #{dataMap.service_name},
- #{dataMap.src_ip_asset_group},
- #{dataMap.srcip_organization_name},
- #{dataMap.victim_host_asset_name},
- #{dataMap.http_resp_codes}::bigint,
- #{dataMap.victim_organization_name},
- #{dataMap.tc_type},
- #{dataMap.direction},
- #{dataMap.http_req_cookie},
- #{dataMap.http_req_protocol},
- #{dataMap.http_req_header_raw},
- #{dataMap.http_url},
- #{dataMap.uname},
- #{dataMap.origin_hostname},
- #{dataMap.origin_os},
- #{dataMap.origin_agent_mac},
- #{dataMap.origin_host_id},
- #{dataMap.origin_agent_version},
- #{dataMap.origin_agent_id},
- #{dataMap.origin_agent_name},
- #{dataMap.origin_work_group},
- #{dataMap.origin_asset_group},
- #{dataMap.origin_local_port},
- #{dataMap.origin_agent_ip},
- #{dataMap.origin_internal_ip},
- #{dataMap.origin_external_ip},
- #{dataMap.origin_local_addr},
- #{dataMap.agent_id},
- #{dataMap.agent_name},
- #{dataMap.tc_title},
- #{dataMap.log_id},
- #{dataMap.event_date},
- #{dataMap.event_time_ts},
- #{dataMap.event_level},
- #{dataMap.src_ip},
- #{dataMap.src_port},
- #{dataMap.dest_ip},
- #{dataMap.dest_port}::BIGINT,
- #{dataMap.event_time},
- #{dataMap.attacker_country},
- #{dataMap.src_mac},
- #{dataMap.dest_mac},
- #{dataMap.proto},
- #{dataMap.dev_id},
- #{dataMap.created_time},
- #{dataMap.src_country},
- #{dataMap.src_country_code},
- #{dataMap.src_region},
- #{dataMap.src_region_code},
- #{dataMap.src_city},
- #{dataMap.src_lon},
- #{dataMap.http_method},
- #{dataMap.http_host},
- #{dataMap.http_req_header},
- #{dataMap.http_req_body},
- #{dataMap.http_resp_header},
- #{dataMap.http_resp_body},
- #{dataMap.file_type},
- #{dataMap.file_md5},
- #{dataMap.file_size},
- #{dataMap.process},
- #{dataMap.start_time},
- #{dataMap.action},
- #{dataMap.attacker_region},
- #{dataMap.end_time},
- #{dataMap.file_created_time},
- #{dataMap.file_modified_time},
- #{dataMap.tc_miguan_scan_port},
- #{dataMap.process_path},
- #{dataMap.parent_process_path},
- #{dataMap.gname},
- #{dataMap.exe_name},
- #{dataMap.exe_path},
- #{dataMap.login_time},
- #{dataMap.login_times},
- #{dataMap.check_item},
- #{dataMap.check_type},
- #{dataMap.attacker_ip},
- #{dataMap.attacker_port},
- #{dataMap.victim_ip},
- #{dataMap.victim_port},
- #{dataMap.attacker_city},
- #{dataMap.attacker_lon},
- #{dataMap.attacker_lat},
- #{dataMap.victim_country},
- #{dataMap.victim_region},
- #{dataMap.victim_city},
- #{dataMap.victim_lon},
- #{dataMap.victim_lat},
- #{dataMap.origin_event_id},
- #{dataMap.origin_event_name},
- #{dataMap.origin_event_category},
- #{dataMap.origin_event_level},
- #{dataMap.origin_attack_chain},
- #{dataMap.engine_type},
- #{dataMap.evil_payload},
- #{dataMap.http_resp_status},
- #{dataMap.dns_query},
- #{dataMap.dns_query_type},
- #{dataMap.dns_ttl},
- #{dataMap.dns_answer},
- #{dataMap.dns_subdomains},
- #{dataMap.file_sha256},
- #{dataMap.file_ssdeep},
- #{dataMap.victim_country_code},
- #{dataMap.http_xff_ip},
- #{dataMap.tc_miguan_class},
- #{dataMap.pid},
- #{dataMap.ppid},
- #{dataMap.process_name},
- #{dataMap.backdoor_type},
- #{dataMap.tty},
- #{dataMap.sudo_user},
- #{dataMap.sudo_group},
- #{dataMap.origin_event_type},
- #{dataMap.dest_domain},
- #{dataMap.shell_cmdline},
- #{dataMap.parent_cmdline},
- #{dataMap.attack_chain},
- #{dataMap.process_tree},
- #{dataMap.host_file_sha256},
- #{dataMap.host_file_md5},
- #{dataMap.host_file_size},
- #{dataMap.host_file_type},
- #{dataMap.dest_country},
- #{dataMap.dest_country_code},
- #{dataMap.log_origin},
- #{dataMap.dest_region},
- #{dataMap.src_lat},
- #{dataMap.dest_region_code},
- #{dataMap.dest_city},
- #{dataMap.dest_lon},
- #{dataMap.dest_lat},
- #{dataMap.event_category},
- #{dataMap.attack_result},
- #{dataMap.probe_ip},
- #{dataMap.device_ip},
- #{dataMap.device_manufacturer},
- #{dataMap.device_name},
- #{dataMap.product_name},
- #{dataMap.__id},
- #{dataMap.__count},
- #{dataMap.__count_reason},
- #{dataMap.event_type},
- #{dataMap.protocol},
- #{dataMap.shell_cmd},
- #{dataMap.parent_name},
- #{dataMap.host_file_path},
- #{dataMap.uid},
- #{dataMap.fall},
- #{dataMap.tc_miguan_server_ip},
- #{dataMap.dev_type},
- #{dataMap.collect_method},
- #{dataMap.field_cate_id},
- #{dataMap.device_type},
- #{dataMap.tc_miguan_client_ip},
- #{dataMap.tc_miguan_name},
- #{dataMap.origin_total_packages},
- #{dataMap.origin_total_bytes},
- #{dataMap.origin_peak_packages_rate},
- #{dataMap.origin_peak_bytes_rate},
- #{dataMap.origin_peak_flows_rate},
- #{dataMap.apt_orgname},
- #{dataMap.apt_orgmsg},
- #{dataMap.mail_message_id},
- #{dataMap.mail_bcc},
- #{dataMap.mail_size},
- #{dataMap.mail_attach_hashcode},
- #{dataMap.mail_url},
- #{dataMap.mail_cc},
- #{dataMap.algorithm},
- #{dataMap.miningpool_ip},
- #{dataMap.process_md5},
- #{dataMap.pprocess_md5},
- #{dataMap.source_servername},
- #{dataMap.origin_source_servername},
- #{dataMap.mail_filename},
- #{dataMap.dst_upload_appname},
- #{dataMap.target_port},
- #{dataMap.gid},
- #{dataMap.origin_uid},
- #{dataMap.origin_gid},
- #{dataMap.target_ports},
- #{dataMap.tc_miguan_name1},
- #{dataMap.tc_miguan_class1},
- #{dataMap.etl_time},
- #{dataMap.tc_miguan_scan_port2},
- #{dataMap.desip_security_scope},
- #{dataMap.srcip_security_scope},
- #{dataMap.collect_time_ts},
- #{dataMap.tc_miguan_scan_port1},
- #{dataMap.src_dev_name},
- #{dataMap.collect_protocol},
- #{dataMap.destination_system_type},
- #{dataMap.destination_system},
- #{dataMap.etl_host},
- #{dataMap.normalize_rule_id},
- #{dataMap.normalize_rule_name},
- #{dataMap.syslog_uuid},
- #{dataMap.syslog_topic},
+ #{dataMap.id},
+ #{dataMap.created_at},
+ #{dataMap.log_time},
+ #{dataMap.device_id},
+ #{dataMap.webshell_type},
+ #{dataMap.vuirs_type},
+ #{dataMap.vuirs_url},
+ #{dataMap.class_filename},
+ #{dataMap.class_path},
+ #{dataMap.parent_class},
+ #{dataMap.jar_path},
+ #{dataMap.class_md5},
+ #{dataMap.class_loader},
+ #{dataMap.class_hashcode},
+ #{dataMap.class_loader_hashcode},
+ #{dataMap.tc_nameip},
+ #{dataMap.perform_sql},
+ #{dataMap.tc_account},
+ #{dataMap.tc_appname},
+ #{dataMap.process_uname},
+ #{dataMap.p_process_uname},
+ #{dataMap.container_name},
+ #{dataMap.container_id},
+ #{dataMap.http_resp_server},
+ #{dataMap.srcip_id},
+ #{dataMap.cdnip},
+ #{dataMap.natip},
+ #{dataMap.mail_sender},
+ #{dataMap.mail_receiver},
+ #{dataMap.vpn_mac},
+ #{dataMap.vpn_os},
+ #{dataMap.vpn_user},
+ #{dataMap.vpn_groupname},
+ #{dataMap.vpn_access_ip},
+ #{dataMap.dest_ip_apt},
+ #{dataMap.origin_attack_result},
+ #{dataMap.description},
+ #{dataMap.solution},
+ #{dataMap.attack_cause},
+ #{dataMap.username},
+ #{dataMap.tc_flow_id},
+ #{dataMap.login_result},
+ #{dataMap.cmdline},
+ #{dataMap.origin_attack_action},
+ #{dataMap.victim_domain},
+ #{dataMap.vpn_deviceid},
+ #{dataMap.vpn_access_action},
+ #{dataMap.file_access_time},
+ #{dataMap.file_name},
+ #{dataMap.tc_class},
+ #{dataMap.tc_name2},
+ #{dataMap.login_lasttime},
+ #{dataMap.origin_permissions},
+ #{dataMap.begin_permissions},
+ #{dataMap.print_time},
+ #{dataMap.printer},
+ #{dataMap.printer_type},
+ #{dataMap.print_pages},
+ #{dataMap.print_copies},
+ #{dataMap.src_device},
+ #{dataMap.dst_device},
+ #{dataMap.src_file},
+ #{dataMap.src_file_type},
+ #{dataMap.src_file_path},
+ #{dataMap.dst_file},
+ #{dataMap.dst_file_type},
+ #{dataMap.dst_file_path},
+ #{dataMap.dlp_policy_name},
+ #{dataMap.dlp_policy_type},
+ #{dataMap.dst_upload_url},
+ #{dataMap.process_uuid},
+ #{dataMap.p_process_uuid},
+ #{dataMap.env},
+ #{dataMap.brute_force_service},
+ #{dataMap.vuirs_name},
+ #{dataMap.http_req_length},
+ #{dataMap.http_req_content_type},
+ #{dataMap.tc_scan_port},
+ #{dataMap.tc_labels},
+ #{dataMap.http_resp_content_type},
+ #{dataMap.dns_msg_type},
+ #{dataMap.dns_answer_length},
+ #{dataMap.dns_ioc},
+ #{dataMap.tx_bytes},
+ #{dataMap.rx_bytes},
+ #{dataMap.all_bytes},
+ #{dataMap.duration_time},
+ #{dataMap.mail_attach_name},
+ #{dataMap.mail_subject},
+ #{dataMap.mail_message},
+ #{dataMap.mail_send_server},
+ #{dataMap.mail_agent},
+ #{dataMap.tls_version},
+ #{dataMap.tls_server_cert},
+ #{dataMap.tls_server_suite},
+ #{dataMap.tls_client_suites_len},
+ #{dataMap.tls_ja3},
+ #{dataMap.tls_ja3s},
+ #{dataMap.vpn_access_port},
+ #{dataMap.log_topic},
+ #{dataMap.collect_time},
+ #{dataMap.src_is_intranetip},
+ #{dataMap.src_ip_ioc},
+ #{dataMap.src_ip_apt},
+ #{dataMap.srcip_name},
+ #{dataMap.tc_client},
+ #{dataMap.srcip_organization_id},
+ #{dataMap.dest_ip_intranetip},
+ #{dataMap.dest_ip_ioc},
+ #{dataMap.desip_id},
+ #{dataMap.desip_name},
+ #{dataMap.tc_hostip},
+ #{dataMap.desip_organization_id},
+ #{dataMap.origin_confidence},
+ #{dataMap.origin_malscore},
+ #{dataMap.attacker_icampaign},
+ #{dataMap.attacker_host_asset_id},
+ #{dataMap.attacker_organization_id},
+ #{dataMap.victim_host_asset_id},
+ #{dataMap.victim_organization_id},
+ #{dataMap.logout_time},
+ #{dataMap.http_req_line},
+ #{dataMap.desip_security_scope_id},
+ #{dataMap.srcip_security_scope_id},
+ #{dataMap.http_resp_length},
+ #{dataMap.tc_attack_type},
+ #{dataMap.tc_realip},
+ #{dataMap.attacker_ip_lists},
+ #{dataMap.login_password},
+ #{dataMap.detail},
+ #{dataMap.attacker_country_code},
+ #{dataMap.attacker_region_code},
+ #{dataMap.victim_region_code},
+ #{dataMap.payload},
+ #{dataMap.http_referer},
+ #{dataMap.http_user_agent},
+ #{dataMap.http_session},
+ #{dataMap.http_query_string},
+ #{dataMap.file_path},
+ #{dataMap.file_permission},
+ #{dataMap.login_abnormal_type},
+ #{dataMap.file_tag},
+ #{dataMap.file_platform},
+ #{dataMap.target_ip},
+ #{dataMap.collect_date},
+ #{dataMap.tc_client_ip},
+ #{dataMap.tc_server_ip},
+ #{dataMap.tc_externalip},
+ #{dataMap.http_status_code},
+ #{dataMap.device_domian},
+ #{dataMap.src_ip_str},
+ #{dataMap.src_port_str},
+ #{dataMap.dest_ip_str} ,
+ CAST(#{dataMap.dest_port_str} AS text),
+ #{dataMap.pcap},
+ #{dataMap.ioc},
+ #{dataMap.malicious_family},
+ #{dataMap.vuln_cve},
+ #{dataMap.aliyun_type},
+ #{dataMap.attacker_host_asset_name},
+ #{dataMap.attacker_organization_name},
+ #{dataMap.ct_id},
+ #{dataMap.cve_list},
+ #{dataMap.desip_organization_name},
+ #{dataMap.dest_ip_group},
+ #{dataMap.file_gid},
+ #{dataMap.file_owner},
+ #{dataMap.file_ownergroup},
+ #{dataMap.file_uid},
+ #{dataMap.http_resp_cookie},
+ #{dataMap.origin_rule_id},
+ #{dataMap.origin_rule_name},
+ #{dataMap.service_name},
+ #{dataMap.src_ip_asset_group},
+ #{dataMap.srcip_organization_name},
+ #{dataMap.victim_host_asset_name},
+ #{dataMap.http_resp_codes}::bigint,
+ #{dataMap.victim_organization_name},
+ #{dataMap.tc_type},
+ #{dataMap.direction},
+ #{dataMap.http_req_cookie},
+ #{dataMap.http_req_protocol},
+ #{dataMap.http_req_header_raw},
+ #{dataMap.http_url},
+ #{dataMap.uname},
+ #{dataMap.origin_hostname},
+ #{dataMap.origin_os},
+ #{dataMap.origin_agent_mac},
+ #{dataMap.origin_host_id},
+ #{dataMap.origin_agent_version},
+ #{dataMap.origin_agent_id},
+ #{dataMap.origin_agent_name},
+ #{dataMap.origin_work_group},
+ #{dataMap.origin_asset_group},
+ #{dataMap.origin_local_port},
+ #{dataMap.origin_agent_ip},
+ #{dataMap.origin_internal_ip},
+ #{dataMap.origin_external_ip},
+ #{dataMap.origin_local_addr},
+ #{dataMap.agent_id},
+ #{dataMap.agent_name},
+ #{dataMap.tc_title},
+ #{dataMap.log_id},
+ #{dataMap.event_date},
+ #{dataMap.event_time_ts},
+ #{dataMap.event_level},
+ #{dataMap.src_ip},
+ #{dataMap.src_port},
+ #{dataMap.dest_ip},
+ #{dataMap.dest_port}::BIGINT,
+ #{dataMap.event_time},
+ #{dataMap.attacker_country},
+ #{dataMap.src_mac},
+ #{dataMap.dest_mac},
+ #{dataMap.proto},
+ #{dataMap.dev_id},
+ #{dataMap.created_time},
+ #{dataMap.src_country},
+ #{dataMap.src_country_code},
+ #{dataMap.src_region},
+ #{dataMap.src_region_code},
+ #{dataMap.src_city},
+ #{dataMap.src_lon},
+ #{dataMap.http_method},
+ #{dataMap.http_host},
+ #{dataMap.http_req_header},
+ #{dataMap.http_req_body},
+ #{dataMap.http_resp_header},
+ #{dataMap.http_resp_body},
+ #{dataMap.file_type},
+ #{dataMap.file_md5},
+ #{dataMap.file_size},
+ #{dataMap.process},
+ #{dataMap.start_time},
+ #{dataMap.action},
+ #{dataMap.attacker_region},
+ #{dataMap.end_time},
+ #{dataMap.file_created_time},
+ #{dataMap.file_modified_time},
+ #{dataMap.tc_miguan_scan_port},
+ #{dataMap.process_path},
+ #{dataMap.parent_process_path},
+ #{dataMap.gname},
+ #{dataMap.exe_name},
+ #{dataMap.exe_path},
+ #{dataMap.login_time},
+ #{dataMap.login_times},
+ #{dataMap.check_item},
+ #{dataMap.check_type},
+ #{dataMap.attacker_ip},
+ #{dataMap.attacker_port},
+ #{dataMap.victim_ip},
+ #{dataMap.victim_port},
+ #{dataMap.attacker_city},
+ #{dataMap.attacker_lon},
+ #{dataMap.attacker_lat},
+ #{dataMap.victim_country},
+ #{dataMap.victim_region},
+ #{dataMap.victim_city},
+ #{dataMap.victim_lon},
+ #{dataMap.victim_lat},
+ #{dataMap.origin_event_id},
+ #{dataMap.origin_event_name},
+ #{dataMap.origin_event_category},
+ #{dataMap.origin_event_level},
+ #{dataMap.origin_attack_chain},
+ #{dataMap.engine_type},
+ #{dataMap.evil_payload},
+ #{dataMap.http_resp_status},
+ #{dataMap.dns_query},
+ #{dataMap.dns_query_type},
+ #{dataMap.dns_ttl},
+ #{dataMap.dns_answer},
+ #{dataMap.dns_subdomains},
+ #{dataMap.file_sha256},
+ #{dataMap.file_ssdeep},
+ #{dataMap.victim_country_code},
+ #{dataMap.http_xff_ip},
+ #{dataMap.tc_miguan_class},
+ #{dataMap.pid},
+ #{dataMap.ppid},
+ #{dataMap.process_name},
+ #{dataMap.backdoor_type},
+ #{dataMap.tty},
+ #{dataMap.sudo_user},
+ #{dataMap.sudo_group},
+ #{dataMap.origin_event_type},
+ #{dataMap.dest_domain},
+ #{dataMap.shell_cmdline},
+ #{dataMap.parent_cmdline},
+ #{dataMap.attack_chain},
+ #{dataMap.process_tree},
+ #{dataMap.host_file_sha256},
+ #{dataMap.host_file_md5},
+ #{dataMap.host_file_size},
+ #{dataMap.host_file_type},
+ #{dataMap.dest_country},
+ #{dataMap.dest_country_code},
+ #{dataMap.log_origin},
+ #{dataMap.dest_region},
+ #{dataMap.src_lat},
+ #{dataMap.dest_region_code},
+ #{dataMap.dest_city},
+ #{dataMap.dest_lon},
+ #{dataMap.dest_lat},
+ #{dataMap.event_category},
+ #{dataMap.attack_result},
+ #{dataMap.probe_ip},
+ #{dataMap.device_ip},
+ #{dataMap.device_manufacturer},
+ #{dataMap.device_name},
+ #{dataMap.product_name},
+ #{dataMap.__id},
+ #{dataMap.__count},
+ #{dataMap.__count_reason},
+ #{dataMap.event_type},
+ #{dataMap.protocol},
+ #{dataMap.shell_cmd},
+ #{dataMap.parent_name},
+ #{dataMap.host_file_path},
+ #{dataMap.uid},
+ #{dataMap.fall},
+ #{dataMap.tc_miguan_server_ip},
+ #{dataMap.dev_type},
+ #{dataMap.collect_method},
+ #{dataMap.field_cate_id},
+ #{dataMap.device_type},
+ #{dataMap.tc_miguan_client_ip},
+ #{dataMap.tc_miguan_name},
+ #{dataMap.origin_total_packages},
+ #{dataMap.origin_total_bytes},
+ #{dataMap.origin_peak_packages_rate},
+ #{dataMap.origin_peak_bytes_rate},
+ #{dataMap.origin_peak_flows_rate},
+ #{dataMap.apt_orgname},
+ #{dataMap.apt_orgmsg},
+ #{dataMap.mail_message_id},
+ #{dataMap.mail_bcc},
+ #{dataMap.mail_size},
+ #{dataMap.mail_attach_hashcode},
+ #{dataMap.mail_url},
+ #{dataMap.mail_cc},
+ #{dataMap.algorithm},
+ #{dataMap.miningpool_ip},
+ #{dataMap.process_md5},
+ #{dataMap.pprocess_md5},
+ #{dataMap.source_servername},
+ #{dataMap.origin_source_servername},
+ #{dataMap.mail_filename},
+ #{dataMap.dst_upload_appname},
+ #{dataMap.target_port},
+ #{dataMap.gid},
+ #{dataMap.origin_uid},
+ #{dataMap.origin_gid},
+ #{dataMap.target_ports},
+ #{dataMap.tc_miguan_name1},
+ #{dataMap.tc_miguan_class1},
+ #{dataMap.etl_time},
+ #{dataMap.tc_miguan_scan_port2},
+ #{dataMap.desip_security_scope},
+ #{dataMap.srcip_security_scope},
+ #{dataMap.collect_time_ts},
+ #{dataMap.tc_miguan_scan_port1},
+ #{dataMap.src_dev_name},
+ #{dataMap.collect_protocol},
+ #{dataMap.destination_system_type},
+ #{dataMap.destination_system},
+ #{dataMap.etl_host},
+ #{dataMap.normalize_rule_id},
+ #{dataMap.normalize_rule_name},
+ #{dataMap.syslog_uuid},
+ #{dataMap.syslog_topic},
+ #{dataMap.rule_version},
@@ -776,14 +779,4 @@
-
-
- INSERT INTO syslog_normal_alarm
- (id, log_time, src_ip, dest_ip, event_level)
- VALUES
-
- (#{item.id}, #{item.log_time}, #{item.src_ip}, #{item.dest_ip}, #{item.event_level})
-
-
-
diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml
index 89306b3..45ae9a0 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml
+++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml
@@ -141,756 +141,759 @@
-
+
+
INSERT INTO syslog_normal_data
- id,
- created_at,
- log_time,
- device_id,
- webshell_type,
- vuirs_type,
- vuirs_url,
- class_filename,
- class_path,
- parent_class,
- jar_path,
- class_md5,
- class_loader,
- class_hashcode,
- class_loader_hashcode,
- tc_nameip,
- perform_sql,
- tc_account,
- tc_appname,
- process_uname,
- p_process_uname,
- container_name,
- container_id,
- http_resp_server,
- srcip_id,
- cdnip,
- natip,
- mail_sender,
- mail_receiver,
- vpn_mac,
- vpn_os,
- vpn_user,
- vpn_groupname,
- vpn_access_ip,
- dest_ip_apt,
- origin_attack_result,
- description,
- solution,
- attack_cause,
- username,
- tc_flow_id,
- login_result,
- cmdline,
- origin_attack_action,
- victim_domain,
- vpn_deviceid,
- vpn_access_action,
- file_access_time,
- file_name,
- tc_class,
- tc_name2,
- login_lasttime,
- origin_permissions,
- begin_permissions,
- print_time,
- printer,
- printer_type,
- print_pages,
- print_copies,
- src_device,
- dst_device,
- src_file,
- src_file_type,
- src_file_path,
- dst_file,
- dst_file_type,
- dst_file_path,
- dlp_policy_name,
- dlp_policy_type,
- dst_upload_url,
- process_uuid,
- p_process_uuid,
- env,
- brute_force_service,
- vuirs_name,
- http_req_length,
- http_req_content_type,
- tc_scan_port,
- tc_labels,
- http_resp_content_type,
- dns_msg_type,
- dns_answer_length,
- dns_ioc,
- tx_bytes,
- rx_bytes,
- all_bytes,
- duration_time,
- mail_attach_name,
- mail_subject,
- mail_message,
- mail_send_server,
- mail_agent,
- tls_version,
- tls_server_cert,
- tls_server_suite,
- tls_client_suites_len,
- tls_ja3,
- tls_ja3s,
- vpn_access_port,
- log_topic,
- collect_time,
- src_is_intranetip,
- src_ip_ioc,
- src_ip_apt,
- srcip_name,
- tc_client,
- srcip_organization_id,
- dest_ip_intranetip,
- dest_ip_ioc,
- desip_id,
- desip_name,
- tc_hostip,
- desip_organization_id,
- origin_confidence,
- origin_malscore,
- attacker_icampaign,
- attacker_host_asset_id,
- attacker_organization_id,
- victim_host_asset_id,
- victim_organization_id,
- logout_time,
- http_req_line,
- desip_security_scope_id,
- srcip_security_scope_id,
- http_resp_length,
- tc_attack_type,
- tc_realip,
- attacker_ip_lists,
- login_password,
- detail,
- attacker_country_code,
- attacker_region_code,
- victim_region_code,
- payload,
- http_referer,
- http_user_agent,
- http_session,
- http_query_string,
- file_path,
- file_permission,
- login_abnormal_type,
- file_tag,
- file_platform,
- target_ip,
- collect_date,
- tc_client_ip,
- tc_server_ip,
- tc_externalip,
- http_status_code,
- device_domian,
- src_ip_str,
- src_port_str,
- dest_ip_str,
- dest_port_str,
- pcap,
- ioc,
- malicious_family,
- vuln_cve,
- aliyun_type,
- attacker_host_asset_name,
- attacker_organization_name,
- ct_id,
- cve_list,
- desip_organization_name,
- dest_ip_group,
- file_gid,
- file_owner,
- file_ownergroup,
- file_uid,
- http_resp_cookie,
- origin_rule_id,
- origin_rule_name,
- service_name,
- src_ip_asset_group,
- srcip_organization_name,
- victim_host_asset_name,
- http_resp_codes,
- victim_organization_name,
- tc_type,
- direction,
- http_req_cookie,
- http_req_protocol,
- http_req_header_raw,
- http_url,
- uname,
- origin_hostname,
- origin_os,
- origin_agent_mac,
- origin_host_id,
- origin_agent_version,
- origin_agent_id,
- origin_agent_name,
- origin_work_group,
- origin_asset_group,
- origin_local_port,
- origin_agent_ip,
- origin_internal_ip,
- origin_external_ip,
- origin_local_addr,
- agent_id,
- agent_name,
- tc_title,
- log_id,
- event_date,
- event_time_ts,
- event_level,
- src_ip ,
- src_port,
- dest_ip,
- dest_port,
- event_time,
- attacker_country,
- src_mac,
- dest_mac,
- proto,
- dev_id,
- created_time,
- src_country,
- src_country_code,
- src_region,
- src_region_code,
- src_city,
- src_lon,
- http_method,
- http_host,
- http_req_header,
- http_req_body,
- http_resp_header,
- http_resp_body,
- file_type,
- file_md5,
- file_size,
- process,
- start_time,
- action,
- attacker_region,
- end_time,
- file_created_time,
- file_modified_time,
- tc_miguan_scan_port,
- process_path,
- parent_process_path,
- gname,
- exe_name,
- exe_path,
- login_time,
- login_times,
- check_item,
- check_type,
- attacker_ip,
- attacker_port,
- victim_ip,
- victim_port,
- attacker_city,
- attacker_lon,
- attacker_lat,
- victim_country,
- victim_region,
- victim_city,
- victim_lon,
- victim_lat,
- origin_event_id,
- origin_event_name,
- origin_event_category,
- origin_event_level,
- origin_attack_chain,
- engine_type,
- evil_payload,
- http_resp_status,
- dns_query,
- dns_query_type,
- dns_ttl,
- dns_answer,
- dns_subdomains,
- file_sha256,
- file_ssdeep,
- victim_country_code,
- http_xff_ip,
- tc_miguan_class,
- pid,
- ppid,
- process_name,
- backdoor_type,
- tty,
- sudo_user,
- sudo_group,
- origin_event_type,
- dest_domain,
- shell_cmdline,
- parent_cmdline,
- attack_chain,
- process_tree,
- host_file_sha256,
- host_file_md5,
- host_file_size,
- host_file_type,
- dest_country,
- dest_country_code,
- log_origin,
- dest_region,
- src_lat,
- dest_region_code,
- dest_city,
- dest_lon,
- dest_lat,
- event_category,
- attack_result,
- probe_ip,
- device_ip,
- device_manufacturer,
- device_name,
- product_name,
- __id,
- __count,
- __count_reason,
- event_type,
- protocol,
- shell_cmd,
- parent_name,
- host_file_path,
- uid,
- fall,
- tc_miguan_server_ip,
- dev_type,
- collect_method,
- field_cate_id,
- device_type,
- tc_miguan_client_ip,
- tc_miguan_name,
- origin_total_packages,
- origin_total_bytes,
- origin_peak_packages_rate,
- origin_peak_bytes_rate,
- origin_peak_flows_rate,
- apt_orgname,
- apt_orgmsg,
- mail_message_id,
- mail_bcc,
- mail_size,
- mail_attach_hashcode,
- mail_url,
- mail_cc,
- algorithm,
- miningpool_ip,
- process_md5,
- pprocess_md5,
- source_servername,
- origin_source_servername,
- mail_filename,
- dst_upload_appname,
- target_port,
- gid,
- origin_uid,
- origin_gid,
- target_ports,
- tc_miguan_name1,
- tc_miguan_class1,
- etl_time,
- tc_miguan_scan_port2,
- desip_security_scope,
- srcip_security_scope,
- collect_time_ts,
- tc_miguan_scan_port1,
- src_dev_name,
- collect_protocol,
- destination_system_type,
- destination_system,
- etl_host,
- normalize_rule_id,
- normalize_rule_name,
- syslog_uuid,
- syslog_topic,
+ id,
+ created_at,
+ log_time,
+ device_id,
+ webshell_type,
+ vuirs_type,
+ vuirs_url,
+ class_filename,
+ class_path,
+ parent_class,
+ jar_path,
+ class_md5,
+ class_loader,
+ class_hashcode,
+ class_loader_hashcode,
+ tc_nameip,
+ perform_sql,
+ tc_account,
+ tc_appname,
+ process_uname,
+ p_process_uname,
+ container_name,
+ container_id,
+ http_resp_server,
+ srcip_id,
+ cdnip,
+ natip,
+ mail_sender,
+ mail_receiver,
+ vpn_mac,
+ vpn_os,
+ vpn_user,
+ vpn_groupname,
+ vpn_access_ip,
+ dest_ip_apt,
+ origin_attack_result,
+ description,
+ solution,
+ attack_cause,
+ username,
+ tc_flow_id,
+ login_result,
+ cmdline,
+ origin_attack_action,
+ victim_domain,
+ vpn_deviceid,
+ vpn_access_action,
+ file_access_time,
+ file_name,
+ tc_class,
+ tc_name2,
+ login_lasttime,
+ origin_permissions,
+ begin_permissions,
+ print_time,
+ printer,
+ printer_type,
+ print_pages,
+ print_copies,
+ src_device,
+ dst_device,
+ src_file,
+ src_file_type,
+ src_file_path,
+ dst_file,
+ dst_file_type,
+ dst_file_path,
+ dlp_policy_name,
+ dlp_policy_type,
+ dst_upload_url,
+ process_uuid,
+ p_process_uuid,
+ env,
+ brute_force_service,
+ vuirs_name,
+ http_req_length,
+ http_req_content_type,
+ tc_scan_port,
+ tc_labels,
+ http_resp_content_type,
+ dns_msg_type,
+ dns_answer_length,
+ dns_ioc,
+ tx_bytes,
+ rx_bytes,
+ all_bytes,
+ duration_time,
+ mail_attach_name,
+ mail_subject,
+ mail_message,
+ mail_send_server,
+ mail_agent,
+ tls_version,
+ tls_server_cert,
+ tls_server_suite,
+ tls_client_suites_len,
+ tls_ja3,
+ tls_ja3s,
+ vpn_access_port,
+ log_topic,
+ collect_time,
+ src_is_intranetip,
+ src_ip_ioc,
+ src_ip_apt,
+ srcip_name,
+ tc_client,
+ srcip_organization_id,
+ dest_ip_intranetip,
+ dest_ip_ioc,
+ desip_id,
+ desip_name,
+ tc_hostip,
+ desip_organization_id,
+ origin_confidence,
+ origin_malscore,
+ attacker_icampaign,
+ attacker_host_asset_id,
+ attacker_organization_id,
+ victim_host_asset_id,
+ victim_organization_id,
+ logout_time,
+ http_req_line,
+ desip_security_scope_id,
+ srcip_security_scope_id,
+ http_resp_length,
+ tc_attack_type,
+ tc_realip,
+ attacker_ip_lists,
+ login_password,
+ detail,
+ attacker_country_code,
+ attacker_region_code,
+ victim_region_code,
+ payload,
+ http_referer,
+ http_user_agent,
+ http_session,
+ http_query_string,
+ file_path,
+ file_permission,
+ login_abnormal_type,
+ file_tag,
+ file_platform,
+ target_ip,
+ collect_date,
+ tc_client_ip,
+ tc_server_ip,
+ tc_externalip,
+ http_status_code,
+ device_domian,
+ src_ip_str,
+ src_port_str,
+ dest_ip_str,
+ dest_port_str,
+ pcap,
+ ioc,
+ malicious_family,
+ vuln_cve,
+ aliyun_type,
+ attacker_host_asset_name,
+ attacker_organization_name,
+ ct_id,
+ cve_list,
+ desip_organization_name,
+ dest_ip_group,
+ file_gid,
+ file_owner,
+ file_ownergroup,
+ file_uid,
+ http_resp_cookie,
+ origin_rule_id,
+ origin_rule_name,
+ service_name,
+ src_ip_asset_group,
+ srcip_organization_name,
+ victim_host_asset_name,
+ http_resp_codes,
+ victim_organization_name,
+ tc_type,
+ direction,
+ http_req_cookie,
+ http_req_protocol,
+ http_req_header_raw,
+ http_url,
+ uname,
+ origin_hostname,
+ origin_os,
+ origin_agent_mac,
+ origin_host_id,
+ origin_agent_version,
+ origin_agent_id,
+ origin_agent_name,
+ origin_work_group,
+ origin_asset_group,
+ origin_local_port,
+ origin_agent_ip,
+ origin_internal_ip,
+ origin_external_ip,
+ origin_local_addr,
+ agent_id,
+ agent_name,
+ tc_title,
+ log_id,
+ event_date,
+ event_time_ts,
+ event_level,
+ src_ip ,
+ src_port,
+ dest_ip,
+ dest_port,
+ event_time,
+ attacker_country,
+ src_mac,
+ dest_mac,
+ proto,
+ dev_id,
+ created_time,
+ src_country,
+ src_country_code,
+ src_region,
+ src_region_code,
+ src_city,
+ src_lon,
+ http_method,
+ http_host,
+ http_req_header,
+ http_req_body,
+ http_resp_header,
+ http_resp_body,
+ file_type,
+ file_md5,
+ file_size,
+ process,
+ start_time,
+ action,
+ attacker_region,
+ end_time,
+ file_created_time,
+ file_modified_time,
+ tc_miguan_scan_port,
+ process_path,
+ parent_process_path,
+ gname,
+ exe_name,
+ exe_path,
+ login_time,
+ login_times,
+ check_item,
+ check_type,
+ attacker_ip,
+ attacker_port,
+ victim_ip,
+ victim_port,
+ attacker_city,
+ attacker_lon,
+ attacker_lat,
+ victim_country,
+ victim_region,
+ victim_city,
+ victim_lon,
+ victim_lat,
+ origin_event_id,
+ origin_event_name,
+ origin_event_category,
+ origin_event_level,
+ origin_attack_chain,
+ engine_type,
+ evil_payload,
+ http_resp_status,
+ dns_query,
+ dns_query_type,
+ dns_ttl,
+ dns_answer,
+ dns_subdomains,
+ file_sha256,
+ file_ssdeep,
+ victim_country_code,
+ http_xff_ip,
+ tc_miguan_class,
+ pid,
+ ppid,
+ process_name,
+ backdoor_type,
+ tty,
+ sudo_user,
+ sudo_group,
+ origin_event_type,
+ dest_domain,
+ shell_cmdline,
+ parent_cmdline,
+ attack_chain,
+ process_tree,
+ host_file_sha256,
+ host_file_md5,
+ host_file_size,
+ host_file_type,
+ dest_country,
+ dest_country_code,
+ log_origin,
+ dest_region,
+ src_lat,
+ dest_region_code,
+ dest_city,
+ dest_lon,
+ dest_lat,
+ event_category,
+ attack_result,
+ probe_ip,
+ device_ip,
+ device_manufacturer,
+ device_name,
+ product_name,
+ __id,
+ __count,
+ __count_reason,
+ event_type,
+ protocol,
+ shell_cmd,
+ parent_name,
+ host_file_path,
+ uid,
+ fall,
+ tc_miguan_server_ip,
+ dev_type,
+ collect_method,
+ field_cate_id,
+ device_type,
+ tc_miguan_client_ip,
+ tc_miguan_name,
+ origin_total_packages,
+ origin_total_bytes,
+ origin_peak_packages_rate,
+ origin_peak_bytes_rate,
+ origin_peak_flows_rate,
+ apt_orgname,
+ apt_orgmsg,
+ mail_message_id,
+ mail_bcc,
+ mail_size,
+ mail_attach_hashcode,
+ mail_url,
+ mail_cc,
+ algorithm,
+ miningpool_ip,
+ process_md5,
+ pprocess_md5,
+ source_servername,
+ origin_source_servername,
+ mail_filename,
+ dst_upload_appname,
+ target_port,
+ gid,
+ origin_uid,
+ origin_gid,
+ target_ports,
+ tc_miguan_name1,
+ tc_miguan_class1,
+ etl_time,
+ tc_miguan_scan_port2,
+ desip_security_scope,
+ srcip_security_scope,
+ collect_time_ts,
+ tc_miguan_scan_port1,
+ src_dev_name,
+ collect_protocol,
+ destination_system_type,
+ destination_system,
+ etl_host,
+ normalize_rule_id,
+ normalize_rule_name,
+ syslog_uuid,
+ syslog_topic,
+ rule_version,
VALUES
- #{dataMap.id},
- #{dataMap.created_at},
- #{dataMap.log_time},
- #{dataMap.device_id},
- #{dataMap.webshell_type},
- #{dataMap.vuirs_type},
- #{dataMap.vuirs_url},
- #{dataMap.class_filename},
- #{dataMap.class_path},
- #{dataMap.parent_class},
- #{dataMap.jar_path},
- #{dataMap.class_md5},
- #{dataMap.class_loader},
- #{dataMap.class_hashcode},
- #{dataMap.class_loader_hashcode},
- #{dataMap.tc_nameip},
- #{dataMap.perform_sql},
- #{dataMap.tc_account},
- #{dataMap.tc_appname},
- #{dataMap.process_uname},
- #{dataMap.p_process_uname},
- #{dataMap.container_name},
- #{dataMap.container_id},
- #{dataMap.http_resp_server},
- #{dataMap.srcip_id},
- #{dataMap.cdnip},
- #{dataMap.natip},
- #{dataMap.mail_sender},
- #{dataMap.mail_receiver},
- #{dataMap.vpn_mac},
- #{dataMap.vpn_os},
- #{dataMap.vpn_user},
- #{dataMap.vpn_groupname},
- #{dataMap.vpn_access_ip},
- #{dataMap.dest_ip_apt},
- #{dataMap.origin_attack_result},
- #{dataMap.description},
- #{dataMap.solution},
- #{dataMap.attack_cause},
- #{dataMap.username},
- #{dataMap.tc_flow_id},
- #{dataMap.login_result},
- #{dataMap.cmdline},
- #{dataMap.origin_attack_action},
- #{dataMap.victim_domain},
- #{dataMap.vpn_deviceid},
- #{dataMap.vpn_access_action},
- #{dataMap.file_access_time},
- #{dataMap.file_name},
- #{dataMap.tc_class},
- #{dataMap.tc_name2},
- #{dataMap.login_lasttime},
- #{dataMap.origin_permissions},
- #{dataMap.begin_permissions},
- #{dataMap.print_time},
- #{dataMap.printer},
- #{dataMap.printer_type},
- #{dataMap.print_pages},
- #{dataMap.print_copies},
- #{dataMap.src_device},
- #{dataMap.dst_device},
- #{dataMap.src_file},
- #{dataMap.src_file_type},
- #{dataMap.src_file_path},
- #{dataMap.dst_file},
- #{dataMap.dst_file_type},
- #{dataMap.dst_file_path},
- #{dataMap.dlp_policy_name},
- #{dataMap.dlp_policy_type},
- #{dataMap.dst_upload_url},
- #{dataMap.process_uuid},
- #{dataMap.p_process_uuid},
- #{dataMap.env},
- #{dataMap.brute_force_service},
- #{dataMap.vuirs_name},
- #{dataMap.http_req_length},
- #{dataMap.http_req_content_type},
- #{dataMap.tc_scan_port},
- #{dataMap.tc_labels},
- #{dataMap.http_resp_content_type},
- #{dataMap.dns_msg_type},
- #{dataMap.dns_answer_length},
- #{dataMap.dns_ioc},
- #{dataMap.tx_bytes},
- #{dataMap.rx_bytes},
- #{dataMap.all_bytes},
- #{dataMap.duration_time},
- #{dataMap.mail_attach_name},
- #{dataMap.mail_subject},
- #{dataMap.mail_message},
- #{dataMap.mail_send_server},
- #{dataMap.mail_agent},
- #{dataMap.tls_version},
- #{dataMap.tls_server_cert},
- #{dataMap.tls_server_suite},
- #{dataMap.tls_client_suites_len},
- #{dataMap.tls_ja3},
- #{dataMap.tls_ja3s},
- #{dataMap.vpn_access_port},
- #{dataMap.log_topic},
- #{dataMap.collect_time},
- #{dataMap.src_is_intranetip},
- #{dataMap.src_ip_ioc},
- #{dataMap.src_ip_apt},
- #{dataMap.srcip_name},
- #{dataMap.tc_client},
- #{dataMap.srcip_organization_id},
- #{dataMap.dest_ip_intranetip},
- #{dataMap.dest_ip_ioc},
- #{dataMap.desip_id},
- #{dataMap.desip_name},
- #{dataMap.tc_hostip},
- #{dataMap.desip_organization_id},
- #{dataMap.origin_confidence},
- #{dataMap.origin_malscore},
- #{dataMap.attacker_icampaign},
- #{dataMap.attacker_host_asset_id},
- #{dataMap.attacker_organization_id},
- #{dataMap.victim_host_asset_id},
- #{dataMap.victim_organization_id},
- #{dataMap.logout_time},
- #{dataMap.http_req_line},
- #{dataMap.desip_security_scope_id},
- #{dataMap.srcip_security_scope_id},
- #{dataMap.http_resp_length},
- #{dataMap.tc_attack_type},
- #{dataMap.tc_realip},
- #{dataMap.attacker_ip_lists},
- #{dataMap.login_password},
- #{dataMap.detail},
- #{dataMap.attacker_country_code},
- #{dataMap.attacker_region_code},
- #{dataMap.victim_region_code},
- #{dataMap.payload},
- #{dataMap.http_referer},
- #{dataMap.http_user_agent},
- #{dataMap.http_session},
- #{dataMap.http_query_string},
- #{dataMap.file_path},
- #{dataMap.file_permission},
- #{dataMap.login_abnormal_type},
- #{dataMap.file_tag},
- #{dataMap.file_platform},
- #{dataMap.target_ip},
- #{dataMap.collect_date},
- #{dataMap.tc_client_ip},
- #{dataMap.tc_server_ip},
- #{dataMap.tc_externalip},
- #{dataMap.http_status_code},
- #{dataMap.device_domian},
- #{dataMap.src_ip_str},
- #{dataMap.src_port_str},
- #{dataMap.dest_ip_str} ,
- CAST(#{dataMap.dest_port_str} AS text),
- #{dataMap.pcap},
- #{dataMap.ioc},
- #{dataMap.malicious_family},
- #{dataMap.vuln_cve},
- #{dataMap.aliyun_type},
- #{dataMap.attacker_host_asset_name},
- #{dataMap.attacker_organization_name},
- #{dataMap.ct_id},
- #{dataMap.cve_list},
- #{dataMap.desip_organization_name},
- #{dataMap.dest_ip_group},
- #{dataMap.file_gid},
- #{dataMap.file_owner},
- #{dataMap.file_ownergroup},
- #{dataMap.file_uid},
- #{dataMap.http_resp_cookie},
- #{dataMap.origin_rule_id},
- #{dataMap.origin_rule_name},
- #{dataMap.service_name},
- #{dataMap.src_ip_asset_group},
- #{dataMap.srcip_organization_name},
- #{dataMap.victim_host_asset_name},
- #{dataMap.http_resp_codes}::bigint,
- #{dataMap.victim_organization_name},
- #{dataMap.tc_type},
- #{dataMap.direction},
- #{dataMap.http_req_cookie},
- #{dataMap.http_req_protocol},
- #{dataMap.http_req_header_raw},
- #{dataMap.http_url},
- #{dataMap.uname},
- #{dataMap.origin_hostname},
- #{dataMap.origin_os},
- #{dataMap.origin_agent_mac},
- #{dataMap.origin_host_id},
- #{dataMap.origin_agent_version},
- #{dataMap.origin_agent_id},
- #{dataMap.origin_agent_name},
- #{dataMap.origin_work_group},
- #{dataMap.origin_asset_group},
- #{dataMap.origin_local_port},
- #{dataMap.origin_agent_ip},
- #{dataMap.origin_internal_ip},
- #{dataMap.origin_external_ip},
- #{dataMap.origin_local_addr},
- #{dataMap.agent_id},
- #{dataMap.agent_name},
- #{dataMap.tc_title},
- #{dataMap.log_id},
- #{dataMap.event_date},
- #{dataMap.event_time_ts},
- #{dataMap.event_level},
- #{dataMap.src_ip},
- #{dataMap.src_port},
- #{dataMap.dest_ip},
- #{dataMap.dest_port}::BIGINT,
- #{dataMap.event_time},
- #{dataMap.attacker_country},
- #{dataMap.src_mac},
- #{dataMap.dest_mac},
- #{dataMap.proto},
- #{dataMap.dev_id},
- #{dataMap.created_time},
- #{dataMap.src_country},
- #{dataMap.src_country_code},
- #{dataMap.src_region},
- #{dataMap.src_region_code},
- #{dataMap.src_city},
- #{dataMap.src_lon},
- #{dataMap.http_method},
- #{dataMap.http_host},
- #{dataMap.http_req_header},
- #{dataMap.http_req_body},
- #{dataMap.http_resp_header},
- #{dataMap.http_resp_body},
- #{dataMap.file_type},
- #{dataMap.file_md5},
- #{dataMap.file_size},
- #{dataMap.process},
- #{dataMap.start_time},
- #{dataMap.action},
- #{dataMap.attacker_region},
- #{dataMap.end_time},
- #{dataMap.file_created_time},
- #{dataMap.file_modified_time},
- #{dataMap.tc_miguan_scan_port},
- #{dataMap.process_path},
- #{dataMap.parent_process_path},
- #{dataMap.gname},
- #{dataMap.exe_name},
- #{dataMap.exe_path},
- #{dataMap.login_time},
- #{dataMap.login_times},
- #{dataMap.check_item},
- #{dataMap.check_type},
- #{dataMap.attacker_ip},
- #{dataMap.attacker_port},
- #{dataMap.victim_ip},
- #{dataMap.victim_port},
- #{dataMap.attacker_city},
- #{dataMap.attacker_lon},
- #{dataMap.attacker_lat},
- #{dataMap.victim_country},
- #{dataMap.victim_region},
- #{dataMap.victim_city},
- #{dataMap.victim_lon},
- #{dataMap.victim_lat},
- #{dataMap.origin_event_id},
- #{dataMap.origin_event_name},
- #{dataMap.origin_event_category},
- #{dataMap.origin_event_level},
- #{dataMap.origin_attack_chain},
- #{dataMap.engine_type},
- #{dataMap.evil_payload},
- #{dataMap.http_resp_status},
- #{dataMap.dns_query},
- #{dataMap.dns_query_type},
- #{dataMap.dns_ttl},
- #{dataMap.dns_answer},
- #{dataMap.dns_subdomains},
- #{dataMap.file_sha256},
- #{dataMap.file_ssdeep},
- #{dataMap.victim_country_code},
- #{dataMap.http_xff_ip},
- #{dataMap.tc_miguan_class},
- #{dataMap.pid},
- #{dataMap.ppid},
- #{dataMap.process_name},
- #{dataMap.backdoor_type},
- #{dataMap.tty},
- #{dataMap.sudo_user},
- #{dataMap.sudo_group},
- #{dataMap.origin_event_type},
- #{dataMap.dest_domain},
- #{dataMap.shell_cmdline},
- #{dataMap.parent_cmdline},
- #{dataMap.attack_chain},
- #{dataMap.process_tree},
- #{dataMap.host_file_sha256},
- #{dataMap.host_file_md5},
- #{dataMap.host_file_size},
- #{dataMap.host_file_type},
- #{dataMap.dest_country},
- #{dataMap.dest_country_code},
- #{dataMap.log_origin},
- #{dataMap.dest_region},
- #{dataMap.src_lat},
- #{dataMap.dest_region_code},
- #{dataMap.dest_city},
- #{dataMap.dest_lon},
- #{dataMap.dest_lat},
- #{dataMap.event_category},
- #{dataMap.attack_result},
- #{dataMap.probe_ip},
- #{dataMap.device_ip},
- #{dataMap.device_manufacturer},
- #{dataMap.device_name},
- #{dataMap.product_name},
- #{dataMap.__id},
- #{dataMap.__count},
- #{dataMap.__count_reason},
- #{dataMap.event_type},
- #{dataMap.protocol},
- #{dataMap.shell_cmd},
- #{dataMap.parent_name},
- #{dataMap.host_file_path},
- #{dataMap.uid},
- #{dataMap.fall},
- #{dataMap.tc_miguan_server_ip},
- #{dataMap.dev_type},
- #{dataMap.collect_method},
- #{dataMap.field_cate_id},
- #{dataMap.device_type},
- #{dataMap.tc_miguan_client_ip},
- #{dataMap.tc_miguan_name},
- #{dataMap.origin_total_packages},
- #{dataMap.origin_total_bytes},
- #{dataMap.origin_peak_packages_rate},
- #{dataMap.origin_peak_bytes_rate},
- #{dataMap.origin_peak_flows_rate},
- #{dataMap.apt_orgname},
- #{dataMap.apt_orgmsg},
- #{dataMap.mail_message_id},
- #{dataMap.mail_bcc},
- #{dataMap.mail_size},
- #{dataMap.mail_attach_hashcode},
- #{dataMap.mail_url},
- #{dataMap.mail_cc},
- #{dataMap.algorithm},
- #{dataMap.miningpool_ip},
- #{dataMap.process_md5},
- #{dataMap.pprocess_md5},
- #{dataMap.source_servername},
- #{dataMap.origin_source_servername},
- #{dataMap.mail_filename},
- #{dataMap.dst_upload_appname},
- #{dataMap.target_port},
- #{dataMap.gid},
- #{dataMap.origin_uid},
- #{dataMap.origin_gid},
- #{dataMap.target_ports},
- #{dataMap.tc_miguan_name1},
- #{dataMap.tc_miguan_class1},
- #{dataMap.etl_time},
- #{dataMap.tc_miguan_scan_port2},
- #{dataMap.desip_security_scope},
- #{dataMap.srcip_security_scope},
- #{dataMap.collect_time_ts},
- #{dataMap.tc_miguan_scan_port1},
- #{dataMap.src_dev_name},
- #{dataMap.collect_protocol},
- #{dataMap.destination_system_type},
- #{dataMap.destination_system},
- #{dataMap.etl_host},
- #{dataMap.normalize_rule_id},
- #{dataMap.normalize_rule_name},
- #{dataMap.syslog_uuid},
- #{dataMap.syslog_topic},
+ #{dataMap.id},
+ #{dataMap.created_at},
+ #{dataMap.log_time},
+ #{dataMap.device_id},
+ #{dataMap.webshell_type},
+ #{dataMap.vuirs_type},
+ #{dataMap.vuirs_url},
+ #{dataMap.class_filename},
+ #{dataMap.class_path},
+ #{dataMap.parent_class},
+ #{dataMap.jar_path},
+ #{dataMap.class_md5},
+ #{dataMap.class_loader},
+ #{dataMap.class_hashcode},
+ #{dataMap.class_loader_hashcode},
+ #{dataMap.tc_nameip},
+ #{dataMap.perform_sql},
+ #{dataMap.tc_account},
+ #{dataMap.tc_appname},
+ #{dataMap.process_uname},
+ #{dataMap.p_process_uname},
+ #{dataMap.container_name},
+ #{dataMap.container_id},
+ #{dataMap.http_resp_server},
+ #{dataMap.srcip_id},
+ #{dataMap.cdnip},
+ #{dataMap.natip},
+ #{dataMap.mail_sender},
+ #{dataMap.mail_receiver},
+ #{dataMap.vpn_mac},
+ #{dataMap.vpn_os},
+ #{dataMap.vpn_user},
+ #{dataMap.vpn_groupname},
+ #{dataMap.vpn_access_ip},
+ #{dataMap.dest_ip_apt},
+ #{dataMap.origin_attack_result},
+ #{dataMap.description},
+ #{dataMap.solution},
+ #{dataMap.attack_cause},
+ #{dataMap.username},
+ #{dataMap.tc_flow_id},
+ #{dataMap.login_result},
+ #{dataMap.cmdline},
+ #{dataMap.origin_attack_action},
+ #{dataMap.victim_domain},
+ #{dataMap.vpn_deviceid},
+ #{dataMap.vpn_access_action},
+ #{dataMap.file_access_time},
+ #{dataMap.file_name},
+ #{dataMap.tc_class},
+ #{dataMap.tc_name2},
+ #{dataMap.login_lasttime},
+ #{dataMap.origin_permissions},
+ #{dataMap.begin_permissions},
+ #{dataMap.print_time},
+ #{dataMap.printer},
+ #{dataMap.printer_type},
+ #{dataMap.print_pages},
+ #{dataMap.print_copies},
+ #{dataMap.src_device},
+ #{dataMap.dst_device},
+ #{dataMap.src_file},
+ #{dataMap.src_file_type},
+ #{dataMap.src_file_path},
+ #{dataMap.dst_file},
+ #{dataMap.dst_file_type},
+ #{dataMap.dst_file_path},
+ #{dataMap.dlp_policy_name},
+ #{dataMap.dlp_policy_type},
+ #{dataMap.dst_upload_url},
+ #{dataMap.process_uuid},
+ #{dataMap.p_process_uuid},
+ #{dataMap.env},
+ #{dataMap.brute_force_service},
+ #{dataMap.vuirs_name},
+ #{dataMap.http_req_length},
+ #{dataMap.http_req_content_type},
+ #{dataMap.tc_scan_port},
+ #{dataMap.tc_labels},
+ #{dataMap.http_resp_content_type},
+ #{dataMap.dns_msg_type},
+ #{dataMap.dns_answer_length},
+ #{dataMap.dns_ioc},
+ #{dataMap.tx_bytes},
+ #{dataMap.rx_bytes},
+ #{dataMap.all_bytes},
+ #{dataMap.duration_time},
+ #{dataMap.mail_attach_name},
+ #{dataMap.mail_subject},
+ #{dataMap.mail_message},
+ #{dataMap.mail_send_server},
+ #{dataMap.mail_agent},
+ #{dataMap.tls_version},
+ #{dataMap.tls_server_cert},
+ #{dataMap.tls_server_suite},
+ #{dataMap.tls_client_suites_len},
+ #{dataMap.tls_ja3},
+ #{dataMap.tls_ja3s},
+ #{dataMap.vpn_access_port},
+ #{dataMap.log_topic},
+ #{dataMap.collect_time},
+ #{dataMap.src_is_intranetip},
+ #{dataMap.src_ip_ioc},
+ #{dataMap.src_ip_apt},
+ #{dataMap.srcip_name},
+ #{dataMap.tc_client},
+ #{dataMap.srcip_organization_id},
+ #{dataMap.dest_ip_intranetip},
+ #{dataMap.dest_ip_ioc},
+ #{dataMap.desip_id},
+ #{dataMap.desip_name},
+ #{dataMap.tc_hostip},
+ #{dataMap.desip_organization_id},
+ #{dataMap.origin_confidence},
+ #{dataMap.origin_malscore},
+ #{dataMap.attacker_icampaign},
+ #{dataMap.attacker_host_asset_id},
+ #{dataMap.attacker_organization_id},
+ #{dataMap.victim_host_asset_id},
+ #{dataMap.victim_organization_id},
+ #{dataMap.logout_time},
+ #{dataMap.http_req_line},
+ #{dataMap.desip_security_scope_id},
+ #{dataMap.srcip_security_scope_id},
+ #{dataMap.http_resp_length},
+ #{dataMap.tc_attack_type},
+ #{dataMap.tc_realip},
+ #{dataMap.attacker_ip_lists},
+ #{dataMap.login_password},
+ #{dataMap.detail},
+ #{dataMap.attacker_country_code},
+ #{dataMap.attacker_region_code},
+ #{dataMap.victim_region_code},
+ #{dataMap.payload},
+ #{dataMap.http_referer},
+ #{dataMap.http_user_agent},
+ #{dataMap.http_session},
+ #{dataMap.http_query_string},
+ #{dataMap.file_path},
+ #{dataMap.file_permission},
+ #{dataMap.login_abnormal_type},
+ #{dataMap.file_tag},
+ #{dataMap.file_platform},
+ #{dataMap.target_ip},
+ #{dataMap.collect_date},
+ #{dataMap.tc_client_ip},
+ #{dataMap.tc_server_ip},
+ #{dataMap.tc_externalip},
+ #{dataMap.http_status_code},
+ #{dataMap.device_domian},
+ #{dataMap.src_ip_str},
+ #{dataMap.src_port_str},
+ #{dataMap.dest_ip_str} ,
+ CAST(#{dataMap.dest_port_str} AS text),
+ #{dataMap.pcap},
+ #{dataMap.ioc},
+ #{dataMap.malicious_family},
+ #{dataMap.vuln_cve},
+ #{dataMap.aliyun_type},
+ #{dataMap.attacker_host_asset_name},
+ #{dataMap.attacker_organization_name},
+ #{dataMap.ct_id},
+ #{dataMap.cve_list},
+ #{dataMap.desip_organization_name},
+ #{dataMap.dest_ip_group},
+ #{dataMap.file_gid},
+ #{dataMap.file_owner},
+ #{dataMap.file_ownergroup},
+ #{dataMap.file_uid},
+ #{dataMap.http_resp_cookie},
+ #{dataMap.origin_rule_id},
+ #{dataMap.origin_rule_name},
+ #{dataMap.service_name},
+ #{dataMap.src_ip_asset_group},
+ #{dataMap.srcip_organization_name},
+ #{dataMap.victim_host_asset_name},
+ #{dataMap.http_resp_codes}::bigint,
+ #{dataMap.victim_organization_name},
+ #{dataMap.tc_type},
+ #{dataMap.direction},
+ #{dataMap.http_req_cookie},
+ #{dataMap.http_req_protocol},
+ #{dataMap.http_req_header_raw},
+ #{dataMap.http_url},
+ #{dataMap.uname},
+ #{dataMap.origin_hostname},
+ #{dataMap.origin_os},
+ #{dataMap.origin_agent_mac},
+ #{dataMap.origin_host_id},
+ #{dataMap.origin_agent_version},
+ #{dataMap.origin_agent_id},
+ #{dataMap.origin_agent_name},
+ #{dataMap.origin_work_group},
+ #{dataMap.origin_asset_group},
+ #{dataMap.origin_local_port},
+ #{dataMap.origin_agent_ip},
+ #{dataMap.origin_internal_ip},
+ #{dataMap.origin_external_ip},
+ #{dataMap.origin_local_addr},
+ #{dataMap.agent_id},
+ #{dataMap.agent_name},
+ #{dataMap.tc_title},
+ #{dataMap.log_id},
+ #{dataMap.event_date},
+ #{dataMap.event_time_ts},
+ #{dataMap.event_level},
+ #{dataMap.src_ip},
+ #{dataMap.src_port},
+ #{dataMap.dest_ip},
+ #{dataMap.dest_port}::BIGINT,
+ #{dataMap.event_time},
+ #{dataMap.attacker_country},
+ #{dataMap.src_mac},
+ #{dataMap.dest_mac},
+ #{dataMap.proto},
+ #{dataMap.dev_id},
+ #{dataMap.created_time},
+ #{dataMap.src_country},
+ #{dataMap.src_country_code},
+ #{dataMap.src_region},
+ #{dataMap.src_region_code},
+ #{dataMap.src_city},
+ #{dataMap.src_lon},
+ #{dataMap.http_method},
+ #{dataMap.http_host},
+ #{dataMap.http_req_header},
+ #{dataMap.http_req_body},
+ #{dataMap.http_resp_header},
+ #{dataMap.http_resp_body},
+ #{dataMap.file_type},
+ #{dataMap.file_md5},
+ #{dataMap.file_size},
+ #{dataMap.process},
+ #{dataMap.start_time},
+ #{dataMap.action},
+ #{dataMap.attacker_region},
+ #{dataMap.end_time},
+ #{dataMap.file_created_time},
+ #{dataMap.file_modified_time},
+ #{dataMap.tc_miguan_scan_port},
+ #{dataMap.process_path},
+ #{dataMap.parent_process_path},
+ #{dataMap.gname},
+ #{dataMap.exe_name},
+ #{dataMap.exe_path},
+ #{dataMap.login_time},
+ #{dataMap.login_times},
+ #{dataMap.check_item},
+ #{dataMap.check_type},
+ #{dataMap.attacker_ip},
+ #{dataMap.attacker_port},
+ #{dataMap.victim_ip},
+ #{dataMap.victim_port},
+ #{dataMap.attacker_city},
+ #{dataMap.attacker_lon},
+ #{dataMap.attacker_lat},
+ #{dataMap.victim_country},
+ #{dataMap.victim_region},
+ #{dataMap.victim_city},
+ #{dataMap.victim_lon},
+ #{dataMap.victim_lat},
+ #{dataMap.origin_event_id},
+ #{dataMap.origin_event_name},
+ #{dataMap.origin_event_category},
+ #{dataMap.origin_event_level},
+ #{dataMap.origin_attack_chain},
+ #{dataMap.engine_type},
+ #{dataMap.evil_payload},
+ #{dataMap.http_resp_status},
+ #{dataMap.dns_query},
+ #{dataMap.dns_query_type},
+ #{dataMap.dns_ttl},
+ #{dataMap.dns_answer},
+ #{dataMap.dns_subdomains},
+ #{dataMap.file_sha256},
+ #{dataMap.file_ssdeep},
+ #{dataMap.victim_country_code},
+ #{dataMap.http_xff_ip},
+ #{dataMap.tc_miguan_class},
+ #{dataMap.pid},
+ #{dataMap.ppid},
+ #{dataMap.process_name},
+ #{dataMap.backdoor_type},
+ #{dataMap.tty},
+ #{dataMap.sudo_user},
+ #{dataMap.sudo_group},
+ #{dataMap.origin_event_type},
+ #{dataMap.dest_domain},
+ #{dataMap.shell_cmdline},
+ #{dataMap.parent_cmdline},
+ #{dataMap.attack_chain},
+ #{dataMap.process_tree},
+ #{dataMap.host_file_sha256},
+ #{dataMap.host_file_md5},
+ #{dataMap.host_file_size},
+ #{dataMap.host_file_type},
+ #{dataMap.dest_country},
+ #{dataMap.dest_country_code},
+ #{dataMap.log_origin},
+ #{dataMap.dest_region},
+ #{dataMap.src_lat},
+ #{dataMap.dest_region_code},
+ #{dataMap.dest_city},
+ #{dataMap.dest_lon},
+ #{dataMap.dest_lat},
+ #{dataMap.event_category},
+ #{dataMap.attack_result},
+ #{dataMap.probe_ip},
+ #{dataMap.device_ip},
+ #{dataMap.device_manufacturer},
+ #{dataMap.device_name},
+ #{dataMap.product_name},
+ #{dataMap.__id},
+ #{dataMap.__count},
+ #{dataMap.__count_reason},
+ #{dataMap.event_type},
+ #{dataMap.protocol},
+ #{dataMap.shell_cmd},
+ #{dataMap.parent_name},
+ #{dataMap.host_file_path},
+ #{dataMap.uid},
+ #{dataMap.fall},
+ #{dataMap.tc_miguan_server_ip},
+ #{dataMap.dev_type},
+ #{dataMap.collect_method},
+ #{dataMap.field_cate_id},
+ #{dataMap.device_type},
+ #{dataMap.tc_miguan_client_ip},
+ #{dataMap.tc_miguan_name},
+ #{dataMap.origin_total_packages},
+ #{dataMap.origin_total_bytes},
+ #{dataMap.origin_peak_packages_rate},
+ #{dataMap.origin_peak_bytes_rate},
+ #{dataMap.origin_peak_flows_rate},
+ #{dataMap.apt_orgname},
+ #{dataMap.apt_orgmsg},
+ #{dataMap.mail_message_id},
+ #{dataMap.mail_bcc},
+ #{dataMap.mail_size},
+ #{dataMap.mail_attach_hashcode},
+ #{dataMap.mail_url},
+ #{dataMap.mail_cc},
+ #{dataMap.algorithm},
+ #{dataMap.miningpool_ip},
+ #{dataMap.process_md5},
+ #{dataMap.pprocess_md5},
+ #{dataMap.source_servername},
+ #{dataMap.origin_source_servername},
+ #{dataMap.mail_filename},
+ #{dataMap.dst_upload_appname},
+ #{dataMap.target_port},
+ #{dataMap.gid},
+ #{dataMap.origin_uid},
+ #{dataMap.origin_gid},
+ #{dataMap.target_ports},
+ #{dataMap.tc_miguan_name1},
+ #{dataMap.tc_miguan_class1},
+ #{dataMap.etl_time},
+ #{dataMap.tc_miguan_scan_port2},
+ #{dataMap.desip_security_scope},
+ #{dataMap.srcip_security_scope},
+ #{dataMap.collect_time_ts},
+ #{dataMap.tc_miguan_scan_port1},
+ #{dataMap.src_dev_name},
+ #{dataMap.collect_protocol},
+ #{dataMap.destination_system_type},
+ #{dataMap.destination_system},
+ #{dataMap.etl_host},
+ #{dataMap.normalize_rule_id},
+ #{dataMap.normalize_rule_name},
+ #{dataMap.syslog_uuid},
+ #{dataMap.syslog_topic},
+ #{dataMap.rule_version},
@@ -914,14 +917,4 @@
-
-
- INSERT INTO syslog_normal_data
- (id, log_time, src_ip, dest_ip, event_level)
- VALUES
-
- (#{item.id}, #{item.log_time}, #{item.src_ip}, #{item.dest_ip}, #{item.event_level})
-
-
-
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java
index 7a7ba61..560917c 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java
@@ -4,6 +4,7 @@ import com.Modules.NormalData.compiled.BatchEnrichmentContext;
import com.Modules.NormalData.compiled.BatchRuleSnapshot;
import com.Modules.NormalData.compiled.CompiledRuleCache;
import com.Modules.NormalData.compiled.CompiledRuleCompiler;
+import com.Modules.NormalData.compiled.PreparedNormalizationInput;
import com.Modules.NormalData.compiled.RuleExecutionMetrics;
import com.Modules.NormalData.compiled.RuleSource;
import com.common.entity.SyslogNonNormalMessage;
@@ -90,6 +91,20 @@ class LogNormalProcessorRuleSelectionTest {
verify(fixture.nonNormalService).saveMessage(any(SyslogNonNormalMessage.class));
}
+ @Test
+ void activeKafkaPreparationDefersNonNormalPersistence() {
+ Fixture fixture = fixture(Arrays.asList(
+ regexRule(10L, "first", "first=(\\S+)"),
+ regexRule(20L, "second", "second=(\\S+)")), config(true));
+
+ LogNormalProcessor.NormalizationResult result = fixture.prepareActive("nothing matches");
+
+ assertEquals(LogNormalProcessor.ProcessingOutcome.FILTERED, result.getOutcome());
+ assertEquals(1, result.getNonNormalMessages().size());
+ assertEquals("未命中规则", result.getNonNormalMessages().get(0).getReason());
+ verify(fixture.nonNormalService, never()).saveMessage(any());
+ }
+
@Test
void nonNormalSaveFailureRemainsRetryable() {
Fixture fixture = fixture(Arrays.asList(
@@ -232,5 +247,23 @@ class LogNormalProcessorRuleSelectionTest {
new BatchEnrichmentContext(false, taskService, deviceService),
new RuleExecutionMetrics());
}
+
+ private LogNormalProcessor.NormalizationResult prepareActive(String body) {
+ LocalDateTime receivedAt = LocalDateTime.of(2026, 8, 21, 15, 0);
+ Map deviceInfo = new LinkedHashMap<>();
+ deviceInfo.put("receive_time", "20260821150000000");
+ deviceInfo.put("event_id", "event-selection");
+ deviceInfo.put("device_id", "7");
+ deviceInfo.put("device_name", "fw");
+ deviceInfo.put("vendor", "vendor");
+ deviceInfo.put("data_type", "json");
+ deviceInfo.put("device_collect_id", "3");
+ PreparedNormalizationInput input = new PreparedNormalizationInput(
+ "event-selection", "raw", receivedAt, receivedAt,
+ 7L, 3L, "json", body, body, "", deviceInfo);
+ return processor.prepare(input, snapshot,
+ new BatchEnrichmentContext(false, taskService, deviceService),
+ new RuleExecutionMetrics());
+ }
}
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/QuanxiNormalizationTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/QuanxiNormalizationTest.java
index 288f322..23fbea9 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/QuanxiNormalizationTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/QuanxiNormalizationTest.java
@@ -59,7 +59,7 @@ class QuanxiNormalizationTest {
assertTrue(f.store.isAlarmData(row));
assertEquals(1, f.store.insertDynamic(row));
}
- verify(f.alarm, times(20)).insertDynamic(anyMap());
+ verify(f.alarm, times(20)).insertBatchPlanned(anyMap());
verifyNoInteractions(f.normal);
}
}
@@ -107,7 +107,9 @@ class QuanxiNormalizationTest {
@Test void ruleDestinationsAreActualAlarmSqlColumnsAndHttpPathsAreExact() throws Exception {
JsonNode rule = JSON.readTree(file("normalization-rule.json").toFile());
String sql = new String(Files.readAllBytes(file("../../../syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml")), "UTF-8");
- for (JsonNode m : rule.path("mappers")) assertTrue(sql.contains("dataMap." + m.path("dest_field").asText() + " "));
+ for (JsonNode m : rule.path("mappers")) {
+ assertTrue(sql.contains("dataMap." + m.path("dest_field").asText()));
+ }
for (String[] field : HTTP_FIELDS) {
boolean found = false;
for (JsonNode m : rule.path("mappers")) if (field[1].equals(m.path("dest_field").asText())) {
@@ -138,7 +140,7 @@ class QuanxiNormalizationTest {
ReflectionTestUtils.setField(store, "syslogNormalDataMapper", normal);
ReflectionTestUtils.setField(store, "statsChangeCaptureService", mock(StatsChangeCaptureService.class));
ReflectionTestUtils.setField(store, "directStatsProjectionWriter", mock(DirectStatsProjectionWriter.class));
- when(alarm.insertDynamic(anyMap())).thenReturn(1);
+ when(alarm.insertBatchPlanned(anyMap())).thenReturn(1);
NormalizationOptimizationProperties p = new NormalizationOptimizationProperties();
p.getCompiledRule().setEnabled(compiled); p.getCompiledRule().setShadowCompareEnabled(false);
p.getRuleSelection().setEnabled(true); p.getRuleSelection().setShadowEnabled(false);
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java
index 9e46944..ed5e055 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java
@@ -3,9 +3,11 @@ package com.Modules.NormalData;
import com.Modules.NormalData.compiled.BatchEnrichmentContext;
import com.Modules.NormalData.compiled.BatchRuleSnapshot;
import com.Modules.NormalData.compiled.PreparedNormalizationInput;
+import com.common.entity.SyslogNonNormalMessage;
import com.common.service.DeviceCollectTaskService;
import com.common.service.DeviceDeviceService;
import com.common.service.DmStandardizedBatchWriter;
+import com.common.service.SyslogNonNormalMessageService;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -19,12 +21,18 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -32,23 +40,29 @@ import static org.mockito.Mockito.when;
class SysLogProcessorParallelTest {
private ExecutorService executor;
+ private ExecutorService nonNormalExecutor;
private SysLogProcessor processor;
private LogNormalProcessor normalProcessor;
private SyslogFailureRouter failureRouter;
private DmStandardizedBatchWriter batchWriter;
+ private SyslogNonNormalMessageService nonNormalMessageService;
@BeforeEach
void setUp() {
executor = Executors.newFixedThreadPool(4);
+ nonNormalExecutor = Executors.newSingleThreadExecutor();
processor = new SysLogProcessor();
normalProcessor = mock(LogNormalProcessor.class);
failureRouter = mock(SyslogFailureRouter.class);
batchWriter = mock(DmStandardizedBatchWriter.class);
+ nonNormalMessageService = mock(SyslogNonNormalMessageService.class);
ReflectionTestUtils.setField(processor, "taskExecutor", executor);
ReflectionTestUtils.setField(processor, "processTimeoutMs", 5000L);
ReflectionTestUtils.setField(processor, "logNormalProcessor", normalProcessor);
ReflectionTestUtils.setField(processor, "failureRouter", failureRouter);
ReflectionTestUtils.setField(processor, "standardizedBatchWriter", batchWriter);
+ ReflectionTestUtils.setField(processor, "nonNormalMessageService", nonNormalMessageService);
+ ReflectionTestUtils.setField(processor, "nonNormalWriterExecutor", nonNormalExecutor);
when(batchWriter.write(any())).thenReturn(new DmStandardizedBatchWriter.WriteReport<>());
when(normalProcessor.captureRuleSnapshot(any())).thenReturn(
new BatchRuleSnapshot(Collections.emptyMap(), System.currentTimeMillis()));
@@ -59,7 +73,9 @@ class SysLogProcessorParallelTest {
@AfterEach
void tearDown() throws InterruptedException {
executor.shutdownNow();
+ nonNormalExecutor.shutdownNow();
executor.awaitTermination(5, TimeUnit.SECONDS);
+ nonNormalExecutor.awaitTermination(5, TimeUnit.SECONDS);
}
@Test
@@ -121,6 +137,73 @@ class SysLogProcessorParallelTest {
"达梦写入耗时应独立覆盖批写调用");
}
+ @Test
+ void deferredNonNormalMessagesAreSavedOncePerKafkaBatch() {
+ when(normalProcessor.prepare(any(PreparedNormalizationInput.class),
+ any(), any(), any())).thenAnswer(invocation -> {
+ SyslogNonNormalMessage message = new SyslogNonNormalMessage();
+ message.setId(java.util.UUID.randomUUID().toString());
+ return LogNormalProcessor.NormalizationResult.filtered(
+ Collections.singletonList(message));
+ });
+ when(nonNormalMessageService.saveBatchMessages(anyList()))
+ .thenAnswer(invocation -> ((List>) invocation.getArgument(0)).size());
+
+ SysLogProcessor.BatchMetrics metrics = new SysLogProcessor.BatchMetrics();
+ processor.normalizePreparedMessages(messages(8), metrics);
+
+ @SuppressWarnings("unchecked")
+ org.mockito.ArgumentCaptor> captor =
+ org.mockito.ArgumentCaptor.forClass(List.class);
+ verify(nonNormalMessageService, timeout(1000)).saveBatchMessages(captor.capture());
+ assertEquals(8, captor.getValue().size());
+ assertEquals(8, metrics.nonNormalQueued());
+ assertEquals(0, metrics.nonNormalRejected());
+ assertEquals(0, metrics.successCount());
+ }
+
+ @Test
+ void asynchronousNonNormalFailureDoesNotFailKafkaBatch() {
+ when(normalProcessor.prepare(any(PreparedNormalizationInput.class),
+ any(), any(), any())).thenAnswer(invocation -> {
+ SyslogNonNormalMessage message = new SyslogNonNormalMessage();
+ message.setId(java.util.UUID.randomUUID().toString());
+ return LogNormalProcessor.NormalizationResult.filtered(
+ Collections.singletonList(message));
+ });
+ when(nonNormalMessageService.saveBatchMessages(anyList()))
+ .thenThrow(new IllegalStateException("dm unavailable"));
+
+ SysLogProcessor.BatchMetrics metrics = new SysLogProcessor.BatchMetrics();
+ processor.normalizePreparedMessages(messages(2), metrics);
+
+ verify(nonNormalMessageService, timeout(1000)).saveBatchMessages(anyList());
+ assertEquals(2, metrics.nonNormalQueued());
+ assertEquals(2, metrics.filteredCount());
+ }
+
+ @Test
+ void rejectedNonNormalTaskDoesNotFailKafkaBatch() {
+ when(normalProcessor.prepare(any(PreparedNormalizationInput.class),
+ any(), any(), any())).thenAnswer(invocation -> {
+ SyslogNonNormalMessage message = new SyslogNonNormalMessage();
+ return LogNormalProcessor.NormalizationResult.filtered(
+ Collections.singletonList(message));
+ });
+ ReflectionTestUtils.setField(processor, "nonNormalWriterExecutor",
+ (java.util.concurrent.Executor) command -> {
+ throw new RejectedExecutionException("queue full");
+ });
+
+ SysLogProcessor.BatchMetrics metrics = new SysLogProcessor.BatchMetrics();
+ processor.normalizePreparedMessages(messages(3), metrics);
+
+ verify(nonNormalMessageService, never()).saveBatchMessages(anyList());
+ assertEquals(0, metrics.nonNormalQueued());
+ assertEquals(3, metrics.nonNormalRejected());
+ assertEquals(3, metrics.filteredCount());
+ }
+
private List messages(int count) {
List messages = new ArrayList<>();
for (int i = 0; i < count; i++) {
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/mapper/FixedColumnBatchMapperTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/mapper/FixedColumnBatchMapperTest.java
new file mode 100644
index 0000000..af7eefb
--- /dev/null
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/mapper/FixedColumnBatchMapperTest.java
@@ -0,0 +1,127 @@
+package com.common.mapper;
+
+import org.apache.ibatis.builder.xml.XMLMapperBuilder;
+import org.apache.ibatis.mapping.MappedStatement;
+import org.apache.ibatis.session.Configuration;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class FixedColumnBatchMapperTest {
+
+ @Test
+ void normalDataUsesWhitelistedBatchUnionTemplate() {
+ assertBatchUnionMapper(
+ "/mapper/SyslogNormalDataMapper.xml", "syslog_normal_data");
+ }
+
+ @Test
+ void normalAlarmUsesWhitelistedBatchUnionTemplate() {
+ assertBatchUnionMapper(
+ "/mapper/SyslogNormalAlarmMapper.xml", "syslog_normal_alarm");
+ }
+
+ @Test
+ void alignedRowsProduceIdenticalMyBatisSqlEvenWhenValuesAreNull() {
+ assertSameBoundSql(
+ "/mapper/SyslogNormalDataMapper.xml",
+ "com.common.mapper.SyslogNormalDataMapper.insertBatchPlanned");
+ assertSameBoundSql(
+ "/mapper/SyslogNormalAlarmMapper.xml",
+ "com.common.mapper.SyslogNormalAlarmMapper.insertBatchPlanned");
+ }
+
+ private void assertBatchUnionMapper(String resourcePath, String table) {
+ String xml = resource(resourcePath);
+ Matcher statement = Pattern.compile(
+ "\\s*VALUES\\s*\\s*",
+ Pattern.DOTALL).matcher(xml);
+
+ assertTrue(statement.find(), "缺少 " + table + " 批次列并集 INSERT");
+ String insert = statement.group(0);
+ assertTrue(insert.contains("dataMap.containsKey('src_ip')"));
+ assertTrue(insert.contains("dataMap.containsKey('dest_ip')"));
+ assertTrue(insert.contains("dataMap.containsKey('rule_version')"));
+ assertFalse(insert.contains("dataMap.src_ip != null"),
+ "列选择必须依据调用方补齐后的统一键集合,不能再按各行值判断");
+
+ Matcher conditions = Pattern.compile("dataMap\\.containsKey\\('([a-zA-Z0-9_]+)'\\)")
+ .matcher(insert);
+ java.util.Map occurrences = new java.util.HashMap<>();
+ while (conditions.find()) {
+ occurrences.merge(conditions.group(1), 1, Integer::sum);
+ }
+ assertTrue(occurrences.size() > 300, "白名单必须保留历史标准化候选字段");
+ assertTrue(occurrences.values().stream().allMatch(count -> count == 2),
+ "每个允许字段必须同时出现在列清单和值清单中");
+ }
+
+ private void assertSameBoundSql(String resourcePath, String statementId) {
+ String xml = resource(resourcePath);
+ Configuration configuration = new Configuration();
+ try (InputStream stream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) {
+ new XMLMapperBuilder(stream, configuration, resourcePath,
+ configuration.getSqlFragments()).parse();
+ } catch (Exception parseFailure) {
+ throw new AssertionError("Mapper XML 解析失败: " + resourcePath, parseFailure);
+ }
+ MappedStatement statement = configuration.getMappedStatement(statementId);
+
+ Map first = new HashMap<>();
+ first.put("id", "row-1");
+ first.put("src_ip", "10.0.0.1");
+ first.put("dest_ip", null);
+ first.put("rule_version", "v1");
+ Map second = new HashMap<>();
+ second.put("id", "row-2");
+ second.put("src_ip", null);
+ second.put("dest_ip", "10.0.0.2");
+ second.put("rule_version", "v1");
+
+ String firstSql = normalizedSql(statement, first);
+ String secondSql = normalizedSql(statement, second);
+ assertEquals(firstSql, secondSql, "同一字段并集的各行必须生成完全相同的 JDBC Batch SQL");
+ assertTrue(firstSql.contains("src_ip"));
+ assertTrue(firstSql.contains("dest_ip"));
+ assertFalse(firstSql.contains("webshell_type"));
+ }
+
+ private String normalizedSql(MappedStatement statement, Map row) {
+ Map parameter = new HashMap<>();
+ parameter.put("dataMap", row);
+ return statement.getBoundSql(parameter).getSql().replaceAll("\\s+", " ").trim();
+ }
+
+ @Test
+ void nonNormalDataUsesFixedColumnsAndMultiRowValues() {
+ String xml = resource("/mapper/SyslogNonNormalMessageMapper.xml");
+ Matcher statement = Pattern.compile(
+ "", Pattern.DOTALL).matcher(xml);
+
+ assertTrue(statement.find(), "缺少 syslog_non_normal_message 批量 INSERT");
+ assertFalse(statement.group(0).contains(" first = new HashMap<>();
+ first.put("id", "row-1");
+ first.put("src_ip", "10.0.0.1");
+ first.put("unused_null", null);
+
+ Map second = new HashMap<>();
+ second.put("id", "row-2");
+ second.put("dest_ip", "10.0.0.2");
+
+ List> aligned =
+ DmStandardizedBatchExecutor.alignBatchColumns(Arrays.asList(first, second));
+
+ assertEquals(aligned.get(0).keySet(), aligned.get(1).keySet());
+ assertEquals(Arrays.asList("dest_ip", "id", "src_ip"),
+ aligned.get(0).keySet().stream().sorted().collect(java.util.stream.Collectors.toList()));
+ assertEquals("10.0.0.1", aligned.get(0).get("src_ip"));
+ assertNull(aligned.get(0).get("dest_ip"));
+ assertNull(aligned.get(1).get("src_ip"));
+ assertEquals("10.0.0.2", aligned.get(1).get("dest_ip"));
+ assertFalse(aligned.get(0).containsKey("unused_null"));
+ assertFalse(first.containsKey("dest_ip"), "不得修改上游标准化结果 Map");
+ assertFalse(second.containsKey("src_ip"), "不得修改上游标准化结果 Map");
+ }
+}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmStandardizedBatchWriterTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmStandardizedBatchWriterTest.java
index 628a737..b893dd4 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmStandardizedBatchWriterTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmStandardizedBatchWriterTest.java
@@ -100,6 +100,31 @@ class DmStandardizedBatchWriterTest {
verify(normalDataService, never()).insertDynamic(any());
}
+ @Test
+ void normalDataWithDifferentPopulatedColumnsSharesOneUnionPlannedBatch() {
+ List> requests = requests(2, -1);
+ requests.get(0).getData().put("src_ip", "10.0.0.1");
+ requests.get(1).getData().put("dest_ip", "10.0.0.2");
+
+ DmStandardizedBatchWriter.WriteReport report = writer.write(requests);
+
+ assertEquals(2, report.getSuccessfulRows());
+ verify(batchExecutor, times(1)).execute(anyBoolean(), anyList(), anyInt());
+ }
+
+ @Test
+ void alarmDataWithDifferentPopulatedColumnsSharesOneUnionPlannedBatch() {
+ when(normalDataService.isAlarmData(any())).thenReturn(true);
+ List> requests = requests(2, -1);
+ requests.get(0).getData().put("src_ip", "10.0.0.1");
+ requests.get(1).getData().put("dest_ip", "10.0.0.2");
+
+ DmStandardizedBatchWriter.WriteReport report = writer.write(requests);
+
+ assertEquals(2, report.getSuccessfulRows());
+ verify(batchExecutor, times(1)).execute(anyBoolean(), anyList(), anyInt());
+ }
+
private List> requests(int count, int badIndex) {
List> requests = new ArrayList<>();
for (int i = 0; i < count; i++) {
@@ -108,7 +133,7 @@ class DmStandardizedBatchWriterTest {
row.put("rule_version", "legacy");
row.put("created_at", LocalDateTime.of(2026, 8, 11, 3, 55));
row.put("log_time", LocalDateTime.of(2026, 8, 11, 3, 55));
- // 所有行都保留相同列签名;坏行只改变值,确保它真的混在同一个 JDBC batch 中。
+ // 普通日志按当前小批次字段并集生成模板;额外键不会再造成批次拆分。
row.put("bad", i == badIndex);
requests.add(new DmStandardizedBatchWriter.WriteRequest<>("source-" + i, row));
}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNonNormalMessageServiceBatchTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNonNormalMessageServiceBatchTest.java
new file mode 100644
index 0000000..19e5178
--- /dev/null
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNonNormalMessageServiceBatchTest.java
@@ -0,0 +1,61 @@
+package com.common.service;
+
+import com.common.entity.SyslogNonNormalMessage;
+import com.common.mapper.SyslogNonNormalMessageMapper;
+import com.common.service.impl.SyslogNonNormalMessageServiceImpl;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class SyslogNonNormalMessageServiceBatchTest {
+
+ private SyslogNonNormalMessageServiceImpl service;
+ private SyslogNonNormalMessageMapper mapper;
+ private DirectStatsProjectionWriter directStatsProjectionWriter;
+
+ @BeforeEach
+ void setUp() {
+ service = new SyslogNonNormalMessageServiceImpl();
+ mapper = mock(SyslogNonNormalMessageMapper.class);
+ directStatsProjectionWriter = mock(DirectStatsProjectionWriter.class);
+ ReflectionTestUtils.setField(service, "messageMapper", mapper);
+ ReflectionTestUtils.setField(service, "statsChangeCaptureService",
+ mock(StatsChangeCaptureService.class));
+ ReflectionTestUtils.setField(service, "directStatsProjectionWriter",
+ directStatsProjectionWriter);
+ ReflectionTestUtils.setField(service, "nonNormalBatchSize", 100);
+ when(mapper.batchInsert(anyList()))
+ .thenAnswer(invocation -> ((List>) invocation.getArgument(0)).size());
+ }
+
+ @Test
+ void chunksLargePollInsideOneServiceTransactionAndAggregatesStatsOnce() {
+ List messages = new ArrayList<>();
+ for (int i = 0; i < 250; i++) {
+ SyslogNonNormalMessage message = new SyslogNonNormalMessage();
+ message.setId("non-normal-" + i);
+ messages.add(message);
+ }
+
+ assertEquals(250, service.saveBatchMessages(messages));
+
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class);
+ verify(mapper, times(3)).batchInsert(captor.capture());
+ assertEquals(100, captor.getAllValues().get(0).size());
+ assertEquals(100, captor.getAllValues().get(1).size());
+ assertEquals(50, captor.getAllValues().get(2).size());
+ verify(directStatsProjectionWriter).writeNonNormalRows(messages);
+ }
+}
diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNormalDataServiceIdempotencyTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNormalDataServiceIdempotencyTest.java
index b608ae5..2575989 100644
--- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNormalDataServiceIdempotencyTest.java
+++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNormalDataServiceIdempotencyTest.java
@@ -38,7 +38,7 @@ class SyslogNormalDataServiceIdempotencyTest {
@Test
void dmUniqueViolationWithExistingNormalResultIsIdempotentSuccess() {
Map data = idempotencyData(false);
- when(normalDataMapper.insertDynamic(anyMap())).thenThrow(dmUniqueViolation());
+ when(normalDataMapper.insertBatchPlanned(anyMap())).thenThrow(dmUniqueViolation());
when(normalDataMapper.countByLogTimeAndId(
(LocalDateTime) data.get("log_time"), (String) data.get("id")))
.thenReturn(1);
@@ -53,7 +53,7 @@ class SyslogNormalDataServiceIdempotencyTest {
@Test
void dmUniqueViolationWithExistingAlarmResultIsIdempotentSuccess() {
Map data = idempotencyData(true);
- when(normalAlarmMapper.insertDynamic(anyMap())).thenThrow(dmUniqueViolation());
+ when(normalAlarmMapper.insertBatchPlanned(anyMap())).thenThrow(dmUniqueViolation());
when(normalAlarmMapper.countByLogTimeAndId(
(LocalDateTime) data.get("log_time"), (String) data.get("id")))
.thenReturn(1);
@@ -70,7 +70,7 @@ class SyslogNormalDataServiceIdempotencyTest {
Map data = idempotencyData(false);
DataIntegrityViolationException invalidData = new DataIntegrityViolationException(
"value too long", new SQLException("data too long"));
- when(normalDataMapper.insertDynamic(anyMap())).thenThrow(invalidData);
+ when(normalDataMapper.insertBatchPlanned(anyMap())).thenThrow(invalidData);
assertThrows(DataIntegrityViolationException.class, () -> service.insertDynamic(data));
diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java
index cab9855..8e1153b 100644
--- a/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java
+++ b/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java
@@ -23,9 +23,9 @@ class KafkaProducerEventIdTest {
void maxRequestSizeIsPassedToTheManuallyConstructedKafkaProducer() {
Properties properties = new Properties();
- kafkaProducer.applyMaxRequestSize(properties, 4 * 1024 * 1024);
+ kafkaProducer.applyMaxRequestSize(properties, 8 * 1024 * 1024);
- assertEquals(4 * 1024 * 1024, properties.get(ProducerConfig.MAX_REQUEST_SIZE_CONFIG));
+ assertEquals(8 * 1024 * 1024, properties.get(ProducerConfig.MAX_REQUEST_SIZE_CONFIG));
}
@Test
diff --git a/haobang-security-dm/upgrade-doc/v1/kafka/02-skip-raw-backlog.sh b/haobang-security-dm/upgrade-doc/v1/kafka/02-skip-raw-backlog.sh
new file mode 100755
index 0000000..eb299a0
--- /dev/null
+++ b/haobang-security-dm/upgrade-doc/v1/kafka/02-skip-raw-backlog.sh
@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# 将指定消费组在原始 Topic 上的 offset 重置到指定 Kafka 消息时间。
+# 适用于“丢弃截止时间之前的历史积压,保留截止时间之后消息”的现场操作。
+#
+# 本脚本不会停止/启动应用,也不会删除 Topic 或消息:
+# - 执行前必须由运维停止该消费组的全部 syslog-consumer-rule 实例;
+# - 自动导出原 offset,Kafka 数据仍在 retention 内时可用于人工回退;
+# - 先执行 dry-run,只有输入精确确认串才执行真正的 offset 重置。
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+BOOTSTRAP="${BOOTSTRAP:-10.141.121.35:9092,10.141.121.50:9092,10.141.121.59:9092}"
+GROUP_ID="${GROUP_ID:-agent-01-syslog-group}"
+RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}"
+CUTOFF_DATETIME="${CUTOFF_DATETIME:-2026-09-11T00:00:00.000}"
+KAFKA_TIMEZONE="${KAFKA_TIMEZONE:-Asia/Shanghai}"
+
+# 优先使用调用方显式指定的目录;否则自动识别脚本同目录、现场 Kafka 安装目录或 PATH。
+if [[ -z "${KAFKA_BIN:-}" ]]; then
+ for CANDIDATE_BIN in \
+ "${SCRIPT_DIR}" \
+ "/opt/kafka/kafka_2.13-3.6.1/bin"; do
+ if [[ -x "${CANDIDATE_BIN}/kafka-consumer-groups.sh" ]] \
+ && [[ -x "${CANDIDATE_BIN}/kafka-topics.sh" ]]; then
+ KAFKA_BIN="${CANDIDATE_BIN}"
+ break
+ fi
+ done
+fi
+
+if [[ -z "${KAFKA_BIN:-}" ]] && command -v kafka-consumer-groups.sh >/dev/null 2>&1; then
+ KAFKA_BIN=$(dirname -- "$(command -v kafka-consumer-groups.sh)")
+fi
+
+if [[ -z "${KAFKA_BIN:-}" ]]; then
+ echo "ERROR: 未找到 Kafka CLI;现场预期路径为 /opt/kafka/kafka_2.13-3.6.1/bin" >&2
+ exit 1
+fi
+
+CONSUMER_GROUPS_CMD="${KAFKA_BIN}/kafka-consumer-groups.sh"
+TOPICS_CMD="${KAFKA_BIN}/kafka-topics.sh"
+
+if [[ ! -x "${CONSUMER_GROUPS_CMD}" ]] || [[ ! -x "${TOPICS_CMD}" ]]; then
+ echo "ERROR: ${KAFKA_BIN} 下缺少可执行的 kafka-consumer-groups.sh 或 kafka-topics.sh" >&2
+ exit 1
+fi
+
+if [[ ! "${CUTOFF_DATETIME}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}$ ]]; then
+ echo "ERROR: CUTOFF_DATETIME 格式必须是 YYYY-MM-DDTHH:mm:ss.SSS" >&2
+ exit 1
+fi
+
+COMMON_ARGS=(--bootstrap-server "${BOOTSTRAP}")
+if [[ -n "${COMMAND_CONFIG:-}" ]]; then
+ if [[ ! -r "${COMMAND_CONFIG}" ]]; then
+ echo "ERROR: 无法读取认证配置 ${COMMAND_CONFIG}" >&2
+ exit 1
+ fi
+ COMMON_ARGS+=(--command-config "${COMMAND_CONFIG}")
+fi
+
+# Kafka 3.6 的 --to-datetime 不携带时区,显式固定 JVM 时区,避免宿主机差异造成 8 小时偏移。
+export KAFKA_OPTS="${KAFKA_OPTS:-} -Duser.timezone=${KAFKA_TIMEZONE}"
+
+RUN_ID=$(date '+%Y%m%d_%H%M%S')
+AUDIT_DIR="${OFFSET_BACKUP_DIR:-./kafka-offset-reset-${RUN_ID}}"
+mkdir -p "${AUDIT_DIR}"
+
+BEFORE_FILE="${AUDIT_DIR}/before-describe.txt"
+RESTORE_FILE="${AUDIT_DIR}/restore-current-offsets.csv"
+DRY_RUN_FILE="${AUDIT_DIR}/reset-dry-run.txt"
+EXECUTE_FILE="${AUDIT_DIR}/reset-execute.txt"
+AFTER_FILE="${AUDIT_DIR}/after-describe.txt"
+
+echo "目标 Kafka:${BOOTSTRAP}"
+echo "目标消费组:${GROUP_ID}"
+echo "目标 Topic:${RAW_TOPIC}"
+echo "保留起点:${CUTOFF_DATETIME} (${KAFKA_TIMEZONE})"
+echo "审计目录:${AUDIT_DIR}"
+
+echo "[1/6] 检查 Topic"
+TOPIC_INFO=$("${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}")
+printf '%s\n' "${TOPIC_INFO}"
+
+PARTITION_COUNT=$(printf '%s\n' "${TOPIC_INFO}" | awk '
+ /PartitionCount:/ {
+ for (i=1; i<=NF; i++) if ($i=="PartitionCount:") { print $(i+1); exit }
+ }
+')
+if [[ ! "${PARTITION_COUNT}" =~ ^[1-9][0-9]*$ ]]; then
+ echo "ERROR: 无法确认 ${RAW_TOPIC} 的分区数" >&2
+ exit 1
+fi
+
+echo "[2/6] 检查消费组是否已停止,并保存执行前快照"
+"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \
+ --describe --group "${GROUP_ID}" 2>&1 | tee "${BEFORE_FILE}"
+
+MATCHED_ROWS=$(awk -v topic="${RAW_TOPIC}" '
+ $2==topic && $3~/^[0-9]+$/ && $4~/^[0-9]+$/ && $5~/^[0-9]+$/ {count++}
+ END {print count+0}
+' "${BEFORE_FILE}")
+if [[ "${MATCHED_ROWS}" -ne "${PARTITION_COUNT}" ]]; then
+ echo "ERROR: 快照只识别到 ${MATCHED_ROWS}/${PARTITION_COUNT} 个目标分区,拒绝继续" >&2
+ exit 1
+fi
+
+if awk -v topic="${RAW_TOPIC}" '
+ $2==topic && $3~/^[0-9]+$/ && $7!="-" {active=1}
+ END {exit(active ? 0 : 1)}
+' "${BEFORE_FILE}"; then
+ echo "ERROR: 消费组 ${GROUP_ID} 仍有在线成员。请先停止全部 syslog-consumer-rule 实例。" >&2
+ exit 1
+fi
+
+CURRENT_LAG=$(awk -v topic="${RAW_TOPIC}" '
+ $2==topic && $6~/^[0-9]+$/ {sum+=$6}
+ END {printf "%.0f", sum+0}
+' "${BEFORE_FILE}")
+echo "当前积压:${CURRENT_LAG} 条"
+
+echo "[3/6] 导出原 offset 回退文件"
+"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \
+ --group "${GROUP_ID}" \
+ --topic "${RAW_TOPIC}" \
+ --reset-offsets \
+ --to-current \
+ --dry-run \
+ --export > "${RESTORE_FILE}"
+
+RESTORE_ROWS=$(awk -F, -v topic="${RAW_TOPIC}" '
+ $1==topic && $2~/^[0-9]+$/ && $3~/^[0-9]+$/ {count++}
+ END {print count+0}
+' "${RESTORE_FILE}")
+if [[ "${RESTORE_ROWS}" -ne "${PARTITION_COUNT}" ]]; then
+ echo "ERROR: 回退文件只包含 ${RESTORE_ROWS}/${PARTITION_COUNT} 个分区,拒绝继续" >&2
+ exit 1
+fi
+echo "原 offset 已保存:${RESTORE_FILE}"
+
+echo "[4/6] 预演按时间重置"
+"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \
+ --group "${GROUP_ID}" \
+ --topic "${RAW_TOPIC}" \
+ --reset-offsets \
+ --to-datetime "${CUTOFF_DATETIME}" \
+ --dry-run | tee "${DRY_RUN_FILE}"
+
+PLAN_ROWS=$(awk -v topic="${RAW_TOPIC}" '
+ $1==topic && $2~/^[0-9]+$/ && $3~/^[0-9]+$/ {count++; next}
+ $2==topic && $3~/^[0-9]+$/ && $4~/^[0-9]+$/ {count++}
+ END {print count+0}
+' "${DRY_RUN_FILE}")
+if [[ "${PLAN_ROWS}" -ne "${PARTITION_COUNT}" ]]; then
+ echo "ERROR: dry-run 只生成 ${PLAN_ROWS}/${PARTITION_COUNT} 个分区,拒绝执行" >&2
+ exit 1
+fi
+
+EXPECTED_CONFIRM="RESET:${GROUP_ID}:${RAW_TOPIC}:${CUTOFF_DATETIME}"
+echo "[5/6] 等待人工确认"
+echo "即将永久跳过 ${CUTOFF_DATETIME} (${KAFKA_TIMEZONE}) 之前尚未消费的消息。"
+echo "原 offset 回退文件:${RESTORE_FILE}"
+echo "如确认执行,请完整输入:${EXPECTED_CONFIRM}"
+
+if [[ -n "${CONFIRM_RESET:-}" ]]; then
+ CONFIRM_INPUT="${CONFIRM_RESET}"
+elif [[ -t 0 ]]; then
+ read -r CONFIRM_INPUT
+else
+ echo "ERROR: 当前不是交互终端;必须通过 CONFIRM_RESET 提供精确确认串" >&2
+ exit 1
+fi
+
+if [[ "${CONFIRM_INPUT}" != "${EXPECTED_CONFIRM}" ]]; then
+ echo "CANCELLED: 确认串不匹配,未修改任何 offset。" >&2
+ exit 1
+fi
+
+echo "[6/6] 执行 offset 重置"
+"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \
+ --group "${GROUP_ID}" \
+ --topic "${RAW_TOPIC}" \
+ --reset-offsets \
+ --to-datetime "${CUTOFF_DATETIME}" \
+ --execute | tee "${EXECUTE_FILE}"
+
+echo "保存执行后快照"
+"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \
+ --describe --group "${GROUP_ID}" 2>&1 | tee "${AFTER_FILE}"
+
+echo "PASS: 已将 ${GROUP_ID}/${RAW_TOPIC} 重置到 ${CUTOFF_DATETIME} (${KAFKA_TIMEZONE})。"
+echo "未删除 Topic;现在可以启动 syslog-consumer-rule 并检查 LAG。"
+echo "回退文件:${RESTORE_FILE}(仅在 Kafka 原消息仍处于 retention 内时有效)。"
diff --git a/haobang-security-dm/upgrade-doc/v1/kafka/kafka-cluster-upgrade.sh b/haobang-security-dm/upgrade-doc/v1/kafka/kafka-cluster-upgrade.sh
new file mode 100755
index 0000000..100b1ff
--- /dev/null
+++ b/haobang-security-dm/upgrade-doc/v1/kafka/kafka-cluster-upgrade.sh
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Kafka 集群只需在一台装有 Kafka CLI、能够访问集群的服务器上执行一次。
+# 示例:
+# export KAFKA_BIN=<现场Kafka客户端bin目录>
+# export BOOTSTRAP=<现场broker1地址>:<端口>,<现场broker2地址>:<端口>
+# export RAW_TOPIC=<现场原始Topic名称>
+# export COMMAND_CONFIG=<现场client.properties路径> # 无认证时不设置
+# ./kafka-cluster-upgrade.sh
+# BOOTSTRAP 必须使用客户 Kafka 运维提供的地址;集群建议填写两个以上 broker。
+
+: "${KAFKA_BIN:?请根据现场环境设置 KAFKA_BIN}"
+: "${BOOTSTRAP:?请填写客户 Kafka 运维提供的 broker 地址和端口}"
+: "${RAW_TOPIC:?请填写现场实际使用的原始 Topic 名称}"
+
+TOPICS_CMD="${KAFKA_BIN}/kafka-topics.sh"
+CONFIGS_CMD="${KAFKA_BIN}/kafka-configs.sh"
+
+if [[ ! -x "${TOPICS_CMD}" ]] || [[ ! -x "${CONFIGS_CMD}" ]]; then
+ echo "ERROR: ${KAFKA_BIN} 下缺少 kafka-topics.sh 或 kafka-configs.sh" >&2
+ exit 1
+fi
+
+COMMON_ARGS=(--bootstrap-server "${BOOTSTRAP}")
+if [[ -n "${COMMAND_CONFIG:-}" ]]; then
+ if [[ ! -r "${COMMAND_CONFIG}" ]]; then
+ echo "ERROR: 无法读取认证配置 ${COMMAND_CONFIG}" >&2
+ exit 1
+ fi
+ COMMON_ARGS+=(--command-config "${COMMAND_CONFIG}")
+fi
+
+echo "[1/4] 测试 Kafka 集群连接:${BOOTSTRAP}"
+"${TOPICS_CMD}" "${COMMON_ARGS[@]}" --list >/dev/null
+
+echo "[2/4] 读取原始 Topic:${RAW_TOPIC}"
+RAW_INFO=$("${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}")
+printf '%s\n' "${RAW_INFO}"
+
+PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk '
+ /PartitionCount:/ {
+ for (i = 1; i <= NF; i++) if ($i == "PartitionCount:") { print $(i + 1); exit }
+ }
+')
+REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk '
+ /ReplicationFactor:/ {
+ for (i = 1; i <= NF; i++) if ($i == "ReplicationFactor:") { print $(i + 1); exit }
+ }
+')
+
+if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then
+ echo "ERROR: 无法读取原始 Topic 的分区数或副本数" >&2
+ exit 1
+fi
+
+echo "新 Topic 使用分区数=${PARTITIONS},副本数=${REPLICAS}"
+
+SUFFIXES=("RETRY.1M" "RETRY.10M" "RETRY.1H" "DLT" "REPLAY")
+RETENTIONS=(259200000 604800000 604800000 7776000000 604800000)
+
+echo "[3/4] 创建或更新升级 Topic"
+for i in 0 1 2 3 4; do
+ TOPIC="${RAW_TOPIC}.${SUFFIXES[$i]}"
+ RETENTION_MS="${RETENTIONS[$i]}"
+
+ echo "处理 ${TOPIC}"
+ "${TOPICS_CMD}" "${COMMON_ARGS[@]}" \
+ --create --if-not-exists \
+ --topic "${TOPIC}" \
+ --partitions "${PARTITIONS}" \
+ --replication-factor "${REPLICAS}" \
+ --config cleanup.policy=delete \
+ --config "retention.ms=${RETENTION_MS}"
+
+ "${CONFIGS_CMD}" "${COMMON_ARGS[@]}" \
+ --alter --entity-type topics --entity-name "${TOPIC}" \
+ --add-config "cleanup.policy=delete,retention.ms=${RETENTION_MS}"
+done
+
+echo "[4/4] 显示升级结果"
+for SUFFIX in "${SUFFIXES[@]}"; do
+ "${TOPICS_CMD}" "${COMMON_ARGS[@]}" \
+ --describe --topic "${RAW_TOPIC}.${SUFFIX}"
+done
+
+echo "PASS: Kafka 集群 Topic 已创建或更新;未删除 Topic,未修改消费组 offset。"
diff --git a/haobang-security-dm/upgrade-doc/v2/influxdb/01-ensure-v2-bucket.sh b/haobang-security-dm/upgrade-doc/v2/influxdb/01-ensure-v2-bucket.sh
index da13f37..51b2d7d 100755
--- a/haobang-security-dm/upgrade-doc/v2/influxdb/01-ensure-v2-bucket.sh
+++ b/haobang-security-dm/upgrade-doc/v2/influxdb/01-ensure-v2-bucket.sh
@@ -7,7 +7,8 @@ set -euo pipefail
INFLUX_CONFIG_PATH="${INFLUX_CONFIG_PATH:-/etc/influxdb2/influx-configs}"
V2_BUCKET="${V2_BUCKET:-yelangbucket_v2}"
-RETENTION="${RETENTION:-720h}"
+# 现场要求原始日志保留 180 天;仍可通过 RETENTION 环境变量覆盖。
+RETENTION="${RETENTION:-180d}"
SHARD_GROUP_DURATION="${SHARD_GROUP_DURATION:-24h}"
command -v docker >/dev/null 2>&1 || {