diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java index d51e83d..7355ad6 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/LogNormalProcessor.java @@ -50,18 +50,31 @@ public class LogNormalProcessor { public static final class NormalizationResult { private final ProcessingOutcome outcome; private final List> rows; + private final List nonNormalMessages; - private NormalizationResult(ProcessingOutcome outcome, List> rows) { + private NormalizationResult(ProcessingOutcome outcome, List> rows, + List nonNormalMessages) { this.outcome = outcome; this.rows = Collections.unmodifiableList(new ArrayList<>(rows)); + this.nonNormalMessages = Collections.unmodifiableList(new ArrayList<>(nonNormalMessages)); } static NormalizationResult success(List> rows) { - return new NormalizationResult(ProcessingOutcome.SUCCESS, rows); + return success(rows, Collections.emptyList()); + } + + static NormalizationResult success(List> rows, + List nonNormalMessages) { + return new NormalizationResult(ProcessingOutcome.SUCCESS, rows, nonNormalMessages); } static NormalizationResult filtered() { - return new NormalizationResult(ProcessingOutcome.FILTERED, Collections.emptyList()); + return filtered(Collections.emptyList()); + } + + static NormalizationResult filtered(List nonNormalMessages) { + return new NormalizationResult(ProcessingOutcome.FILTERED, + Collections.emptyList(), nonNormalMessages); } public ProcessingOutcome getOutcome() { @@ -71,6 +84,14 @@ public class LogNormalProcessor { public List> getRows() { return rows; } + + /** + * 活动 Kafka 批处理链路中需要集中落库的非标日志。 + * 历史单条入口仍会立即保存,因此其结果通常为空。 + */ + public List getNonNormalMessages() { + return nonNormalMessages; + } } private static final ZoneId RECEIVE_TIME_ZONE = ZoneId.of("Asia/Shanghai"); @@ -126,7 +147,9 @@ public class LogNormalProcessor { RuleExecutionMetrics executionMetrics; MessageParseContext messageParseContext; boolean shadowMode; + boolean deferNonNormalPersistence; final List> normalizedRows = new ArrayList<>(); + final List deferredNonNormalMessages = new ArrayList<>(); } // ==================== 构造器注入(Spring 自动完成)==================== @@ -254,7 +277,7 @@ public class LogNormalProcessor { * 只执行标准化计算并返回待写入行,不在本方法中写两张标准化表。 * *

这是批量写入的关键边界:并行线程只负责 CPU/规则计算,达梦事务由 Kafka 批次汇总后的 - * 批写器统一管理。非标日志仍沿用原有独立保存路径,不属于本次两张标准化表的改造范围。

+ * 批写器统一管理。活动 Kafka 链路产生的非标日志也随结果返回,在 poll 边界集中批写。

*/ public NormalizationResult prepare(String logMsg, String syslogUUID, String syslogTopic, LocalDateTime fallbackFirstReceivedAt) { @@ -326,6 +349,8 @@ public class LogNormalProcessor { ctx.executionMetrics = executionMetrics == null ? new RuleExecutionMetrics() : executionMetrics; ctx.messageParseContext = new MessageParseContext( optimizationProperties.getMessageParseCache().isEnabled(), globalMetrics); + // 活动 Kafka 链路必须把非标日志带回 poll 边界集中批写,禁止计算线程逐条开启达梦事务。 + ctx.deferNonNormalPersistence = true; return init(ctx); } @@ -373,7 +398,7 @@ public class LogNormalProcessor { // 原文已在进入标准化前写入 InfluxDB。设备没有绑定规则是可预期的业务终态, // 继续 Retry/DLT 不会凭空产生规则,只会重复消耗 Kafka、线程和日志 I/O。 // 因此只跳过达梦标准化并允许 ACK;系统依赖和规则执行异常仍由下方异常链处理。 - return NormalizationResult.filtered(); + return NormalizationResult.filtered(ctx.deferredNonNormalMessages); } NormalizationOptimizationProperties.RuleSelection selectionConfig = optimizationProperties.getRuleSelection(); @@ -420,9 +445,9 @@ public class LogNormalProcessor { if (!decision.isSelected()) { saveSelectionFailure(ctx, deviceId, "未命中规则", "同设备多规则均未形成可用归属候选"); - // 规则均未命中是确定性的业务终态;非标保存成功后直接 ACK,避免 Retry - // 为同一个 syslog_uuid 重复写入随机 ID 的非标记录。保存失败仍会由上层异常链重试。 - return NormalizationResult.filtered(); + // 规则均未命中是确定性的业务终态;非标记录随结果返回并交给独立线程批写。 + // 按现场临时策略,非标写入结果不参与原 Kafka 消息的 ACK。 + return NormalizationResult.filtered(ctx.deferredNonNormalMessages); } RuleSelectionEngine.Candidate selected = decision.getWinner(); @@ -442,7 +467,7 @@ public class LogNormalProcessor { saveSelectionFailure(ctx, deviceId, "多规则归属歧义", "候选评分相同但业务结果不同, candidateRuleIds=" + candidateRuleIds(decision.getTiedCandidates())); - return NormalizationResult.filtered(); + return NormalizationResult.filtered(ctx.deferredNonNormalMessages); } globalMetrics.ruleSelection("equivalent_tie"); prepared = byRule.get(selected.getSource().getId()); @@ -716,8 +741,12 @@ public class LogNormalProcessor { } private NormalizationResult finish(ProcessContext ctx) { - if (ctx.savedCount > 0) return NormalizationResult.success(ctx.normalizedRows); - if (ctx.filteredCount > 0 && ctx.rejectedCount == 0) return NormalizationResult.filtered(); + if (ctx.savedCount > 0) { + return NormalizationResult.success(ctx.normalizedRows, ctx.deferredNonNormalMessages); + } + if (ctx.filteredCount > 0 && ctx.rejectedCount == 0) { + return NormalizationResult.filtered(ctx.deferredNonNormalMessages); + } throw new IllegalArgumentException("Syslog 未产生可用标准化结果: eventId=" + ctx.strSyslogUUID); } @@ -1133,7 +1162,9 @@ public class LogNormalProcessor { normalMessage.setCollectTaskId(collect_id); if (collectTask != null) normalMessage.setCollectTaskName(collectTask.getTaskName()); - if (!this.messageService.saveMessage(normalMessage)) { + if (ctx.deferNonNormalPersistence) { + ctx.deferredNonNormalMessages.add(normalMessage); + } else if (!this.messageService.saveMessage(normalMessage)) { throw new IllegalStateException("非标日志服务返回保存失败"); } ctx.isSaveNonNormal = true; diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java index 7bc34d4..f60c1c3 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/Modules/NormalData/SysLogProcessor.java @@ -9,7 +9,9 @@ import com.config.AppProperties; import com.config.KafkaConsumerProperties; import com.common.service.DmStandardizedBatchWriter; import com.common.service.IngressStatsProjectionWriter; +import com.common.service.SyslogNonNormalMessageService; import com.common.entity.IngressStatsEvent; +import com.common.entity.SyslogNonNormalMessage; import com.influxdb.client.write.Point; import com.influx.raw.InfluxRawPointFactory; import com.influx.raw.PointTimeResolver; @@ -71,6 +73,13 @@ public class SysLogProcessor { @Autowired private DmStandardizedBatchWriter standardizedBatchWriter; + @Autowired + private SyslogNonNormalMessageService nonNormalMessageService; + + @Autowired + @Qualifier("nonNormalWriterExecutor") + private Executor nonNormalWriterExecutor; + @Autowired private IngressStatsProjectionWriter ingressStatsProjectionWriter; @@ -83,6 +92,9 @@ public class SysLogProcessor { @Autowired private RawInfluxMetrics rawInfluxMetrics; private final AtomicInteger totalProcessed = new AtomicInteger(0); + private final AtomicLong nonNormalAsyncCompletedTotal = new AtomicLong(); + private final AtomicLong nonNormalAsyncFailedTotal = new AtomicLong(); + private final AtomicLong nonNormalAsyncRejectedTotal = new AtomicLong(); // [SYSLOG-ALARM][KACK][已修复-主链路停用] 历史辅助方法仍保留此计数用于兼容,但唯一活动 listener 不再用全局计数决定 ACK。 private final AtomicInteger currentBatchCount = new AtomicInteger(0); @@ -133,8 +145,9 @@ public class SysLogProcessor { } /** - * 统一批处理边界:原文批量写入 InfluxDB 后,有界并行执行标准化; - * 只有整批全部到达可靠终态才提交 offset。 + * 统一批处理边界:原文批量写入 InfluxDB 后,有界并行执行标准化。 + * 正常消息仍需到达可靠终态才提交 offset;非标消息按现场临时策略异步落库, + * 其写库结果不参与 Kafka ACK。 */ private void consumeBatch(List> records, Acknowledgment ack, boolean enforceRetryDue, String lane) { @@ -180,7 +193,9 @@ public class SysLogProcessor { + "ruleComputeMs={}, ruleSnapshotMs={}, messageParseMs={}, fieldMappingMs={}, " + "filterCompleteMs={}, enrichmentMs={}, ruleAssembleMs={}, compiledRuleHit={}, " + "compiledRuleMiss={}, messageParseHit={}, messageParseMiss={}, deviceUnique={}, " - + "collectTaskUnique={}, shadowCompared={}, shadowMismatch={}, dmWriteMs={}, totalMs={}", + + "collectTaskUnique={}, shadowCompared={}, shadowMismatch={}, dmWriteMs={}, " + + "nonNormalQueued={}, nonNormalRejected={}, nonNormalAsyncCompletedTotal={}, " + + "nonNormalAsyncFailedTotal={}, nonNormalAsyncRejectedTotal={}, totalMs={}", lane, topicPartitions(records), records.size(), preparedMessages.size(), persistedMessages.size(), metrics.success.get(), metrics.filtered.get(), metrics.retry.get(), metrics.dlt.get(), metrics.normalizedRows.get(), metrics.dmSuccessfulRows.get(), metrics.dmFailedMessages.get(), ingressGroups, @@ -190,6 +205,9 @@ public class SysLogProcessor { metrics.compiledRuleMisses(), metrics.messageParseHits(), metrics.messageParseMisses(), metrics.deviceUnique(), metrics.collectTaskUnique(), metrics.shadowCompared(), metrics.shadowDifferences(), metrics.dmWriteMs(), + metrics.nonNormalQueued(), metrics.nonNormalRejected(), + nonNormalAsyncCompletedTotal.get(), nonNormalAsyncFailedTotal.get(), + nonNormalAsyncRejectedTotal.get(), System.currentTimeMillis() - batchStartedAt); } @@ -492,15 +510,18 @@ public class SysLogProcessor { metrics.recordRuleExecution(executionMetrics, enrichmentContext); List> writeRequests = new ArrayList<>(); + List nonNormalMessages = new ArrayList<>(); Set pendingDatabaseMessages = new HashSet<>(); + int filteredMessages = 0; for (CompletableFuture future : futures) { PreparedNormalization prepared = future.join(); if (prepared.terminalState != null) { metrics.record(prepared.terminalState); continue; } + nonNormalMessages.addAll(prepared.result.getNonNormalMessages()); if (prepared.result.getOutcome() == LogNormalProcessor.ProcessingOutcome.FILTERED) { - metrics.record(SyslogFailureRouter.TerminalState.FILTERED); + filteredMessages++; continue; } @@ -528,6 +549,14 @@ public class SysLogProcessor { metrics.record(SyslogFailureRouter.TerminalState.SUCCESS); } } + for (int i = 0; i < filteredMessages; i++) { + metrics.record(SyslogFailureRouter.TerminalState.FILTERED); + } + + // 正常链路先达到可靠终态,再把非标日志交给独立单线程,避免正常写入失败重试时 + // 提前制造非标重复数据。这里只提交任务,不等待结果,非标写入不参与 Kafka ACK。 + // 队列满、写库失败或进程异常退出都可能丢非标数据,这是现场选择的性能优先策略。 + submitNonNormalMessages(nonNormalMessages, metrics); } catch (TimeoutException timeout) { cancelOutstanding(futures); throw new IllegalStateException("标准化批次处理超时,禁止提交 offset", timeout); @@ -544,6 +573,48 @@ public class SysLogProcessor { } } + /** + * 把非标批次交给独立线程后立即返回。任何提交或执行失败只记录,不传播到 Kafka ACK 边界。 + */ + private void submitNonNormalMessages(List messages, BatchMetrics metrics) { + if (messages.isEmpty()) { + return; + } + List batch = new ArrayList<>(messages); + try { + nonNormalWriterExecutor.execute(() -> { + long startedAt = System.nanoTime(); + try { + int inserted = nonNormalMessageService.saveBatchMessages(batch); + if (inserted != batch.size()) { + nonNormalAsyncFailedTotal.addAndGet(batch.size()); + log.error("非标日志异步批写结果异常,Kafka offset 不回退: rows={}, inserted={}, " + + "durationMs={}, failedTotal={}", + batch.size(), inserted, elapsedMillis(startedAt), + nonNormalAsyncFailedTotal.get()); + return; + } + nonNormalAsyncCompletedTotal.addAndGet(inserted); + log.debug("非标日志异步批写完成: rows={}, durationMs={}, completedTotal={}", + inserted, elapsedMillis(startedAt), nonNormalAsyncCompletedTotal.get()); + } catch (Exception failure) { + nonNormalAsyncFailedTotal.addAndGet(batch.size()); + log.error("非标日志异步批写失败,Kafka offset 不回退,当前非标数据可能丢失: " + + "rows={}, durationMs={}, failedTotal={}", + batch.size(), elapsedMillis(startedAt), + nonNormalAsyncFailedTotal.get(), failure); + } + }); + metrics.recordNonNormalSubmission(batch.size(), 0); + } catch (RuntimeException rejected) { + nonNormalAsyncRejectedTotal.addAndGet(batch.size()); + metrics.recordNonNormalSubmission(0, batch.size()); + log.error("非标日志异步队列提交失败,当前非标数据已丢弃且不阻塞 Kafka ACK: " + + "rows={}, rejectedTotal={}", + batch.size(), nonNormalAsyncRejectedTotal.get(), rejected); + } + } + private static long elapsedMillis(long startedAtNanos) { return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAtNanos); } @@ -681,6 +752,8 @@ public class SysLogProcessor { private final AtomicLong ruleComputeMs = new AtomicLong(); private final AtomicLong ruleSnapshotMs = new AtomicLong(); private final AtomicLong dmWriteMs = new AtomicLong(); + private final AtomicInteger nonNormalQueued = new AtomicInteger(); + private final AtomicInteger nonNormalRejected = new AtomicInteger(); private volatile RuleExecutionMetrics ruleExecutionMetrics = new RuleExecutionMetrics(); private volatile int deviceUnique; private volatile int collectTaskUnique; @@ -703,6 +776,10 @@ public class SysLogProcessor { return success.get(); } + int filteredCount() { + return filtered.get(); + } + long ruleComputeMs() { return ruleComputeMs.get(); } @@ -711,6 +788,9 @@ public class SysLogProcessor { return dmWriteMs.get(); } + int nonNormalQueued() { return nonNormalQueued.get(); } + int nonNormalRejected() { return nonNormalRejected.get(); } + long ruleSnapshotMs() { return ruleSnapshotMs.get(); } long messageParseMs() { return ruleExecutionMetrics.getMessageParseMs(); } long fieldMappingMs() { return ruleExecutionMetrics.getFieldMappingMs(); } @@ -747,6 +827,11 @@ public class SysLogProcessor { dmWriteMs.set(Math.max(0L, elapsedMs)); } + void recordNonNormalSubmission(int queued, int rejected) { + nonNormalQueued.addAndGet(Math.max(0, queued)); + nonNormalRejected.addAndGet(Math.max(0, rejected)); + } + /** 仅用于批次汇总观测,不参与 ACK 判断;ACK 仍由每条消息的可靠终态决定。 */ void recordDmWrite(int rows, int successfulRows, int failedMessages) { this.normalizedRows.addAndGet(rows); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java index 5e3bc32..1232e47 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalAlarmMapper.java @@ -15,21 +15,16 @@ import java.util.Map; public interface SyslogNormalAlarmMapper { /** - * 动态插入数据 - 使用 Map + * 使用当前小批次的字段并集模板插入告警日志。调用方保证同批 Map 具有相同键集合, + * 缺失值绑定为 NULL,使 MyBatis ExecutorType.BATCH 能复用 PreparedStatement。 */ - int insertDynamic(@Param("dataMap") Map dataMap); + int insertBatchPlanned(@Param("dataMap") Map dataMap); /** * 动态插入数据 - 使用实体类 */ int insertByEntity(SyslogNormalAlarm entity); - /** - * 批量插入数据 - */ - int batchInsert(@Param("dataList") List> dataList); - - /** * 统计24小时内分组数据量 */ diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java index 3a64ed5..d65c29d 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/mapper/SyslogNormalDataMapper.java @@ -12,21 +12,16 @@ import java.util.List; public interface SyslogNormalDataMapper { /** - * 动态插入数据 - 使用 Map + * 使用当前小批次的字段并集模板插入普通日志。调用方保证同批 Map 具有相同键集合, + * 缺失值绑定为 NULL,使 MyBatis ExecutorType.BATCH 能复用 PreparedStatement。 */ - int insertDynamic(@Param("dataMap") Map dataMap); + int insertBatchPlanned(@Param("dataMap") Map dataMap); /** * 动态插入数据 - 使用实体类 */ int insertByEntity(SyslogNormalData entity); - /** - * 批量插入数据 - */ - int batchInsert(@Param("dataList") java.util.List> dataList); - - /** * 统计24小时内分组数据量 */ diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java index deacf5d..ea560a3 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchExecutor.java @@ -13,13 +13,18 @@ import org.springframework.transaction.support.TransactionSynchronizationAdapter import org.springframework.transaction.support.TransactionSynchronizationManager; import java.util.List; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.Map; +import java.util.Set; /** * 在一个达梦小事务中执行真正的 MyBatis JDBC batch。 * *

这个类只负责数据库执行,不负责错误分类、拆分或 Kafka 状态。上层写入器保证传入的所有行 - * 属于同一张表和同一个动态列签名,因此 MyBatis BatchExecutor 可以复用同一个 PreparedStatement。 + * 普通日志和告警日志分别使用当前小批次的非空字段并集模板,因此同一目标表、同一分区日期内 + * MyBatis BatchExecutor 可以复用同一个 PreparedStatement。 * 每个 {@link #execute(boolean, List, int)} 调用都会创建独立事务;失败时整个小批次回滚,上层随后 * 决定二分定位还是按系统故障停止。

*/ @@ -63,7 +68,7 @@ public class DmStandardizedBatchExecutor { * 执行一个同构小批次。 * * @param alarmTable true 写 syslog_normal_alarm,false 写 syslog_normal_data - * @param rows 已按相同动态列集合分组的行 + * @param rows 已按相同目标表和分区日期分组的行 * @param transactionTimeoutSeconds 当前小事务硬超时 */ public void execute(boolean alarmTable, List> rows, @@ -72,10 +77,11 @@ public class DmStandardizedBatchExecutor { return; } + List> alignedRows = alignBatchColumns(rows); int attempt = 0; while (true) { try { - executeTransaction(alarmTable, rows, transactionTimeoutSeconds, attempt); + executeTransaction(alarmTable, alignedRows, transactionTimeoutSeconds, attempt); return; } catch (DirectStatsProjectionWriter.ConcurrentStatsWriteException concurrentFailure) { if (attempt >= Math.max(0, maxConcurrencyRetries)) { @@ -89,6 +95,39 @@ public class DmStandardizedBatchExecutor { } } + /** + * 计算当前小批次所有非空字段的并集,并为每行建立相同键集合的浅拷贝。 + * + *

Mapper XML 只允许声明过的列,额外业务键即使出现在 Map 中也不会进入 SQL。 + * 同一键在某行不存在时,Map#get 返回 null,最终由 JDBC 按 SQL NULL 绑定。

+ */ + static List> alignBatchColumns(List> rows) { + Set columnUnion = new LinkedHashSet<>(); + for (Map row : rows) { + if (row == null) { + throw new IllegalArgumentException("达梦批写行不能为空"); + } + for (Map.Entry entry : row.entrySet()) { + if (entry.getKey() != null && entry.getValue() != null) { + columnUnion.add(entry.getKey()); + } + } + } + if (columnUnion.isEmpty()) { + throw new IllegalArgumentException("达梦批写没有可插入的非空字段"); + } + + List> alignedRows = new ArrayList<>(rows.size()); + for (Map row : rows) { + Map aligned = new LinkedHashMap<>(columnUnion.size()); + for (String column : columnUnion) { + aligned.put(column, row.get(column)); + } + alignedRows.add(aligned); + } + return alignedRows; + } + private void executeTransaction(boolean alarmTable, List> rows, int transactionTimeoutSeconds, int retries) { @@ -112,12 +151,12 @@ public class DmStandardizedBatchExecutor { SyslogNormalAlarmMapper mapper = batchSqlSession.getMapper(SyslogNormalAlarmMapper.class); for (Map row : rows) { // BatchExecutor 下这里不会逐条提交;返回值也不是最终影响行数,不能据此判断成功。 - mapper.insertDynamic(row); + mapper.insertBatchPlanned(row); } } else { SyslogNormalDataMapper mapper = batchSqlSession.getMapper(SyslogNormalDataMapper.class); for (Map row : rows) { - mapper.insertDynamic(row); + mapper.insertBatchPlanned(row); } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java index f568935..6b0a947 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/DmStandardizedBatchWriter.java @@ -32,8 +32,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; @@ -48,7 +46,7 @@ import java.util.concurrent.atomic.AtomicInteger; *

写入器遵守三条不能破坏的可靠性规则:

* *
    - *
  1. 正常行按目标表、分区日期和动态列签名分组后批量提交;
  2. + *
  3. 普通日志和告警日志分别按目标表、分区日期计算批次字段并集后批量提交;
  4. *
  5. 数据类错误回滚当前小批次并有限二分,最终只把坏行交给 Retry/DLT;
  6. *
  7. 连接、超时、提交未知等系统故障绝不降级为大量单条请求,而是立即失败并保留 Kafka offset。
  8. *
@@ -67,7 +65,7 @@ public class DmStandardizedBatchWriter { @Value("${dm.standardized-write.mode:BATCH}") private String mode; - @Value("${dm.standardized-write.batch-size:100}") + @Value("${dm.standardized-write.batch-size:500}") private int batchSize; @Value("${dm.standardized-write.transaction-timeout-seconds:45}") @@ -161,7 +159,7 @@ public class DmStandardizedBatchWriter { } /** - * 相同目标表、log_time 日期和非空列集合才会进入同一批次。 + * 两张标准化表在每个小批次计算字段并集,只按目标表和 log_time 日期分组。 * LinkedHashMap 保持稳定遍历顺序,便于测试和现场日志复现。 */ private void writeGroupedBatches(List> requests, WriteReport report, @@ -173,7 +171,7 @@ public class DmStandardizedBatchWriter { groups.computeIfAbsent(key, ignored -> new ArrayList<>()).add(request); } catch (RuntimeException invalidGroupingKey) { report.recordFailure(request.getSource(), new PermanentRowWriteException( - "标准化结果缺少稳定分区时间或列签名无效", invalidGroupingKey)); + "标准化结果缺少稳定分区时间", invalidGroupingKey)); } } @@ -294,18 +292,7 @@ public class DmStandardizedBatchWriter { boolean alarmTable = normalDataService.isAlarmData(data); LocalDate partitionDate = partitionDate(data.get("log_time")); - // MyBatis XML 的动态列判断只包含非 null 值,所以签名也只记录非 null 列。 - // TreeSet 提供固定顺序,避免 HashMap 插入顺序不同却被错误拆成多个批次。 - Set columns = new TreeSet<>(); - for (Map.Entry entry : data.entrySet()) { - if (entry.getValue() != null) { - columns.add(entry.getKey()); - } - } - if (columns.isEmpty()) { - throw new IllegalArgumentException("标准化结果没有可写入列"); - } - return new BatchKey(alarmTable, partitionDate, String.join(",", columns)); + return new BatchKey(alarmTable, partitionDate); } private LocalDate partitionDate(Object value) { @@ -499,12 +486,10 @@ public class DmStandardizedBatchWriter { private static final class BatchKey { private final boolean alarmTable; private final LocalDate partitionDate; - private final String columnSignature; - private BatchKey(boolean alarmTable, LocalDate partitionDate, String columnSignature) { + private BatchKey(boolean alarmTable, LocalDate partitionDate) { this.alarmTable = alarmTable; this.partitionDate = partitionDate; - this.columnSignature = columnSignature; } @Override @@ -513,13 +498,12 @@ public class DmStandardizedBatchWriter { if (!(other instanceof BatchKey)) return false; BatchKey that = (BatchKey) other; return alarmTable == that.alarmTable - && Objects.equals(partitionDate, that.partitionDate) - && Objects.equals(columnSignature, that.columnSignature); + && Objects.equals(partitionDate, that.partitionDate); } @Override public int hashCode() { - return Objects.hash(alarmTable, partitionDate, columnSignature); + return Objects.hash(alarmTable, partitionDate); } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java index 91f9921..a679909 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/SyslogNormalDataService.java @@ -15,6 +15,7 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.List; +import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; import java.time.LocalDateTime; @@ -35,21 +36,22 @@ public class SyslogNormalDataService { /** - * 动态插入单条数据 - Map方式 + * 标准化单条兼容入口:按 event_level 路由后,两张目标表均使用受白名单约束的动态列模板。 */ @Transactional public int insertDynamic(Map dataMap) { // [流程节点7] 根据标准化结果分类落库:安全事件写入 syslog_normal_alarm,访问事件写入 syslog_normal_data。 // [SYSLOG-ALARM][DBBATCH] 每条标准化结果单独开启事务并执行动态 INSERT,会产生大量数据库往返、SQL 解析和提交开销; - // 高峰期 DB 变慢又会放大 Kafka 批次超时。建议按目标表和固定列集合聚合批次写入,并用 event_id + 规则版本唯一键保证重试幂等。 + // 高峰期 DB 变慢又会放大 Kafka 批次超时。建议按目标表和批次字段并集聚合写入,并用 event_id + 规则版本唯一键保证重试幂等。 prepareForInsert(dataMap); // [流程节点7.2] event_level 非空走安全事件表;为空走访问事件表,这是两条后续分析链路的分叉点。 boolean alarmData = isAlarmData(dataMap); + Map plannedData = nonNullColumnMap(dataMap); try { int inserted = alarmData - ? syslogNormalAlarmMapper.insertDynamic(dataMap) - : syslogNormalDataMapper.insertDynamic(dataMap); + ? syslogNormalAlarmMapper.insertBatchPlanned(plannedData) + : syslogNormalDataMapper.insertBatchPlanned(plannedData); if (inserted > 0) { if (alarmData && lateAlarmBucketWriter != null) { lateAlarmBucketWriter.record(java.util.Collections.singletonList(dataMap)); @@ -72,6 +74,17 @@ public class SyslogNormalDataService { } } + /** 单条兼容路径也只把实际非空字段交给列模板,避免原 Map 中的空键误触发列选择。 */ + private Map nonNullColumnMap(Map dataMap) { + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : dataMap.entrySet()) { + if (entry.getKey() != null && entry.getValue() != null) { + result.put(entry.getKey(), entry.getValue()); + } + } + return result; + } + /** * 在单条和批量写入之前执行完全相同的数据准备与关键字段校验。 * @@ -220,9 +233,8 @@ public class SyslogNormalDataService { data.put("log_time", new java.util.Date()); } } - // 旧 mapper 的 batchInsert 只写固定的少数字段,DIRECT 下会造成“统计有 device/rule, - // 事实行却丢了 device/rule”的永久差异。主 Kafka 链路使用 DmStandardizedBatchExecutor; - // 这个低频兼容 API 逐条复用完整动态 INSERT,但仍处于本方法的同一个 Spring 事务中。 + // 主 Kafka 链路使用 DmStandardizedBatchExecutor 执行真正的 JDBC batch;这个低频兼容 + // API 在同一个 Spring 事务内复用受白名单约束的列模板,不能再调用旧的五字段 batch SQL。 int inserted = 0; for (Map data : dataList) { inserted += insertDynamic(data); diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java index 6bd33f3..c49fc72 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/impl/SyslogNonNormalMessageServiceImpl.java @@ -11,12 +11,14 @@ import com.common.service.StatsChangeCaptureService; import com.common.service.DirectStatsProjectionWriter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.ZoneOffset; +import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -38,6 +40,9 @@ public class SyslogNonNormalMessageServiceImpl @Autowired private DirectStatsProjectionWriter directStatsProjectionWriter; + @Value("${dm.non-normal-write.batch-size:500}") + private int nonNormalBatchSize; + @Override @Transactional(rollbackFor = Exception.class) public boolean saveMessage(SyslogNonNormalMessage message) { @@ -125,8 +130,20 @@ public class SyslogNonNormalMessageServiceImpl } } - // 批量插入 - int inserted = messageMapper.batchInsert(messages); + // 同一个服务事务内分块执行固定列多行 INSERT,避免单条提交,也避免一个 Kafka poll + // 直接生成过大的 SQL。所有分块成功后才统一写 DIRECT 统计并提交事务。 + int inserted = 0; + int effectiveBatchSize = Math.max(1, nonNormalBatchSize); + for (int start = 0; start < messages.size(); start += effectiveBatchSize) { + int end = Math.min(start + effectiveBatchSize, messages.size()); + int affected = messageMapper.batchInsert( + new ArrayList<>(messages.subList(start, end))); + if (affected <= 0) { + throw new IllegalStateException("非标日志批量 INSERT 未写入记录: start=" + + start + ", end=" + end); + } + inserted += affected; + } if (inserted > 0) { if (statsChangeCaptureService.isEnabled()) { for (SyslogNonNormalMessage message : messages) { diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java index dd5fedd..a57e542 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java +++ b/haobang-security-dm/syslog-consumer-rule/src/main/java/com/config/ThreadPoolConfig.java @@ -31,6 +31,9 @@ public class ThreadPoolConfig { @Value("${dm.standardized-write.writer-queue-capacity:100}") private int dmWriterQueueCapacity; + @Value("${dm.non-normal-write.queue-capacity:500}") + private int nonNormalWriterQueueCapacity; + @Bean("logNormalProcessorExecutor") @Primary public Executor logNormalProcessorExecutor() { @@ -71,4 +74,22 @@ public class ThreadPoolConfig { executor.initialize(); return executor; } + + /** + * 非标日志使用单独的串行写线程,与正常事实批写解耦。 + * listener 不等待该线程,队列溢出或异步写入失败只记录错误,不阻塞 Kafka ACK。 + */ + @Bean("nonNormalWriterExecutor") + public Executor nonNormalWriterExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(1); + executor.setMaxPoolSize(1); + executor.setQueueCapacity(Math.max(1, nonNormalWriterQueueCapacity)); + executor.setThreadNamePrefix("dm-non-normal-writer-"); + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy()); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(60); + executor.initialize(); + return executor; + } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties index efd1c5a..1943d00 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/application.properties @@ -56,8 +56,11 @@ mybatis.type-aliases-package=com.common.entity mybatis.configuration.default-statement-timeout=30 mybatis.configuration.default-fetch-size=1000 mybatis.configuration.map-underscore-to-camel-case=true +# 批次列并集写入会为部分行绑定空字段;达梦对未指定类型的 NULL 参数兼容性较差,统一使用 JDBC NULL。 +mybatis.configuration.jdbc-type-for-null=NULL mybatis.type-handlers-package=com.Modules.etl.handler mybatis-plus.configuration.map-underscore-to-camel-case=true +mybatis-plus.configuration.jdbc-type-for-null=NULL mybatis-plus.type-handlers-package=com.Modules.etl.handler # kafka Configuration @@ -142,10 +145,11 @@ app.processor.batch-size=100 app.processor.process-timeout-ms=120000 # 达梦标准化表默认启用批量写入;现场需要紧急回退时可显式切换为 SINGLE。 -# BATCH 模式按目标表、log_time 日期和动态列集合分组,失败时只对数据类错误有限二分; +# BATCH 模式按目标表和 log_time 日期分组,按小批次实际非空字段并集执行 JDBC Batch; +# 失败时只对数据类错误有限二分; # 连接、超时或提交结果未知会立即停止并保留 Kafka offset,禁止放大为逐条请求风暴。 dm.standardized-write.mode=BATCH -dm.standardized-write.batch-size=100 +dm.standardized-write.batch-size=${DM_STANDARDIZED_WRITE_BATCH_SIZE:500} dm.standardized-write.writer-concurrency=2 dm.standardized-write.writer-queue-capacity=100 dm.standardized-write.transaction-timeout-seconds=45 @@ -154,6 +158,8 @@ dm.standardized-write.max-split-depth=10 dm.standardized-write.max-fallback-operations=100 dm.standardized-write.circuit-breaker-failure-threshold=3 dm.standardized-write.circuit-breaker-backoff-ms=30000 +dm.non-normal-write.batch-size=${DM_NON_NORMAL_WRITE_BATCH_SIZE:500} +dm.non-normal-write.queue-capacity=${DM_NON_NORMAL_WRITE_QUEUE_CAPACITY:500} # ETL配置 etl.batch.page-size=1000 diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml index 9e84f53..ee8f6cf 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml @@ -4,756 +4,759 @@ - + + INSERT INTO syslog_normal_alarm - id, - created_at, - log_time, - device_id, - webshell_type, - vuirs_type, - vuirs_url, - class_filename, - class_path, - parent_class, - jar_path, - class_md5, - class_loader, - class_hashcode, - class_loader_hashcode, - tc_nameip, - perform_sql, - tc_account, - tc_appname, - process_uname, - p_process_uname, - container_name, - container_id, - http_resp_server, - srcip_id, - cdnip, - natip, - mail_sender, - mail_receiver, - vpn_mac, - vpn_os, - vpn_user, - vpn_groupname, - vpn_access_ip, - dest_ip_apt, - origin_attack_result, - description, - solution, - attack_cause, - username, - tc_flow_id, - login_result, - cmdline, - origin_attack_action, - victim_domain, - vpn_deviceid, - vpn_access_action, - file_access_time, - file_name, - tc_class, - tc_name2, - login_lasttime, - origin_permissions, - begin_permissions, - print_time, - printer, - printer_type, - print_pages, - print_copies, - src_device, - dst_device, - src_file, - src_file_type, - src_file_path, - dst_file, - dst_file_type, - dst_file_path, - dlp_policy_name, - dlp_policy_type, - dst_upload_url, - process_uuid, - p_process_uuid, - env, - brute_force_service, - vuirs_name, - http_req_length, - http_req_content_type, - tc_scan_port, - tc_labels, - http_resp_content_type, - dns_msg_type, - dns_answer_length, - dns_ioc, - tx_bytes, - rx_bytes, - all_bytes, - duration_time, - mail_attach_name, - mail_subject, - mail_message, - mail_send_server, - mail_agent, - tls_version, - tls_server_cert, - tls_server_suite, - tls_client_suites_len, - tls_ja3, - tls_ja3s, - vpn_access_port, - log_topic, - collect_time, - src_is_intranetip, - src_ip_ioc, - src_ip_apt, - srcip_name, - tc_client, - srcip_organization_id, - dest_ip_intranetip, - dest_ip_ioc, - desip_id, - desip_name, - tc_hostip, - desip_organization_id, - origin_confidence, - origin_malscore, - attacker_icampaign, - attacker_host_asset_id, - attacker_organization_id, - victim_host_asset_id, - victim_organization_id, - logout_time, - http_req_line, - desip_security_scope_id, - srcip_security_scope_id, - http_resp_length, - tc_attack_type, - tc_realip, - attacker_ip_lists, - login_password, - detail, - attacker_country_code, - attacker_region_code, - victim_region_code, - payload, - http_referer, - http_user_agent, - http_session, - http_query_string, - file_path, - file_permission, - login_abnormal_type, - file_tag, - file_platform, - target_ip, - collect_date, - tc_client_ip, - tc_server_ip, - tc_externalip, - http_status_code, - device_domian, - src_ip_str, - src_port_str, - dest_ip_str, - dest_port_str, - pcap, - ioc, - malicious_family, - vuln_cve, - aliyun_type, - attacker_host_asset_name, - attacker_organization_name, - ct_id, - cve_list, - desip_organization_name, - dest_ip_group, - file_gid, - file_owner, - file_ownergroup, - file_uid, - http_resp_cookie, - origin_rule_id, - origin_rule_name, - service_name, - src_ip_asset_group, - srcip_organization_name, - victim_host_asset_name, - http_resp_codes, - victim_organization_name, - tc_type, - direction, - http_req_cookie, - http_req_protocol, - http_req_header_raw, - http_url, - uname, - origin_hostname, - origin_os, - origin_agent_mac, - origin_host_id, - origin_agent_version, - origin_agent_id, - origin_agent_name, - origin_work_group, - origin_asset_group, - origin_local_port, - origin_agent_ip, - origin_internal_ip, - origin_external_ip, - origin_local_addr, - agent_id, - agent_name, - tc_title, - log_id, - event_date, - event_time_ts, - event_level, - src_ip , - src_port, - dest_ip, - dest_port, - event_time, - attacker_country, - src_mac, - dest_mac, - proto, - dev_id, - created_time, - src_country, - src_country_code, - src_region, - src_region_code, - src_city, - src_lon, - http_method, - http_host, - http_req_header, - http_req_body, - http_resp_header, - http_resp_body, - file_type, - file_md5, - file_size, - process, - start_time, - action, - attacker_region, - end_time, - file_created_time, - file_modified_time, - tc_miguan_scan_port, - process_path, - parent_process_path, - gname, - exe_name, - exe_path, - login_time, - login_times, - check_item, - check_type, - attacker_ip, - attacker_port, - victim_ip, - victim_port, - attacker_city, - attacker_lon, - attacker_lat, - victim_country, - victim_region, - victim_city, - victim_lon, - victim_lat, - origin_event_id, - origin_event_name, - origin_event_category, - origin_event_level, - origin_attack_chain, - engine_type, - evil_payload, - http_resp_status, - dns_query, - dns_query_type, - dns_ttl, - dns_answer, - dns_subdomains, - file_sha256, - file_ssdeep, - victim_country_code, - http_xff_ip, - tc_miguan_class, - pid, - ppid, - process_name, - backdoor_type, - tty, - sudo_user, - sudo_group, - origin_event_type, - dest_domain, - shell_cmdline, - parent_cmdline, - attack_chain, - process_tree, - host_file_sha256, - host_file_md5, - host_file_size, - host_file_type, - dest_country, - dest_country_code, - log_origin, - dest_region, - src_lat, - dest_region_code, - dest_city, - dest_lon, - dest_lat, - event_category, - attack_result, - probe_ip, - device_ip, - device_manufacturer, - device_name, - product_name, - __id, - __count, - __count_reason, - event_type, - protocol, - shell_cmd, - parent_name, - host_file_path, - uid, - fall, - tc_miguan_server_ip, - dev_type, - collect_method, - field_cate_id, - device_type, - tc_miguan_client_ip, - tc_miguan_name, - origin_total_packages, - origin_total_bytes, - origin_peak_packages_rate, - origin_peak_bytes_rate, - origin_peak_flows_rate, - apt_orgname, - apt_orgmsg, - mail_message_id, - mail_bcc, - mail_size, - mail_attach_hashcode, - mail_url, - mail_cc, - algorithm, - miningpool_ip, - process_md5, - pprocess_md5, - source_servername, - origin_source_servername, - mail_filename, - dst_upload_appname, - target_port, - gid, - origin_uid, - origin_gid, - target_ports, - tc_miguan_name1, - tc_miguan_class1, - etl_time, - tc_miguan_scan_port2, - desip_security_scope, - srcip_security_scope, - collect_time_ts, - tc_miguan_scan_port1, - src_dev_name, - collect_protocol, - destination_system_type, - destination_system, - etl_host, - normalize_rule_id, - normalize_rule_name, - syslog_uuid, - syslog_topic, + id, + created_at, + log_time, + device_id, + webshell_type, + vuirs_type, + vuirs_url, + class_filename, + class_path, + parent_class, + jar_path, + class_md5, + class_loader, + class_hashcode, + class_loader_hashcode, + tc_nameip, + perform_sql, + tc_account, + tc_appname, + process_uname, + p_process_uname, + container_name, + container_id, + http_resp_server, + srcip_id, + cdnip, + natip, + mail_sender, + mail_receiver, + vpn_mac, + vpn_os, + vpn_user, + vpn_groupname, + vpn_access_ip, + dest_ip_apt, + origin_attack_result, + description, + solution, + attack_cause, + username, + tc_flow_id, + login_result, + cmdline, + origin_attack_action, + victim_domain, + vpn_deviceid, + vpn_access_action, + file_access_time, + file_name, + tc_class, + tc_name2, + login_lasttime, + origin_permissions, + begin_permissions, + print_time, + printer, + printer_type, + print_pages, + print_copies, + src_device, + dst_device, + src_file, + src_file_type, + src_file_path, + dst_file, + dst_file_type, + dst_file_path, + dlp_policy_name, + dlp_policy_type, + dst_upload_url, + process_uuid, + p_process_uuid, + env, + brute_force_service, + vuirs_name, + http_req_length, + http_req_content_type, + tc_scan_port, + tc_labels, + http_resp_content_type, + dns_msg_type, + dns_answer_length, + dns_ioc, + tx_bytes, + rx_bytes, + all_bytes, + duration_time, + mail_attach_name, + mail_subject, + mail_message, + mail_send_server, + mail_agent, + tls_version, + tls_server_cert, + tls_server_suite, + tls_client_suites_len, + tls_ja3, + tls_ja3s, + vpn_access_port, + log_topic, + collect_time, + src_is_intranetip, + src_ip_ioc, + src_ip_apt, + srcip_name, + tc_client, + srcip_organization_id, + dest_ip_intranetip, + dest_ip_ioc, + desip_id, + desip_name, + tc_hostip, + desip_organization_id, + origin_confidence, + origin_malscore, + attacker_icampaign, + attacker_host_asset_id, + attacker_organization_id, + victim_host_asset_id, + victim_organization_id, + logout_time, + http_req_line, + desip_security_scope_id, + srcip_security_scope_id, + http_resp_length, + tc_attack_type, + tc_realip, + attacker_ip_lists, + login_password, + detail, + attacker_country_code, + attacker_region_code, + victim_region_code, + payload, + http_referer, + http_user_agent, + http_session, + http_query_string, + file_path, + file_permission, + login_abnormal_type, + file_tag, + file_platform, + target_ip, + collect_date, + tc_client_ip, + tc_server_ip, + tc_externalip, + http_status_code, + device_domian, + src_ip_str, + src_port_str, + dest_ip_str, + dest_port_str, + pcap, + ioc, + malicious_family, + vuln_cve, + aliyun_type, + attacker_host_asset_name, + attacker_organization_name, + ct_id, + cve_list, + desip_organization_name, + dest_ip_group, + file_gid, + file_owner, + file_ownergroup, + file_uid, + http_resp_cookie, + origin_rule_id, + origin_rule_name, + service_name, + src_ip_asset_group, + srcip_organization_name, + victim_host_asset_name, + http_resp_codes, + victim_organization_name, + tc_type, + direction, + http_req_cookie, + http_req_protocol, + http_req_header_raw, + http_url, + uname, + origin_hostname, + origin_os, + origin_agent_mac, + origin_host_id, + origin_agent_version, + origin_agent_id, + origin_agent_name, + origin_work_group, + origin_asset_group, + origin_local_port, + origin_agent_ip, + origin_internal_ip, + origin_external_ip, + origin_local_addr, + agent_id, + agent_name, + tc_title, + log_id, + event_date, + event_time_ts, + event_level, + src_ip , + src_port, + dest_ip, + dest_port, + event_time, + attacker_country, + src_mac, + dest_mac, + proto, + dev_id, + created_time, + src_country, + src_country_code, + src_region, + src_region_code, + src_city, + src_lon, + http_method, + http_host, + http_req_header, + http_req_body, + http_resp_header, + http_resp_body, + file_type, + file_md5, + file_size, + process, + start_time, + action, + attacker_region, + end_time, + file_created_time, + file_modified_time, + tc_miguan_scan_port, + process_path, + parent_process_path, + gname, + exe_name, + exe_path, + login_time, + login_times, + check_item, + check_type, + attacker_ip, + attacker_port, + victim_ip, + victim_port, + attacker_city, + attacker_lon, + attacker_lat, + victim_country, + victim_region, + victim_city, + victim_lon, + victim_lat, + origin_event_id, + origin_event_name, + origin_event_category, + origin_event_level, + origin_attack_chain, + engine_type, + evil_payload, + http_resp_status, + dns_query, + dns_query_type, + dns_ttl, + dns_answer, + dns_subdomains, + file_sha256, + file_ssdeep, + victim_country_code, + http_xff_ip, + tc_miguan_class, + pid, + ppid, + process_name, + backdoor_type, + tty, + sudo_user, + sudo_group, + origin_event_type, + dest_domain, + shell_cmdline, + parent_cmdline, + attack_chain, + process_tree, + host_file_sha256, + host_file_md5, + host_file_size, + host_file_type, + dest_country, + dest_country_code, + log_origin, + dest_region, + src_lat, + dest_region_code, + dest_city, + dest_lon, + dest_lat, + event_category, + attack_result, + probe_ip, + device_ip, + device_manufacturer, + device_name, + product_name, + __id, + __count, + __count_reason, + event_type, + protocol, + shell_cmd, + parent_name, + host_file_path, + uid, + fall, + tc_miguan_server_ip, + dev_type, + collect_method, + field_cate_id, + device_type, + tc_miguan_client_ip, + tc_miguan_name, + origin_total_packages, + origin_total_bytes, + origin_peak_packages_rate, + origin_peak_bytes_rate, + origin_peak_flows_rate, + apt_orgname, + apt_orgmsg, + mail_message_id, + mail_bcc, + mail_size, + mail_attach_hashcode, + mail_url, + mail_cc, + algorithm, + miningpool_ip, + process_md5, + pprocess_md5, + source_servername, + origin_source_servername, + mail_filename, + dst_upload_appname, + target_port, + gid, + origin_uid, + origin_gid, + target_ports, + tc_miguan_name1, + tc_miguan_class1, + etl_time, + tc_miguan_scan_port2, + desip_security_scope, + srcip_security_scope, + collect_time_ts, + tc_miguan_scan_port1, + src_dev_name, + collect_protocol, + destination_system_type, + destination_system, + etl_host, + normalize_rule_id, + normalize_rule_name, + syslog_uuid, + syslog_topic, + rule_version, VALUES - #{dataMap.id}, - #{dataMap.created_at}, - #{dataMap.log_time}, - #{dataMap.device_id}, - #{dataMap.webshell_type}, - #{dataMap.vuirs_type}, - #{dataMap.vuirs_url}, - #{dataMap.class_filename}, - #{dataMap.class_path}, - #{dataMap.parent_class}, - #{dataMap.jar_path}, - #{dataMap.class_md5}, - #{dataMap.class_loader}, - #{dataMap.class_hashcode}, - #{dataMap.class_loader_hashcode}, - #{dataMap.tc_nameip}, - #{dataMap.perform_sql}, - #{dataMap.tc_account}, - #{dataMap.tc_appname}, - #{dataMap.process_uname}, - #{dataMap.p_process_uname}, - #{dataMap.container_name}, - #{dataMap.container_id}, - #{dataMap.http_resp_server}, - #{dataMap.srcip_id}, - #{dataMap.cdnip}, - #{dataMap.natip}, - #{dataMap.mail_sender}, - #{dataMap.mail_receiver}, - #{dataMap.vpn_mac}, - #{dataMap.vpn_os}, - #{dataMap.vpn_user}, - #{dataMap.vpn_groupname}, - #{dataMap.vpn_access_ip}, - #{dataMap.dest_ip_apt}, - #{dataMap.origin_attack_result}, - #{dataMap.description}, - #{dataMap.solution}, - #{dataMap.attack_cause}, - #{dataMap.username}, - #{dataMap.tc_flow_id}, - #{dataMap.login_result}, - #{dataMap.cmdline}, - #{dataMap.origin_attack_action}, - #{dataMap.victim_domain}, - #{dataMap.vpn_deviceid}, - #{dataMap.vpn_access_action}, - #{dataMap.file_access_time}, - #{dataMap.file_name}, - #{dataMap.tc_class}, - #{dataMap.tc_name2}, - #{dataMap.login_lasttime}, - #{dataMap.origin_permissions}, - #{dataMap.begin_permissions}, - #{dataMap.print_time}, - #{dataMap.printer}, - #{dataMap.printer_type}, - #{dataMap.print_pages}, - #{dataMap.print_copies}, - #{dataMap.src_device}, - #{dataMap.dst_device}, - #{dataMap.src_file}, - #{dataMap.src_file_type}, - #{dataMap.src_file_path}, - #{dataMap.dst_file}, - #{dataMap.dst_file_type}, - #{dataMap.dst_file_path}, - #{dataMap.dlp_policy_name}, - #{dataMap.dlp_policy_type}, - #{dataMap.dst_upload_url}, - #{dataMap.process_uuid}, - #{dataMap.p_process_uuid}, - #{dataMap.env}, - #{dataMap.brute_force_service}, - #{dataMap.vuirs_name}, - #{dataMap.http_req_length}, - #{dataMap.http_req_content_type}, - #{dataMap.tc_scan_port}, - #{dataMap.tc_labels}, - #{dataMap.http_resp_content_type}, - #{dataMap.dns_msg_type}, - #{dataMap.dns_answer_length}, - #{dataMap.dns_ioc}, - #{dataMap.tx_bytes}, - #{dataMap.rx_bytes}, - #{dataMap.all_bytes}, - #{dataMap.duration_time}, - #{dataMap.mail_attach_name}, - #{dataMap.mail_subject}, - #{dataMap.mail_message}, - #{dataMap.mail_send_server}, - #{dataMap.mail_agent}, - #{dataMap.tls_version}, - #{dataMap.tls_server_cert}, - #{dataMap.tls_server_suite}, - #{dataMap.tls_client_suites_len}, - #{dataMap.tls_ja3}, - #{dataMap.tls_ja3s}, - #{dataMap.vpn_access_port}, - #{dataMap.log_topic}, - #{dataMap.collect_time}, - #{dataMap.src_is_intranetip}, - #{dataMap.src_ip_ioc}, - #{dataMap.src_ip_apt}, - #{dataMap.srcip_name}, - #{dataMap.tc_client}, - #{dataMap.srcip_organization_id}, - #{dataMap.dest_ip_intranetip}, - #{dataMap.dest_ip_ioc}, - #{dataMap.desip_id}, - #{dataMap.desip_name}, - #{dataMap.tc_hostip}, - #{dataMap.desip_organization_id}, - #{dataMap.origin_confidence}, - #{dataMap.origin_malscore}, - #{dataMap.attacker_icampaign}, - #{dataMap.attacker_host_asset_id}, - #{dataMap.attacker_organization_id}, - #{dataMap.victim_host_asset_id}, - #{dataMap.victim_organization_id}, - #{dataMap.logout_time}, - #{dataMap.http_req_line}, - #{dataMap.desip_security_scope_id}, - #{dataMap.srcip_security_scope_id}, - #{dataMap.http_resp_length}, - #{dataMap.tc_attack_type}, - #{dataMap.tc_realip}, - #{dataMap.attacker_ip_lists}, - #{dataMap.login_password}, - #{dataMap.detail}, - #{dataMap.attacker_country_code}, - #{dataMap.attacker_region_code}, - #{dataMap.victim_region_code}, - #{dataMap.payload}, - #{dataMap.http_referer}, - #{dataMap.http_user_agent}, - #{dataMap.http_session}, - #{dataMap.http_query_string}, - #{dataMap.file_path}, - #{dataMap.file_permission}, - #{dataMap.login_abnormal_type}, - #{dataMap.file_tag}, - #{dataMap.file_platform}, - #{dataMap.target_ip}, - #{dataMap.collect_date}, - #{dataMap.tc_client_ip}, - #{dataMap.tc_server_ip}, - #{dataMap.tc_externalip}, - #{dataMap.http_status_code}, - #{dataMap.device_domian}, - #{dataMap.src_ip_str}, - #{dataMap.src_port_str}, - #{dataMap.dest_ip_str} , - CAST(#{dataMap.dest_port_str} AS text), - #{dataMap.pcap}, - #{dataMap.ioc}, - #{dataMap.malicious_family}, - #{dataMap.vuln_cve}, - #{dataMap.aliyun_type}, - #{dataMap.attacker_host_asset_name}, - #{dataMap.attacker_organization_name}, - #{dataMap.ct_id}, - #{dataMap.cve_list}, - #{dataMap.desip_organization_name}, - #{dataMap.dest_ip_group}, - #{dataMap.file_gid}, - #{dataMap.file_owner}, - #{dataMap.file_ownergroup}, - #{dataMap.file_uid}, - #{dataMap.http_resp_cookie}, - #{dataMap.origin_rule_id}, - #{dataMap.origin_rule_name}, - #{dataMap.service_name}, - #{dataMap.src_ip_asset_group}, - #{dataMap.srcip_organization_name}, - #{dataMap.victim_host_asset_name}, - #{dataMap.http_resp_codes}::bigint, - #{dataMap.victim_organization_name}, - #{dataMap.tc_type}, - #{dataMap.direction}, - #{dataMap.http_req_cookie}, - #{dataMap.http_req_protocol}, - #{dataMap.http_req_header_raw}, - #{dataMap.http_url}, - #{dataMap.uname}, - #{dataMap.origin_hostname}, - #{dataMap.origin_os}, - #{dataMap.origin_agent_mac}, - #{dataMap.origin_host_id}, - #{dataMap.origin_agent_version}, - #{dataMap.origin_agent_id}, - #{dataMap.origin_agent_name}, - #{dataMap.origin_work_group}, - #{dataMap.origin_asset_group}, - #{dataMap.origin_local_port}, - #{dataMap.origin_agent_ip}, - #{dataMap.origin_internal_ip}, - #{dataMap.origin_external_ip}, - #{dataMap.origin_local_addr}, - #{dataMap.agent_id}, - #{dataMap.agent_name}, - #{dataMap.tc_title}, - #{dataMap.log_id}, - #{dataMap.event_date}, - #{dataMap.event_time_ts}, - #{dataMap.event_level}, - #{dataMap.src_ip}, - #{dataMap.src_port}, - #{dataMap.dest_ip}, - #{dataMap.dest_port}::BIGINT, - #{dataMap.event_time}, - #{dataMap.attacker_country}, - #{dataMap.src_mac}, - #{dataMap.dest_mac}, - #{dataMap.proto}, - #{dataMap.dev_id}, - #{dataMap.created_time}, - #{dataMap.src_country}, - #{dataMap.src_country_code}, - #{dataMap.src_region}, - #{dataMap.src_region_code}, - #{dataMap.src_city}, - #{dataMap.src_lon}, - #{dataMap.http_method}, - #{dataMap.http_host}, - #{dataMap.http_req_header}, - #{dataMap.http_req_body}, - #{dataMap.http_resp_header}, - #{dataMap.http_resp_body}, - #{dataMap.file_type}, - #{dataMap.file_md5}, - #{dataMap.file_size}, - #{dataMap.process}, - #{dataMap.start_time}, - #{dataMap.action}, - #{dataMap.attacker_region}, - #{dataMap.end_time}, - #{dataMap.file_created_time}, - #{dataMap.file_modified_time}, - #{dataMap.tc_miguan_scan_port}, - #{dataMap.process_path}, - #{dataMap.parent_process_path}, - #{dataMap.gname}, - #{dataMap.exe_name}, - #{dataMap.exe_path}, - #{dataMap.login_time}, - #{dataMap.login_times}, - #{dataMap.check_item}, - #{dataMap.check_type}, - #{dataMap.attacker_ip}, - #{dataMap.attacker_port}, - #{dataMap.victim_ip}, - #{dataMap.victim_port}, - #{dataMap.attacker_city}, - #{dataMap.attacker_lon}, - #{dataMap.attacker_lat}, - #{dataMap.victim_country}, - #{dataMap.victim_region}, - #{dataMap.victim_city}, - #{dataMap.victim_lon}, - #{dataMap.victim_lat}, - #{dataMap.origin_event_id}, - #{dataMap.origin_event_name}, - #{dataMap.origin_event_category}, - #{dataMap.origin_event_level}, - #{dataMap.origin_attack_chain}, - #{dataMap.engine_type}, - #{dataMap.evil_payload}, - #{dataMap.http_resp_status}, - #{dataMap.dns_query}, - #{dataMap.dns_query_type}, - #{dataMap.dns_ttl}, - #{dataMap.dns_answer}, - #{dataMap.dns_subdomains}, - #{dataMap.file_sha256}, - #{dataMap.file_ssdeep}, - #{dataMap.victim_country_code}, - #{dataMap.http_xff_ip}, - #{dataMap.tc_miguan_class}, - #{dataMap.pid}, - #{dataMap.ppid}, - #{dataMap.process_name}, - #{dataMap.backdoor_type}, - #{dataMap.tty}, - #{dataMap.sudo_user}, - #{dataMap.sudo_group}, - #{dataMap.origin_event_type}, - #{dataMap.dest_domain}, - #{dataMap.shell_cmdline}, - #{dataMap.parent_cmdline}, - #{dataMap.attack_chain}, - #{dataMap.process_tree}, - #{dataMap.host_file_sha256}, - #{dataMap.host_file_md5}, - #{dataMap.host_file_size}, - #{dataMap.host_file_type}, - #{dataMap.dest_country}, - #{dataMap.dest_country_code}, - #{dataMap.log_origin}, - #{dataMap.dest_region}, - #{dataMap.src_lat}, - #{dataMap.dest_region_code}, - #{dataMap.dest_city}, - #{dataMap.dest_lon}, - #{dataMap.dest_lat}, - #{dataMap.event_category}, - #{dataMap.attack_result}, - #{dataMap.probe_ip}, - #{dataMap.device_ip}, - #{dataMap.device_manufacturer}, - #{dataMap.device_name}, - #{dataMap.product_name}, - #{dataMap.__id}, - #{dataMap.__count}, - #{dataMap.__count_reason}, - #{dataMap.event_type}, - #{dataMap.protocol}, - #{dataMap.shell_cmd}, - #{dataMap.parent_name}, - #{dataMap.host_file_path}, - #{dataMap.uid}, - #{dataMap.fall}, - #{dataMap.tc_miguan_server_ip}, - #{dataMap.dev_type}, - #{dataMap.collect_method}, - #{dataMap.field_cate_id}, - #{dataMap.device_type}, - #{dataMap.tc_miguan_client_ip}, - #{dataMap.tc_miguan_name}, - #{dataMap.origin_total_packages}, - #{dataMap.origin_total_bytes}, - #{dataMap.origin_peak_packages_rate}, - #{dataMap.origin_peak_bytes_rate}, - #{dataMap.origin_peak_flows_rate}, - #{dataMap.apt_orgname}, - #{dataMap.apt_orgmsg}, - #{dataMap.mail_message_id}, - #{dataMap.mail_bcc}, - #{dataMap.mail_size}, - #{dataMap.mail_attach_hashcode}, - #{dataMap.mail_url}, - #{dataMap.mail_cc}, - #{dataMap.algorithm}, - #{dataMap.miningpool_ip}, - #{dataMap.process_md5}, - #{dataMap.pprocess_md5}, - #{dataMap.source_servername}, - #{dataMap.origin_source_servername}, - #{dataMap.mail_filename}, - #{dataMap.dst_upload_appname}, - #{dataMap.target_port}, - #{dataMap.gid}, - #{dataMap.origin_uid}, - #{dataMap.origin_gid}, - #{dataMap.target_ports}, - #{dataMap.tc_miguan_name1}, - #{dataMap.tc_miguan_class1}, - #{dataMap.etl_time}, - #{dataMap.tc_miguan_scan_port2}, - #{dataMap.desip_security_scope}, - #{dataMap.srcip_security_scope}, - #{dataMap.collect_time_ts}, - #{dataMap.tc_miguan_scan_port1}, - #{dataMap.src_dev_name}, - #{dataMap.collect_protocol}, - #{dataMap.destination_system_type}, - #{dataMap.destination_system}, - #{dataMap.etl_host}, - #{dataMap.normalize_rule_id}, - #{dataMap.normalize_rule_name}, - #{dataMap.syslog_uuid}, - #{dataMap.syslog_topic}, + #{dataMap.id}, + #{dataMap.created_at}, + #{dataMap.log_time}, + #{dataMap.device_id}, + #{dataMap.webshell_type}, + #{dataMap.vuirs_type}, + #{dataMap.vuirs_url}, + #{dataMap.class_filename}, + #{dataMap.class_path}, + #{dataMap.parent_class}, + #{dataMap.jar_path}, + #{dataMap.class_md5}, + #{dataMap.class_loader}, + #{dataMap.class_hashcode}, + #{dataMap.class_loader_hashcode}, + #{dataMap.tc_nameip}, + #{dataMap.perform_sql}, + #{dataMap.tc_account}, + #{dataMap.tc_appname}, + #{dataMap.process_uname}, + #{dataMap.p_process_uname}, + #{dataMap.container_name}, + #{dataMap.container_id}, + #{dataMap.http_resp_server}, + #{dataMap.srcip_id}, + #{dataMap.cdnip}, + #{dataMap.natip}, + #{dataMap.mail_sender}, + #{dataMap.mail_receiver}, + #{dataMap.vpn_mac}, + #{dataMap.vpn_os}, + #{dataMap.vpn_user}, + #{dataMap.vpn_groupname}, + #{dataMap.vpn_access_ip}, + #{dataMap.dest_ip_apt}, + #{dataMap.origin_attack_result}, + #{dataMap.description}, + #{dataMap.solution}, + #{dataMap.attack_cause}, + #{dataMap.username}, + #{dataMap.tc_flow_id}, + #{dataMap.login_result}, + #{dataMap.cmdline}, + #{dataMap.origin_attack_action}, + #{dataMap.victim_domain}, + #{dataMap.vpn_deviceid}, + #{dataMap.vpn_access_action}, + #{dataMap.file_access_time}, + #{dataMap.file_name}, + #{dataMap.tc_class}, + #{dataMap.tc_name2}, + #{dataMap.login_lasttime}, + #{dataMap.origin_permissions}, + #{dataMap.begin_permissions}, + #{dataMap.print_time}, + #{dataMap.printer}, + #{dataMap.printer_type}, + #{dataMap.print_pages}, + #{dataMap.print_copies}, + #{dataMap.src_device}, + #{dataMap.dst_device}, + #{dataMap.src_file}, + #{dataMap.src_file_type}, + #{dataMap.src_file_path}, + #{dataMap.dst_file}, + #{dataMap.dst_file_type}, + #{dataMap.dst_file_path}, + #{dataMap.dlp_policy_name}, + #{dataMap.dlp_policy_type}, + #{dataMap.dst_upload_url}, + #{dataMap.process_uuid}, + #{dataMap.p_process_uuid}, + #{dataMap.env}, + #{dataMap.brute_force_service}, + #{dataMap.vuirs_name}, + #{dataMap.http_req_length}, + #{dataMap.http_req_content_type}, + #{dataMap.tc_scan_port}, + #{dataMap.tc_labels}, + #{dataMap.http_resp_content_type}, + #{dataMap.dns_msg_type}, + #{dataMap.dns_answer_length}, + #{dataMap.dns_ioc}, + #{dataMap.tx_bytes}, + #{dataMap.rx_bytes}, + #{dataMap.all_bytes}, + #{dataMap.duration_time}, + #{dataMap.mail_attach_name}, + #{dataMap.mail_subject}, + #{dataMap.mail_message}, + #{dataMap.mail_send_server}, + #{dataMap.mail_agent}, + #{dataMap.tls_version}, + #{dataMap.tls_server_cert}, + #{dataMap.tls_server_suite}, + #{dataMap.tls_client_suites_len}, + #{dataMap.tls_ja3}, + #{dataMap.tls_ja3s}, + #{dataMap.vpn_access_port}, + #{dataMap.log_topic}, + #{dataMap.collect_time}, + #{dataMap.src_is_intranetip}, + #{dataMap.src_ip_ioc}, + #{dataMap.src_ip_apt}, + #{dataMap.srcip_name}, + #{dataMap.tc_client}, + #{dataMap.srcip_organization_id}, + #{dataMap.dest_ip_intranetip}, + #{dataMap.dest_ip_ioc}, + #{dataMap.desip_id}, + #{dataMap.desip_name}, + #{dataMap.tc_hostip}, + #{dataMap.desip_organization_id}, + #{dataMap.origin_confidence}, + #{dataMap.origin_malscore}, + #{dataMap.attacker_icampaign}, + #{dataMap.attacker_host_asset_id}, + #{dataMap.attacker_organization_id}, + #{dataMap.victim_host_asset_id}, + #{dataMap.victim_organization_id}, + #{dataMap.logout_time}, + #{dataMap.http_req_line}, + #{dataMap.desip_security_scope_id}, + #{dataMap.srcip_security_scope_id}, + #{dataMap.http_resp_length}, + #{dataMap.tc_attack_type}, + #{dataMap.tc_realip}, + #{dataMap.attacker_ip_lists}, + #{dataMap.login_password}, + #{dataMap.detail}, + #{dataMap.attacker_country_code}, + #{dataMap.attacker_region_code}, + #{dataMap.victim_region_code}, + #{dataMap.payload}, + #{dataMap.http_referer}, + #{dataMap.http_user_agent}, + #{dataMap.http_session}, + #{dataMap.http_query_string}, + #{dataMap.file_path}, + #{dataMap.file_permission}, + #{dataMap.login_abnormal_type}, + #{dataMap.file_tag}, + #{dataMap.file_platform}, + #{dataMap.target_ip}, + #{dataMap.collect_date}, + #{dataMap.tc_client_ip}, + #{dataMap.tc_server_ip}, + #{dataMap.tc_externalip}, + #{dataMap.http_status_code}, + #{dataMap.device_domian}, + #{dataMap.src_ip_str}, + #{dataMap.src_port_str}, + #{dataMap.dest_ip_str} , + CAST(#{dataMap.dest_port_str} AS text), + #{dataMap.pcap}, + #{dataMap.ioc}, + #{dataMap.malicious_family}, + #{dataMap.vuln_cve}, + #{dataMap.aliyun_type}, + #{dataMap.attacker_host_asset_name}, + #{dataMap.attacker_organization_name}, + #{dataMap.ct_id}, + #{dataMap.cve_list}, + #{dataMap.desip_organization_name}, + #{dataMap.dest_ip_group}, + #{dataMap.file_gid}, + #{dataMap.file_owner}, + #{dataMap.file_ownergroup}, + #{dataMap.file_uid}, + #{dataMap.http_resp_cookie}, + #{dataMap.origin_rule_id}, + #{dataMap.origin_rule_name}, + #{dataMap.service_name}, + #{dataMap.src_ip_asset_group}, + #{dataMap.srcip_organization_name}, + #{dataMap.victim_host_asset_name}, + #{dataMap.http_resp_codes}::bigint, + #{dataMap.victim_organization_name}, + #{dataMap.tc_type}, + #{dataMap.direction}, + #{dataMap.http_req_cookie}, + #{dataMap.http_req_protocol}, + #{dataMap.http_req_header_raw}, + #{dataMap.http_url}, + #{dataMap.uname}, + #{dataMap.origin_hostname}, + #{dataMap.origin_os}, + #{dataMap.origin_agent_mac}, + #{dataMap.origin_host_id}, + #{dataMap.origin_agent_version}, + #{dataMap.origin_agent_id}, + #{dataMap.origin_agent_name}, + #{dataMap.origin_work_group}, + #{dataMap.origin_asset_group}, + #{dataMap.origin_local_port}, + #{dataMap.origin_agent_ip}, + #{dataMap.origin_internal_ip}, + #{dataMap.origin_external_ip}, + #{dataMap.origin_local_addr}, + #{dataMap.agent_id}, + #{dataMap.agent_name}, + #{dataMap.tc_title}, + #{dataMap.log_id}, + #{dataMap.event_date}, + #{dataMap.event_time_ts}, + #{dataMap.event_level}, + #{dataMap.src_ip}, + #{dataMap.src_port}, + #{dataMap.dest_ip}, + #{dataMap.dest_port}::BIGINT, + #{dataMap.event_time}, + #{dataMap.attacker_country}, + #{dataMap.src_mac}, + #{dataMap.dest_mac}, + #{dataMap.proto}, + #{dataMap.dev_id}, + #{dataMap.created_time}, + #{dataMap.src_country}, + #{dataMap.src_country_code}, + #{dataMap.src_region}, + #{dataMap.src_region_code}, + #{dataMap.src_city}, + #{dataMap.src_lon}, + #{dataMap.http_method}, + #{dataMap.http_host}, + #{dataMap.http_req_header}, + #{dataMap.http_req_body}, + #{dataMap.http_resp_header}, + #{dataMap.http_resp_body}, + #{dataMap.file_type}, + #{dataMap.file_md5}, + #{dataMap.file_size}, + #{dataMap.process}, + #{dataMap.start_time}, + #{dataMap.action}, + #{dataMap.attacker_region}, + #{dataMap.end_time}, + #{dataMap.file_created_time}, + #{dataMap.file_modified_time}, + #{dataMap.tc_miguan_scan_port}, + #{dataMap.process_path}, + #{dataMap.parent_process_path}, + #{dataMap.gname}, + #{dataMap.exe_name}, + #{dataMap.exe_path}, + #{dataMap.login_time}, + #{dataMap.login_times}, + #{dataMap.check_item}, + #{dataMap.check_type}, + #{dataMap.attacker_ip}, + #{dataMap.attacker_port}, + #{dataMap.victim_ip}, + #{dataMap.victim_port}, + #{dataMap.attacker_city}, + #{dataMap.attacker_lon}, + #{dataMap.attacker_lat}, + #{dataMap.victim_country}, + #{dataMap.victim_region}, + #{dataMap.victim_city}, + #{dataMap.victim_lon}, + #{dataMap.victim_lat}, + #{dataMap.origin_event_id}, + #{dataMap.origin_event_name}, + #{dataMap.origin_event_category}, + #{dataMap.origin_event_level}, + #{dataMap.origin_attack_chain}, + #{dataMap.engine_type}, + #{dataMap.evil_payload}, + #{dataMap.http_resp_status}, + #{dataMap.dns_query}, + #{dataMap.dns_query_type}, + #{dataMap.dns_ttl}, + #{dataMap.dns_answer}, + #{dataMap.dns_subdomains}, + #{dataMap.file_sha256}, + #{dataMap.file_ssdeep}, + #{dataMap.victim_country_code}, + #{dataMap.http_xff_ip}, + #{dataMap.tc_miguan_class}, + #{dataMap.pid}, + #{dataMap.ppid}, + #{dataMap.process_name}, + #{dataMap.backdoor_type}, + #{dataMap.tty}, + #{dataMap.sudo_user}, + #{dataMap.sudo_group}, + #{dataMap.origin_event_type}, + #{dataMap.dest_domain}, + #{dataMap.shell_cmdline}, + #{dataMap.parent_cmdline}, + #{dataMap.attack_chain}, + #{dataMap.process_tree}, + #{dataMap.host_file_sha256}, + #{dataMap.host_file_md5}, + #{dataMap.host_file_size}, + #{dataMap.host_file_type}, + #{dataMap.dest_country}, + #{dataMap.dest_country_code}, + #{dataMap.log_origin}, + #{dataMap.dest_region}, + #{dataMap.src_lat}, + #{dataMap.dest_region_code}, + #{dataMap.dest_city}, + #{dataMap.dest_lon}, + #{dataMap.dest_lat}, + #{dataMap.event_category}, + #{dataMap.attack_result}, + #{dataMap.probe_ip}, + #{dataMap.device_ip}, + #{dataMap.device_manufacturer}, + #{dataMap.device_name}, + #{dataMap.product_name}, + #{dataMap.__id}, + #{dataMap.__count}, + #{dataMap.__count_reason}, + #{dataMap.event_type}, + #{dataMap.protocol}, + #{dataMap.shell_cmd}, + #{dataMap.parent_name}, + #{dataMap.host_file_path}, + #{dataMap.uid}, + #{dataMap.fall}, + #{dataMap.tc_miguan_server_ip}, + #{dataMap.dev_type}, + #{dataMap.collect_method}, + #{dataMap.field_cate_id}, + #{dataMap.device_type}, + #{dataMap.tc_miguan_client_ip}, + #{dataMap.tc_miguan_name}, + #{dataMap.origin_total_packages}, + #{dataMap.origin_total_bytes}, + #{dataMap.origin_peak_packages_rate}, + #{dataMap.origin_peak_bytes_rate}, + #{dataMap.origin_peak_flows_rate}, + #{dataMap.apt_orgname}, + #{dataMap.apt_orgmsg}, + #{dataMap.mail_message_id}, + #{dataMap.mail_bcc}, + #{dataMap.mail_size}, + #{dataMap.mail_attach_hashcode}, + #{dataMap.mail_url}, + #{dataMap.mail_cc}, + #{dataMap.algorithm}, + #{dataMap.miningpool_ip}, + #{dataMap.process_md5}, + #{dataMap.pprocess_md5}, + #{dataMap.source_servername}, + #{dataMap.origin_source_servername}, + #{dataMap.mail_filename}, + #{dataMap.dst_upload_appname}, + #{dataMap.target_port}, + #{dataMap.gid}, + #{dataMap.origin_uid}, + #{dataMap.origin_gid}, + #{dataMap.target_ports}, + #{dataMap.tc_miguan_name1}, + #{dataMap.tc_miguan_class1}, + #{dataMap.etl_time}, + #{dataMap.tc_miguan_scan_port2}, + #{dataMap.desip_security_scope}, + #{dataMap.srcip_security_scope}, + #{dataMap.collect_time_ts}, + #{dataMap.tc_miguan_scan_port1}, + #{dataMap.src_dev_name}, + #{dataMap.collect_protocol}, + #{dataMap.destination_system_type}, + #{dataMap.destination_system}, + #{dataMap.etl_host}, + #{dataMap.normalize_rule_id}, + #{dataMap.normalize_rule_name}, + #{dataMap.syslog_uuid}, + #{dataMap.syslog_topic}, + #{dataMap.rule_version}, @@ -776,14 +779,4 @@ - - - INSERT INTO syslog_normal_alarm - (id, log_time, src_ip, dest_ip, event_level) - VALUES - - (#{item.id}, #{item.log_time}, #{item.src_ip}, #{item.dest_ip}, #{item.event_level}) - - - diff --git a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml index 89306b3..45ae9a0 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml +++ b/haobang-security-dm/syslog-consumer-rule/src/main/resources/mapper/SyslogNormalDataMapper.xml @@ -141,756 +141,759 @@ - + + INSERT INTO syslog_normal_data - id, - created_at, - log_time, - device_id, - webshell_type, - vuirs_type, - vuirs_url, - class_filename, - class_path, - parent_class, - jar_path, - class_md5, - class_loader, - class_hashcode, - class_loader_hashcode, - tc_nameip, - perform_sql, - tc_account, - tc_appname, - process_uname, - p_process_uname, - container_name, - container_id, - http_resp_server, - srcip_id, - cdnip, - natip, - mail_sender, - mail_receiver, - vpn_mac, - vpn_os, - vpn_user, - vpn_groupname, - vpn_access_ip, - dest_ip_apt, - origin_attack_result, - description, - solution, - attack_cause, - username, - tc_flow_id, - login_result, - cmdline, - origin_attack_action, - victim_domain, - vpn_deviceid, - vpn_access_action, - file_access_time, - file_name, - tc_class, - tc_name2, - login_lasttime, - origin_permissions, - begin_permissions, - print_time, - printer, - printer_type, - print_pages, - print_copies, - src_device, - dst_device, - src_file, - src_file_type, - src_file_path, - dst_file, - dst_file_type, - dst_file_path, - dlp_policy_name, - dlp_policy_type, - dst_upload_url, - process_uuid, - p_process_uuid, - env, - brute_force_service, - vuirs_name, - http_req_length, - http_req_content_type, - tc_scan_port, - tc_labels, - http_resp_content_type, - dns_msg_type, - dns_answer_length, - dns_ioc, - tx_bytes, - rx_bytes, - all_bytes, - duration_time, - mail_attach_name, - mail_subject, - mail_message, - mail_send_server, - mail_agent, - tls_version, - tls_server_cert, - tls_server_suite, - tls_client_suites_len, - tls_ja3, - tls_ja3s, - vpn_access_port, - log_topic, - collect_time, - src_is_intranetip, - src_ip_ioc, - src_ip_apt, - srcip_name, - tc_client, - srcip_organization_id, - dest_ip_intranetip, - dest_ip_ioc, - desip_id, - desip_name, - tc_hostip, - desip_organization_id, - origin_confidence, - origin_malscore, - attacker_icampaign, - attacker_host_asset_id, - attacker_organization_id, - victim_host_asset_id, - victim_organization_id, - logout_time, - http_req_line, - desip_security_scope_id, - srcip_security_scope_id, - http_resp_length, - tc_attack_type, - tc_realip, - attacker_ip_lists, - login_password, - detail, - attacker_country_code, - attacker_region_code, - victim_region_code, - payload, - http_referer, - http_user_agent, - http_session, - http_query_string, - file_path, - file_permission, - login_abnormal_type, - file_tag, - file_platform, - target_ip, - collect_date, - tc_client_ip, - tc_server_ip, - tc_externalip, - http_status_code, - device_domian, - src_ip_str, - src_port_str, - dest_ip_str, - dest_port_str, - pcap, - ioc, - malicious_family, - vuln_cve, - aliyun_type, - attacker_host_asset_name, - attacker_organization_name, - ct_id, - cve_list, - desip_organization_name, - dest_ip_group, - file_gid, - file_owner, - file_ownergroup, - file_uid, - http_resp_cookie, - origin_rule_id, - origin_rule_name, - service_name, - src_ip_asset_group, - srcip_organization_name, - victim_host_asset_name, - http_resp_codes, - victim_organization_name, - tc_type, - direction, - http_req_cookie, - http_req_protocol, - http_req_header_raw, - http_url, - uname, - origin_hostname, - origin_os, - origin_agent_mac, - origin_host_id, - origin_agent_version, - origin_agent_id, - origin_agent_name, - origin_work_group, - origin_asset_group, - origin_local_port, - origin_agent_ip, - origin_internal_ip, - origin_external_ip, - origin_local_addr, - agent_id, - agent_name, - tc_title, - log_id, - event_date, - event_time_ts, - event_level, - src_ip , - src_port, - dest_ip, - dest_port, - event_time, - attacker_country, - src_mac, - dest_mac, - proto, - dev_id, - created_time, - src_country, - src_country_code, - src_region, - src_region_code, - src_city, - src_lon, - http_method, - http_host, - http_req_header, - http_req_body, - http_resp_header, - http_resp_body, - file_type, - file_md5, - file_size, - process, - start_time, - action, - attacker_region, - end_time, - file_created_time, - file_modified_time, - tc_miguan_scan_port, - process_path, - parent_process_path, - gname, - exe_name, - exe_path, - login_time, - login_times, - check_item, - check_type, - attacker_ip, - attacker_port, - victim_ip, - victim_port, - attacker_city, - attacker_lon, - attacker_lat, - victim_country, - victim_region, - victim_city, - victim_lon, - victim_lat, - origin_event_id, - origin_event_name, - origin_event_category, - origin_event_level, - origin_attack_chain, - engine_type, - evil_payload, - http_resp_status, - dns_query, - dns_query_type, - dns_ttl, - dns_answer, - dns_subdomains, - file_sha256, - file_ssdeep, - victim_country_code, - http_xff_ip, - tc_miguan_class, - pid, - ppid, - process_name, - backdoor_type, - tty, - sudo_user, - sudo_group, - origin_event_type, - dest_domain, - shell_cmdline, - parent_cmdline, - attack_chain, - process_tree, - host_file_sha256, - host_file_md5, - host_file_size, - host_file_type, - dest_country, - dest_country_code, - log_origin, - dest_region, - src_lat, - dest_region_code, - dest_city, - dest_lon, - dest_lat, - event_category, - attack_result, - probe_ip, - device_ip, - device_manufacturer, - device_name, - product_name, - __id, - __count, - __count_reason, - event_type, - protocol, - shell_cmd, - parent_name, - host_file_path, - uid, - fall, - tc_miguan_server_ip, - dev_type, - collect_method, - field_cate_id, - device_type, - tc_miguan_client_ip, - tc_miguan_name, - origin_total_packages, - origin_total_bytes, - origin_peak_packages_rate, - origin_peak_bytes_rate, - origin_peak_flows_rate, - apt_orgname, - apt_orgmsg, - mail_message_id, - mail_bcc, - mail_size, - mail_attach_hashcode, - mail_url, - mail_cc, - algorithm, - miningpool_ip, - process_md5, - pprocess_md5, - source_servername, - origin_source_servername, - mail_filename, - dst_upload_appname, - target_port, - gid, - origin_uid, - origin_gid, - target_ports, - tc_miguan_name1, - tc_miguan_class1, - etl_time, - tc_miguan_scan_port2, - desip_security_scope, - srcip_security_scope, - collect_time_ts, - tc_miguan_scan_port1, - src_dev_name, - collect_protocol, - destination_system_type, - destination_system, - etl_host, - normalize_rule_id, - normalize_rule_name, - syslog_uuid, - syslog_topic, + id, + created_at, + log_time, + device_id, + webshell_type, + vuirs_type, + vuirs_url, + class_filename, + class_path, + parent_class, + jar_path, + class_md5, + class_loader, + class_hashcode, + class_loader_hashcode, + tc_nameip, + perform_sql, + tc_account, + tc_appname, + process_uname, + p_process_uname, + container_name, + container_id, + http_resp_server, + srcip_id, + cdnip, + natip, + mail_sender, + mail_receiver, + vpn_mac, + vpn_os, + vpn_user, + vpn_groupname, + vpn_access_ip, + dest_ip_apt, + origin_attack_result, + description, + solution, + attack_cause, + username, + tc_flow_id, + login_result, + cmdline, + origin_attack_action, + victim_domain, + vpn_deviceid, + vpn_access_action, + file_access_time, + file_name, + tc_class, + tc_name2, + login_lasttime, + origin_permissions, + begin_permissions, + print_time, + printer, + printer_type, + print_pages, + print_copies, + src_device, + dst_device, + src_file, + src_file_type, + src_file_path, + dst_file, + dst_file_type, + dst_file_path, + dlp_policy_name, + dlp_policy_type, + dst_upload_url, + process_uuid, + p_process_uuid, + env, + brute_force_service, + vuirs_name, + http_req_length, + http_req_content_type, + tc_scan_port, + tc_labels, + http_resp_content_type, + dns_msg_type, + dns_answer_length, + dns_ioc, + tx_bytes, + rx_bytes, + all_bytes, + duration_time, + mail_attach_name, + mail_subject, + mail_message, + mail_send_server, + mail_agent, + tls_version, + tls_server_cert, + tls_server_suite, + tls_client_suites_len, + tls_ja3, + tls_ja3s, + vpn_access_port, + log_topic, + collect_time, + src_is_intranetip, + src_ip_ioc, + src_ip_apt, + srcip_name, + tc_client, + srcip_organization_id, + dest_ip_intranetip, + dest_ip_ioc, + desip_id, + desip_name, + tc_hostip, + desip_organization_id, + origin_confidence, + origin_malscore, + attacker_icampaign, + attacker_host_asset_id, + attacker_organization_id, + victim_host_asset_id, + victim_organization_id, + logout_time, + http_req_line, + desip_security_scope_id, + srcip_security_scope_id, + http_resp_length, + tc_attack_type, + tc_realip, + attacker_ip_lists, + login_password, + detail, + attacker_country_code, + attacker_region_code, + victim_region_code, + payload, + http_referer, + http_user_agent, + http_session, + http_query_string, + file_path, + file_permission, + login_abnormal_type, + file_tag, + file_platform, + target_ip, + collect_date, + tc_client_ip, + tc_server_ip, + tc_externalip, + http_status_code, + device_domian, + src_ip_str, + src_port_str, + dest_ip_str, + dest_port_str, + pcap, + ioc, + malicious_family, + vuln_cve, + aliyun_type, + attacker_host_asset_name, + attacker_organization_name, + ct_id, + cve_list, + desip_organization_name, + dest_ip_group, + file_gid, + file_owner, + file_ownergroup, + file_uid, + http_resp_cookie, + origin_rule_id, + origin_rule_name, + service_name, + src_ip_asset_group, + srcip_organization_name, + victim_host_asset_name, + http_resp_codes, + victim_organization_name, + tc_type, + direction, + http_req_cookie, + http_req_protocol, + http_req_header_raw, + http_url, + uname, + origin_hostname, + origin_os, + origin_agent_mac, + origin_host_id, + origin_agent_version, + origin_agent_id, + origin_agent_name, + origin_work_group, + origin_asset_group, + origin_local_port, + origin_agent_ip, + origin_internal_ip, + origin_external_ip, + origin_local_addr, + agent_id, + agent_name, + tc_title, + log_id, + event_date, + event_time_ts, + event_level, + src_ip , + src_port, + dest_ip, + dest_port, + event_time, + attacker_country, + src_mac, + dest_mac, + proto, + dev_id, + created_time, + src_country, + src_country_code, + src_region, + src_region_code, + src_city, + src_lon, + http_method, + http_host, + http_req_header, + http_req_body, + http_resp_header, + http_resp_body, + file_type, + file_md5, + file_size, + process, + start_time, + action, + attacker_region, + end_time, + file_created_time, + file_modified_time, + tc_miguan_scan_port, + process_path, + parent_process_path, + gname, + exe_name, + exe_path, + login_time, + login_times, + check_item, + check_type, + attacker_ip, + attacker_port, + victim_ip, + victim_port, + attacker_city, + attacker_lon, + attacker_lat, + victim_country, + victim_region, + victim_city, + victim_lon, + victim_lat, + origin_event_id, + origin_event_name, + origin_event_category, + origin_event_level, + origin_attack_chain, + engine_type, + evil_payload, + http_resp_status, + dns_query, + dns_query_type, + dns_ttl, + dns_answer, + dns_subdomains, + file_sha256, + file_ssdeep, + victim_country_code, + http_xff_ip, + tc_miguan_class, + pid, + ppid, + process_name, + backdoor_type, + tty, + sudo_user, + sudo_group, + origin_event_type, + dest_domain, + shell_cmdline, + parent_cmdline, + attack_chain, + process_tree, + host_file_sha256, + host_file_md5, + host_file_size, + host_file_type, + dest_country, + dest_country_code, + log_origin, + dest_region, + src_lat, + dest_region_code, + dest_city, + dest_lon, + dest_lat, + event_category, + attack_result, + probe_ip, + device_ip, + device_manufacturer, + device_name, + product_name, + __id, + __count, + __count_reason, + event_type, + protocol, + shell_cmd, + parent_name, + host_file_path, + uid, + fall, + tc_miguan_server_ip, + dev_type, + collect_method, + field_cate_id, + device_type, + tc_miguan_client_ip, + tc_miguan_name, + origin_total_packages, + origin_total_bytes, + origin_peak_packages_rate, + origin_peak_bytes_rate, + origin_peak_flows_rate, + apt_orgname, + apt_orgmsg, + mail_message_id, + mail_bcc, + mail_size, + mail_attach_hashcode, + mail_url, + mail_cc, + algorithm, + miningpool_ip, + process_md5, + pprocess_md5, + source_servername, + origin_source_servername, + mail_filename, + dst_upload_appname, + target_port, + gid, + origin_uid, + origin_gid, + target_ports, + tc_miguan_name1, + tc_miguan_class1, + etl_time, + tc_miguan_scan_port2, + desip_security_scope, + srcip_security_scope, + collect_time_ts, + tc_miguan_scan_port1, + src_dev_name, + collect_protocol, + destination_system_type, + destination_system, + etl_host, + normalize_rule_id, + normalize_rule_name, + syslog_uuid, + syslog_topic, + rule_version, VALUES - #{dataMap.id}, - #{dataMap.created_at}, - #{dataMap.log_time}, - #{dataMap.device_id}, - #{dataMap.webshell_type}, - #{dataMap.vuirs_type}, - #{dataMap.vuirs_url}, - #{dataMap.class_filename}, - #{dataMap.class_path}, - #{dataMap.parent_class}, - #{dataMap.jar_path}, - #{dataMap.class_md5}, - #{dataMap.class_loader}, - #{dataMap.class_hashcode}, - #{dataMap.class_loader_hashcode}, - #{dataMap.tc_nameip}, - #{dataMap.perform_sql}, - #{dataMap.tc_account}, - #{dataMap.tc_appname}, - #{dataMap.process_uname}, - #{dataMap.p_process_uname}, - #{dataMap.container_name}, - #{dataMap.container_id}, - #{dataMap.http_resp_server}, - #{dataMap.srcip_id}, - #{dataMap.cdnip}, - #{dataMap.natip}, - #{dataMap.mail_sender}, - #{dataMap.mail_receiver}, - #{dataMap.vpn_mac}, - #{dataMap.vpn_os}, - #{dataMap.vpn_user}, - #{dataMap.vpn_groupname}, - #{dataMap.vpn_access_ip}, - #{dataMap.dest_ip_apt}, - #{dataMap.origin_attack_result}, - #{dataMap.description}, - #{dataMap.solution}, - #{dataMap.attack_cause}, - #{dataMap.username}, - #{dataMap.tc_flow_id}, - #{dataMap.login_result}, - #{dataMap.cmdline}, - #{dataMap.origin_attack_action}, - #{dataMap.victim_domain}, - #{dataMap.vpn_deviceid}, - #{dataMap.vpn_access_action}, - #{dataMap.file_access_time}, - #{dataMap.file_name}, - #{dataMap.tc_class}, - #{dataMap.tc_name2}, - #{dataMap.login_lasttime}, - #{dataMap.origin_permissions}, - #{dataMap.begin_permissions}, - #{dataMap.print_time}, - #{dataMap.printer}, - #{dataMap.printer_type}, - #{dataMap.print_pages}, - #{dataMap.print_copies}, - #{dataMap.src_device}, - #{dataMap.dst_device}, - #{dataMap.src_file}, - #{dataMap.src_file_type}, - #{dataMap.src_file_path}, - #{dataMap.dst_file}, - #{dataMap.dst_file_type}, - #{dataMap.dst_file_path}, - #{dataMap.dlp_policy_name}, - #{dataMap.dlp_policy_type}, - #{dataMap.dst_upload_url}, - #{dataMap.process_uuid}, - #{dataMap.p_process_uuid}, - #{dataMap.env}, - #{dataMap.brute_force_service}, - #{dataMap.vuirs_name}, - #{dataMap.http_req_length}, - #{dataMap.http_req_content_type}, - #{dataMap.tc_scan_port}, - #{dataMap.tc_labels}, - #{dataMap.http_resp_content_type}, - #{dataMap.dns_msg_type}, - #{dataMap.dns_answer_length}, - #{dataMap.dns_ioc}, - #{dataMap.tx_bytes}, - #{dataMap.rx_bytes}, - #{dataMap.all_bytes}, - #{dataMap.duration_time}, - #{dataMap.mail_attach_name}, - #{dataMap.mail_subject}, - #{dataMap.mail_message}, - #{dataMap.mail_send_server}, - #{dataMap.mail_agent}, - #{dataMap.tls_version}, - #{dataMap.tls_server_cert}, - #{dataMap.tls_server_suite}, - #{dataMap.tls_client_suites_len}, - #{dataMap.tls_ja3}, - #{dataMap.tls_ja3s}, - #{dataMap.vpn_access_port}, - #{dataMap.log_topic}, - #{dataMap.collect_time}, - #{dataMap.src_is_intranetip}, - #{dataMap.src_ip_ioc}, - #{dataMap.src_ip_apt}, - #{dataMap.srcip_name}, - #{dataMap.tc_client}, - #{dataMap.srcip_organization_id}, - #{dataMap.dest_ip_intranetip}, - #{dataMap.dest_ip_ioc}, - #{dataMap.desip_id}, - #{dataMap.desip_name}, - #{dataMap.tc_hostip}, - #{dataMap.desip_organization_id}, - #{dataMap.origin_confidence}, - #{dataMap.origin_malscore}, - #{dataMap.attacker_icampaign}, - #{dataMap.attacker_host_asset_id}, - #{dataMap.attacker_organization_id}, - #{dataMap.victim_host_asset_id}, - #{dataMap.victim_organization_id}, - #{dataMap.logout_time}, - #{dataMap.http_req_line}, - #{dataMap.desip_security_scope_id}, - #{dataMap.srcip_security_scope_id}, - #{dataMap.http_resp_length}, - #{dataMap.tc_attack_type}, - #{dataMap.tc_realip}, - #{dataMap.attacker_ip_lists}, - #{dataMap.login_password}, - #{dataMap.detail}, - #{dataMap.attacker_country_code}, - #{dataMap.attacker_region_code}, - #{dataMap.victim_region_code}, - #{dataMap.payload}, - #{dataMap.http_referer}, - #{dataMap.http_user_agent}, - #{dataMap.http_session}, - #{dataMap.http_query_string}, - #{dataMap.file_path}, - #{dataMap.file_permission}, - #{dataMap.login_abnormal_type}, - #{dataMap.file_tag}, - #{dataMap.file_platform}, - #{dataMap.target_ip}, - #{dataMap.collect_date}, - #{dataMap.tc_client_ip}, - #{dataMap.tc_server_ip}, - #{dataMap.tc_externalip}, - #{dataMap.http_status_code}, - #{dataMap.device_domian}, - #{dataMap.src_ip_str}, - #{dataMap.src_port_str}, - #{dataMap.dest_ip_str} , - CAST(#{dataMap.dest_port_str} AS text), - #{dataMap.pcap}, - #{dataMap.ioc}, - #{dataMap.malicious_family}, - #{dataMap.vuln_cve}, - #{dataMap.aliyun_type}, - #{dataMap.attacker_host_asset_name}, - #{dataMap.attacker_organization_name}, - #{dataMap.ct_id}, - #{dataMap.cve_list}, - #{dataMap.desip_organization_name}, - #{dataMap.dest_ip_group}, - #{dataMap.file_gid}, - #{dataMap.file_owner}, - #{dataMap.file_ownergroup}, - #{dataMap.file_uid}, - #{dataMap.http_resp_cookie}, - #{dataMap.origin_rule_id}, - #{dataMap.origin_rule_name}, - #{dataMap.service_name}, - #{dataMap.src_ip_asset_group}, - #{dataMap.srcip_organization_name}, - #{dataMap.victim_host_asset_name}, - #{dataMap.http_resp_codes}::bigint, - #{dataMap.victim_organization_name}, - #{dataMap.tc_type}, - #{dataMap.direction}, - #{dataMap.http_req_cookie}, - #{dataMap.http_req_protocol}, - #{dataMap.http_req_header_raw}, - #{dataMap.http_url}, - #{dataMap.uname}, - #{dataMap.origin_hostname}, - #{dataMap.origin_os}, - #{dataMap.origin_agent_mac}, - #{dataMap.origin_host_id}, - #{dataMap.origin_agent_version}, - #{dataMap.origin_agent_id}, - #{dataMap.origin_agent_name}, - #{dataMap.origin_work_group}, - #{dataMap.origin_asset_group}, - #{dataMap.origin_local_port}, - #{dataMap.origin_agent_ip}, - #{dataMap.origin_internal_ip}, - #{dataMap.origin_external_ip}, - #{dataMap.origin_local_addr}, - #{dataMap.agent_id}, - #{dataMap.agent_name}, - #{dataMap.tc_title}, - #{dataMap.log_id}, - #{dataMap.event_date}, - #{dataMap.event_time_ts}, - #{dataMap.event_level}, - #{dataMap.src_ip}, - #{dataMap.src_port}, - #{dataMap.dest_ip}, - #{dataMap.dest_port}::BIGINT, - #{dataMap.event_time}, - #{dataMap.attacker_country}, - #{dataMap.src_mac}, - #{dataMap.dest_mac}, - #{dataMap.proto}, - #{dataMap.dev_id}, - #{dataMap.created_time}, - #{dataMap.src_country}, - #{dataMap.src_country_code}, - #{dataMap.src_region}, - #{dataMap.src_region_code}, - #{dataMap.src_city}, - #{dataMap.src_lon}, - #{dataMap.http_method}, - #{dataMap.http_host}, - #{dataMap.http_req_header}, - #{dataMap.http_req_body}, - #{dataMap.http_resp_header}, - #{dataMap.http_resp_body}, - #{dataMap.file_type}, - #{dataMap.file_md5}, - #{dataMap.file_size}, - #{dataMap.process}, - #{dataMap.start_time}, - #{dataMap.action}, - #{dataMap.attacker_region}, - #{dataMap.end_time}, - #{dataMap.file_created_time}, - #{dataMap.file_modified_time}, - #{dataMap.tc_miguan_scan_port}, - #{dataMap.process_path}, - #{dataMap.parent_process_path}, - #{dataMap.gname}, - #{dataMap.exe_name}, - #{dataMap.exe_path}, - #{dataMap.login_time}, - #{dataMap.login_times}, - #{dataMap.check_item}, - #{dataMap.check_type}, - #{dataMap.attacker_ip}, - #{dataMap.attacker_port}, - #{dataMap.victim_ip}, - #{dataMap.victim_port}, - #{dataMap.attacker_city}, - #{dataMap.attacker_lon}, - #{dataMap.attacker_lat}, - #{dataMap.victim_country}, - #{dataMap.victim_region}, - #{dataMap.victim_city}, - #{dataMap.victim_lon}, - #{dataMap.victim_lat}, - #{dataMap.origin_event_id}, - #{dataMap.origin_event_name}, - #{dataMap.origin_event_category}, - #{dataMap.origin_event_level}, - #{dataMap.origin_attack_chain}, - #{dataMap.engine_type}, - #{dataMap.evil_payload}, - #{dataMap.http_resp_status}, - #{dataMap.dns_query}, - #{dataMap.dns_query_type}, - #{dataMap.dns_ttl}, - #{dataMap.dns_answer}, - #{dataMap.dns_subdomains}, - #{dataMap.file_sha256}, - #{dataMap.file_ssdeep}, - #{dataMap.victim_country_code}, - #{dataMap.http_xff_ip}, - #{dataMap.tc_miguan_class}, - #{dataMap.pid}, - #{dataMap.ppid}, - #{dataMap.process_name}, - #{dataMap.backdoor_type}, - #{dataMap.tty}, - #{dataMap.sudo_user}, - #{dataMap.sudo_group}, - #{dataMap.origin_event_type}, - #{dataMap.dest_domain}, - #{dataMap.shell_cmdline}, - #{dataMap.parent_cmdline}, - #{dataMap.attack_chain}, - #{dataMap.process_tree}, - #{dataMap.host_file_sha256}, - #{dataMap.host_file_md5}, - #{dataMap.host_file_size}, - #{dataMap.host_file_type}, - #{dataMap.dest_country}, - #{dataMap.dest_country_code}, - #{dataMap.log_origin}, - #{dataMap.dest_region}, - #{dataMap.src_lat}, - #{dataMap.dest_region_code}, - #{dataMap.dest_city}, - #{dataMap.dest_lon}, - #{dataMap.dest_lat}, - #{dataMap.event_category}, - #{dataMap.attack_result}, - #{dataMap.probe_ip}, - #{dataMap.device_ip}, - #{dataMap.device_manufacturer}, - #{dataMap.device_name}, - #{dataMap.product_name}, - #{dataMap.__id}, - #{dataMap.__count}, - #{dataMap.__count_reason}, - #{dataMap.event_type}, - #{dataMap.protocol}, - #{dataMap.shell_cmd}, - #{dataMap.parent_name}, - #{dataMap.host_file_path}, - #{dataMap.uid}, - #{dataMap.fall}, - #{dataMap.tc_miguan_server_ip}, - #{dataMap.dev_type}, - #{dataMap.collect_method}, - #{dataMap.field_cate_id}, - #{dataMap.device_type}, - #{dataMap.tc_miguan_client_ip}, - #{dataMap.tc_miguan_name}, - #{dataMap.origin_total_packages}, - #{dataMap.origin_total_bytes}, - #{dataMap.origin_peak_packages_rate}, - #{dataMap.origin_peak_bytes_rate}, - #{dataMap.origin_peak_flows_rate}, - #{dataMap.apt_orgname}, - #{dataMap.apt_orgmsg}, - #{dataMap.mail_message_id}, - #{dataMap.mail_bcc}, - #{dataMap.mail_size}, - #{dataMap.mail_attach_hashcode}, - #{dataMap.mail_url}, - #{dataMap.mail_cc}, - #{dataMap.algorithm}, - #{dataMap.miningpool_ip}, - #{dataMap.process_md5}, - #{dataMap.pprocess_md5}, - #{dataMap.source_servername}, - #{dataMap.origin_source_servername}, - #{dataMap.mail_filename}, - #{dataMap.dst_upload_appname}, - #{dataMap.target_port}, - #{dataMap.gid}, - #{dataMap.origin_uid}, - #{dataMap.origin_gid}, - #{dataMap.target_ports}, - #{dataMap.tc_miguan_name1}, - #{dataMap.tc_miguan_class1}, - #{dataMap.etl_time}, - #{dataMap.tc_miguan_scan_port2}, - #{dataMap.desip_security_scope}, - #{dataMap.srcip_security_scope}, - #{dataMap.collect_time_ts}, - #{dataMap.tc_miguan_scan_port1}, - #{dataMap.src_dev_name}, - #{dataMap.collect_protocol}, - #{dataMap.destination_system_type}, - #{dataMap.destination_system}, - #{dataMap.etl_host}, - #{dataMap.normalize_rule_id}, - #{dataMap.normalize_rule_name}, - #{dataMap.syslog_uuid}, - #{dataMap.syslog_topic}, + #{dataMap.id}, + #{dataMap.created_at}, + #{dataMap.log_time}, + #{dataMap.device_id}, + #{dataMap.webshell_type}, + #{dataMap.vuirs_type}, + #{dataMap.vuirs_url}, + #{dataMap.class_filename}, + #{dataMap.class_path}, + #{dataMap.parent_class}, + #{dataMap.jar_path}, + #{dataMap.class_md5}, + #{dataMap.class_loader}, + #{dataMap.class_hashcode}, + #{dataMap.class_loader_hashcode}, + #{dataMap.tc_nameip}, + #{dataMap.perform_sql}, + #{dataMap.tc_account}, + #{dataMap.tc_appname}, + #{dataMap.process_uname}, + #{dataMap.p_process_uname}, + #{dataMap.container_name}, + #{dataMap.container_id}, + #{dataMap.http_resp_server}, + #{dataMap.srcip_id}, + #{dataMap.cdnip}, + #{dataMap.natip}, + #{dataMap.mail_sender}, + #{dataMap.mail_receiver}, + #{dataMap.vpn_mac}, + #{dataMap.vpn_os}, + #{dataMap.vpn_user}, + #{dataMap.vpn_groupname}, + #{dataMap.vpn_access_ip}, + #{dataMap.dest_ip_apt}, + #{dataMap.origin_attack_result}, + #{dataMap.description}, + #{dataMap.solution}, + #{dataMap.attack_cause}, + #{dataMap.username}, + #{dataMap.tc_flow_id}, + #{dataMap.login_result}, + #{dataMap.cmdline}, + #{dataMap.origin_attack_action}, + #{dataMap.victim_domain}, + #{dataMap.vpn_deviceid}, + #{dataMap.vpn_access_action}, + #{dataMap.file_access_time}, + #{dataMap.file_name}, + #{dataMap.tc_class}, + #{dataMap.tc_name2}, + #{dataMap.login_lasttime}, + #{dataMap.origin_permissions}, + #{dataMap.begin_permissions}, + #{dataMap.print_time}, + #{dataMap.printer}, + #{dataMap.printer_type}, + #{dataMap.print_pages}, + #{dataMap.print_copies}, + #{dataMap.src_device}, + #{dataMap.dst_device}, + #{dataMap.src_file}, + #{dataMap.src_file_type}, + #{dataMap.src_file_path}, + #{dataMap.dst_file}, + #{dataMap.dst_file_type}, + #{dataMap.dst_file_path}, + #{dataMap.dlp_policy_name}, + #{dataMap.dlp_policy_type}, + #{dataMap.dst_upload_url}, + #{dataMap.process_uuid}, + #{dataMap.p_process_uuid}, + #{dataMap.env}, + #{dataMap.brute_force_service}, + #{dataMap.vuirs_name}, + #{dataMap.http_req_length}, + #{dataMap.http_req_content_type}, + #{dataMap.tc_scan_port}, + #{dataMap.tc_labels}, + #{dataMap.http_resp_content_type}, + #{dataMap.dns_msg_type}, + #{dataMap.dns_answer_length}, + #{dataMap.dns_ioc}, + #{dataMap.tx_bytes}, + #{dataMap.rx_bytes}, + #{dataMap.all_bytes}, + #{dataMap.duration_time}, + #{dataMap.mail_attach_name}, + #{dataMap.mail_subject}, + #{dataMap.mail_message}, + #{dataMap.mail_send_server}, + #{dataMap.mail_agent}, + #{dataMap.tls_version}, + #{dataMap.tls_server_cert}, + #{dataMap.tls_server_suite}, + #{dataMap.tls_client_suites_len}, + #{dataMap.tls_ja3}, + #{dataMap.tls_ja3s}, + #{dataMap.vpn_access_port}, + #{dataMap.log_topic}, + #{dataMap.collect_time}, + #{dataMap.src_is_intranetip}, + #{dataMap.src_ip_ioc}, + #{dataMap.src_ip_apt}, + #{dataMap.srcip_name}, + #{dataMap.tc_client}, + #{dataMap.srcip_organization_id}, + #{dataMap.dest_ip_intranetip}, + #{dataMap.dest_ip_ioc}, + #{dataMap.desip_id}, + #{dataMap.desip_name}, + #{dataMap.tc_hostip}, + #{dataMap.desip_organization_id}, + #{dataMap.origin_confidence}, + #{dataMap.origin_malscore}, + #{dataMap.attacker_icampaign}, + #{dataMap.attacker_host_asset_id}, + #{dataMap.attacker_organization_id}, + #{dataMap.victim_host_asset_id}, + #{dataMap.victim_organization_id}, + #{dataMap.logout_time}, + #{dataMap.http_req_line}, + #{dataMap.desip_security_scope_id}, + #{dataMap.srcip_security_scope_id}, + #{dataMap.http_resp_length}, + #{dataMap.tc_attack_type}, + #{dataMap.tc_realip}, + #{dataMap.attacker_ip_lists}, + #{dataMap.login_password}, + #{dataMap.detail}, + #{dataMap.attacker_country_code}, + #{dataMap.attacker_region_code}, + #{dataMap.victim_region_code}, + #{dataMap.payload}, + #{dataMap.http_referer}, + #{dataMap.http_user_agent}, + #{dataMap.http_session}, + #{dataMap.http_query_string}, + #{dataMap.file_path}, + #{dataMap.file_permission}, + #{dataMap.login_abnormal_type}, + #{dataMap.file_tag}, + #{dataMap.file_platform}, + #{dataMap.target_ip}, + #{dataMap.collect_date}, + #{dataMap.tc_client_ip}, + #{dataMap.tc_server_ip}, + #{dataMap.tc_externalip}, + #{dataMap.http_status_code}, + #{dataMap.device_domian}, + #{dataMap.src_ip_str}, + #{dataMap.src_port_str}, + #{dataMap.dest_ip_str} , + CAST(#{dataMap.dest_port_str} AS text), + #{dataMap.pcap}, + #{dataMap.ioc}, + #{dataMap.malicious_family}, + #{dataMap.vuln_cve}, + #{dataMap.aliyun_type}, + #{dataMap.attacker_host_asset_name}, + #{dataMap.attacker_organization_name}, + #{dataMap.ct_id}, + #{dataMap.cve_list}, + #{dataMap.desip_organization_name}, + #{dataMap.dest_ip_group}, + #{dataMap.file_gid}, + #{dataMap.file_owner}, + #{dataMap.file_ownergroup}, + #{dataMap.file_uid}, + #{dataMap.http_resp_cookie}, + #{dataMap.origin_rule_id}, + #{dataMap.origin_rule_name}, + #{dataMap.service_name}, + #{dataMap.src_ip_asset_group}, + #{dataMap.srcip_organization_name}, + #{dataMap.victim_host_asset_name}, + #{dataMap.http_resp_codes}::bigint, + #{dataMap.victim_organization_name}, + #{dataMap.tc_type}, + #{dataMap.direction}, + #{dataMap.http_req_cookie}, + #{dataMap.http_req_protocol}, + #{dataMap.http_req_header_raw}, + #{dataMap.http_url}, + #{dataMap.uname}, + #{dataMap.origin_hostname}, + #{dataMap.origin_os}, + #{dataMap.origin_agent_mac}, + #{dataMap.origin_host_id}, + #{dataMap.origin_agent_version}, + #{dataMap.origin_agent_id}, + #{dataMap.origin_agent_name}, + #{dataMap.origin_work_group}, + #{dataMap.origin_asset_group}, + #{dataMap.origin_local_port}, + #{dataMap.origin_agent_ip}, + #{dataMap.origin_internal_ip}, + #{dataMap.origin_external_ip}, + #{dataMap.origin_local_addr}, + #{dataMap.agent_id}, + #{dataMap.agent_name}, + #{dataMap.tc_title}, + #{dataMap.log_id}, + #{dataMap.event_date}, + #{dataMap.event_time_ts}, + #{dataMap.event_level}, + #{dataMap.src_ip}, + #{dataMap.src_port}, + #{dataMap.dest_ip}, + #{dataMap.dest_port}::BIGINT, + #{dataMap.event_time}, + #{dataMap.attacker_country}, + #{dataMap.src_mac}, + #{dataMap.dest_mac}, + #{dataMap.proto}, + #{dataMap.dev_id}, + #{dataMap.created_time}, + #{dataMap.src_country}, + #{dataMap.src_country_code}, + #{dataMap.src_region}, + #{dataMap.src_region_code}, + #{dataMap.src_city}, + #{dataMap.src_lon}, + #{dataMap.http_method}, + #{dataMap.http_host}, + #{dataMap.http_req_header}, + #{dataMap.http_req_body}, + #{dataMap.http_resp_header}, + #{dataMap.http_resp_body}, + #{dataMap.file_type}, + #{dataMap.file_md5}, + #{dataMap.file_size}, + #{dataMap.process}, + #{dataMap.start_time}, + #{dataMap.action}, + #{dataMap.attacker_region}, + #{dataMap.end_time}, + #{dataMap.file_created_time}, + #{dataMap.file_modified_time}, + #{dataMap.tc_miguan_scan_port}, + #{dataMap.process_path}, + #{dataMap.parent_process_path}, + #{dataMap.gname}, + #{dataMap.exe_name}, + #{dataMap.exe_path}, + #{dataMap.login_time}, + #{dataMap.login_times}, + #{dataMap.check_item}, + #{dataMap.check_type}, + #{dataMap.attacker_ip}, + #{dataMap.attacker_port}, + #{dataMap.victim_ip}, + #{dataMap.victim_port}, + #{dataMap.attacker_city}, + #{dataMap.attacker_lon}, + #{dataMap.attacker_lat}, + #{dataMap.victim_country}, + #{dataMap.victim_region}, + #{dataMap.victim_city}, + #{dataMap.victim_lon}, + #{dataMap.victim_lat}, + #{dataMap.origin_event_id}, + #{dataMap.origin_event_name}, + #{dataMap.origin_event_category}, + #{dataMap.origin_event_level}, + #{dataMap.origin_attack_chain}, + #{dataMap.engine_type}, + #{dataMap.evil_payload}, + #{dataMap.http_resp_status}, + #{dataMap.dns_query}, + #{dataMap.dns_query_type}, + #{dataMap.dns_ttl}, + #{dataMap.dns_answer}, + #{dataMap.dns_subdomains}, + #{dataMap.file_sha256}, + #{dataMap.file_ssdeep}, + #{dataMap.victim_country_code}, + #{dataMap.http_xff_ip}, + #{dataMap.tc_miguan_class}, + #{dataMap.pid}, + #{dataMap.ppid}, + #{dataMap.process_name}, + #{dataMap.backdoor_type}, + #{dataMap.tty}, + #{dataMap.sudo_user}, + #{dataMap.sudo_group}, + #{dataMap.origin_event_type}, + #{dataMap.dest_domain}, + #{dataMap.shell_cmdline}, + #{dataMap.parent_cmdline}, + #{dataMap.attack_chain}, + #{dataMap.process_tree}, + #{dataMap.host_file_sha256}, + #{dataMap.host_file_md5}, + #{dataMap.host_file_size}, + #{dataMap.host_file_type}, + #{dataMap.dest_country}, + #{dataMap.dest_country_code}, + #{dataMap.log_origin}, + #{dataMap.dest_region}, + #{dataMap.src_lat}, + #{dataMap.dest_region_code}, + #{dataMap.dest_city}, + #{dataMap.dest_lon}, + #{dataMap.dest_lat}, + #{dataMap.event_category}, + #{dataMap.attack_result}, + #{dataMap.probe_ip}, + #{dataMap.device_ip}, + #{dataMap.device_manufacturer}, + #{dataMap.device_name}, + #{dataMap.product_name}, + #{dataMap.__id}, + #{dataMap.__count}, + #{dataMap.__count_reason}, + #{dataMap.event_type}, + #{dataMap.protocol}, + #{dataMap.shell_cmd}, + #{dataMap.parent_name}, + #{dataMap.host_file_path}, + #{dataMap.uid}, + #{dataMap.fall}, + #{dataMap.tc_miguan_server_ip}, + #{dataMap.dev_type}, + #{dataMap.collect_method}, + #{dataMap.field_cate_id}, + #{dataMap.device_type}, + #{dataMap.tc_miguan_client_ip}, + #{dataMap.tc_miguan_name}, + #{dataMap.origin_total_packages}, + #{dataMap.origin_total_bytes}, + #{dataMap.origin_peak_packages_rate}, + #{dataMap.origin_peak_bytes_rate}, + #{dataMap.origin_peak_flows_rate}, + #{dataMap.apt_orgname}, + #{dataMap.apt_orgmsg}, + #{dataMap.mail_message_id}, + #{dataMap.mail_bcc}, + #{dataMap.mail_size}, + #{dataMap.mail_attach_hashcode}, + #{dataMap.mail_url}, + #{dataMap.mail_cc}, + #{dataMap.algorithm}, + #{dataMap.miningpool_ip}, + #{dataMap.process_md5}, + #{dataMap.pprocess_md5}, + #{dataMap.source_servername}, + #{dataMap.origin_source_servername}, + #{dataMap.mail_filename}, + #{dataMap.dst_upload_appname}, + #{dataMap.target_port}, + #{dataMap.gid}, + #{dataMap.origin_uid}, + #{dataMap.origin_gid}, + #{dataMap.target_ports}, + #{dataMap.tc_miguan_name1}, + #{dataMap.tc_miguan_class1}, + #{dataMap.etl_time}, + #{dataMap.tc_miguan_scan_port2}, + #{dataMap.desip_security_scope}, + #{dataMap.srcip_security_scope}, + #{dataMap.collect_time_ts}, + #{dataMap.tc_miguan_scan_port1}, + #{dataMap.src_dev_name}, + #{dataMap.collect_protocol}, + #{dataMap.destination_system_type}, + #{dataMap.destination_system}, + #{dataMap.etl_host}, + #{dataMap.normalize_rule_id}, + #{dataMap.normalize_rule_name}, + #{dataMap.syslog_uuid}, + #{dataMap.syslog_topic}, + #{dataMap.rule_version}, @@ -914,14 +917,4 @@ - - - INSERT INTO syslog_normal_data - (id, log_time, src_ip, dest_ip, event_level) - VALUES - - (#{item.id}, #{item.log_time}, #{item.src_ip}, #{item.dest_ip}, #{item.event_level}) - - - diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java index 7a7ba61..560917c 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/LogNormalProcessorRuleSelectionTest.java @@ -4,6 +4,7 @@ import com.Modules.NormalData.compiled.BatchEnrichmentContext; import com.Modules.NormalData.compiled.BatchRuleSnapshot; import com.Modules.NormalData.compiled.CompiledRuleCache; import com.Modules.NormalData.compiled.CompiledRuleCompiler; +import com.Modules.NormalData.compiled.PreparedNormalizationInput; import com.Modules.NormalData.compiled.RuleExecutionMetrics; import com.Modules.NormalData.compiled.RuleSource; import com.common.entity.SyslogNonNormalMessage; @@ -90,6 +91,20 @@ class LogNormalProcessorRuleSelectionTest { verify(fixture.nonNormalService).saveMessage(any(SyslogNonNormalMessage.class)); } + @Test + void activeKafkaPreparationDefersNonNormalPersistence() { + Fixture fixture = fixture(Arrays.asList( + regexRule(10L, "first", "first=(\\S+)"), + regexRule(20L, "second", "second=(\\S+)")), config(true)); + + LogNormalProcessor.NormalizationResult result = fixture.prepareActive("nothing matches"); + + assertEquals(LogNormalProcessor.ProcessingOutcome.FILTERED, result.getOutcome()); + assertEquals(1, result.getNonNormalMessages().size()); + assertEquals("未命中规则", result.getNonNormalMessages().get(0).getReason()); + verify(fixture.nonNormalService, never()).saveMessage(any()); + } + @Test void nonNormalSaveFailureRemainsRetryable() { Fixture fixture = fixture(Arrays.asList( @@ -232,5 +247,23 @@ class LogNormalProcessorRuleSelectionTest { new BatchEnrichmentContext(false, taskService, deviceService), new RuleExecutionMetrics()); } + + private LogNormalProcessor.NormalizationResult prepareActive(String body) { + LocalDateTime receivedAt = LocalDateTime.of(2026, 8, 21, 15, 0); + Map deviceInfo = new LinkedHashMap<>(); + deviceInfo.put("receive_time", "20260821150000000"); + deviceInfo.put("event_id", "event-selection"); + deviceInfo.put("device_id", "7"); + deviceInfo.put("device_name", "fw"); + deviceInfo.put("vendor", "vendor"); + deviceInfo.put("data_type", "json"); + deviceInfo.put("device_collect_id", "3"); + PreparedNormalizationInput input = new PreparedNormalizationInput( + "event-selection", "raw", receivedAt, receivedAt, + 7L, 3L, "json", body, body, "", deviceInfo); + return processor.prepare(input, snapshot, + new BatchEnrichmentContext(false, taskService, deviceService), + new RuleExecutionMetrics()); + } } } diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/QuanxiNormalizationTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/QuanxiNormalizationTest.java index 288f322..23fbea9 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/QuanxiNormalizationTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/QuanxiNormalizationTest.java @@ -59,7 +59,7 @@ class QuanxiNormalizationTest { assertTrue(f.store.isAlarmData(row)); assertEquals(1, f.store.insertDynamic(row)); } - verify(f.alarm, times(20)).insertDynamic(anyMap()); + verify(f.alarm, times(20)).insertBatchPlanned(anyMap()); verifyNoInteractions(f.normal); } } @@ -107,7 +107,9 @@ class QuanxiNormalizationTest { @Test void ruleDestinationsAreActualAlarmSqlColumnsAndHttpPathsAreExact() throws Exception { JsonNode rule = JSON.readTree(file("normalization-rule.json").toFile()); String sql = new String(Files.readAllBytes(file("../../../syslog-consumer-rule/src/main/resources/mapper/SyslogNormalAlarmMapper.xml")), "UTF-8"); - for (JsonNode m : rule.path("mappers")) assertTrue(sql.contains("dataMap." + m.path("dest_field").asText() + " ")); + for (JsonNode m : rule.path("mappers")) { + assertTrue(sql.contains("dataMap." + m.path("dest_field").asText())); + } for (String[] field : HTTP_FIELDS) { boolean found = false; for (JsonNode m : rule.path("mappers")) if (field[1].equals(m.path("dest_field").asText())) { @@ -138,7 +140,7 @@ class QuanxiNormalizationTest { ReflectionTestUtils.setField(store, "syslogNormalDataMapper", normal); ReflectionTestUtils.setField(store, "statsChangeCaptureService", mock(StatsChangeCaptureService.class)); ReflectionTestUtils.setField(store, "directStatsProjectionWriter", mock(DirectStatsProjectionWriter.class)); - when(alarm.insertDynamic(anyMap())).thenReturn(1); + when(alarm.insertBatchPlanned(anyMap())).thenReturn(1); NormalizationOptimizationProperties p = new NormalizationOptimizationProperties(); p.getCompiledRule().setEnabled(compiled); p.getCompiledRule().setShadowCompareEnabled(false); p.getRuleSelection().setEnabled(true); p.getRuleSelection().setShadowEnabled(false); diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java index 9e46944..ed5e055 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/Modules/NormalData/SysLogProcessorParallelTest.java @@ -3,9 +3,11 @@ package com.Modules.NormalData; import com.Modules.NormalData.compiled.BatchEnrichmentContext; import com.Modules.NormalData.compiled.BatchRuleSnapshot; import com.Modules.NormalData.compiled.PreparedNormalizationInput; +import com.common.entity.SyslogNonNormalMessage; import com.common.service.DeviceCollectTaskService; import com.common.service.DeviceDeviceService; import com.common.service.DmStandardizedBatchWriter; +import com.common.service.SyslogNonNormalMessageService; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -19,12 +21,18 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -32,23 +40,29 @@ import static org.mockito.Mockito.when; class SysLogProcessorParallelTest { private ExecutorService executor; + private ExecutorService nonNormalExecutor; private SysLogProcessor processor; private LogNormalProcessor normalProcessor; private SyslogFailureRouter failureRouter; private DmStandardizedBatchWriter batchWriter; + private SyslogNonNormalMessageService nonNormalMessageService; @BeforeEach void setUp() { executor = Executors.newFixedThreadPool(4); + nonNormalExecutor = Executors.newSingleThreadExecutor(); processor = new SysLogProcessor(); normalProcessor = mock(LogNormalProcessor.class); failureRouter = mock(SyslogFailureRouter.class); batchWriter = mock(DmStandardizedBatchWriter.class); + nonNormalMessageService = mock(SyslogNonNormalMessageService.class); ReflectionTestUtils.setField(processor, "taskExecutor", executor); ReflectionTestUtils.setField(processor, "processTimeoutMs", 5000L); ReflectionTestUtils.setField(processor, "logNormalProcessor", normalProcessor); ReflectionTestUtils.setField(processor, "failureRouter", failureRouter); ReflectionTestUtils.setField(processor, "standardizedBatchWriter", batchWriter); + ReflectionTestUtils.setField(processor, "nonNormalMessageService", nonNormalMessageService); + ReflectionTestUtils.setField(processor, "nonNormalWriterExecutor", nonNormalExecutor); when(batchWriter.write(any())).thenReturn(new DmStandardizedBatchWriter.WriteReport<>()); when(normalProcessor.captureRuleSnapshot(any())).thenReturn( new BatchRuleSnapshot(Collections.emptyMap(), System.currentTimeMillis())); @@ -59,7 +73,9 @@ class SysLogProcessorParallelTest { @AfterEach void tearDown() throws InterruptedException { executor.shutdownNow(); + nonNormalExecutor.shutdownNow(); executor.awaitTermination(5, TimeUnit.SECONDS); + nonNormalExecutor.awaitTermination(5, TimeUnit.SECONDS); } @Test @@ -121,6 +137,73 @@ class SysLogProcessorParallelTest { "达梦写入耗时应独立覆盖批写调用"); } + @Test + void deferredNonNormalMessagesAreSavedOncePerKafkaBatch() { + when(normalProcessor.prepare(any(PreparedNormalizationInput.class), + any(), any(), any())).thenAnswer(invocation -> { + SyslogNonNormalMessage message = new SyslogNonNormalMessage(); + message.setId(java.util.UUID.randomUUID().toString()); + return LogNormalProcessor.NormalizationResult.filtered( + Collections.singletonList(message)); + }); + when(nonNormalMessageService.saveBatchMessages(anyList())) + .thenAnswer(invocation -> ((List) invocation.getArgument(0)).size()); + + SysLogProcessor.BatchMetrics metrics = new SysLogProcessor.BatchMetrics(); + processor.normalizePreparedMessages(messages(8), metrics); + + @SuppressWarnings("unchecked") + org.mockito.ArgumentCaptor> captor = + org.mockito.ArgumentCaptor.forClass(List.class); + verify(nonNormalMessageService, timeout(1000)).saveBatchMessages(captor.capture()); + assertEquals(8, captor.getValue().size()); + assertEquals(8, metrics.nonNormalQueued()); + assertEquals(0, metrics.nonNormalRejected()); + assertEquals(0, metrics.successCount()); + } + + @Test + void asynchronousNonNormalFailureDoesNotFailKafkaBatch() { + when(normalProcessor.prepare(any(PreparedNormalizationInput.class), + any(), any(), any())).thenAnswer(invocation -> { + SyslogNonNormalMessage message = new SyslogNonNormalMessage(); + message.setId(java.util.UUID.randomUUID().toString()); + return LogNormalProcessor.NormalizationResult.filtered( + Collections.singletonList(message)); + }); + when(nonNormalMessageService.saveBatchMessages(anyList())) + .thenThrow(new IllegalStateException("dm unavailable")); + + SysLogProcessor.BatchMetrics metrics = new SysLogProcessor.BatchMetrics(); + processor.normalizePreparedMessages(messages(2), metrics); + + verify(nonNormalMessageService, timeout(1000)).saveBatchMessages(anyList()); + assertEquals(2, metrics.nonNormalQueued()); + assertEquals(2, metrics.filteredCount()); + } + + @Test + void rejectedNonNormalTaskDoesNotFailKafkaBatch() { + when(normalProcessor.prepare(any(PreparedNormalizationInput.class), + any(), any(), any())).thenAnswer(invocation -> { + SyslogNonNormalMessage message = new SyslogNonNormalMessage(); + return LogNormalProcessor.NormalizationResult.filtered( + Collections.singletonList(message)); + }); + ReflectionTestUtils.setField(processor, "nonNormalWriterExecutor", + (java.util.concurrent.Executor) command -> { + throw new RejectedExecutionException("queue full"); + }); + + SysLogProcessor.BatchMetrics metrics = new SysLogProcessor.BatchMetrics(); + processor.normalizePreparedMessages(messages(3), metrics); + + verify(nonNormalMessageService, never()).saveBatchMessages(anyList()); + assertEquals(0, metrics.nonNormalQueued()); + assertEquals(3, metrics.nonNormalRejected()); + assertEquals(3, metrics.filteredCount()); + } + private List messages(int count) { List messages = new ArrayList<>(); for (int i = 0; i < count; i++) { diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/mapper/FixedColumnBatchMapperTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/mapper/FixedColumnBatchMapperTest.java new file mode 100644 index 0000000..af7eefb --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/mapper/FixedColumnBatchMapperTest.java @@ -0,0 +1,127 @@ +package com.common.mapper; + +import org.apache.ibatis.builder.xml.XMLMapperBuilder; +import org.apache.ibatis.mapping.MappedStatement; +import org.apache.ibatis.session.Configuration; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Scanner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class FixedColumnBatchMapperTest { + + @Test + void normalDataUsesWhitelistedBatchUnionTemplate() { + assertBatchUnionMapper( + "/mapper/SyslogNormalDataMapper.xml", "syslog_normal_data"); + } + + @Test + void normalAlarmUsesWhitelistedBatchUnionTemplate() { + assertBatchUnionMapper( + "/mapper/SyslogNormalAlarmMapper.xml", "syslog_normal_alarm"); + } + + @Test + void alignedRowsProduceIdenticalMyBatisSqlEvenWhenValuesAreNull() { + assertSameBoundSql( + "/mapper/SyslogNormalDataMapper.xml", + "com.common.mapper.SyslogNormalDataMapper.insertBatchPlanned"); + assertSameBoundSql( + "/mapper/SyslogNormalAlarmMapper.xml", + "com.common.mapper.SyslogNormalAlarmMapper.insertBatchPlanned"); + } + + private void assertBatchUnionMapper(String resourcePath, String table) { + String xml = resource(resourcePath); + Matcher statement = Pattern.compile( + "\\s*VALUES\\s*\\s*", + Pattern.DOTALL).matcher(xml); + + assertTrue(statement.find(), "缺少 " + table + " 批次列并集 INSERT"); + String insert = statement.group(0); + assertTrue(insert.contains("dataMap.containsKey('src_ip')")); + assertTrue(insert.contains("dataMap.containsKey('dest_ip')")); + assertTrue(insert.contains("dataMap.containsKey('rule_version')")); + assertFalse(insert.contains("dataMap.src_ip != null"), + "列选择必须依据调用方补齐后的统一键集合,不能再按各行值判断"); + + Matcher conditions = Pattern.compile("dataMap\\.containsKey\\('([a-zA-Z0-9_]+)'\\)") + .matcher(insert); + java.util.Map occurrences = new java.util.HashMap<>(); + while (conditions.find()) { + occurrences.merge(conditions.group(1), 1, Integer::sum); + } + assertTrue(occurrences.size() > 300, "白名单必须保留历史标准化候选字段"); + assertTrue(occurrences.values().stream().allMatch(count -> count == 2), + "每个允许字段必须同时出现在列清单和值清单中"); + } + + private void assertSameBoundSql(String resourcePath, String statementId) { + String xml = resource(resourcePath); + Configuration configuration = new Configuration(); + try (InputStream stream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) { + new XMLMapperBuilder(stream, configuration, resourcePath, + configuration.getSqlFragments()).parse(); + } catch (Exception parseFailure) { + throw new AssertionError("Mapper XML 解析失败: " + resourcePath, parseFailure); + } + MappedStatement statement = configuration.getMappedStatement(statementId); + + Map first = new HashMap<>(); + first.put("id", "row-1"); + first.put("src_ip", "10.0.0.1"); + first.put("dest_ip", null); + first.put("rule_version", "v1"); + Map second = new HashMap<>(); + second.put("id", "row-2"); + second.put("src_ip", null); + second.put("dest_ip", "10.0.0.2"); + second.put("rule_version", "v1"); + + String firstSql = normalizedSql(statement, first); + String secondSql = normalizedSql(statement, second); + assertEquals(firstSql, secondSql, "同一字段并集的各行必须生成完全相同的 JDBC Batch SQL"); + assertTrue(firstSql.contains("src_ip")); + assertTrue(firstSql.contains("dest_ip")); + assertFalse(firstSql.contains("webshell_type")); + } + + private String normalizedSql(MappedStatement statement, Map row) { + Map parameter = new HashMap<>(); + parameter.put("dataMap", row); + return statement.getBoundSql(parameter).getSql().replaceAll("\\s+", " ").trim(); + } + + @Test + void nonNormalDataUsesFixedColumnsAndMultiRowValues() { + String xml = resource("/mapper/SyslogNonNormalMessageMapper.xml"); + Matcher statement = Pattern.compile( + "", Pattern.DOTALL).matcher(xml); + + assertTrue(statement.find(), "缺少 syslog_non_normal_message 批量 INSERT"); + assertFalse(statement.group(0).contains(" first = new HashMap<>(); + first.put("id", "row-1"); + first.put("src_ip", "10.0.0.1"); + first.put("unused_null", null); + + Map second = new HashMap<>(); + second.put("id", "row-2"); + second.put("dest_ip", "10.0.0.2"); + + List> aligned = + DmStandardizedBatchExecutor.alignBatchColumns(Arrays.asList(first, second)); + + assertEquals(aligned.get(0).keySet(), aligned.get(1).keySet()); + assertEquals(Arrays.asList("dest_ip", "id", "src_ip"), + aligned.get(0).keySet().stream().sorted().collect(java.util.stream.Collectors.toList())); + assertEquals("10.0.0.1", aligned.get(0).get("src_ip")); + assertNull(aligned.get(0).get("dest_ip")); + assertNull(aligned.get(1).get("src_ip")); + assertEquals("10.0.0.2", aligned.get(1).get("dest_ip")); + assertFalse(aligned.get(0).containsKey("unused_null")); + assertFalse(first.containsKey("dest_ip"), "不得修改上游标准化结果 Map"); + assertFalse(second.containsKey("src_ip"), "不得修改上游标准化结果 Map"); + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmStandardizedBatchWriterTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmStandardizedBatchWriterTest.java index 628a737..b893dd4 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmStandardizedBatchWriterTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/DmStandardizedBatchWriterTest.java @@ -100,6 +100,31 @@ class DmStandardizedBatchWriterTest { verify(normalDataService, never()).insertDynamic(any()); } + @Test + void normalDataWithDifferentPopulatedColumnsSharesOneUnionPlannedBatch() { + List> requests = requests(2, -1); + requests.get(0).getData().put("src_ip", "10.0.0.1"); + requests.get(1).getData().put("dest_ip", "10.0.0.2"); + + DmStandardizedBatchWriter.WriteReport report = writer.write(requests); + + assertEquals(2, report.getSuccessfulRows()); + verify(batchExecutor, times(1)).execute(anyBoolean(), anyList(), anyInt()); + } + + @Test + void alarmDataWithDifferentPopulatedColumnsSharesOneUnionPlannedBatch() { + when(normalDataService.isAlarmData(any())).thenReturn(true); + List> requests = requests(2, -1); + requests.get(0).getData().put("src_ip", "10.0.0.1"); + requests.get(1).getData().put("dest_ip", "10.0.0.2"); + + DmStandardizedBatchWriter.WriteReport report = writer.write(requests); + + assertEquals(2, report.getSuccessfulRows()); + verify(batchExecutor, times(1)).execute(anyBoolean(), anyList(), anyInt()); + } + private List> requests(int count, int badIndex) { List> requests = new ArrayList<>(); for (int i = 0; i < count; i++) { @@ -108,7 +133,7 @@ class DmStandardizedBatchWriterTest { row.put("rule_version", "legacy"); row.put("created_at", LocalDateTime.of(2026, 8, 11, 3, 55)); row.put("log_time", LocalDateTime.of(2026, 8, 11, 3, 55)); - // 所有行都保留相同列签名;坏行只改变值,确保它真的混在同一个 JDBC batch 中。 + // 普通日志按当前小批次字段并集生成模板;额外键不会再造成批次拆分。 row.put("bad", i == badIndex); requests.add(new DmStandardizedBatchWriter.WriteRequest<>("source-" + i, row)); } diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNonNormalMessageServiceBatchTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNonNormalMessageServiceBatchTest.java new file mode 100644 index 0000000..19e5178 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNonNormalMessageServiceBatchTest.java @@ -0,0 +1,61 @@ +package com.common.service; + +import com.common.entity.SyslogNonNormalMessage; +import com.common.mapper.SyslogNonNormalMessageMapper; +import com.common.service.impl.SyslogNonNormalMessageServiceImpl; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class SyslogNonNormalMessageServiceBatchTest { + + private SyslogNonNormalMessageServiceImpl service; + private SyslogNonNormalMessageMapper mapper; + private DirectStatsProjectionWriter directStatsProjectionWriter; + + @BeforeEach + void setUp() { + service = new SyslogNonNormalMessageServiceImpl(); + mapper = mock(SyslogNonNormalMessageMapper.class); + directStatsProjectionWriter = mock(DirectStatsProjectionWriter.class); + ReflectionTestUtils.setField(service, "messageMapper", mapper); + ReflectionTestUtils.setField(service, "statsChangeCaptureService", + mock(StatsChangeCaptureService.class)); + ReflectionTestUtils.setField(service, "directStatsProjectionWriter", + directStatsProjectionWriter); + ReflectionTestUtils.setField(service, "nonNormalBatchSize", 100); + when(mapper.batchInsert(anyList())) + .thenAnswer(invocation -> ((List) invocation.getArgument(0)).size()); + } + + @Test + void chunksLargePollInsideOneServiceTransactionAndAggregatesStatsOnce() { + List messages = new ArrayList<>(); + for (int i = 0; i < 250; i++) { + SyslogNonNormalMessage message = new SyslogNonNormalMessage(); + message.setId("non-normal-" + i); + messages.add(message); + } + + assertEquals(250, service.saveBatchMessages(messages)); + + @SuppressWarnings("unchecked") + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(mapper, times(3)).batchInsert(captor.capture()); + assertEquals(100, captor.getAllValues().get(0).size()); + assertEquals(100, captor.getAllValues().get(1).size()); + assertEquals(50, captor.getAllValues().get(2).size()); + verify(directStatsProjectionWriter).writeNonNormalRows(messages); + } +} diff --git a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNormalDataServiceIdempotencyTest.java b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNormalDataServiceIdempotencyTest.java index b608ae5..2575989 100644 --- a/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNormalDataServiceIdempotencyTest.java +++ b/haobang-security-dm/syslog-consumer-rule/src/test/java/com/common/service/SyslogNormalDataServiceIdempotencyTest.java @@ -38,7 +38,7 @@ class SyslogNormalDataServiceIdempotencyTest { @Test void dmUniqueViolationWithExistingNormalResultIsIdempotentSuccess() { Map data = idempotencyData(false); - when(normalDataMapper.insertDynamic(anyMap())).thenThrow(dmUniqueViolation()); + when(normalDataMapper.insertBatchPlanned(anyMap())).thenThrow(dmUniqueViolation()); when(normalDataMapper.countByLogTimeAndId( (LocalDateTime) data.get("log_time"), (String) data.get("id"))) .thenReturn(1); @@ -53,7 +53,7 @@ class SyslogNormalDataServiceIdempotencyTest { @Test void dmUniqueViolationWithExistingAlarmResultIsIdempotentSuccess() { Map data = idempotencyData(true); - when(normalAlarmMapper.insertDynamic(anyMap())).thenThrow(dmUniqueViolation()); + when(normalAlarmMapper.insertBatchPlanned(anyMap())).thenThrow(dmUniqueViolation()); when(normalAlarmMapper.countByLogTimeAndId( (LocalDateTime) data.get("log_time"), (String) data.get("id"))) .thenReturn(1); @@ -70,7 +70,7 @@ class SyslogNormalDataServiceIdempotencyTest { Map data = idempotencyData(false); DataIntegrityViolationException invalidData = new DataIntegrityViolationException( "value too long", new SQLException("data too long")); - when(normalDataMapper.insertDynamic(anyMap())).thenThrow(invalidData); + when(normalDataMapper.insertBatchPlanned(anyMap())).thenThrow(invalidData); assertThrows(DataIntegrityViolationException.class, () -> service.insertDynamic(data)); diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java index cab9855..8e1153b 100644 --- a/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java +++ b/haobang-security-dm/syslog-serve/src/test/java/com/kafka/KafkaProducerEventIdTest.java @@ -23,9 +23,9 @@ class KafkaProducerEventIdTest { void maxRequestSizeIsPassedToTheManuallyConstructedKafkaProducer() { Properties properties = new Properties(); - kafkaProducer.applyMaxRequestSize(properties, 4 * 1024 * 1024); + kafkaProducer.applyMaxRequestSize(properties, 8 * 1024 * 1024); - assertEquals(4 * 1024 * 1024, properties.get(ProducerConfig.MAX_REQUEST_SIZE_CONFIG)); + assertEquals(8 * 1024 * 1024, properties.get(ProducerConfig.MAX_REQUEST_SIZE_CONFIG)); } @Test diff --git a/haobang-security-dm/upgrade-doc/v1/kafka/02-skip-raw-backlog.sh b/haobang-security-dm/upgrade-doc/v1/kafka/02-skip-raw-backlog.sh new file mode 100755 index 0000000..eb299a0 --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v1/kafka/02-skip-raw-backlog.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# 将指定消费组在原始 Topic 上的 offset 重置到指定 Kafka 消息时间。 +# 适用于“丢弃截止时间之前的历史积压,保留截止时间之后消息”的现场操作。 +# +# 本脚本不会停止/启动应用,也不会删除 Topic 或消息: +# - 执行前必须由运维停止该消费组的全部 syslog-consumer-rule 实例; +# - 自动导出原 offset,Kafka 数据仍在 retention 内时可用于人工回退; +# - 先执行 dry-run,只有输入精确确认串才执行真正的 offset 重置。 + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +BOOTSTRAP="${BOOTSTRAP:-10.141.121.35:9092,10.141.121.50:9092,10.141.121.59:9092}" +GROUP_ID="${GROUP_ID:-agent-01-syslog-group}" +RAW_TOPIC="${RAW_TOPIC:-agent-01-syslog-topic}" +CUTOFF_DATETIME="${CUTOFF_DATETIME:-2026-09-11T00:00:00.000}" +KAFKA_TIMEZONE="${KAFKA_TIMEZONE:-Asia/Shanghai}" + +# 优先使用调用方显式指定的目录;否则自动识别脚本同目录、现场 Kafka 安装目录或 PATH。 +if [[ -z "${KAFKA_BIN:-}" ]]; then + for CANDIDATE_BIN in \ + "${SCRIPT_DIR}" \ + "/opt/kafka/kafka_2.13-3.6.1/bin"; do + if [[ -x "${CANDIDATE_BIN}/kafka-consumer-groups.sh" ]] \ + && [[ -x "${CANDIDATE_BIN}/kafka-topics.sh" ]]; then + KAFKA_BIN="${CANDIDATE_BIN}" + break + fi + done +fi + +if [[ -z "${KAFKA_BIN:-}" ]] && command -v kafka-consumer-groups.sh >/dev/null 2>&1; then + KAFKA_BIN=$(dirname -- "$(command -v kafka-consumer-groups.sh)") +fi + +if [[ -z "${KAFKA_BIN:-}" ]]; then + echo "ERROR: 未找到 Kafka CLI;现场预期路径为 /opt/kafka/kafka_2.13-3.6.1/bin" >&2 + exit 1 +fi + +CONSUMER_GROUPS_CMD="${KAFKA_BIN}/kafka-consumer-groups.sh" +TOPICS_CMD="${KAFKA_BIN}/kafka-topics.sh" + +if [[ ! -x "${CONSUMER_GROUPS_CMD}" ]] || [[ ! -x "${TOPICS_CMD}" ]]; then + echo "ERROR: ${KAFKA_BIN} 下缺少可执行的 kafka-consumer-groups.sh 或 kafka-topics.sh" >&2 + exit 1 +fi + +if [[ ! "${CUTOFF_DATETIME}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}$ ]]; then + echo "ERROR: CUTOFF_DATETIME 格式必须是 YYYY-MM-DDTHH:mm:ss.SSS" >&2 + exit 1 +fi + +COMMON_ARGS=(--bootstrap-server "${BOOTSTRAP}") +if [[ -n "${COMMAND_CONFIG:-}" ]]; then + if [[ ! -r "${COMMAND_CONFIG}" ]]; then + echo "ERROR: 无法读取认证配置 ${COMMAND_CONFIG}" >&2 + exit 1 + fi + COMMON_ARGS+=(--command-config "${COMMAND_CONFIG}") +fi + +# Kafka 3.6 的 --to-datetime 不携带时区,显式固定 JVM 时区,避免宿主机差异造成 8 小时偏移。 +export KAFKA_OPTS="${KAFKA_OPTS:-} -Duser.timezone=${KAFKA_TIMEZONE}" + +RUN_ID=$(date '+%Y%m%d_%H%M%S') +AUDIT_DIR="${OFFSET_BACKUP_DIR:-./kafka-offset-reset-${RUN_ID}}" +mkdir -p "${AUDIT_DIR}" + +BEFORE_FILE="${AUDIT_DIR}/before-describe.txt" +RESTORE_FILE="${AUDIT_DIR}/restore-current-offsets.csv" +DRY_RUN_FILE="${AUDIT_DIR}/reset-dry-run.txt" +EXECUTE_FILE="${AUDIT_DIR}/reset-execute.txt" +AFTER_FILE="${AUDIT_DIR}/after-describe.txt" + +echo "目标 Kafka:${BOOTSTRAP}" +echo "目标消费组:${GROUP_ID}" +echo "目标 Topic:${RAW_TOPIC}" +echo "保留起点:${CUTOFF_DATETIME} (${KAFKA_TIMEZONE})" +echo "审计目录:${AUDIT_DIR}" + +echo "[1/6] 检查 Topic" +TOPIC_INFO=$("${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}") +printf '%s\n' "${TOPIC_INFO}" + +PARTITION_COUNT=$(printf '%s\n' "${TOPIC_INFO}" | awk ' + /PartitionCount:/ { + for (i=1; i<=NF; i++) if ($i=="PartitionCount:") { print $(i+1); exit } + } +') +if [[ ! "${PARTITION_COUNT}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: 无法确认 ${RAW_TOPIC} 的分区数" >&2 + exit 1 +fi + +echo "[2/6] 检查消费组是否已停止,并保存执行前快照" +"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \ + --describe --group "${GROUP_ID}" 2>&1 | tee "${BEFORE_FILE}" + +MATCHED_ROWS=$(awk -v topic="${RAW_TOPIC}" ' + $2==topic && $3~/^[0-9]+$/ && $4~/^[0-9]+$/ && $5~/^[0-9]+$/ {count++} + END {print count+0} +' "${BEFORE_FILE}") +if [[ "${MATCHED_ROWS}" -ne "${PARTITION_COUNT}" ]]; then + echo "ERROR: 快照只识别到 ${MATCHED_ROWS}/${PARTITION_COUNT} 个目标分区,拒绝继续" >&2 + exit 1 +fi + +if awk -v topic="${RAW_TOPIC}" ' + $2==topic && $3~/^[0-9]+$/ && $7!="-" {active=1} + END {exit(active ? 0 : 1)} +' "${BEFORE_FILE}"; then + echo "ERROR: 消费组 ${GROUP_ID} 仍有在线成员。请先停止全部 syslog-consumer-rule 实例。" >&2 + exit 1 +fi + +CURRENT_LAG=$(awk -v topic="${RAW_TOPIC}" ' + $2==topic && $6~/^[0-9]+$/ {sum+=$6} + END {printf "%.0f", sum+0} +' "${BEFORE_FILE}") +echo "当前积压:${CURRENT_LAG} 条" + +echo "[3/6] 导出原 offset 回退文件" +"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \ + --group "${GROUP_ID}" \ + --topic "${RAW_TOPIC}" \ + --reset-offsets \ + --to-current \ + --dry-run \ + --export > "${RESTORE_FILE}" + +RESTORE_ROWS=$(awk -F, -v topic="${RAW_TOPIC}" ' + $1==topic && $2~/^[0-9]+$/ && $3~/^[0-9]+$/ {count++} + END {print count+0} +' "${RESTORE_FILE}") +if [[ "${RESTORE_ROWS}" -ne "${PARTITION_COUNT}" ]]; then + echo "ERROR: 回退文件只包含 ${RESTORE_ROWS}/${PARTITION_COUNT} 个分区,拒绝继续" >&2 + exit 1 +fi +echo "原 offset 已保存:${RESTORE_FILE}" + +echo "[4/6] 预演按时间重置" +"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \ + --group "${GROUP_ID}" \ + --topic "${RAW_TOPIC}" \ + --reset-offsets \ + --to-datetime "${CUTOFF_DATETIME}" \ + --dry-run | tee "${DRY_RUN_FILE}" + +PLAN_ROWS=$(awk -v topic="${RAW_TOPIC}" ' + $1==topic && $2~/^[0-9]+$/ && $3~/^[0-9]+$/ {count++; next} + $2==topic && $3~/^[0-9]+$/ && $4~/^[0-9]+$/ {count++} + END {print count+0} +' "${DRY_RUN_FILE}") +if [[ "${PLAN_ROWS}" -ne "${PARTITION_COUNT}" ]]; then + echo "ERROR: dry-run 只生成 ${PLAN_ROWS}/${PARTITION_COUNT} 个分区,拒绝执行" >&2 + exit 1 +fi + +EXPECTED_CONFIRM="RESET:${GROUP_ID}:${RAW_TOPIC}:${CUTOFF_DATETIME}" +echo "[5/6] 等待人工确认" +echo "即将永久跳过 ${CUTOFF_DATETIME} (${KAFKA_TIMEZONE}) 之前尚未消费的消息。" +echo "原 offset 回退文件:${RESTORE_FILE}" +echo "如确认执行,请完整输入:${EXPECTED_CONFIRM}" + +if [[ -n "${CONFIRM_RESET:-}" ]]; then + CONFIRM_INPUT="${CONFIRM_RESET}" +elif [[ -t 0 ]]; then + read -r CONFIRM_INPUT +else + echo "ERROR: 当前不是交互终端;必须通过 CONFIRM_RESET 提供精确确认串" >&2 + exit 1 +fi + +if [[ "${CONFIRM_INPUT}" != "${EXPECTED_CONFIRM}" ]]; then + echo "CANCELLED: 确认串不匹配,未修改任何 offset。" >&2 + exit 1 +fi + +echo "[6/6] 执行 offset 重置" +"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \ + --group "${GROUP_ID}" \ + --topic "${RAW_TOPIC}" \ + --reset-offsets \ + --to-datetime "${CUTOFF_DATETIME}" \ + --execute | tee "${EXECUTE_FILE}" + +echo "保存执行后快照" +"${CONSUMER_GROUPS_CMD}" "${COMMON_ARGS[@]}" \ + --describe --group "${GROUP_ID}" 2>&1 | tee "${AFTER_FILE}" + +echo "PASS: 已将 ${GROUP_ID}/${RAW_TOPIC} 重置到 ${CUTOFF_DATETIME} (${KAFKA_TIMEZONE})。" +echo "未删除 Topic;现在可以启动 syslog-consumer-rule 并检查 LAG。" +echo "回退文件:${RESTORE_FILE}(仅在 Kafka 原消息仍处于 retention 内时有效)。" diff --git a/haobang-security-dm/upgrade-doc/v1/kafka/kafka-cluster-upgrade.sh b/haobang-security-dm/upgrade-doc/v1/kafka/kafka-cluster-upgrade.sh new file mode 100755 index 0000000..100b1ff --- /dev/null +++ b/haobang-security-dm/upgrade-doc/v1/kafka/kafka-cluster-upgrade.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Kafka 集群只需在一台装有 Kafka CLI、能够访问集群的服务器上执行一次。 +# 示例: +# export KAFKA_BIN=<现场Kafka客户端bin目录> +# export BOOTSTRAP=<现场broker1地址>:<端口>,<现场broker2地址>:<端口> +# export RAW_TOPIC=<现场原始Topic名称> +# export COMMAND_CONFIG=<现场client.properties路径> # 无认证时不设置 +# ./kafka-cluster-upgrade.sh +# BOOTSTRAP 必须使用客户 Kafka 运维提供的地址;集群建议填写两个以上 broker。 + +: "${KAFKA_BIN:?请根据现场环境设置 KAFKA_BIN}" +: "${BOOTSTRAP:?请填写客户 Kafka 运维提供的 broker 地址和端口}" +: "${RAW_TOPIC:?请填写现场实际使用的原始 Topic 名称}" + +TOPICS_CMD="${KAFKA_BIN}/kafka-topics.sh" +CONFIGS_CMD="${KAFKA_BIN}/kafka-configs.sh" + +if [[ ! -x "${TOPICS_CMD}" ]] || [[ ! -x "${CONFIGS_CMD}" ]]; then + echo "ERROR: ${KAFKA_BIN} 下缺少 kafka-topics.sh 或 kafka-configs.sh" >&2 + exit 1 +fi + +COMMON_ARGS=(--bootstrap-server "${BOOTSTRAP}") +if [[ -n "${COMMAND_CONFIG:-}" ]]; then + if [[ ! -r "${COMMAND_CONFIG}" ]]; then + echo "ERROR: 无法读取认证配置 ${COMMAND_CONFIG}" >&2 + exit 1 + fi + COMMON_ARGS+=(--command-config "${COMMAND_CONFIG}") +fi + +echo "[1/4] 测试 Kafka 集群连接:${BOOTSTRAP}" +"${TOPICS_CMD}" "${COMMON_ARGS[@]}" --list >/dev/null + +echo "[2/4] 读取原始 Topic:${RAW_TOPIC}" +RAW_INFO=$("${TOPICS_CMD}" "${COMMON_ARGS[@]}" --describe --topic "${RAW_TOPIC}") +printf '%s\n' "${RAW_INFO}" + +PARTITIONS=$(printf '%s\n' "${RAW_INFO}" | awk ' + /PartitionCount:/ { + for (i = 1; i <= NF; i++) if ($i == "PartitionCount:") { print $(i + 1); exit } + } +') +REPLICAS=$(printf '%s\n' "${RAW_INFO}" | awk ' + /ReplicationFactor:/ { + for (i = 1; i <= NF; i++) if ($i == "ReplicationFactor:") { print $(i + 1); exit } + } +') + +if [[ ! "${PARTITIONS}" =~ ^[1-9][0-9]*$ ]] || [[ ! "${REPLICAS}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: 无法读取原始 Topic 的分区数或副本数" >&2 + exit 1 +fi + +echo "新 Topic 使用分区数=${PARTITIONS},副本数=${REPLICAS}" + +SUFFIXES=("RETRY.1M" "RETRY.10M" "RETRY.1H" "DLT" "REPLAY") +RETENTIONS=(259200000 604800000 604800000 7776000000 604800000) + +echo "[3/4] 创建或更新升级 Topic" +for i in 0 1 2 3 4; do + TOPIC="${RAW_TOPIC}.${SUFFIXES[$i]}" + RETENTION_MS="${RETENTIONS[$i]}" + + echo "处理 ${TOPIC}" + "${TOPICS_CMD}" "${COMMON_ARGS[@]}" \ + --create --if-not-exists \ + --topic "${TOPIC}" \ + --partitions "${PARTITIONS}" \ + --replication-factor "${REPLICAS}" \ + --config cleanup.policy=delete \ + --config "retention.ms=${RETENTION_MS}" + + "${CONFIGS_CMD}" "${COMMON_ARGS[@]}" \ + --alter --entity-type topics --entity-name "${TOPIC}" \ + --add-config "cleanup.policy=delete,retention.ms=${RETENTION_MS}" +done + +echo "[4/4] 显示升级结果" +for SUFFIX in "${SUFFIXES[@]}"; do + "${TOPICS_CMD}" "${COMMON_ARGS[@]}" \ + --describe --topic "${RAW_TOPIC}.${SUFFIX}" +done + +echo "PASS: Kafka 集群 Topic 已创建或更新;未删除 Topic,未修改消费组 offset。" diff --git a/haobang-security-dm/upgrade-doc/v2/influxdb/01-ensure-v2-bucket.sh b/haobang-security-dm/upgrade-doc/v2/influxdb/01-ensure-v2-bucket.sh index da13f37..51b2d7d 100755 --- a/haobang-security-dm/upgrade-doc/v2/influxdb/01-ensure-v2-bucket.sh +++ b/haobang-security-dm/upgrade-doc/v2/influxdb/01-ensure-v2-bucket.sh @@ -7,7 +7,8 @@ set -euo pipefail INFLUX_CONFIG_PATH="${INFLUX_CONFIG_PATH:-/etc/influxdb2/influx-configs}" V2_BUCKET="${V2_BUCKET:-yelangbucket_v2}" -RETENTION="${RETENTION:-720h}" +# 现场要求原始日志保留 180 天;仍可通过 RETENTION 环境变量覆盖。 +RETENTION="${RETENTION:-180d}" SHARD_GROUP_DURATION="${SHARD_GROUP_DURATION:-24h}" command -v docker >/dev/null 2>&1 || {