ai-security-xdr/haobang-security-dm/syslog-consumer-rule/src/main/java/com/common/service/IngressStatsConsumer.java

package com.common.service;

import com.Modules.NormalData.SyslogFailureRouter;
import com.common.entity.IngressStatsEvent;
import com.common.util.Sm4Util;
import com.common.util.SyslogParser;
import com.config.AppProperties;
import com.influx.raw.ReceiveTimeParser;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;

/** 独立消费原始Topic，只维护接收时间；不执行规则、采样或事实表写入。 */
@Component
@Slf4j
public class IngressStatsConsumer {
    private final IngressStatsProjectionWriter writer;
    private final SyslogFailureRouter failureRouter;
    private final AppProperties appProperties;
    private final AtomicLong batches = new AtomicLong();
    private final AtomicLong invalidAttempts = new AtomicLong();
    private final AtomicLong writeFailures = new AtomicLong();

    @Value("${spring.kafka.consumer.group-id}") private String rawGroup;
    @Value("${syslog.large-listener.group-id:${spring.kafka.consumer.group-id}-large}")
    private String largeRawGroup;
    @Value("${stats.ingress-metadata.group-id:${spring.kafka.consumer.group-id}-ingress-v1}")
    private String group;
    @Value("${stats.ingress-metadata.large-group-id:${stats.ingress-metadata.group-id}-large}")
    private String largeGroup;

    public IngressStatsConsumer(IngressStatsProjectionWriter writer,
                                SyslogFailureRouter failureRouter, AppProperties appProperties) {
        this.writer = writer;
        this.failureRouter = failureRouter;
        this.appProperties = appProperties;
    }

    @PostConstruct
    public void validateGroups() {
        if (group == null || group.trim().isEmpty() || largeGroup == null || largeGroup.trim().isEmpty()
                || group.equals(largeGroup) || group.equals(rawGroup) || group.equals(largeRawGroup)
                || largeGroup.equals(rawGroup) || largeGroup.equals(largeRawGroup)) {
            throw new IllegalStateException("入口统计必须使用与主消费隔离的非空 Kafka group-id");
        }
        log.info("Ingress stats consumer ready: group={}, largeGroup={}, mode=INDEPENDENT_KAFKA_DIRECT_MERGE",
                group, largeGroup);
    }

    @KafkaListener(id = "syslog-ingress-stats", groupId = "${stats.ingress-metadata.group-id}",
            topics = "${spring.kafka.consumer.topic}",
            autoStartup = "${stats.ingress-metadata.enabled:true}",
            containerFactory = "ingressStatsKafkaListenerContainerFactory")
    public void consume(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
        consumeBatch(records, ack, "RAW");
    }

    @KafkaListener(id = "syslog-ingress-stats-large", groupId = "${stats.ingress-metadata.large-group-id}",
            topics = "${spring.kafka.consumer.large-topic}",
            autoStartup = "#{${stats.ingress-metadata.enabled:true} && ${syslog.large-listener.enabled:true}}",
            containerFactory = "largeIngressStatsKafkaListenerContainerFactory")
    public void consumeLarge(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
        consumeBatch(records, ack, "RAW_LARGE");
    }

    private void consumeBatch(List<ConsumerRecord<String, String>> records, Acknowledgment ack, String lane) {
        long started = System.nanoTime();
        List<IngressStatsEvent> events = new ArrayList<>(records.size());
        int invalid = 0;
        String firstInvalid = null;
        for (ConsumerRecord<String, String> record : records) {
            try {
                events.add(decode(record));
            } catch (RuntimeException malformed) {
                invalid++;
                if (firstInvalid == null) firstInvalid = record.topic() + "-" + record.partition()
                        + "@" + record.offset() + ":" + malformed.getClass().getSimpleName();
            }
        }
        if (invalid > 0) {
            invalidAttempts.addAndGet(invalid);
            // 原始消息仍留在Kafka；坏头不会永久挡住其它设备接收状态，也不向业务Retry重复投递。
            log.error("[INGRESS_STATS_INVALID] lane={}, invalidRecords={}, firstInvalid={}, "
                    + "action=SKIP_INVALID_METADATA", lane, invalid, firstInvalid);
        }
        int groups;
        try {
            groups = writer.writeBatch(events);
        } catch (RuntimeException failure) {
            writeFailures.incrementAndGet();
            log.error("[INGRESS_STATS_WRITE_FAILED] lane={}, records={}, action=RETRY_STATS_GROUP",
                    lane, records.size(), failure);
            throw failure;
        }
        // MIN/MAX更新和达梦提交均已完成，才推进独立统计组；重复消费不重复增加事实数量。
        ack.acknowledge();
        batches.incrementAndGet();
        log.info("[INGRESS_STATS_BATCH] lane={}, records={}, valid={}, invalid={}, groups={}, totalMs={}",
                lane, records.size(), events.size(), invalid, groups, (System.nanoTime() - started) / 1_000_000L);
    }

    IngressStatsEvent decode(ConsumerRecord<String, String> record) {
        SyslogFailureRouter.MessageContext context = failureRouter.resolve(record);
        String decrypted = Sm4Util.decryptCbc(context.getPayload(), appProperties.getSm4Key());
        if (decrypted.indexOf(']') < 0) throw new IllegalArgumentException("缺少设备元数据头");
        Map<String, String> header = SyslogParser.parseKeyValuePairs(
                SyslogParser.substringBeforeFirstChar(decrypted, ']'));
        Long device = positiveId(header.get("device_id"));
        Long collector = positiveId(header.get("device_collect_id"));
        if (device == null && collector == null) throw new IllegalArgumentException("缺少有效设备/采集任务ID");
        return new IngressStatsEvent(context.getEventId(), device, collector,
                ReceiveTimeParser.parse(header.get("receive_time")).getLocalDateTime());
    }

    private static Long positiveId(String value) {
        if (value == null || value.trim().isEmpty()) return null;
        long parsed = Long.parseLong(value);
        return parsed > 0L ? parsed : null;
    }

    public Map<String, Object> status() {
        Map<String, Object> result = new LinkedHashMap<>();
        result.put("group_id", group);
        result.put("large_group_id", largeGroup);
        result.put("successful_batches", batches.get());
        result.put("invalid_record_attempts", invalidAttempts.get());
        result.put("write_failures", writeFailures.get());
        return result;
    }
}