package com.common.service; import com.Modules.NormalData.SyslogFailureRouter; import com.common.entity.IngressStatsEvent; import com.common.util.Sm4Util; import com.common.util.SyslogParser; import com.config.AppProperties; import com.influx.raw.ReceiveTimeParser; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.springframework.beans.factory.annotation.Value; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.support.Acknowledgment; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; /** 独立消费原始Topic,只维护接收时间;不执行规则、采样或事实表写入。 */ @Component @Slf4j public class IngressStatsConsumer { private final IngressStatsProjectionWriter writer; private final SyslogFailureRouter failureRouter; private final AppProperties appProperties; private final AtomicLong batches = new AtomicLong(); private final AtomicLong invalidAttempts = new AtomicLong(); private final AtomicLong writeFailures = new AtomicLong(); @Value("${spring.kafka.consumer.group-id}") private String rawGroup; @Value("${syslog.large-listener.group-id:${spring.kafka.consumer.group-id}-large}") private String largeRawGroup; @Value("${stats.ingress-metadata.group-id:${spring.kafka.consumer.group-id}-ingress-v1}") private String group; @Value("${stats.ingress-metadata.large-group-id:${stats.ingress-metadata.group-id}-large}") private String largeGroup; public IngressStatsConsumer(IngressStatsProjectionWriter writer, SyslogFailureRouter failureRouter, AppProperties appProperties) { this.writer = writer; this.failureRouter = failureRouter; this.appProperties = appProperties; } @PostConstruct public void validateGroups() { if (group == null || group.trim().isEmpty() || largeGroup == null || largeGroup.trim().isEmpty() || group.equals(largeGroup) || group.equals(rawGroup) || group.equals(largeRawGroup) || largeGroup.equals(rawGroup) || largeGroup.equals(largeRawGroup)) { throw new IllegalStateException("入口统计必须使用与主消费隔离的非空 Kafka group-id"); } log.info("Ingress stats consumer ready: group={}, largeGroup={}, mode=INDEPENDENT_KAFKA_DIRECT_MERGE", group, largeGroup); } @KafkaListener(id = "syslog-ingress-stats", groupId = "${stats.ingress-metadata.group-id}", topics = "${spring.kafka.consumer.topic}", autoStartup = "${stats.ingress-metadata.enabled:true}", containerFactory = "ingressStatsKafkaListenerContainerFactory") public void consume(List> records, Acknowledgment ack) { consumeBatch(records, ack, "RAW"); } @KafkaListener(id = "syslog-ingress-stats-large", groupId = "${stats.ingress-metadata.large-group-id}", topics = "${spring.kafka.consumer.large-topic}", autoStartup = "#{${stats.ingress-metadata.enabled:true} && ${syslog.large-listener.enabled:true}}", containerFactory = "largeIngressStatsKafkaListenerContainerFactory") public void consumeLarge(List> records, Acknowledgment ack) { consumeBatch(records, ack, "RAW_LARGE"); } private void consumeBatch(List> records, Acknowledgment ack, String lane) { long started = System.nanoTime(); List events = new ArrayList<>(records.size()); int invalid = 0; String firstInvalid = null; for (ConsumerRecord record : records) { try { events.add(decode(record)); } catch (RuntimeException malformed) { invalid++; if (firstInvalid == null) firstInvalid = record.topic() + "-" + record.partition() + "@" + record.offset() + ":" + malformed.getClass().getSimpleName(); } } if (invalid > 0) { invalidAttempts.addAndGet(invalid); // 原始消息仍留在Kafka;坏头不会永久挡住其它设备接收状态,也不向业务Retry重复投递。 log.error("[INGRESS_STATS_INVALID] lane={}, invalidRecords={}, firstInvalid={}, " + "action=SKIP_INVALID_METADATA", lane, invalid, firstInvalid); } int groups; try { groups = writer.writeBatch(events); } catch (RuntimeException failure) { writeFailures.incrementAndGet(); log.error("[INGRESS_STATS_WRITE_FAILED] lane={}, records={}, action=RETRY_STATS_GROUP", lane, records.size(), failure); throw failure; } // MIN/MAX更新和达梦提交均已完成,才推进独立统计组;重复消费不重复增加事实数量。 ack.acknowledge(); batches.incrementAndGet(); log.info("[INGRESS_STATS_BATCH] lane={}, records={}, valid={}, invalid={}, groups={}, totalMs={}", lane, records.size(), events.size(), invalid, groups, (System.nanoTime() - started) / 1_000_000L); } IngressStatsEvent decode(ConsumerRecord record) { SyslogFailureRouter.MessageContext context = failureRouter.resolve(record); String decrypted = Sm4Util.decryptCbc(context.getPayload(), appProperties.getSm4Key()); if (decrypted.indexOf(']') < 0) throw new IllegalArgumentException("缺少设备元数据头"); Map header = SyslogParser.parseKeyValuePairs( SyslogParser.substringBeforeFirstChar(decrypted, ']')); Long device = positiveId(header.get("device_id")); Long collector = positiveId(header.get("device_collect_id")); if (device == null && collector == null) throw new IllegalArgumentException("缺少有效设备/采集任务ID"); return new IngressStatsEvent(context.getEventId(), device, collector, ReceiveTimeParser.parse(header.get("receive_time")).getLocalDateTime()); } private static Long positiveId(String value) { if (value == null || value.trim().isEmpty()) return null; long parsed = Long.parseLong(value); return parsed > 0L ? parsed : null; } public Map status() { Map result = new LinkedHashMap<>(); result.put("group_id", group); result.put("large_group_id", largeGroup); result.put("successful_batches", batches.get()); result.put("invalid_record_attempts", invalidAttempts.get()); result.put("write_failures", writeFailures.get()); return result; } }