package com.kafka; import com.common.entity.DeviceDevice; import com.haobang.config.AppProperties; import com.haobang.util.DeviceInfoUtil; import com.haobang.util.Sm4Util; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.serialization.StringSerializer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import javax.annotation.PreDestroy; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.Locale; import java.util.Properties; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; /** * 进程级单例 Kafka Producer。 * *

保留历史类名以避免影响现有调用方,但实例由 Spring 托管。KafkaProducer 本身线程安全, * 长生命周期复用才能使用连接、RecordAccumulator、批量和压缩;逐条 new/close 会破坏这些机制。

*/ @Slf4j @Component public class kafkaProducer { private static final ZoneId EVENT_ID_ZONE = ZoneId.of("Asia/Shanghai"); private static final DateTimeFormatter EVENT_ID_DATE = DateTimeFormatter.ofPattern("yyyyMMdd"); private final AppProperties appProperties; private final Producer producer; private final PartitionKeyStrategy partitionKeyStrategy; @Autowired public kafkaProducer(AppProperties appProperties, @Value("${spring.kafka.producer.properties.retries:10}") int retries, @Value("${spring.kafka.producer.properties.delivery.timeout.ms:120000}") int deliveryTimeoutMs, @Value("${spring.kafka.producer.properties.request.timeout.ms:30000}") int requestTimeoutMs, @Value("${spring.kafka.producer.properties.linger.ms:5}") int lingerMs, @Value("${spring.kafka.producer.properties.batch.size:65536}") int batchSize, @Value("${spring.kafka.producer.compression-type:lz4}") String compressionType, @Value("${spring.kafka.producer.properties.max.block.ms:10000}") int maxBlockMs, @Value("${spring.kafka.producer.properties.buffer.memory:134217728}") long bufferMemory, @Value("${spring.kafka.producer.properties.max.request.size:8388608}") int maxRequestSize, @Value("${syslog.kafka.partition-key-strategy:event_id}") String partitionKeyStrategy) { this.appProperties = appProperties; this.partitionKeyStrategy = PartitionKeyStrategy.parse(partitionKeyStrategy); Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProperties.getKafkaProducerBootstrap()); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); // [SYSLOG-ALARM][KPROD][已修复] acks=all 与幂等发送共同避免 Leader 切换和客户端重试造成假成功或重复。 props.put(ProducerConfig.ACKS_CONFIG, "all"); props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true); props.put(ProducerConfig.RETRIES_CONFIG, retries); props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 5); props.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, deliveryTimeoutMs); props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, requestTimeoutMs); props.put(ProducerConfig.LINGER_MS_CONFIG, lingerMs); props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize); props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, compressionType); // Dispatcher 已限制全局在途数;Producer 缓冲耗尽时快速返回失败,让事件留在 WAL 重试。 props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, maxBlockMs); props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory); applyMaxRequestSize(props, maxRequestSize); // [SYSLOG-ALARM][KPROD][已修复] linger、batch 与压缩恢复跨消息批量能力,吞吐优化不改变 Broker ACK 边界。 // [SYSLOG-ALARM][KPROD][已修复] Producer 只在应用启动时构造一次,单条热路径不再创建网络线程和连接。 this.producer = new KafkaProducer<>(props); log.info("Kafka Producer 已启用分区策略: strategy={}", this.partitionKeyStrategy.configValue); } static void applyMaxRequestSize(Properties props, int maxRequestSize) { if (maxRequestSize <= 0) { throw new IllegalArgumentException("spring.kafka.producer.properties.max.request.size 必须大于 0"); } props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, maxRequestSize); } /** 测试入口:复用真实的分区键选择和 ProducerRecord 构造逻辑,不建立 Kafka 网络连接。 */ kafkaProducer(AppProperties appProperties, Producer producer, String partitionKeyStrategy) { this.appProperties = appProperties; this.producer = producer; this.partitionKeyStrategy = PartitionKeyStrategy.parse(partitionKeyStrategy); } /** * [流程节点3] 为原始 Syslog 生成稳定 event_id,加密并异步投递 Kafka。 * 返回 Future 只有在 Broker 按 acks=all 确认后成功完成,调用方必须据此更新接收审计。 */ public CompletableFuture messagePush(DeviceDevice deviceDevice, String syslog, String receiveTime) { return messagePush(deviceDevice, syslog, receiveTime, newEventId()); } public CompletableFuture messagePush(DeviceDevice deviceDevice, String syslog, String receiveTime, String eventId) { return messagePush(deviceDevice, syslog, receiveTime, eventId, null); } public CompletableFuture messagePush(DeviceDevice deviceDevice, String syslog, String receiveTime, String eventId, Long pointTimeNs) { return messagePush(deviceDevice, syslog, receiveTime, receiveTime, eventId, pointTimeNs); } /** * 显式区分接收时间和事件时间。TCP/UDP 调用旧签名时两者相同, * 后续 API 拉取可以在不改 Kafka 头结构的前提下传入设备事件时间。 */ public CompletableFuture messagePush(DeviceDevice deviceDevice, String syslog, String receiveTime, String eventTime, String eventId, Long pointTimeNs) { // [流程节点3.1] event_id 在接入边界生成并写入加密载荷,Retry/Replay 不再重新生成。 String fullMessage = DeviceInfoUtil.getFullLogString( deviceDevice, syslog, receiveTime, eventTime, eventId, pointTimeNs, appProperties); // [流程节点3.2] Kafka 中保存携带 event_id 的完整密文,原始正文不写普通应用日志。 String encrypted = Sm4Util.encryptCbc(fullMessage, appProperties.getSm4Key()); ProducerRecord record = buildRecord(deviceDevice.getId(), eventId, encrypted); CompletableFuture result = new CompletableFuture<>(); // [SYSLOG-ALARM][KPROD][已修复] 回调成功才表示 Broker 接管;send() 返回本身不再记录 PRODUCED。 producer.send(record, (metadata, exception) -> { if (exception != null) { result.completeExceptionally(exception); return; } result.complete(new DeliveryResult(eventId, metadata)); }); return result; } ProducerRecord buildRecord(Integer deviceId, String eventId, String encrypted) { String partitionKey = partitionKeyStrategy.partitionKey(deviceId, eventId); return new ProducerRecord<>(appProperties.getKafkaProducerTopic(), partitionKey, encrypted); } public static String newEventId() { return newEventId(Instant.now()); } /** event_id 日期固定按接入业务时区计算,不依赖容器或 JVM 默认时区。 */ public static String newEventId(Instant receivedAt) { return EVENT_ID_DATE.format(receivedAt.atZone(EVENT_ID_ZONE)) + "-" + UUID.randomUUID(); } /** 仅用于运维探针的同步健康验证,不用于逐条热路径。 */ public DeliveryResult sendAndAwait(DeviceDevice deviceDevice, String syslog, String receiveTime, long timeoutMs) throws Exception { return messagePush(deviceDevice, syslog, receiveTime).get(timeoutMs, TimeUnit.MILLISECONDS); } @PreDestroy public void close() { // [SYSLOG-ALARM][KPROD][已修复] 只在优雅停机阶段 flush/close,确保缓冲中的已接收消息完成发送。 try { producer.flush(); } finally { producer.close(java.time.Duration.ofSeconds(10)); } } enum PartitionKeyStrategy { EVENT_ID("event_id") { @Override String partitionKey(Integer deviceId, String eventId) { if (eventId == null || eventId.trim().isEmpty()) { throw new IllegalArgumentException("event_id 分区策略要求 eventId 非空"); } return eventId; } }, DEVICE_ID("device_id") { @Override String partitionKey(Integer deviceId, String eventId) { if (deviceId == null) { throw new IllegalArgumentException("device_id 分区策略要求 deviceId 非空"); } return deviceId.toString(); } }; private final String configValue; PartitionKeyStrategy(String configValue) { this.configValue = configValue; } abstract String partitionKey(Integer deviceId, String eventId); static PartitionKeyStrategy parse(String configuredValue) { String normalized = configuredValue == null ? "" : configuredValue.trim().toLowerCase(Locale.ROOT); for (PartitionKeyStrategy strategy : values()) { if (strategy.configValue.equals(normalized)) return strategy; } throw new IllegalArgumentException( "不支持的 Kafka 分区键策略: " + configuredValue + ",仅支持 event_id 或 device_id"); } } @Getter public static class DeliveryResult { private final String eventId; private final String topic; private final int partition; private final long offset; DeliveryResult(String eventId, RecordMetadata metadata) { this.eventId = eventId; this.topic = metadata.topic(); this.partition = metadata.partition(); this.offset = metadata.offset(); } } }