feat(syslog): 完成高吞吐可靠链路与统计存储优化
- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
王绍宏
@@ -1,178 +1,213 @@
|
||||
package com.kafka;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.util.Sm4Util;
|
||||
import org.apache.kafka.clients.producer.*;
|
||||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.haobang.config.AppConfig;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.util.Map;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import com.haobang.util.DeviceInfoUtil;
|
||||
import com.haobang.util.SpringContextUtil;
|
||||
import com.haobang.util.Sm4Util;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.apache.kafka.clients.producer.Producer;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.clients.producer.ProducerRecord;
|
||||
import org.apache.kafka.clients.producer.RecordMetadata;
|
||||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
import java.util.Properties;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 进程级单例 Kafka Producer。
|
||||
*
|
||||
* <p>保留历史类名以避免影响现有调用方，但实例由 Spring 托管。KafkaProducer 本身线程安全，
|
||||
* 长生命周期复用才能使用连接、RecordAccumulator、批量和压缩；逐条 new/close 会破坏这些机制。</p>
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class kafkaProducer {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(kafkaProducer.class);
|
||||
private static final ZoneId EVENT_ID_ZONE = ZoneId.of("Asia/Shanghai");
|
||||
private static final DateTimeFormatter EVENT_ID_DATE = DateTimeFormatter.ofPattern("yyyyMMdd");
|
||||
|
||||
private static AppProperties getProps() {
|
||||
return SpringContextUtil.getBean(AppProperties.class);
|
||||
}
|
||||
public static void main(String[] args) {
|
||||
AppProperties appProps = getProps();
|
||||
// 配置生产者属性
|
||||
private final AppProperties appProperties;
|
||||
private final Producer<String, String> producer;
|
||||
private final PartitionKeyStrategy partitionKeyStrategy;
|
||||
|
||||
@Autowired
|
||||
public kafkaProducer(AppProperties appProperties,
|
||||
@Value("${spring.kafka.producer.properties.retries:10}") int retries,
|
||||
@Value("${spring.kafka.producer.properties.delivery.timeout.ms:120000}") int deliveryTimeoutMs,
|
||||
@Value("${spring.kafka.producer.properties.request.timeout.ms:30000}") int requestTimeoutMs,
|
||||
@Value("${spring.kafka.producer.properties.linger.ms:5}") int lingerMs,
|
||||
@Value("${spring.kafka.producer.properties.batch.size:65536}") int batchSize,
|
||||
@Value("${spring.kafka.producer.compression-type:lz4}") String compressionType,
|
||||
@Value("${spring.kafka.producer.properties.max.block.ms:10000}") int maxBlockMs,
|
||||
@Value("${spring.kafka.producer.properties.buffer.memory:134217728}") long bufferMemory,
|
||||
@Value("${syslog.kafka.partition-key-strategy:event_id}") String partitionKeyStrategy) {
|
||||
this.appProperties = appProperties;
|
||||
this.partitionKeyStrategy = PartitionKeyStrategy.parse(partitionKeyStrategy);
|
||||
Properties props = new Properties();
|
||||
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProps.getKafkaProducerBootstrap());
|
||||
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProperties.getKafkaProducerBootstrap());
|
||||
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
||||
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
||||
|
||||
// 可选配置：提高可靠性
|
||||
// [SYSLOG-ALARM][KPROD][已修复] acks=all 与幂等发送共同避免 Leader 切换和客户端重试造成假成功或重复。
|
||||
props.put(ProducerConfig.ACKS_CONFIG, "all");
|
||||
props.put(ProducerConfig.RETRIES_CONFIG, 3);
|
||||
props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true);
|
||||
props.put(ProducerConfig.RETRIES_CONFIG, retries);
|
||||
props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 5);
|
||||
props.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, deliveryTimeoutMs);
|
||||
props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, requestTimeoutMs);
|
||||
props.put(ProducerConfig.LINGER_MS_CONFIG, lingerMs);
|
||||
props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
|
||||
props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, compressionType);
|
||||
// Dispatcher 已限制全局在途数；Producer 缓冲耗尽时快速返回失败，让事件留在 WAL 重试。
|
||||
props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, maxBlockMs);
|
||||
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);
|
||||
// [SYSLOG-ALARM][KPROD][已修复] linger、batch 与压缩恢复跨消息批量能力，吞吐优化不改变 Broker ACK 边界。
|
||||
// [SYSLOG-ALARM][KPROD][已修复] Producer 只在应用启动时构造一次，单条热路径不再创建网络线程和连接。
|
||||
this.producer = new KafkaProducer<>(props);
|
||||
log.info("Kafka Producer 已启用分区策略: strategy={}", this.partitionKeyStrategy.configValue);
|
||||
}
|
||||
|
||||
// 创建生产者实例
|
||||
Producer<String, String> producer = new KafkaProducer<>(props);
|
||||
|
||||
try {
|
||||
// 发送10条消息
|
||||
//for (int i = 0; i < 1; i++) {
|
||||
String key = "key-" + "1";
|
||||
String value = "message-" + "1" + " at " + System.currentTimeMillis();
|
||||
|
||||
// 创建生产者记录
|
||||
ProducerRecord<String, String> record =
|
||||
new ProducerRecord<>(appProps.getKafkaProducerTopic(), key, value);
|
||||
|
||||
// 发送消息（异步方式）
|
||||
producer.send(record, new Callback() {
|
||||
@Override
|
||||
public void onCompletion(RecordMetadata metadata, Exception exception) {
|
||||
if (exception == null) {
|
||||
System.out.println("消息发送成功: " +
|
||||
"主题=" + metadata.topic() +
|
||||
", 分区=" + metadata.partition() +
|
||||
", 偏移量=" + metadata.offset());
|
||||
} else {
|
||||
System.err.println("消息发送失败: " + exception.getMessage());
|
||||
}
|
||||
}
|
||||
});
|
||||
// 同步发送方式（如果需要）
|
||||
// RecordMetadata metadata = producer.send(record).get();
|
||||
// System.out.println("同步发送成功: " + metadata.offset());
|
||||
// }
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
// 关闭生产者
|
||||
producer.close();
|
||||
}
|
||||
/** 测试入口：复用真实的分区键选择和 ProducerRecord 构造逻辑，不建立 Kafka 网络连接。 */
|
||||
kafkaProducer(AppProperties appProperties, Producer<String, String> producer, String partitionKeyStrategy) {
|
||||
this.appProperties = appProperties;
|
||||
this.producer = producer;
|
||||
this.partitionKeyStrategy = PartitionKeyStrategy.parse(partitionKeyStrategy);
|
||||
}
|
||||
|
||||
/**
|
||||
* * syslog 文本消息
|
||||
* * @param strSyslog
|
||||
* [流程节点3] 为原始 Syslog 生成稳定 event_id，加密并异步投递 Kafka。
|
||||
* 返回 Future 只有在 Broker 按 acks=all 确认后成功完成，调用方必须据此更新接收审计。
|
||||
*/
|
||||
public CompletableFuture<DeliveryResult> messagePush(DeviceDevice deviceDevice,
|
||||
String syslog,
|
||||
String receiveTime) {
|
||||
return messagePush(deviceDevice, syslog, receiveTime, newEventId());
|
||||
}
|
||||
|
||||
public static void messagePush(DeviceDevice deviceDevice, String strSyslog,String strReceiveTime)
|
||||
{
|
||||
AppProperties appProps = getProps();
|
||||
// 配置生产者属性
|
||||
Properties props = new Properties();
|
||||
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProps.getKafkaProducerBootstrap());
|
||||
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
||||
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
||||
public CompletableFuture<DeliveryResult> messagePush(DeviceDevice deviceDevice,
|
||||
String syslog,
|
||||
String receiveTime,
|
||||
String eventId) {
|
||||
return messagePush(deviceDevice, syslog, receiveTime, eventId, null);
|
||||
}
|
||||
|
||||
// 可选配置：提高可靠性
|
||||
props.put(ProducerConfig.ACKS_CONFIG, "all");
|
||||
props.put(ProducerConfig.RETRIES_CONFIG, 3);
|
||||
props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true);
|
||||
public CompletableFuture<DeliveryResult> messagePush(DeviceDevice deviceDevice,
|
||||
String syslog,
|
||||
String receiveTime,
|
||||
String eventId,
|
||||
Long pointTimeNs) {
|
||||
// [流程节点3.1] event_id 在接入边界生成并写入加密载荷，Retry/Replay 不再重新生成。
|
||||
String fullMessage = DeviceInfoUtil.getFullLogString(
|
||||
deviceDevice, syslog, receiveTime, eventId, pointTimeNs, appProperties);
|
||||
// [流程节点3.2] Kafka 中保存携带 event_id 的完整密文，原始正文不写普通应用日志。
|
||||
String encrypted = Sm4Util.encryptCbc(fullMessage, appProperties.getSm4Key());
|
||||
ProducerRecord<String, String> record = buildRecord(deviceDevice.getId(), eventId, encrypted);
|
||||
|
||||
// 创建生产者实例
|
||||
Producer<String, String> producer = new KafkaProducer<>(props);
|
||||
CompletableFuture<DeliveryResult> result = new CompletableFuture<>();
|
||||
// [SYSLOG-ALARM][KPROD][已修复] 回调成功才表示 Broker 接管；send() 返回本身不再记录 PRODUCED。
|
||||
producer.send(record, (metadata, exception) -> {
|
||||
if (exception != null) {
|
||||
result.completeExceptionally(exception);
|
||||
return;
|
||||
}
|
||||
result.complete(new DeliveryResult(eventId, metadata));
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
ProducerRecord<String, String> buildRecord(Integer deviceId, String eventId, String encrypted) {
|
||||
String partitionKey = partitionKeyStrategy.partitionKey(deviceId, eventId);
|
||||
return new ProducerRecord<>(appProperties.getKafkaProducerTopic(), partitionKey, encrypted);
|
||||
}
|
||||
|
||||
public static String newEventId() {
|
||||
return newEventId(Instant.now());
|
||||
}
|
||||
|
||||
/** event_id 日期固定按接入业务时区计算，不依赖容器或 JVM 默认时区。 */
|
||||
public static String newEventId(Instant receivedAt) {
|
||||
return EVENT_ID_DATE.format(receivedAt.atZone(EVENT_ID_ZONE)) + "-" + UUID.randomUUID();
|
||||
}
|
||||
|
||||
/** 仅用于运维探针的同步健康验证，不用于逐条热路径。 */
|
||||
public DeliveryResult sendAndAwait(DeviceDevice deviceDevice, String syslog, String receiveTime,
|
||||
long timeoutMs) throws Exception {
|
||||
return messagePush(deviceDevice, syslog, receiveTime).get(timeoutMs, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void close() {
|
||||
// [SYSLOG-ALARM][KPROD][已修复] 只在优雅停机阶段 flush/close，确保缓冲中的已接收消息完成发送。
|
||||
try {
|
||||
|
||||
String key = "key-" + System.currentTimeMillis();
|
||||
//String value = DeviceInfoUtil.getFullLogString(strSyslog);
|
||||
//采用动态获取Syslog 请求的设备信息
|
||||
String value = DeviceInfoUtil.getFullLogString(deviceDevice,strSyslog,strReceiveTime);
|
||||
|
||||
//Message进行SM4加密写入kafka
|
||||
String Sm4message= Sm4Util.encryptCbc(value, appProps.getSm4Key());
|
||||
//System.out.println("Sm4message:"+Sm4message);
|
||||
|
||||
// 创建生产者记录
|
||||
ProducerRecord<String, String> record =
|
||||
new ProducerRecord<>(appProps.getKafkaProducerTopic(), key, Sm4message);
|
||||
|
||||
// 发送消息（异步方式）
|
||||
producer.send(record, new Callback() {
|
||||
@Override
|
||||
public void onCompletion(RecordMetadata metadata, Exception exception) {
|
||||
if (exception == null) {
|
||||
System.out.println("消息发送成功: " +
|
||||
"主题=" + metadata.topic() +
|
||||
", 分区=" + metadata.partition() +
|
||||
", 偏移量=" + metadata.offset());
|
||||
} else {
|
||||
System.err.println("消息发送失败: " + exception.getMessage());
|
||||
logger.error("消息发送失败: " + exception.getMessage());
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 同步发送方式（如果需要）
|
||||
// RecordMetadata metadata = producer.send(record).get();
|
||||
// System.out.println("同步发送成功: " + metadata.offset());
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
producer.flush();
|
||||
} finally {
|
||||
// 关闭生产者
|
||||
producer.close();
|
||||
producer.close(java.time.Duration.ofSeconds(10));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成json字符串，补充设备配置信息
|
||||
* @param strSyslog
|
||||
* @return
|
||||
*/
|
||||
private static String getLogJsonString(String strSyslog)
|
||||
{
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
// 创建设备配置信息MAP
|
||||
Map<String, Object> map = new LinkedHashMap<>();
|
||||
AppProperties appProps = getProps();
|
||||
map.put("device_id", appProps.getAppServiceDeviceId());
|
||||
map.put("device_name", appProps.getAppServiceDeviceName());
|
||||
map.put("vendor", appProps.getAppServiceVendor());
|
||||
map.put("data_type", appProps.getAppServiceDataType());
|
||||
// 添加syslogMessage
|
||||
map.put("syslogMessage", strSyslog);
|
||||
try {
|
||||
//生成json消息字符串
|
||||
String json = objectMapper.writeValueAsString(map);
|
||||
return json;
|
||||
enum PartitionKeyStrategy {
|
||||
EVENT_ID("event_id") {
|
||||
@Override
|
||||
String partitionKey(Integer deviceId, String eventId) {
|
||||
if (eventId == null || eventId.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("event_id 分区策略要求 eventId 非空");
|
||||
}
|
||||
return eventId;
|
||||
}
|
||||
},
|
||||
DEVICE_ID("device_id") {
|
||||
@Override
|
||||
String partitionKey(Integer deviceId, String eventId) {
|
||||
if (deviceId == null) {
|
||||
throw new IllegalArgumentException("device_id 分区策略要求 deviceId 非空");
|
||||
}
|
||||
return deviceId.toString();
|
||||
}
|
||||
};
|
||||
|
||||
private final String configValue;
|
||||
|
||||
PartitionKeyStrategy(String configValue) {
|
||||
this.configValue = configValue;
|
||||
}
|
||||
catch(Exception ex)
|
||||
{
|
||||
logger.error("kafkaProducer getLogJsonString(）生成json字符串 异常: syslog: "+ strSyslog);
|
||||
logger.error("Exception: : "+ ex.getMessage());
|
||||
|
||||
abstract String partitionKey(Integer deviceId, String eventId);
|
||||
|
||||
static PartitionKeyStrategy parse(String configuredValue) {
|
||||
String normalized = configuredValue == null
|
||||
? "" : configuredValue.trim().toLowerCase(Locale.ROOT);
|
||||
for (PartitionKeyStrategy strategy : values()) {
|
||||
if (strategy.configValue.equals(normalized)) return strategy;
|
||||
}
|
||||
throw new IllegalArgumentException(
|
||||
"不支持的 Kafka 分区键策略: " + configuredValue + "，仅支持 event_id 或 device_id");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Getter
|
||||
public static class DeliveryResult {
|
||||
private final String eventId;
|
||||
private final String topic;
|
||||
private final int partition;
|
||||
private final long offset;
|
||||
|
||||
|
||||
|
||||
DeliveryResult(String eventId, RecordMetadata metadata) {
|
||||
this.eventId = eventId;
|
||||
this.topic = metadata.topic();
|
||||
this.partition = metadata.partition();
|
||||
this.offset = metadata.offset();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user