feat(syslog): 完成高吞吐可靠链路与统计存储优化
- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
王绍宏
+4
-2
@@ -134,9 +134,11 @@ public class DeviceProcess {
|
||||
|
||||
|
||||
public boolean saveDeviceReceiveLog(String syslog ,boolean isSuccess ) {
|
||||
return saveDeviceReceiveLog(syslog, isSuccess, LocalDateTime.now());
|
||||
}
|
||||
|
||||
// 获取当前时间
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
/** 使用网络接收时刻写审计，避免等待 Broker ACK 后把处理完成时间误当成接收时间。 */
|
||||
public boolean saveDeviceReceiveLog(String syslog, boolean isSuccess, LocalDateTime now) {
|
||||
// 定义格式化器，包含毫秒
|
||||
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS");
|
||||
// 转换为字符串
|
||||
|
||||
@@ -1,21 +1,9 @@
|
||||
package com;
|
||||
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.config.SyslogConfig;
|
||||
import com.kafka.kafkaProducer;
|
||||
import org.mybatis.spring.annotation.MapperScan;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.core.env.Environment;
|
||||
import com.netty.SyslogServer;
|
||||
import org.springframework.cache.annotation.EnableCaching;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import com.netty.SyslogServerBoth;
|
||||
|
||||
|
||||
@MapperScan("com.common.mapper")
|
||||
@@ -23,24 +11,7 @@ import com.netty.SyslogServerBoth;
|
||||
@EnableScheduling
|
||||
public class SyslogServeMainApp {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(SyslogServeMainApp.class);
|
||||
|
||||
public static void main(String[] args) {
|
||||
ConfigurableApplicationContext context = SpringApplication.run(SyslogServeMainApp.class, args);
|
||||
Environment env = context.getEnvironment();
|
||||
try {
|
||||
int tcpPort = args.length > 0 ? Integer.parseInt(args[0])
|
||||
: Integer.parseInt(env.getProperty("syslog.tcp.port", "514"));
|
||||
int udpPort = args.length > 1 ? Integer.parseInt(args[1])
|
||||
: Integer.parseInt(env.getProperty("syslog.udp.port", "515"));
|
||||
SyslogServerBoth server = new SyslogServerBoth(tcpPort, udpPort);
|
||||
logger.info("Application SyslogServer start !");
|
||||
server.start();
|
||||
|
||||
}catch ( Exception ex)
|
||||
{
|
||||
System.out.println("Application Catch Error message: "+ex.getMessage());
|
||||
logger.error("Application Catch Error message: "+ex.getMessage());
|
||||
}
|
||||
SpringApplication.run(SyslogServeMainApp.class, args);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,147 +0,0 @@
|
||||
package com.haobang.config;
|
||||
|
||||
import com.typesafe.config.Config;
|
||||
import com.typesafe.config.ConfigFactory;
|
||||
import java.io.File;
|
||||
import com.typesafe.config.ConfigValueFactory;
|
||||
import java.nio.charset.Charset;
|
||||
|
||||
import java.util.Map;
|
||||
public class AppConfig {
|
||||
|
||||
private static final Config config;
|
||||
|
||||
static {
|
||||
// 加载配置文件
|
||||
File configFile = new File("application.properties");
|
||||
Config loadedConfig;
|
||||
|
||||
if (configFile.exists()) {
|
||||
loadedConfig = ConfigFactory.parseFile(configFile);
|
||||
} else {
|
||||
loadedConfig = ConfigFactory.load("application.properties");
|
||||
}
|
||||
|
||||
// 解析环境变量占位符
|
||||
config = resolveEnvironmentVariables(loadedConfig);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 解析环境变量占位符
|
||||
*/
|
||||
private static Config resolveEnvironmentVariables(Config originalConfig) {
|
||||
Config resolvedConfig = originalConfig;
|
||||
|
||||
// 遍历所有配置项，查找需要解析的占位符
|
||||
for (Map.Entry<String, com.typesafe.config.ConfigValue> entry : originalConfig.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
String value = originalConfig.getString(key);
|
||||
|
||||
if (value.contains("${")) {
|
||||
String resolvedValue = resolvePlaceholder(value);
|
||||
resolvedConfig = resolvedConfig.withValue(
|
||||
key,
|
||||
ConfigValueFactory.fromAnyRef(resolvedValue)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return resolvedConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析单个占位符值
|
||||
* 格式: ${ENV_VAR:default_value}
|
||||
*/
|
||||
private static String resolvePlaceholder(String value) {
|
||||
if (!value.startsWith("${") || !value.endsWith("}")) {
|
||||
return value;
|
||||
}
|
||||
|
||||
String placeholder = value.substring(2, value.length() - 1);
|
||||
String[] parts = placeholder.split(":");
|
||||
|
||||
if (parts.length == 0) {
|
||||
return value; // 无效格式，返回原值
|
||||
}
|
||||
|
||||
String envVarName = parts[0].trim();
|
||||
String defaultValue = parts.length > 1 ? parts[1].trim() : "";
|
||||
|
||||
// 1. 从系统环境变量获取
|
||||
String envValue = System.getenv(envVarName);
|
||||
if (envValue != null && !envValue.trim().isEmpty()) {
|
||||
return envValue.trim();
|
||||
}
|
||||
|
||||
// 2. 从系统属性获取 (java -D参数)
|
||||
String sysValue = System.getProperty(envVarName);
|
||||
if (sysValue != null && !sysValue.trim().isEmpty()) {
|
||||
return sysValue.trim();
|
||||
}
|
||||
|
||||
// 3. 返回默认值
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
// Syslog 配置
|
||||
public static int getSyslogTcpPort() {
|
||||
return config.getInt("syslog.tcp.port");
|
||||
}
|
||||
|
||||
public static int getSyslogUdpPort() {
|
||||
return config.getInt("syslog.udp.port");
|
||||
}
|
||||
|
||||
public static int getSyslogMaxFrameLength() {
|
||||
return config.getInt("syslog.max.frame.length");
|
||||
}
|
||||
|
||||
public static int getSyslogBufferSize() {
|
||||
return config.getInt("syslog.buffer.size");
|
||||
}
|
||||
|
||||
/**
|
||||
* syslog 消息字符编码，默认 GBK（国内安全设备普遍使用 GBK）
|
||||
* 配置示例: syslog.charset=GBK
|
||||
*/
|
||||
public static Charset getSyslogCharset() {
|
||||
if (config.hasPath("syslog.charset")) {
|
||||
return Charset.forName(config.getString("syslog.charset"));
|
||||
}
|
||||
return Charset.forName("GBK");
|
||||
}
|
||||
|
||||
// app service 配置
|
||||
public static String getAppServieDeviceId() {
|
||||
return config.getString("app.service.device_id");
|
||||
}
|
||||
public static String getAppServieDeviceName() { return config.getString("app.service.device_name");
|
||||
}
|
||||
public static String getAppServieVendor() {
|
||||
return config.getString("app.service.vendor");
|
||||
}
|
||||
public static String getAppServieProductName() {
|
||||
return config.getString("app.service.product_name");
|
||||
}
|
||||
public static String getAppServieDataType() {
|
||||
return config.getString("app.service.data_type");
|
||||
}
|
||||
// kafka 配置
|
||||
public static String getKafkaProducerBootstrap() {
|
||||
return config.getString("spring.kafka.producer.bootstrap-servers");
|
||||
}
|
||||
public static String getKafkaProducerTopic() {
|
||||
return config.getString("spring.kafka.producer.topic");
|
||||
}
|
||||
|
||||
public static int getDeviceCollectId() {
|
||||
return config.getInt("app.service.device_collect_id");
|
||||
}
|
||||
|
||||
public static String getSM4Key() {
|
||||
return config.getString("syslog.sm4.generateKey");
|
||||
}
|
||||
|
||||
}
|
||||
+108
-1
@@ -5,7 +5,7 @@ import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 应用配置属性 - 替代静态 AppConfig（syslog-serve 模块）
|
||||
* syslog-serve 统一应用配置属性
|
||||
* 使用 Spring Boot 标准 @Value 机制，支持 docker-compose 环境变量覆盖
|
||||
*/
|
||||
@Data
|
||||
@@ -25,9 +25,110 @@ public class AppProperties {
|
||||
@Value("${syslog.buffer.size:1024}")
|
||||
private int syslogBufferSize;
|
||||
|
||||
@Value("${syslog.tcp.backlog:4096}")
|
||||
private int syslogTcpBacklog;
|
||||
|
||||
@Value("${syslog.tcp.receive-buffer-bytes:4194304}")
|
||||
private int syslogTcpReceiveBufferBytes;
|
||||
|
||||
@Value("${syslog.udp.receive-buffer-bytes:16777216}")
|
||||
private int syslogUdpReceiveBufferBytes;
|
||||
|
||||
@Value("${syslog.charset:GBK}")
|
||||
private String syslogCharset;
|
||||
|
||||
// ========== 接入 WAL 配置 ==========
|
||||
/** WAL 是 Kafka 接管前的可靠性边界，正式链路不允许关闭。 */
|
||||
@Value("${syslog.wal.enabled:true}")
|
||||
private boolean syslogWalEnabled;
|
||||
|
||||
/** 必须挂载到宿主机持久化磁盘；兼容原归档目录环境变量。 */
|
||||
@Value("${syslog.wal.directory:${SYSLOG_WAL_DIRECTORY:${SYSLOG_RAW_ARCHIVE_DIRECTORY:/data/syslog-wal}}}")
|
||||
private String syslogWalDirectory;
|
||||
|
||||
/** Netty 到 WAL 单写线程之间的有界队列。 */
|
||||
@Value("${syslog.wal.queue-capacity:10000}")
|
||||
private int syslogWalQueueCapacity;
|
||||
|
||||
@Value("${syslog.wal.high-watermark-percent:70}")
|
||||
private int syslogWalHighWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.low-watermark-percent:40}")
|
||||
private int syslogWalLowWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.retry-interval-ms:10}")
|
||||
private long syslogWalRetryIntervalMs;
|
||||
|
||||
@Value("${syslog.wal.disk-retry-interval-ms:1000}")
|
||||
private long syslogWalDiskRetryIntervalMs;
|
||||
|
||||
/** 累计记录数或时间达到任一条件即 force，Future 只在 force 后视为持久化。 */
|
||||
@Value("${syslog.wal.flush-records:200}")
|
||||
private int syslogWalFlushRecords;
|
||||
|
||||
@Value("${syslog.wal.flush-interval-ms:200}")
|
||||
private long syslogWalFlushIntervalMs;
|
||||
|
||||
@Value("${syslog.wal.segment-max-bytes:268435456}")
|
||||
private long syslogWalSegmentMaxBytes;
|
||||
|
||||
/** 已完成段的保留天数；0 表示不自动删除。 */
|
||||
@Value("${syslog.wal.retention-days:0}")
|
||||
private int syslogWalRetentionDays;
|
||||
|
||||
@Value("${syslog.wal.shutdown-timeout-ms:30000}")
|
||||
private long syslogWalShutdownTimeoutMs;
|
||||
|
||||
/** ACK 游标允许批量刷盘；崩溃时最多重复投递该批，event_id 保证下游幂等。 */
|
||||
@Value("${syslog.wal.checkpoint-flush-records:200}")
|
||||
private int syslogWalCheckpointFlushRecords;
|
||||
|
||||
@Value("${syslog.wal.checkpoint-flush-interval-ms:200}")
|
||||
private long syslogWalCheckpointFlushIntervalMs;
|
||||
|
||||
@Value("${syslog.wal.disk-high-watermark-percent:80}")
|
||||
private int syslogWalDiskHighWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.disk-critical-watermark-percent:90}")
|
||||
private int syslogWalDiskCriticalWatermarkPercent;
|
||||
|
||||
@Value("${syslog.wal.min-free-bytes:10737418240}")
|
||||
private long syslogWalMinFreeBytes;
|
||||
|
||||
/** 兼容部署阶段关闭；开启后新接入事件必须以 WAL schema v3 固化 point_time_ns。 */
|
||||
@Value("${syslog.point-time.write-enabled:false}")
|
||||
private boolean syslogPointTimeWriteEnabled;
|
||||
|
||||
// ========== WAL Dispatcher 配置 ==========
|
||||
@Value("${syslog.dispatch.worker-threads:4}")
|
||||
private int syslogDispatchWorkerThreads;
|
||||
|
||||
@Value("${syslog.dispatch.max-inflight:5000}")
|
||||
private int syslogDispatchMaxInflight;
|
||||
|
||||
@Value("${syslog.dispatch.retry-backoff-ms:1000}")
|
||||
private long syslogDispatchRetryBackoffMs;
|
||||
|
||||
@Value("${syslog.dispatch.shutdown-timeout-ms:30000}")
|
||||
private long syslogDispatchShutdownTimeoutMs;
|
||||
|
||||
@Value("${syslog.device-context.cache-ttl-ms:300000}")
|
||||
private long syslogDeviceContextCacheTtlMs;
|
||||
|
||||
@Value("${syslog.status-log-interval-ms:60000}")
|
||||
private long syslogStatusLogIntervalMs;
|
||||
|
||||
// ========== 旧 v1 原文归档精确导入（默认关闭） ==========
|
||||
@Value("${syslog.wal.legacy-import.enabled:false}")
|
||||
private boolean syslogWalLegacyImportEnabled;
|
||||
|
||||
@Value("${syslog.wal.legacy-import.archive-directory:}")
|
||||
private String syslogWalLegacyImportArchiveDirectory;
|
||||
|
||||
/** 一行可写一个 event_id，也可直接粘贴包含 event_id 的拒绝日志。 */
|
||||
@Value("${syslog.wal.legacy-import.event-id-file:}")
|
||||
private String syslogWalLegacyImportEventIdFile;
|
||||
|
||||
// ========== App Service 配置（探针设备信息）==========
|
||||
@Value("${app.service.device_id:}")
|
||||
private String appServiceDeviceId;
|
||||
@@ -47,6 +148,9 @@ public class AppProperties {
|
||||
@Value("${app.service.device_collect_id:1}")
|
||||
private int appServiceDeviceCollectId;
|
||||
|
||||
@Value("${app.service.device_collect_name:DevCollect-01}")
|
||||
private String appServiceDeviceCollectName;
|
||||
|
||||
// ========== Kafka Producer 配置 ==========
|
||||
@Value("${spring.kafka.producer.bootstrap-servers:localhost:9092}")
|
||||
private String kafkaProducerBootstrap;
|
||||
@@ -54,6 +158,9 @@ public class AppProperties {
|
||||
@Value("${spring.kafka.producer.topic:agent-syslog-topic}")
|
||||
private String kafkaProducerTopic;
|
||||
|
||||
@Value("${spring.kafka.producer.ack-timeout-ms:130000}")
|
||||
private long kafkaProducerAckTimeoutMs;
|
||||
|
||||
// ========== SM4 加密配置 ==========
|
||||
@Value("${syslog.sm4.generateKey:}")
|
||||
private String sm4Key;
|
||||
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
package com.haobang.dispatch;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.entity.DeviceUnknown;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.common.service.DeviceUnknownService;
|
||||
import com.haobang.config.AppProperties;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
|
||||
/**
|
||||
* 来源 IP 到设备接入上下文的进程内缓存。
|
||||
*
|
||||
* <p>原实现每条日志至少查询两次设备表；该缓存使正常热路径不访问达梦。缓存失效时同一
|
||||
* source_ip 使用条带锁合并并发查询，避免恢复洪峰击穿数据库。</p>
|
||||
*/
|
||||
@Component
|
||||
public class DeviceContextResolver {
|
||||
private static final Logger log = LoggerFactory.getLogger(DeviceContextResolver.class);
|
||||
private final DeviceDeviceService deviceService;
|
||||
private final DeviceUnknownService unknownService;
|
||||
private final AppProperties properties;
|
||||
private final ConcurrentHashMap<String, CacheEntry> cache = new ConcurrentHashMap<>();
|
||||
private final Object[] locks = new Object[64];
|
||||
private final LongAdder hits = new LongAdder();
|
||||
private final LongAdder misses = new LongAdder();
|
||||
|
||||
public DeviceContextResolver(DeviceDeviceService deviceService, DeviceUnknownService unknownService,
|
||||
AppProperties properties) {
|
||||
this.deviceService = deviceService;
|
||||
this.unknownService = unknownService;
|
||||
this.properties = properties;
|
||||
for (int i = 0; i < locks.length; i++) locks[i] = new Object();
|
||||
}
|
||||
|
||||
public Resolution resolve(String sourceIp, String protocol) {
|
||||
long now = System.currentTimeMillis();
|
||||
CacheEntry cached = cache.get(sourceIp);
|
||||
if (cached != null && cached.expiresAt > now) {
|
||||
hits.increment();
|
||||
return cached.resolution;
|
||||
}
|
||||
Object lock = locks[(sourceIp.hashCode() & Integer.MAX_VALUE) % locks.length];
|
||||
synchronized (lock) {
|
||||
cached = cache.get(sourceIp);
|
||||
if (cached != null && cached.expiresAt > now) {
|
||||
hits.increment();
|
||||
return cached.resolution;
|
||||
}
|
||||
misses.increment();
|
||||
List<DeviceDevice> devices = deviceService.getByIpSafely(sourceIp);
|
||||
Resolution resolution;
|
||||
if (devices == null || devices.isEmpty()) {
|
||||
recordUnknown(sourceIp, protocol);
|
||||
resolution = Resolution.terminal(Type.UNKNOWN_DEVICE, null,
|
||||
"来源 IP 未注册: " + sourceIp);
|
||||
} else if (devices.size() != 1) {
|
||||
resolution = Resolution.terminal(Type.DUPLICATE_DEVICE, null,
|
||||
"来源 IP 存在多条设备记录: " + sourceIp);
|
||||
} else {
|
||||
DeviceDevice device = devices.get(0);
|
||||
Integer assignedCollector = device.getDeviceCollectId();
|
||||
if (assignedCollector == null
|
||||
|| assignedCollector != properties.getAppServiceDeviceCollectId()) {
|
||||
resolution = Resolution.terminal(Type.NOT_ASSIGNED, device,
|
||||
"设备不属于当前采集探针: deviceId=" + device.getId());
|
||||
} else {
|
||||
resolution = Resolution.registered(device);
|
||||
}
|
||||
}
|
||||
cache.put(sourceIp, new CacheEntry(resolution,
|
||||
now + properties.getSyslogDeviceContextCacheTtlMs()));
|
||||
if (resolution.type != Type.REGISTERED) {
|
||||
log.warn("Syslog 设备接入终态: type={}, sourceIp={}, reason={}",
|
||||
resolution.type, sourceIp, resolution.reason);
|
||||
}
|
||||
return resolution;
|
||||
}
|
||||
}
|
||||
|
||||
public void invalidate(String sourceIp) {
|
||||
if (sourceIp != null) cache.remove(sourceIp);
|
||||
}
|
||||
|
||||
public void invalidateAll() { cache.clear(); }
|
||||
public long getHitCount() { return hits.sum(); }
|
||||
public long getMissCount() { return misses.sum(); }
|
||||
public int getCacheSize() { return cache.size(); }
|
||||
|
||||
/** 未知设备仍保留原业务登记能力，但最多每个本地缓存 TTL 执行一次，不占用网络线程。 */
|
||||
private void recordUnknown(String sourceIp, String protocol) {
|
||||
List<DeviceUnknown> existing = unknownService.getDevicesByIp(sourceIp);
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
if (existing != null && !existing.isEmpty()) {
|
||||
unknownService.updateLastTime(existing.get(0).getId(), now);
|
||||
return;
|
||||
}
|
||||
DeviceUnknown unknown = new DeviceUnknown();
|
||||
unknown.setDeviceCollectId(properties.getAppServiceDeviceCollectId());
|
||||
unknown.setDeviceCollectName(properties.getAppServiceDeviceCollectName());
|
||||
unknown.setDeviceIp(sourceIp);
|
||||
unknown.setFirstTime(now);
|
||||
unknown.setLastTime(now);
|
||||
unknown.setOrganizationId(0);
|
||||
unknown.setNetworkProtocol(protocol);
|
||||
unknown.setSourceMethod("SYSTEM");
|
||||
unknownService.createDevice(unknown);
|
||||
}
|
||||
|
||||
private static final class CacheEntry {
|
||||
private final Resolution resolution;
|
||||
private final long expiresAt;
|
||||
private CacheEntry(Resolution resolution, long expiresAt) {
|
||||
this.resolution = resolution;
|
||||
this.expiresAt = expiresAt;
|
||||
}
|
||||
}
|
||||
|
||||
public enum Type { REGISTERED, UNKNOWN_DEVICE, DUPLICATE_DEVICE, NOT_ASSIGNED }
|
||||
|
||||
public static final class Resolution {
|
||||
private final Type type;
|
||||
private final DeviceDevice device;
|
||||
private final String reason;
|
||||
|
||||
private Resolution(Type type, DeviceDevice device, String reason) {
|
||||
this.type = type;
|
||||
this.device = device;
|
||||
this.reason = reason;
|
||||
}
|
||||
static Resolution registered(DeviceDevice device) {
|
||||
return new Resolution(Type.REGISTERED, device, null);
|
||||
}
|
||||
static Resolution terminal(Type type, DeviceDevice device, String reason) {
|
||||
return new Resolution(type, device, reason);
|
||||
}
|
||||
public boolean isRegistered() { return type == Type.REGISTERED; }
|
||||
public Type getType() { return type; }
|
||||
public DeviceDevice getDevice() { return device; }
|
||||
public String getReason() { return reason; }
|
||||
}
|
||||
}
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
package com.haobang.dispatch;
|
||||
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.wal.SyslogWalRecord;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import com.kafka.kafkaProducer;
|
||||
import com.netty.SyslogMessageHandler;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
|
||||
/** 从 WAL 有界并发异步投递 Kafka，并只按连续 ACK 推进 WAL 游标。 */
|
||||
@Component
|
||||
public class SyslogWalDispatcher {
|
||||
private static final Logger log = LoggerFactory.getLogger(SyslogWalDispatcher.class);
|
||||
private static final DateTimeFormatter WAL_TIME = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS");
|
||||
private static final DateTimeFormatter HEADER_TIME = DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS");
|
||||
|
||||
private final SyslogWalService walService;
|
||||
private final DeviceContextResolver deviceResolver;
|
||||
private final kafkaProducer producer;
|
||||
private final AppProperties properties;
|
||||
private final AtomicBoolean running = new AtomicBoolean();
|
||||
private final Object completionMonitor = new Object();
|
||||
private final Map<Long, SyslogWalRecord> completedOutOfOrder = new HashMap<>();
|
||||
private final LongAdder kafkaAcked = new LongAdder();
|
||||
private final LongAdder terminalFiltered = new LongAdder();
|
||||
private final LongAdder retryCount = new LongAdder();
|
||||
private final AtomicLong inflight = new AtomicLong();
|
||||
|
||||
private Semaphore permits;
|
||||
private ExecutorService workers;
|
||||
private ScheduledExecutorService scheduler;
|
||||
private Thread readerThread;
|
||||
private SyslogWalService.WalCursor cursor;
|
||||
private long nextCommitSequence;
|
||||
private long nextReadSequence;
|
||||
|
||||
public SyslogWalDispatcher(SyslogWalService walService,
|
||||
DeviceContextResolver deviceResolver,
|
||||
kafkaProducer producer,
|
||||
AppProperties properties) {
|
||||
this.walService = walService;
|
||||
this.deviceResolver = deviceResolver;
|
||||
this.producer = producer;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void start() throws Exception {
|
||||
validateConfiguration();
|
||||
permits = new Semaphore(properties.getSyslogDispatchMaxInflight());
|
||||
workers = Executors.newFixedThreadPool(properties.getSyslogDispatchWorkerThreads(),
|
||||
namedFactory("syslog-dispatch-worker"));
|
||||
scheduler = Executors.newScheduledThreadPool(2, namedFactory("syslog-dispatch-scheduler"));
|
||||
cursor = walService.openCursor();
|
||||
nextCommitSequence = walService.getCheckpointSequence() + 1L;
|
||||
nextReadSequence = nextCommitSequence;
|
||||
running.set(true);
|
||||
readerThread = new Thread(this::readLoop, "syslog-wal-dispatch-reader");
|
||||
readerThread.setDaemon(true);
|
||||
readerThread.start();
|
||||
scheduler.scheduleWithFixedDelay(this::logStatus,
|
||||
properties.getSyslogStatusLogIntervalMs(), properties.getSyslogStatusLogIntervalMs(),
|
||||
TimeUnit.MILLISECONDS);
|
||||
log.info("Syslog WAL Dispatcher 已启动: workerThreads={}, maxInflight={}, checkpointSequence={}",
|
||||
properties.getSyslogDispatchWorkerThreads(), properties.getSyslogDispatchMaxInflight(),
|
||||
walService.getCheckpointSequence());
|
||||
}
|
||||
|
||||
private void readLoop() {
|
||||
try {
|
||||
while (running.get()) {
|
||||
permits.acquire();
|
||||
SyslogWalRecord record = cursor.next(1000L);
|
||||
if (record == null) {
|
||||
permits.release();
|
||||
continue;
|
||||
}
|
||||
if (record.getSequence() != nextReadSequence) {
|
||||
permits.release();
|
||||
throw new IllegalStateException("WAL sequence 不连续，拒绝越过: expected="
|
||||
+ nextReadSequence + ", actual=" + record.getSequence());
|
||||
}
|
||||
nextReadSequence++;
|
||||
inflight.incrementAndGet();
|
||||
workers.execute(() -> dispatch(record, 1));
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
} catch (Throwable failure) {
|
||||
running.set(false);
|
||||
log.error("WAL Dispatcher 读取失败，停止推进游标以保护数据", failure);
|
||||
}
|
||||
}
|
||||
|
||||
private void dispatch(SyslogWalRecord record, int attempt) {
|
||||
if (!running.get()) return;
|
||||
if (record.isQuarantinedInvalid()) {
|
||||
log.error("损坏 WAL 已隔离并推进游标: sequence={}, eventId={}, reason={}",
|
||||
record.getSequence(), record.getEventId(), record.getValidationError());
|
||||
markComplete(record);
|
||||
return;
|
||||
}
|
||||
final DeviceContextResolver.Resolution resolution;
|
||||
try {
|
||||
resolution = deviceResolver.resolve(record.getSourceIp(), record.getProtocol());
|
||||
} catch (Throwable failure) {
|
||||
scheduleRetry(record, attempt, "DEVICE_LOOKUP", failure);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!resolution.isRegistered()) {
|
||||
try {
|
||||
walService.recordTerminal(record, resolution.getType().name(), resolution.getReason());
|
||||
terminalFiltered.increment();
|
||||
markComplete(record);
|
||||
} catch (Throwable failure) {
|
||||
scheduleRetry(record, attempt, "TERMINAL_LEDGER", failure);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
final String receiveTime;
|
||||
try {
|
||||
receiveTime = LocalDateTime.parse(record.getReceiveTime(), WAL_TIME).format(HEADER_TIME);
|
||||
producer.messagePush(resolution.getDevice(), record.getRawMessage(), receiveTime,
|
||||
record.getEventId(), record.getPointTimeNs())
|
||||
.whenComplete((delivery, failure) -> {
|
||||
if (failure == null) {
|
||||
kafkaAcked.increment();
|
||||
markComplete(record);
|
||||
} else {
|
||||
scheduleRetry(record, attempt, "KAFKA_ACK", unwrap(failure));
|
||||
}
|
||||
});
|
||||
} catch (Throwable failure) {
|
||||
// 加密、配置或 Producer 本地错误都不能越过 WAL；修复依赖后沿用同一 event_id 重试。
|
||||
scheduleRetry(record, attempt, "KAFKA_PREPARE", failure);
|
||||
}
|
||||
}
|
||||
|
||||
private void scheduleRetry(SyslogWalRecord record, int attempt, String stage, Throwable failure) {
|
||||
retryCount.increment();
|
||||
long retries = retryCount.sum();
|
||||
if (attempt == 1 || retries % 100 == 0) {
|
||||
log.warn("WAL 事件投递失败，将保留原 event_id 重试: eventId={}, sequence={}, stage={}, "
|
||||
+ "attempt={}, retryTotal={}, error={}",
|
||||
record.getEventId(), record.getSequence(), stage, attempt, retries,
|
||||
failure == null ? "unknown" : failure.toString());
|
||||
}
|
||||
if (!running.get()) return;
|
||||
scheduler.schedule(() -> {
|
||||
if (running.get()) workers.execute(() -> dispatch(record, attempt + 1));
|
||||
}, properties.getSyslogDispatchRetryBackoffMs(), TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
private void markComplete(SyslogWalRecord record) {
|
||||
synchronized (completionMonitor) {
|
||||
if (record.getSequence() < nextCommitSequence) return;
|
||||
completedOutOfOrder.put(record.getSequence(), record);
|
||||
SyslogWalRecord latest = null;
|
||||
int released = 0;
|
||||
while (true) {
|
||||
SyslogWalRecord contiguous = completedOutOfOrder.remove(nextCommitSequence);
|
||||
if (contiguous == null) break;
|
||||
latest = contiguous;
|
||||
nextCommitSequence++;
|
||||
released++;
|
||||
}
|
||||
if (latest != null) walService.markCheckpoint(latest);
|
||||
if (released > 0) {
|
||||
inflight.addAndGet(-released);
|
||||
permits.release(released);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void logStatus() {
|
||||
if (!running.get()) return;
|
||||
log.info("Syslog WAL状态: networkReceived={}, appended={}, walPendingApprox={}, walQueue={}/{}, durableSequence={}, "
|
||||
+ "checkpointSequence={}, kafkaInflight={}, kafkaAcked={}, terminalFiltered={}, "
|
||||
+ "retry={}, tcpPauseTotal={}, udpDropped={}, deviceCacheSize={}, deviceCacheHits={}, "
|
||||
+ "deviceCacheMisses={}",
|
||||
SyslogMessageHandler.getNetworkReceivedCount(), walService.getAppendedCount(),
|
||||
walService.getPendingRecordsApprox(),
|
||||
walService.getQueueSize(), walService.getQueueCapacity(), walService.getDurableSequence(),
|
||||
walService.getCheckpointSequence(), inflight.get(), kafkaAcked.sum(), terminalFiltered.sum(),
|
||||
retryCount.sum(), SyslogMessageHandler.getTcpPauseCount(),
|
||||
SyslogMessageHandler.getUdpDroppedCount(), deviceResolver.getCacheSize(),
|
||||
deviceResolver.getHitCount(), deviceResolver.getMissCount());
|
||||
}
|
||||
|
||||
private Throwable unwrap(Throwable failure) {
|
||||
if (failure instanceof java.util.concurrent.CompletionException && failure.getCause() != null) {
|
||||
return failure.getCause();
|
||||
}
|
||||
return failure;
|
||||
}
|
||||
|
||||
private ThreadFactory namedFactory(String prefix) {
|
||||
AtomicLong sequence = new AtomicLong();
|
||||
return runnable -> {
|
||||
Thread thread = new Thread(runnable, prefix + "-" + sequence.incrementAndGet());
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
};
|
||||
}
|
||||
|
||||
private void validateConfiguration() {
|
||||
if (properties.getSyslogDispatchWorkerThreads() <= 0
|
||||
|| properties.getSyslogDispatchMaxInflight() <= 0
|
||||
|| properties.getSyslogDispatchRetryBackoffMs() <= 0
|
||||
|| properties.getSyslogDispatchShutdownTimeoutMs() <= 0
|
||||
|| properties.getSyslogDeviceContextCacheTtlMs() <= 0
|
||||
|| properties.getSyslogStatusLogIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("Dispatcher 线程、在途、重试、缓存和状态日志参数必须大于 0");
|
||||
}
|
||||
}
|
||||
|
||||
public long getInflight() { return inflight.get(); }
|
||||
public long getKafkaAcked() { return kafkaAcked.sum(); }
|
||||
public long getTerminalFiltered() { return terminalFiltered.sum(); }
|
||||
public long getRetryCount() { return retryCount.sum(); }
|
||||
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
if (!running.compareAndSet(true, false)) return;
|
||||
if (readerThread != null) readerThread.interrupt();
|
||||
if (scheduler != null) scheduler.shutdownNow();
|
||||
if (workers != null) workers.shutdown();
|
||||
long deadline = System.nanoTime()
|
||||
+ TimeUnit.MILLISECONDS.toNanos(properties.getSyslogDispatchShutdownTimeoutMs());
|
||||
try {
|
||||
while (inflight.get() > 0 && System.nanoTime() < deadline) Thread.sleep(20L);
|
||||
if (workers != null) {
|
||||
long remaining = Math.max(1L, deadline - System.nanoTime());
|
||||
workers.awaitTermination(remaining, TimeUnit.NANOSECONDS);
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
if (workers != null && !workers.isTerminated()) workers.shutdownNow();
|
||||
try { if (cursor != null) cursor.close(); }
|
||||
catch (Exception failure) { log.warn("关闭 WAL cursor 失败", failure); }
|
||||
log.info("Syslog WAL Dispatcher 已停止: inflight={}, checkpointSequence={}",
|
||||
inflight.get(), walService.getCheckpointSequence());
|
||||
}
|
||||
}
|
||||
+31
-11
@@ -78,25 +78,45 @@ public class DeviceInfoUtil {
|
||||
|
||||
public static String getFullLogString(DeviceDevice deviceDevice,String strSyslog,String receive_time)
|
||||
{
|
||||
AppProperties appProps = getProps();
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
// 创建设备配置信息MAP
|
||||
// 获取当前时间（精度到毫秒）
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
// 定义格式化器，包含毫秒
|
||||
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS");
|
||||
// 转换为字符串
|
||||
String timeString = now.format(formatter);
|
||||
return getFullLogString(deviceDevice, strSyslog, receive_time, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* [流程节点3.1] 将接入边界生成的 event_id 固化到消息头；后续 Kafka 重试和人工回放沿用该值。
|
||||
*/
|
||||
public static String getFullLogString(DeviceDevice deviceDevice, String strSyslog,
|
||||
String receive_time, String eventId)
|
||||
{
|
||||
return getFullLogString(deviceDevice, strSyslog, receive_time, eventId, getProps());
|
||||
}
|
||||
|
||||
/** Dispatcher 使用显式配置对象，避免 WAL 恢复线程依赖静态 SpringContext 初始化时序。 */
|
||||
public static String getFullLogString(DeviceDevice deviceDevice, String strSyslog,
|
||||
String receive_time, String eventId,
|
||||
AppProperties appProps)
|
||||
{
|
||||
return getFullLogString(deviceDevice, strSyslog, receive_time, eventId, null, appProps);
|
||||
}
|
||||
|
||||
/** Kafka 业务头只新增 point_time_ns；WAL schema_version 绝不进入业务载荷。 */
|
||||
public static String getFullLogString(DeviceDevice deviceDevice, String strSyslog,
|
||||
String receive_time, String eventId, Long pointTimeNs,
|
||||
AppProperties appProps)
|
||||
{
|
||||
Map<String, String> map = new LinkedHashMap<>();
|
||||
map.put("receive_time",receive_time);
|
||||
if (eventId != null && !eventId.isEmpty()) {
|
||||
map.put("event_id", eventId);
|
||||
}
|
||||
if (pointTimeNs != null) {
|
||||
map.put("point_time_ns", Long.toString(pointTimeNs));
|
||||
}
|
||||
map.put("device_id",deviceDevice.getId().toString());
|
||||
map.put("device_name", deviceDevice.getName());
|
||||
map.put("vendor", deviceDevice.getVendor());
|
||||
map.put("data_type", appProps.getAppServiceDataType());
|
||||
map.put("device_collect_id", Long.toString(appProps.getAppServiceDeviceCollectId()));
|
||||
String formattedString = formatDeviceInfo(map);
|
||||
//return formattedString + "" + strSyslog.substring(34);
|
||||
//测试环境截取34位之后字符串
|
||||
return formattedString + strSyslog;
|
||||
}
|
||||
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
|
||||
/** 在 receive_time 所属毫秒内分配 InfluxDB Point 的纳秒尾数。 */
|
||||
public final class PointTimeAllocator {
|
||||
public static final long NANOS_PER_MILLI = 1_000_000L;
|
||||
|
||||
private final AtomicLong sequence;
|
||||
private final LongAdder assigned = new LongAdder();
|
||||
|
||||
public PointTimeAllocator() {
|
||||
this(0L);
|
||||
}
|
||||
|
||||
PointTimeAllocator(long initialSequence) {
|
||||
this.sequence = new AtomicLong(initialSequence);
|
||||
}
|
||||
|
||||
public long allocate(long receiveTimeMillis) {
|
||||
long suffix = Math.floorMod(sequence.getAndIncrement(), NANOS_PER_MILLI);
|
||||
long pointTimeNs = Math.addExact(
|
||||
Math.multiplyExact(receiveTimeMillis, NANOS_PER_MILLI), suffix);
|
||||
assigned.increment();
|
||||
return pointTimeNs;
|
||||
}
|
||||
|
||||
public long getAssignedCount() {
|
||||
return assigned.sum();
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/** 网络入口已经解帧、但尚未写入 WAL 的不可变事件。 */
|
||||
public final class SyslogIngressEvent {
|
||||
private final String eventId;
|
||||
private final LocalDateTime receivedAt;
|
||||
private final String sourceIp;
|
||||
private final int sourcePort;
|
||||
private final String protocol;
|
||||
private final String rawMessage;
|
||||
private final Long pointTimeNs;
|
||||
|
||||
public SyslogIngressEvent(String eventId, LocalDateTime receivedAt, String sourceIp,
|
||||
int sourcePort, String protocol, String rawMessage) {
|
||||
this(eventId, receivedAt, sourceIp, sourcePort, protocol, rawMessage, null);
|
||||
}
|
||||
|
||||
public SyslogIngressEvent(String eventId, LocalDateTime receivedAt, String sourceIp,
|
||||
int sourcePort, String protocol, String rawMessage, Long pointTimeNs) {
|
||||
this.eventId = eventId;
|
||||
this.receivedAt = receivedAt;
|
||||
this.sourceIp = sourceIp;
|
||||
this.sourcePort = sourcePort;
|
||||
this.protocol = protocol;
|
||||
this.rawMessage = rawMessage;
|
||||
this.pointTimeNs = pointTimeNs;
|
||||
}
|
||||
|
||||
public String getEventId() { return eventId; }
|
||||
public LocalDateTime getReceivedAt() { return receivedAt; }
|
||||
public String getSourceIp() { return sourceIp; }
|
||||
public int getSourcePort() { return sourcePort; }
|
||||
public String getProtocol() { return protocol; }
|
||||
public String getRawMessage() { return rawMessage; }
|
||||
public Long getPointTimeNs() { return pointTimeNs; }
|
||||
|
||||
public SyslogIngressEvent withPointTimeNs(long value) {
|
||||
return new SyslogIngressEvent(eventId, receivedAt, sourceIp, sourcePort, protocol, rawMessage, value);
|
||||
}
|
||||
}
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.config.AppProperties;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.io.BufferedReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.DirectoryStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 把旧 schema_version=1 归档中明确列出的拒绝事件导入 WAL v2。
|
||||
* 默认关闭，且必须提供 event_id 清单；绝不自动重放整个历史归档。
|
||||
*/
|
||||
@Component
|
||||
public class SyslogLegacyArchiveImporter {
|
||||
private static final Logger log = LoggerFactory.getLogger(SyslogLegacyArchiveImporter.class);
|
||||
private static final Pattern EVENT_ID = Pattern.compile("(20\\d{6}-[0-9a-fA-F-]{36})");
|
||||
private final AppProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final SyslogWalService walService;
|
||||
|
||||
public SyslogLegacyArchiveImporter(AppProperties properties, ObjectMapper objectMapper,
|
||||
SyslogWalService walService) {
|
||||
this.properties = properties;
|
||||
this.objectMapper = objectMapper;
|
||||
this.walService = walService;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void importIfRequested() throws Exception {
|
||||
if (!properties.isSyslogWalLegacyImportEnabled()) return;
|
||||
Path archiveDirectory = requiredPath(properties.getSyslogWalLegacyImportArchiveDirectory(),
|
||||
"syslog.wal.legacy-import.archive-directory");
|
||||
Path eventIdFile = requiredPath(properties.getSyslogWalLegacyImportEventIdFile(),
|
||||
"syslog.wal.legacy-import.event-id-file");
|
||||
if (!Files.isDirectory(archiveDirectory) || !Files.isRegularFile(eventIdFile)) {
|
||||
throw new IllegalArgumentException("旧归档导入目录或 event_id 文件不存在");
|
||||
}
|
||||
|
||||
byte[] requestBytes = Files.readAllBytes(eventIdFile);
|
||||
String requestHash = sha256(requestBytes);
|
||||
Path marker = Paths.get(properties.getSyslogWalDirectory()).toAbsolutePath().normalize()
|
||||
.resolve("checkpoint").resolve("legacy-import-" + requestHash + ".completed");
|
||||
if (Files.exists(marker)) {
|
||||
log.info("旧归档导入请求已完成，跳过重复扫描: marker={}", marker);
|
||||
return;
|
||||
}
|
||||
|
||||
Set<String> requested = extractEventIds(new String(requestBytes, StandardCharsets.UTF_8));
|
||||
if (requested.isEmpty()) throw new IllegalArgumentException("旧归档导入清单中没有合法 event_id");
|
||||
Set<String> remaining = new HashSet<>(requested);
|
||||
long appendedBefore = walService.getAppendedCount();
|
||||
int accepted = 0;
|
||||
for (Path segment : listLegacySegments(archiveDirectory)) {
|
||||
try (BufferedReader reader = Files.newBufferedReader(segment, StandardCharsets.UTF_8)) {
|
||||
String line;
|
||||
while (!remaining.isEmpty() && (line = reader.readLine()) != null) {
|
||||
JsonNode node = objectMapper.readTree(line);
|
||||
String eventId = node.path("event_id").asText();
|
||||
if (!remaining.remove(eventId)) continue;
|
||||
if (node.path("schema_version").asInt() != 1) {
|
||||
throw new IllegalStateException("旧归档导入仅支持 schema_version=1: " + segment);
|
||||
}
|
||||
SyslogIngressEvent event = new SyslogIngressEvent(
|
||||
eventId,
|
||||
LocalDateTime.parse(node.path("receive_time").asText()),
|
||||
node.path("source_ip").asText(),
|
||||
node.path("source_port").asInt(),
|
||||
node.path("protocol").asText(),
|
||||
node.path("raw_message").asText());
|
||||
appendWithoutDropping(event);
|
||||
accepted++;
|
||||
}
|
||||
}
|
||||
if (remaining.isEmpty()) break;
|
||||
}
|
||||
if (!remaining.isEmpty()) {
|
||||
throw new IllegalStateException("旧归档中未找到 " + remaining.size() + " 个 event_id，示例: "
|
||||
+ remaining.iterator().next());
|
||||
}
|
||||
final long expected = appendedBefore + accepted;
|
||||
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60);
|
||||
while (walService.getAppendedCount() < expected && System.nanoTime() < deadline) Thread.sleep(20L);
|
||||
if (walService.getAppendedCount() < expected) {
|
||||
throw new IllegalStateException("旧归档事件未在超时内完成 WAL force，拒绝写完成标记");
|
||||
}
|
||||
Files.write(marker, ("request_hash=" + requestHash + "\ncount=" + accepted + "\n")
|
||||
.getBytes(StandardCharsets.UTF_8));
|
||||
log.info("旧 v1 归档事件已精确导入 WAL: requested={}, imported={}, marker={}",
|
||||
requested.size(), accepted, marker);
|
||||
}
|
||||
|
||||
private void appendWithoutDropping(SyslogIngressEvent event) throws Exception {
|
||||
event = walService.prepareForWal(event);
|
||||
while (true) {
|
||||
SyslogWalService.AdmissionResult result = walService.tryAppend(event);
|
||||
if (result == SyslogWalService.AdmissionResult.ACCEPTED) return;
|
||||
if (result == SyslogWalService.AdmissionResult.UNAVAILABLE) {
|
||||
throw new IllegalStateException("WAL 不可用，无法导入旧归档 eventId=" + event.getEventId(),
|
||||
walService.getFatalFailure());
|
||||
}
|
||||
Thread.sleep(walService.getRetryIntervalMs());
|
||||
}
|
||||
}
|
||||
|
||||
private Set<String> extractEventIds(String text) {
|
||||
Set<String> ids = new HashSet<>();
|
||||
Matcher matcher = EVENT_ID.matcher(text);
|
||||
while (matcher.find()) ids.add(matcher.group(1));
|
||||
return ids;
|
||||
}
|
||||
|
||||
private List<Path> listLegacySegments(Path directory) throws Exception {
|
||||
List<Path> result = new ArrayList<>();
|
||||
try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory, "syslog-raw-*.jsonl")) {
|
||||
for (Path path : stream) result.add(path);
|
||||
}
|
||||
result.sort((left, right) -> left.getFileName().toString().compareTo(right.getFileName().toString()));
|
||||
return result;
|
||||
}
|
||||
|
||||
private Path requiredPath(String value, String name) {
|
||||
if (value == null || value.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException(name + " 不能为空");
|
||||
}
|
||||
return Paths.get(value).toAbsolutePath().normalize();
|
||||
}
|
||||
|
||||
private String sha256(byte[] value) throws Exception {
|
||||
byte[] digest = MessageDigest.getInstance("SHA-256").digest(value);
|
||||
StringBuilder text = new StringBuilder();
|
||||
for (byte item : digest) text.append(String.format("%02x", item & 0xff));
|
||||
return text.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
/**
|
||||
* WAL v2/v3 磁盘记录；segment/offset 仅用于运行时确认游标，不写入 JSON 正文。
|
||||
*
|
||||
* <p>早期 v2 记录错误复用了 Redis 多态 ObjectMapper，因而额外包含 "@class"。这里仅忽略这个
|
||||
* 已知历史字段，使修复后的专用 WAL ObjectMapper 能无损恢复既有数据；其他未知字段仍保持
|
||||
* Jackson 默认的严格失败语义，避免悄悄接受不可识别的 WAL schema。</p>
|
||||
*/
|
||||
@JsonIgnoreProperties(value = "@class")
|
||||
public class SyslogWalRecord {
|
||||
@JsonProperty("schema_version")
|
||||
private int schemaVersion = 2;
|
||||
private long sequence;
|
||||
@JsonProperty("event_id")
|
||||
private String eventId;
|
||||
@JsonProperty("receive_time")
|
||||
private String receiveTime;
|
||||
@JsonProperty("point_time_ns")
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
private Long pointTimeNs;
|
||||
@JsonProperty("source_ip")
|
||||
private String sourceIp;
|
||||
@JsonProperty("source_port")
|
||||
private int sourcePort;
|
||||
private String protocol;
|
||||
private String charset;
|
||||
@JsonProperty("raw_length")
|
||||
private int rawLength;
|
||||
@JsonProperty("raw_sha256")
|
||||
private String rawSha256;
|
||||
@JsonProperty("raw_message")
|
||||
private String rawMessage;
|
||||
|
||||
@JsonIgnore
|
||||
private String segment;
|
||||
@JsonIgnore
|
||||
private long startOffset;
|
||||
@JsonIgnore
|
||||
private long endOffset;
|
||||
@JsonIgnore
|
||||
private boolean quarantinedInvalid;
|
||||
@JsonIgnore
|
||||
private String validationError;
|
||||
|
||||
public int getSchemaVersion() { return schemaVersion; }
|
||||
public void setSchemaVersion(int schemaVersion) { this.schemaVersion = schemaVersion; }
|
||||
public long getSequence() { return sequence; }
|
||||
public void setSequence(long sequence) { this.sequence = sequence; }
|
||||
public String getEventId() { return eventId; }
|
||||
public void setEventId(String eventId) { this.eventId = eventId; }
|
||||
public String getReceiveTime() { return receiveTime; }
|
||||
public void setReceiveTime(String receiveTime) { this.receiveTime = receiveTime; }
|
||||
public Long getPointTimeNs() { return pointTimeNs; }
|
||||
public void setPointTimeNs(Long pointTimeNs) { this.pointTimeNs = pointTimeNs; }
|
||||
public String getSourceIp() { return sourceIp; }
|
||||
public void setSourceIp(String sourceIp) { this.sourceIp = sourceIp; }
|
||||
public int getSourcePort() { return sourcePort; }
|
||||
public void setSourcePort(int sourcePort) { this.sourcePort = sourcePort; }
|
||||
public String getProtocol() { return protocol; }
|
||||
public void setProtocol(String protocol) { this.protocol = protocol; }
|
||||
public String getCharset() { return charset; }
|
||||
public void setCharset(String charset) { this.charset = charset; }
|
||||
public int getRawLength() { return rawLength; }
|
||||
public void setRawLength(int rawLength) { this.rawLength = rawLength; }
|
||||
public String getRawSha256() { return rawSha256; }
|
||||
public void setRawSha256(String rawSha256) { this.rawSha256 = rawSha256; }
|
||||
public String getRawMessage() { return rawMessage; }
|
||||
public void setRawMessage(String rawMessage) { this.rawMessage = rawMessage; }
|
||||
public String getSegment() { return segment; }
|
||||
public void setSegment(String segment) { this.segment = segment; }
|
||||
public long getStartOffset() { return startOffset; }
|
||||
public void setStartOffset(long startOffset) { this.startOffset = startOffset; }
|
||||
public long getEndOffset() { return endOffset; }
|
||||
public void setEndOffset(long endOffset) { this.endOffset = endOffset; }
|
||||
public boolean isQuarantinedInvalid() { return quarantinedInvalid; }
|
||||
public void setQuarantinedInvalid(boolean quarantinedInvalid) { this.quarantinedInvalid = quarantinedInvalid; }
|
||||
public String getValidationError() { return validationError; }
|
||||
public void setValidationError(String validationError) { this.validationError = validationError; }
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.format.ResolverStyle;
|
||||
|
||||
/** WAL 所有扫描、恢复和派发路径共用的 v2/v3 严格校验。 */
|
||||
final class SyslogWalRecordValidator {
|
||||
private static final ZoneId RECEIVE_ZONE = ZoneId.of("Asia/Shanghai");
|
||||
private static final DateTimeFormatter RECEIVE_FORMAT = DateTimeFormatter
|
||||
.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS").withResolverStyle(ResolverStyle.STRICT);
|
||||
|
||||
private SyslogWalRecordValidator() { }
|
||||
|
||||
static void validateNode(JsonNode node) {
|
||||
JsonNode versionNode = node.get("schema_version");
|
||||
if (versionNode == null || !versionNode.isIntegralNumber() || !versionNode.canConvertToInt()) {
|
||||
throw new IllegalArgumentException("schema_version 缺失或不是整数");
|
||||
}
|
||||
int version = versionNode.intValue();
|
||||
JsonNode pointTimeNode = node.get("point_time_ns");
|
||||
if (version == 2) {
|
||||
if (pointTimeNode != null && !pointTimeNode.isNull()) {
|
||||
throw new IllegalArgumentException("WAL v2 禁止携带 point_time_ns");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (version != 3) {
|
||||
throw new IllegalArgumentException("不支持的 WAL 版本: " + version);
|
||||
}
|
||||
if (pointTimeNode == null || !pointTimeNode.isIntegralNumber() || !pointTimeNode.canConvertToLong()) {
|
||||
throw new IllegalArgumentException("WAL v3 point_time_ns 缺失、非数字或溢出");
|
||||
}
|
||||
validatePointTime(pointTimeNode.longValue(), text(node, "receive_time"));
|
||||
}
|
||||
|
||||
static void validateRecord(SyslogWalRecord record) {
|
||||
if (record.getSchemaVersion() == 2) {
|
||||
if (record.getPointTimeNs() != null) {
|
||||
throw new IllegalArgumentException("WAL v2 禁止携带 point_time_ns");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (record.getSchemaVersion() != 3) {
|
||||
throw new IllegalArgumentException("不支持的 WAL 版本: " + record.getSchemaVersion());
|
||||
}
|
||||
if (record.getPointTimeNs() == null) {
|
||||
throw new IllegalArgumentException("WAL v3 缺少 point_time_ns");
|
||||
}
|
||||
validatePointTime(record.getPointTimeNs(), record.getReceiveTime());
|
||||
}
|
||||
|
||||
static long receiveTimeMillis(String receiveTime) {
|
||||
if (receiveTime == null || receiveTime.isEmpty()) {
|
||||
throw new IllegalArgumentException("receive_time 不能为空");
|
||||
}
|
||||
return LocalDateTime.parse(receiveTime, RECEIVE_FORMAT)
|
||||
.atZone(RECEIVE_ZONE).toInstant().toEpochMilli();
|
||||
}
|
||||
|
||||
private static void validatePointTime(long pointTimeNs, String receiveTime) {
|
||||
if (pointTimeNs <= 0L) {
|
||||
throw new IllegalArgumentException("point_time_ns 必须为正整数");
|
||||
}
|
||||
long receiveTimeMillis = receiveTimeMillis(receiveTime);
|
||||
if (Math.floorDiv(pointTimeNs, PointTimeAllocator.NANOS_PER_MILLI) != receiveTimeMillis) {
|
||||
throw new IllegalArgumentException("point_time_ns 毫秒部分与 receive_time 不一致");
|
||||
}
|
||||
}
|
||||
|
||||
private static String text(JsonNode node, String name) {
|
||||
JsonNode value = node.get(name);
|
||||
if (value == null || !value.isTextual()) {
|
||||
throw new IllegalArgumentException(name + " 缺失或不是字符串");
|
||||
}
|
||||
return value.textValue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,834 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.config.AppProperties;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.file.AtomicMoveNotSupportedException;
|
||||
import java.nio.file.DirectoryStream;
|
||||
import java.nio.file.FileStore;
|
||||
import java.nio.file.FileAlreadyExistsException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.nio.file.attribute.PosixFilePermission;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.EnumSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
|
||||
/**
|
||||
* Kafka 接管前的本地持久化 WAL。
|
||||
*
|
||||
* <p>网络线程只向有界队列提交事件；单写线程顺序追加并批量 force。Dispatcher 只读取已经
|
||||
* force 的记录，Kafka ACK 后按连续 sequence 推进持久化游标。崩溃恢复可能重复投递游标后
|
||||
* 的少量记录，但 event_id 不变，由下游幂等约束消除业务重复。</p>
|
||||
*/
|
||||
@Component
|
||||
public class SyslogWalService {
|
||||
private static final Logger log = LoggerFactory.getLogger(SyslogWalService.class);
|
||||
private static final DateTimeFormatter RECORD_TIME = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS");
|
||||
private static final DateTimeFormatter FILE_TIME = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmssSSS");
|
||||
private static final String SEGMENT_GLOB = "syslog-wal-*.jsonl";
|
||||
|
||||
public enum AdmissionResult { ACCEPTED, BACKPRESSURE, UNAVAILABLE }
|
||||
|
||||
private final AppProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final Charset syslogCharset;
|
||||
private final PointTimeAllocator pointTimeAllocator = new PointTimeAllocator();
|
||||
private final AtomicBoolean initialized = new AtomicBoolean();
|
||||
private final AtomicLong nextSequence = new AtomicLong();
|
||||
private final AtomicLong durableSequence = new AtomicLong();
|
||||
private final LongAdder appended = new LongAdder();
|
||||
private final LongAdder admissionBackpressure = new LongAdder();
|
||||
private final LongAdder quarantined = new LongAdder();
|
||||
private final LongAdder terminalRecorded = new LongAdder();
|
||||
private final Object durableMonitor = new Object();
|
||||
private final Object checkpointMonitor = new Object();
|
||||
|
||||
private volatile boolean running;
|
||||
private volatile Throwable fatalFailure;
|
||||
private volatile Checkpoint persistedCheckpoint = new Checkpoint();
|
||||
private volatile Checkpoint pendingCheckpoint = new Checkpoint();
|
||||
private volatile long checkpointDirtyRecords;
|
||||
private volatile long lastCheckpointFlushNanos;
|
||||
private volatile DiskState diskState = DiskState.NORMAL;
|
||||
private volatile long lastDiskCheckNanos;
|
||||
|
||||
private ArrayBlockingQueue<SyslogIngressEvent> appendQueue;
|
||||
private Thread writerThread;
|
||||
private Path rootDirectory;
|
||||
private Path activeDirectory;
|
||||
private Path completedDirectory;
|
||||
private Path quarantineDirectory;
|
||||
private Path terminalDirectory;
|
||||
private Path checkpointDirectory;
|
||||
private Path checkpointFile;
|
||||
private FileChannel writerChannel;
|
||||
private Path writerPath;
|
||||
private LocalDate writerDate;
|
||||
private long writerBytes;
|
||||
private int recordsSinceForce;
|
||||
private long lastWrittenSequence;
|
||||
private long lastForceNanos;
|
||||
|
||||
public SyslogWalService(AppProperties properties, ObjectMapper objectMapper) {
|
||||
this.properties = properties;
|
||||
/*
|
||||
* WAL 是内部、固定 schema 的持久化协议，不能直接复用 Redis 缓存的多态 ObjectMapper。
|
||||
*
|
||||
* CacheConfig 为 Redis 的 LinkedHashMap/实体反序列化开启了 default typing。直接复用它会在
|
||||
* WAL JSON 中写入 "@class"，但它的安全白名单又不允许 com.haobang.wal.SyslogWalRecord，
|
||||
* 最终造成“自己写出的第一条 WAL 自己无法读取”，Dispatcher 永久停在 offset=0。
|
||||
*
|
||||
* copy() 保留 JavaTimeModule 等通用模块，但 deactivateDefaultTyping() 明确移除 Redis 专用的
|
||||
* 多态协议。这样新 WAL 不再依赖 Java 类名，也不会通过磁盘 JSON 触发任意类型解析。
|
||||
* SyslogWalRecord 对历史 "@class" 字段做了精确忽略，因此现场已经落盘的旧记录仍可恢复。
|
||||
*/
|
||||
this.objectMapper = objectMapper.copy().deactivateDefaultTyping();
|
||||
this.syslogCharset = resolveSyslogCharset(properties.getSyslogCharset());
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void initialize() {
|
||||
if (!initialized.compareAndSet(false, true)) return;
|
||||
validateConfiguration();
|
||||
try {
|
||||
rootDirectory = Paths.get(properties.getSyslogWalDirectory()).toAbsolutePath().normalize();
|
||||
activeDirectory = rootDirectory.resolve("active");
|
||||
completedDirectory = rootDirectory.resolve("completed");
|
||||
quarantineDirectory = rootDirectory.resolve("quarantine");
|
||||
terminalDirectory = rootDirectory.resolve("terminal");
|
||||
checkpointDirectory = rootDirectory.resolve("checkpoint");
|
||||
checkpointFile = checkpointDirectory.resolve("dispatch-checkpoint.json");
|
||||
Files.createDirectories(activeDirectory);
|
||||
Files.createDirectories(completedDirectory);
|
||||
Files.createDirectories(quarantineDirectory);
|
||||
Files.createDirectories(terminalDirectory);
|
||||
Files.createDirectories(checkpointDirectory);
|
||||
applyDirectoryPermissions(rootDirectory);
|
||||
applyDirectoryPermissions(activeDirectory);
|
||||
applyDirectoryPermissions(completedDirectory);
|
||||
applyDirectoryPermissions(quarantineDirectory);
|
||||
applyDirectoryPermissions(terminalDirectory);
|
||||
applyDirectoryPermissions(checkpointDirectory);
|
||||
|
||||
repairIncompleteTails();
|
||||
persistedCheckpoint = loadCheckpoint();
|
||||
pendingCheckpoint = persistedCheckpoint.copy();
|
||||
long maxSequence = Math.max(persistedCheckpoint.sequence, scanMaxSequence());
|
||||
nextSequence.set(maxSequence);
|
||||
durableSequence.set(maxSequence);
|
||||
cleanupCompletedSegments();
|
||||
refreshDiskState(true);
|
||||
} catch (Exception failure) {
|
||||
fatalFailure = failure;
|
||||
throw new IllegalStateException("Syslog WAL 初始化失败，拒绝开放接入端口: "
|
||||
+ properties.getSyslogWalDirectory(), failure);
|
||||
}
|
||||
|
||||
appendQueue = new ArrayBlockingQueue<>(properties.getSyslogWalQueueCapacity());
|
||||
running = true;
|
||||
lastForceNanos = System.nanoTime();
|
||||
lastCheckpointFlushNanos = System.nanoTime();
|
||||
writerThread = new Thread(this::writerLoop, "syslog-wal-writer");
|
||||
writerThread.setDaemon(true);
|
||||
writerThread.start();
|
||||
log.info("Syslog WAL 已启用: directory={}, charset={}, queueCapacity={}, highWatermark={}%, lowWatermark={}%, "
|
||||
+ "flushRecords={}, flushIntervalMs={}, checkpointSequence={}",
|
||||
rootDirectory, syslogCharset.name(), properties.getSyslogWalQueueCapacity(),
|
||||
properties.getSyslogWalHighWatermarkPercent(), properties.getSyslogWalLowWatermarkPercent(),
|
||||
properties.getSyslogWalFlushRecords(), properties.getSyslogWalFlushIntervalMs(),
|
||||
persistedCheckpoint.sequence);
|
||||
}
|
||||
|
||||
public AdmissionResult tryAppend(SyslogIngressEvent event) {
|
||||
if (properties.isSyslogPointTimeWriteEnabled() && event.getPointTimeNs() == null) {
|
||||
throw new IllegalArgumentException("point_time 写入已开启，SyslogIngressEvent 必须先固化 point_time_ns");
|
||||
}
|
||||
refreshDiskState(false);
|
||||
if (!running || fatalFailure != null || diskState == DiskState.CRITICAL) {
|
||||
return AdmissionResult.UNAVAILABLE;
|
||||
}
|
||||
if (!appendQueue.offer(event)) {
|
||||
admissionBackpressure.increment();
|
||||
return AdmissionResult.BACKPRESSURE;
|
||||
}
|
||||
return AdmissionResult.ACCEPTED;
|
||||
}
|
||||
|
||||
public boolean shouldPauseTcp() {
|
||||
refreshDiskState(false);
|
||||
if (fatalFailure != null || diskState != DiskState.NORMAL) return true;
|
||||
int high = percentageCount(properties.getSyslogWalHighWatermarkPercent());
|
||||
return appendQueue != null && appendQueue.size() >= high;
|
||||
}
|
||||
|
||||
public boolean canResumeTcp() {
|
||||
refreshDiskState(false);
|
||||
if (fatalFailure != null || diskState != DiskState.NORMAL) return false;
|
||||
int low = percentageCount(properties.getSyslogWalLowWatermarkPercent());
|
||||
return appendQueue != null && appendQueue.size() <= low;
|
||||
}
|
||||
|
||||
public long getRetryIntervalMs() {
|
||||
refreshDiskState(false);
|
||||
return diskState == DiskState.NORMAL && fatalFailure == null
|
||||
? properties.getSyslogWalRetryIntervalMs()
|
||||
: properties.getSyslogWalDiskRetryIntervalMs();
|
||||
}
|
||||
public int getQueueSize() { return appendQueue == null ? 0 : appendQueue.size(); }
|
||||
public int getQueueCapacity() { return properties.getSyslogWalQueueCapacity(); }
|
||||
public long getAppendedCount() { return appended.sum(); }
|
||||
public long getBackpressureCount() { return admissionBackpressure.sum(); }
|
||||
public long getQuarantinedCount() { return quarantined.sum(); }
|
||||
public long getTerminalRecordedCount() { return terminalRecorded.sum(); }
|
||||
public long getIngressPointTimeAssignedCount() { return pointTimeAllocator.getAssignedCount(); }
|
||||
public long getDurableSequence() { return durableSequence.get(); }
|
||||
public long getCheckpointSequence() { return persistedCheckpoint.sequence; }
|
||||
public long getPendingRecordsApprox() {
|
||||
return Math.max(0L, nextSequence.get() - persistedCheckpoint.sequence);
|
||||
}
|
||||
public Throwable getFatalFailure() { return fatalFailure; }
|
||||
public boolean isDiskCritical() { refreshDiskState(false); return diskState == DiskState.CRITICAL; }
|
||||
|
||||
/** 每个事件只调用一次；TCP 背压重试必须复用返回对象，不能重新分配 Point 身份。 */
|
||||
public SyslogIngressEvent prepareForWal(SyslogIngressEvent event) {
|
||||
if (!properties.isSyslogPointTimeWriteEnabled() || event.getPointTimeNs() != null) {
|
||||
return event;
|
||||
}
|
||||
long receiveTimeMillis = event.getReceivedAt().atZone(ZoneId.of("Asia/Shanghai"))
|
||||
.toInstant().toEpochMilli();
|
||||
return event.withPointTimeNs(pointTimeAllocator.allocate(receiveTimeMillis));
|
||||
}
|
||||
|
||||
public WalCursor openCursor() throws IOException {
|
||||
return new WalCursor(persistedCheckpoint.copy());
|
||||
}
|
||||
|
||||
/** 仅由 Dispatcher 在连续 ACK 区间推进后调用。 */
|
||||
public void markCheckpoint(SyslogWalRecord record) {
|
||||
synchronized (checkpointMonitor) {
|
||||
if (record.getSequence() <= pendingCheckpoint.sequence) return;
|
||||
pendingCheckpoint = new Checkpoint(record.getSequence(), record.getSegment(), record.getEndOffset());
|
||||
checkpointDirtyRecords++;
|
||||
long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastCheckpointFlushNanos);
|
||||
if (checkpointDirtyRecords >= properties.getSyslogWalCheckpointFlushRecords()
|
||||
|| elapsedMs >= properties.getSyslogWalCheckpointFlushIntervalMs()) {
|
||||
flushCheckpointLocked();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 永久异常必须先持久化隔离记录，调用方才能把对应 sequence 视为完成。 */
|
||||
public void quarantine(SyslogWalRecord record, String reason, Throwable failure) throws IOException {
|
||||
Map<String, Object> envelope = new LinkedHashMap<>();
|
||||
envelope.put("quarantined_at", Instant.now().toString());
|
||||
envelope.put("reason", reason);
|
||||
envelope.put("error_type", failure == null ? null : failure.getClass().getName());
|
||||
envelope.put("error_message", failure == null ? null : failure.getMessage());
|
||||
envelope.put("wal_segment", record.getSegment());
|
||||
envelope.put("wal_offset", record.getStartOffset());
|
||||
envelope.put("record", record);
|
||||
byte[] json = objectMapper.writeValueAsBytes(envelope);
|
||||
Path target = quarantineDirectory.resolve(String.format("quarantine-%020d-%s.json",
|
||||
record.getSequence(), record.getEventId()));
|
||||
try {
|
||||
try (FileChannel channel = FileChannel.open(target, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
|
||||
writeFully(channel, json);
|
||||
channel.force(true);
|
||||
}
|
||||
applyFilePermissions(target);
|
||||
} catch (FileAlreadyExistsException alreadyPersisted) {
|
||||
// 进程可能在隔离文件 force 后、游标推进前退出；重复恢复视为已持久化。
|
||||
return;
|
||||
}
|
||||
quarantined.increment();
|
||||
}
|
||||
|
||||
/** 业务终态先持久化原因，随后才允许 Dispatcher 推进连续 ACK 游标。 */
|
||||
public void recordTerminal(SyslogWalRecord record, String type, String reason) throws IOException {
|
||||
Map<String, Object> envelope = new LinkedHashMap<>();
|
||||
envelope.put("completed_at", Instant.now().toString());
|
||||
envelope.put("type", type);
|
||||
envelope.put("reason", reason);
|
||||
envelope.put("event_id", record.getEventId());
|
||||
envelope.put("sequence", record.getSequence());
|
||||
envelope.put("source_ip", record.getSourceIp());
|
||||
envelope.put("wal_segment", record.getSegment());
|
||||
envelope.put("wal_offset", record.getStartOffset());
|
||||
byte[] json = objectMapper.writeValueAsBytes(envelope);
|
||||
Path target = terminalDirectory.resolve(String.format("terminal-%020d-%s.json",
|
||||
record.getSequence(), record.getEventId()));
|
||||
try {
|
||||
try (FileChannel channel = FileChannel.open(target, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
|
||||
writeFully(channel, json);
|
||||
channel.force(true);
|
||||
}
|
||||
applyFilePermissions(target);
|
||||
} catch (FileAlreadyExistsException alreadyPersisted) {
|
||||
return;
|
||||
}
|
||||
terminalRecorded.increment();
|
||||
}
|
||||
|
||||
private void writerLoop() {
|
||||
try {
|
||||
while (running || !appendQueue.isEmpty()) {
|
||||
SyslogIngressEvent event = appendQueue.poll(properties.getSyslogWalFlushIntervalMs(), TimeUnit.MILLISECONDS);
|
||||
if (event != null) appendEvent(event);
|
||||
long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastForceNanos);
|
||||
if (recordsSinceForce > 0 && (recordsSinceForce >= properties.getSyslogWalFlushRecords()
|
||||
|| elapsedMs >= properties.getSyslogWalFlushIntervalMs())) {
|
||||
forceWriter();
|
||||
}
|
||||
flushCheckpointIfDue();
|
||||
}
|
||||
forceWriter();
|
||||
synchronized (checkpointMonitor) { flushCheckpointLocked(); }
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
failWal(interrupted);
|
||||
} catch (Throwable failure) {
|
||||
failWal(failure);
|
||||
} finally {
|
||||
closeWriter();
|
||||
}
|
||||
}
|
||||
|
||||
private void appendEvent(SyslogIngressEvent event) throws Exception {
|
||||
long sequence = nextSequence.incrementAndGet();
|
||||
SyslogWalRecord record = toRecord(sequence, event);
|
||||
byte[] json = objectMapper.writeValueAsBytes(record);
|
||||
byte[] line = new byte[json.length + 1];
|
||||
System.arraycopy(json, 0, line, 0, json.length);
|
||||
line[line.length - 1] = (byte) '\n';
|
||||
|
||||
LocalDate receiveDate = event.getReceivedAt().toLocalDate();
|
||||
if (writerChannel == null) openWriter(sequence, receiveDate);
|
||||
if (writerBytes > 0 && (writerBytes + line.length > properties.getSyslogWalSegmentMaxBytes()
|
||||
|| !receiveDate.equals(writerDate))) {
|
||||
forceWriter();
|
||||
closeWriter();
|
||||
openWriter(sequence, receiveDate);
|
||||
}
|
||||
writeFully(writerChannel, line);
|
||||
writerBytes += line.length;
|
||||
recordsSinceForce++;
|
||||
lastWrittenSequence = sequence;
|
||||
}
|
||||
|
||||
private SyslogWalRecord toRecord(long sequence, SyslogIngressEvent event) throws Exception {
|
||||
byte[] raw = event.getRawMessage().getBytes(syslogCharset);
|
||||
SyslogWalRecord record = new SyslogWalRecord();
|
||||
if (properties.isSyslogPointTimeWriteEnabled()) {
|
||||
record.setSchemaVersion(3);
|
||||
record.setPointTimeNs(event.getPointTimeNs());
|
||||
} else {
|
||||
record.setSchemaVersion(2);
|
||||
}
|
||||
record.setSequence(sequence);
|
||||
record.setEventId(event.getEventId());
|
||||
record.setReceiveTime(event.getReceivedAt().format(RECORD_TIME));
|
||||
record.setSourceIp(event.getSourceIp());
|
||||
record.setSourcePort(event.getSourcePort());
|
||||
record.setProtocol(event.getProtocol());
|
||||
record.setCharset(syslogCharset.name());
|
||||
record.setRawLength(raw.length);
|
||||
record.setRawSha256(sha256(raw));
|
||||
record.setRawMessage(event.getRawMessage());
|
||||
SyslogWalRecordValidator.validateRecord(record);
|
||||
return record;
|
||||
}
|
||||
|
||||
private void forceWriter() throws IOException {
|
||||
if (writerChannel == null || recordsSinceForce == 0) return;
|
||||
writerChannel.force(false);
|
||||
durableSequence.set(lastWrittenSequence);
|
||||
appended.add(recordsSinceForce);
|
||||
recordsSinceForce = 0;
|
||||
lastForceNanos = System.nanoTime();
|
||||
synchronized (durableMonitor) { durableMonitor.notifyAll(); }
|
||||
}
|
||||
|
||||
private void openWriter(long startSequence, LocalDate date) throws IOException {
|
||||
String name = String.format("syslog-wal-%020d-%s.jsonl", startSequence,
|
||||
LocalDateTime.now().format(FILE_TIME));
|
||||
writerPath = activeDirectory.resolve(name);
|
||||
writerChannel = FileChannel.open(writerPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
|
||||
applyFilePermissions(writerPath);
|
||||
writerDate = date;
|
||||
writerBytes = 0L;
|
||||
}
|
||||
|
||||
private void closeWriter() {
|
||||
if (writerChannel == null) return;
|
||||
try { writerChannel.close(); }
|
||||
catch (IOException failure) { log.warn("关闭 WAL 分段失败: file={}", writerPath, failure); }
|
||||
finally { writerChannel = null; }
|
||||
}
|
||||
|
||||
private void failWal(Throwable failure) {
|
||||
fatalFailure = failure;
|
||||
running = false;
|
||||
synchronized (durableMonitor) { durableMonitor.notifyAll(); }
|
||||
log.error("Syslog WAL 发生致命故障，接入已进入 fail-closed 状态", failure);
|
||||
}
|
||||
|
||||
private int percentageCount(int percent) {
|
||||
return Math.max(1, (int) Math.floor(properties.getSyslogWalQueueCapacity() * percent / 100.0));
|
||||
}
|
||||
|
||||
private void validateConfiguration() {
|
||||
if (!properties.isSyslogWalEnabled()) {
|
||||
throw new IllegalArgumentException("syslog.wal.enabled=false 会破坏可靠接入边界，正式链路禁止关闭 WAL");
|
||||
}
|
||||
if (properties.getSyslogWalQueueCapacity() <= 0 || properties.getSyslogWalFlushRecords() <= 0
|
||||
|| properties.getSyslogWalFlushIntervalMs() <= 0 || properties.getSyslogWalSegmentMaxBytes() <= 0
|
||||
|| properties.getSyslogWalShutdownTimeoutMs() <= 0
|
||||
|| properties.getSyslogWalCheckpointFlushRecords() <= 0
|
||||
|| properties.getSyslogWalCheckpointFlushIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("WAL 队列、刷盘、分段、游标和停机参数必须大于 0");
|
||||
}
|
||||
if (properties.getSyslogWalRetryIntervalMs() <= 0
|
||||
|| properties.getSyslogWalDiskRetryIntervalMs() <= 0) {
|
||||
throw new IllegalArgumentException("WAL 队列和磁盘重试间隔必须大于 0");
|
||||
}
|
||||
if (properties.getSyslogWalLowWatermarkPercent() < 0
|
||||
|| properties.getSyslogWalHighWatermarkPercent() <= properties.getSyslogWalLowWatermarkPercent()
|
||||
|| properties.getSyslogWalHighWatermarkPercent() >= 100) {
|
||||
throw new IllegalArgumentException("WAL 水位必须满足 0 <= low < high < 100");
|
||||
}
|
||||
if (properties.getSyslogWalDiskHighWatermarkPercent() <= 0
|
||||
|| properties.getSyslogWalDiskCriticalWatermarkPercent()
|
||||
<= properties.getSyslogWalDiskHighWatermarkPercent()
|
||||
|| properties.getSyslogWalDiskCriticalWatermarkPercent() >= 100) {
|
||||
throw new IllegalArgumentException("WAL 磁盘水位必须满足 0 < high < critical < 100");
|
||||
}
|
||||
if (properties.getSyslogWalRetentionDays() < 0 || properties.getSyslogWalMinFreeBytes() < 0) {
|
||||
throw new IllegalArgumentException("WAL 保留天数和最小剩余字节不能小于 0");
|
||||
}
|
||||
}
|
||||
|
||||
public Charset getSyslogCharset() {
|
||||
return syslogCharset;
|
||||
}
|
||||
|
||||
private Charset resolveSyslogCharset(String configuredCharset) {
|
||||
if (configuredCharset == null || configuredCharset.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("syslog.charset 不能为空");
|
||||
}
|
||||
try {
|
||||
return Charset.forName(configuredCharset.trim());
|
||||
} catch (Exception invalidCharset) {
|
||||
throw new IllegalArgumentException("不支持的 syslog.charset: " + configuredCharset, invalidCharset);
|
||||
}
|
||||
}
|
||||
|
||||
private void repairIncompleteTails() throws IOException {
|
||||
for (Path path : listSegments(activeDirectory)) {
|
||||
long size = Files.size(path);
|
||||
if (size == 0) continue;
|
||||
try (FileChannel channel = FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
|
||||
ByteBuffer one = ByteBuffer.allocate(1);
|
||||
channel.position(size - 1);
|
||||
channel.read(one);
|
||||
one.flip();
|
||||
if (one.get() == (byte) '\n') continue;
|
||||
long position = size - 1;
|
||||
while (position >= 0) {
|
||||
one.clear();
|
||||
channel.position(position);
|
||||
channel.read(one);
|
||||
one.flip();
|
||||
if (one.get() == (byte) '\n') break;
|
||||
position--;
|
||||
}
|
||||
long truncateAt = position < 0 ? 0 : position + 1;
|
||||
channel.truncate(truncateAt);
|
||||
channel.force(true);
|
||||
log.warn("WAL 启动恢复已截断未完成尾记录: file={}, oldBytes={}, newBytes={}",
|
||||
path.getFileName(), size, truncateAt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private long scanMaxSequence() throws IOException {
|
||||
long max = 0L;
|
||||
for (Path path : listSegments(activeDirectory)) {
|
||||
try (java.io.BufferedReader reader = Files.newBufferedReader(path, java.nio.charset.StandardCharsets.UTF_8)) {
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
if (line.trim().isEmpty()) continue;
|
||||
JsonNode node = objectMapper.readTree(line);
|
||||
JsonNode sequenceNode = node.get("sequence");
|
||||
if (sequenceNode == null || !sequenceNode.isIntegralNumber()
|
||||
|| !sequenceNode.canConvertToLong() || sequenceNode.longValue() <= 0L) {
|
||||
throw new IOException("WAL sequence 缺失或非法: " + path);
|
||||
}
|
||||
try {
|
||||
SyslogWalRecordValidator.validateNode(node);
|
||||
} catch (IllegalArgumentException invalidRecord) {
|
||||
log.error("WAL 扫描发现损坏记录，启动后将由游标隔离: file={}, sequence={}, reason={}",
|
||||
path, sequenceNode.longValue(), invalidRecord.getMessage());
|
||||
}
|
||||
max = Math.max(max, sequenceNode.longValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
private Checkpoint loadCheckpoint() throws IOException {
|
||||
if (!Files.exists(checkpointFile)) return new Checkpoint();
|
||||
return objectMapper.readValue(Files.readAllBytes(checkpointFile), Checkpoint.class);
|
||||
}
|
||||
|
||||
private void flushCheckpointIfDue() {
|
||||
synchronized (checkpointMonitor) {
|
||||
if (checkpointDirtyRecords == 0) return;
|
||||
long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastCheckpointFlushNanos);
|
||||
if (elapsedMs >= properties.getSyslogWalCheckpointFlushIntervalMs()) flushCheckpointLocked();
|
||||
}
|
||||
}
|
||||
|
||||
private void flushCheckpointLocked() {
|
||||
if (checkpointDirtyRecords == 0 || pendingCheckpoint.sequence <= persistedCheckpoint.sequence) return;
|
||||
Path temporary = checkpointDirectory.resolve("dispatch-checkpoint.tmp");
|
||||
try {
|
||||
byte[] json = objectMapper.writeValueAsBytes(pendingCheckpoint);
|
||||
try (FileChannel channel = FileChannel.open(temporary, StandardOpenOption.CREATE,
|
||||
StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
|
||||
writeFully(channel, json);
|
||||
channel.force(true);
|
||||
}
|
||||
try {
|
||||
Files.move(temporary, checkpointFile, StandardCopyOption.REPLACE_EXISTING,
|
||||
StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (AtomicMoveNotSupportedException unsupported) {
|
||||
Files.move(temporary, checkpointFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
applyFilePermissions(checkpointFile);
|
||||
forceDirectoryBestEffort(checkpointDirectory);
|
||||
persistedCheckpoint = pendingCheckpoint.copy();
|
||||
checkpointDirtyRecords = 0L;
|
||||
lastCheckpointFlushNanos = System.nanoTime();
|
||||
moveCompletedSegments();
|
||||
} catch (IOException failure) {
|
||||
fatalFailure = failure;
|
||||
running = false;
|
||||
log.error("持久化 WAL ACK 游标失败，接入已停止", failure);
|
||||
}
|
||||
}
|
||||
|
||||
private void moveCompletedSegments() throws IOException {
|
||||
String checkpointSegment = persistedCheckpoint.segment;
|
||||
if (checkpointSegment == null) return;
|
||||
for (Path segment : listSegments(activeDirectory)) {
|
||||
String name = segment.getFileName().toString();
|
||||
if (name.compareTo(checkpointSegment) >= 0 || segment.equals(writerPath)) continue;
|
||||
Files.move(segment, completedDirectory.resolve(name), StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
cleanupCompletedSegments();
|
||||
}
|
||||
|
||||
private void cleanupCompletedSegments() throws IOException {
|
||||
int retentionDays = properties.getSyslogWalRetentionDays();
|
||||
if (retentionDays <= 0) return;
|
||||
Instant cutoff = LocalDate.now().minusDays(retentionDays).atStartOfDay(ZoneId.systemDefault()).toInstant();
|
||||
for (Path path : listSegments(completedDirectory)) {
|
||||
if (Files.getLastModifiedTime(path).toInstant().isBefore(cutoff)) Files.deleteIfExists(path);
|
||||
}
|
||||
}
|
||||
|
||||
private List<Path> listSegments(Path directory) throws IOException {
|
||||
if (directory == null || !Files.exists(directory)) return Collections.emptyList();
|
||||
List<Path> paths = new ArrayList<>();
|
||||
try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory, SEGMENT_GLOB)) {
|
||||
for (Path path : stream) paths.add(path);
|
||||
}
|
||||
paths.sort(Comparator.comparing(path -> path.getFileName().toString()));
|
||||
return paths;
|
||||
}
|
||||
|
||||
private void refreshDiskState(boolean force) {
|
||||
if (rootDirectory == null) return;
|
||||
long now = System.nanoTime();
|
||||
if (!force && TimeUnit.NANOSECONDS.toMillis(now - lastDiskCheckNanos) < 1000) return;
|
||||
lastDiskCheckNanos = now;
|
||||
try {
|
||||
FileStore store = Files.getFileStore(rootDirectory);
|
||||
long total = store.getTotalSpace();
|
||||
long usable = store.getUsableSpace();
|
||||
long usedPercent = total <= 0 ? 0 : ((total - usable) * 100L / total);
|
||||
DiskState next = usedPercent >= properties.getSyslogWalDiskCriticalWatermarkPercent()
|
||||
|| usable < properties.getSyslogWalMinFreeBytes() ? DiskState.CRITICAL
|
||||
: usedPercent >= properties.getSyslogWalDiskHighWatermarkPercent()
|
||||
? DiskState.HIGH : DiskState.NORMAL;
|
||||
if (next != diskState) {
|
||||
log.warn("WAL 磁盘状态变化: previous={}, current={}, usedPercent={}%, usableBytes={}",
|
||||
diskState, next, usedPercent, usable);
|
||||
}
|
||||
diskState = next;
|
||||
} catch (IOException failure) {
|
||||
diskState = DiskState.CRITICAL;
|
||||
fatalFailure = failure;
|
||||
log.error("无法检查 WAL 磁盘容量，按 fail-closed 处理", failure);
|
||||
}
|
||||
}
|
||||
|
||||
private String sha256(byte[] value) throws Exception {
|
||||
byte[] digest = MessageDigest.getInstance("SHA-256").digest(value);
|
||||
StringBuilder hex = new StringBuilder(digest.length * 2);
|
||||
for (byte item : digest) hex.append(String.format("%02x", item & 0xff));
|
||||
return hex.toString();
|
||||
}
|
||||
|
||||
private void applyFilePermissions(Path path) {
|
||||
try {
|
||||
Files.setPosixFilePermissions(path, EnumSet.of(
|
||||
PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE));
|
||||
} catch (UnsupportedOperationException | IOException failure) {
|
||||
log.warn("当前文件系统不支持 WAL 文件 POSIX 0600 权限，请检查目录 ACL: file={}", path);
|
||||
}
|
||||
}
|
||||
|
||||
private void applyDirectoryPermissions(Path path) {
|
||||
try {
|
||||
Set<PosixFilePermission> permissions = EnumSet.of(
|
||||
PosixFilePermission.OWNER_READ,
|
||||
PosixFilePermission.OWNER_WRITE,
|
||||
PosixFilePermission.OWNER_EXECUTE);
|
||||
Files.setPosixFilePermissions(path, permissions);
|
||||
} catch (UnsupportedOperationException | IOException failure) {
|
||||
log.warn("当前文件系统不支持 WAL 目录 POSIX 0700 权限，请检查目录 ACL: directory={}", path);
|
||||
}
|
||||
}
|
||||
|
||||
private void forceDirectoryBestEffort(Path directory) {
|
||||
try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) {
|
||||
channel.force(true);
|
||||
} catch (UnsupportedOperationException | IOException failure) {
|
||||
// ATOMIC_MOVE 仍保证不会读取半个 checkpoint；Linux 生产环境应支持目录 force。
|
||||
log.warn("当前文件系统不支持 checkpoint 目录 force，请确认文件系统掉电语义: directory={}",
|
||||
directory);
|
||||
}
|
||||
}
|
||||
|
||||
private static void writeFully(FileChannel channel, byte[] bytes) throws IOException {
|
||||
ByteBuffer buffer = ByteBuffer.wrap(bytes);
|
||||
while (buffer.hasRemaining()) channel.write(buffer);
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
running = false;
|
||||
synchronized (durableMonitor) { durableMonitor.notifyAll(); }
|
||||
if (writerThread != null) {
|
||||
try {
|
||||
writerThread.join(properties.getSyslogWalShutdownTimeoutMs());
|
||||
if (writerThread.isAlive()) {
|
||||
log.error("WAL 写线程未在超时内退出: remaining={}", appendQueue.size());
|
||||
writerThread.interrupt();
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
synchronized (checkpointMonitor) { flushCheckpointLocked(); }
|
||||
}
|
||||
|
||||
public final class WalCursor implements AutoCloseable {
|
||||
private String currentSegment;
|
||||
private long position;
|
||||
private FileChannel channel;
|
||||
|
||||
private WalCursor(Checkpoint checkpoint) throws IOException {
|
||||
this.currentSegment = checkpoint.segment;
|
||||
this.position = checkpoint.offset;
|
||||
if (currentSegment != null) {
|
||||
Path path = activeDirectory.resolve(currentSegment);
|
||||
if (!Files.exists(path)) {
|
||||
throw new IOException("WAL checkpoint 指向的活动分段不存在，拒绝跳过: " + path);
|
||||
}
|
||||
channel = FileChannel.open(path, StandardOpenOption.READ);
|
||||
}
|
||||
}
|
||||
|
||||
public SyslogWalRecord next(long waitMs) throws IOException, InterruptedException {
|
||||
long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(waitMs);
|
||||
while (running || getPendingRecordsApprox() > 0) {
|
||||
ensureChannel();
|
||||
if (channel == null) {
|
||||
awaitDurable(deadline);
|
||||
if (System.nanoTime() >= deadline) return null;
|
||||
continue;
|
||||
}
|
||||
long start = position;
|
||||
byte[] line = readLine();
|
||||
if (line == null) {
|
||||
if (openNextSegment()) continue;
|
||||
awaitDurable(deadline);
|
||||
if (System.nanoTime() >= deadline) return null;
|
||||
continue;
|
||||
}
|
||||
SyslogWalRecord record;
|
||||
try {
|
||||
JsonNode node = objectMapper.readTree(line);
|
||||
try {
|
||||
SyslogWalRecordValidator.validateNode(node);
|
||||
record = objectMapper.treeToValue(node, SyslogWalRecord.class);
|
||||
SyslogWalRecordValidator.validateRecord(record);
|
||||
} catch (Exception invalidRecord) {
|
||||
record = invalidRecordTombstone(node, invalidRecord);
|
||||
}
|
||||
} catch (Exception failure) {
|
||||
throw new IOException("WAL 中间记录损坏，拒绝越过: segment=" + currentSegment
|
||||
+ ", offset=" + start, failure);
|
||||
}
|
||||
if (record.getSequence() > durableSequence.get()) {
|
||||
position = start;
|
||||
awaitDurable(deadline);
|
||||
if (System.nanoTime() >= deadline) return null;
|
||||
continue;
|
||||
}
|
||||
record.setSegment(currentSegment);
|
||||
record.setStartOffset(start);
|
||||
record.setEndOffset(position);
|
||||
if (record.isQuarantinedInvalid()) {
|
||||
quarantine(record, "INVALID_WAL_RECORD", new IllegalArgumentException(record.getValidationError()));
|
||||
}
|
||||
return record;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private SyslogWalRecord invalidRecordTombstone(JsonNode node, Exception failure) throws IOException {
|
||||
JsonNode sequenceNode = node == null ? null : node.get("sequence");
|
||||
if (sequenceNode == null || !sequenceNode.isIntegralNumber()
|
||||
|| !sequenceNode.canConvertToLong() || sequenceNode.longValue() <= 0L) {
|
||||
throw new IOException("损坏 WAL 无法取得合法 sequence，拒绝越过", failure);
|
||||
}
|
||||
SyslogWalRecord record = new SyslogWalRecord();
|
||||
record.setSequence(sequenceNode.longValue());
|
||||
record.setSchemaVersion(node.path("schema_version").asInt(-1));
|
||||
record.setEventId(node.path("event_id").asText("unknown-" + record.getSequence()));
|
||||
record.setReceiveTime(node.path("receive_time").asText(null));
|
||||
record.setQuarantinedInvalid(true);
|
||||
record.setValidationError(failure.getMessage() == null
|
||||
? failure.getClass().getName() : failure.getMessage());
|
||||
return record;
|
||||
}
|
||||
|
||||
private void ensureChannel() throws IOException {
|
||||
if (channel != null) return;
|
||||
List<Path> segments = listSegments(activeDirectory);
|
||||
if (segments.isEmpty()) return;
|
||||
Path first = segments.get(0);
|
||||
currentSegment = first.getFileName().toString();
|
||||
position = 0L;
|
||||
channel = FileChannel.open(first, StandardOpenOption.READ);
|
||||
}
|
||||
|
||||
private boolean openNextSegment() throws IOException {
|
||||
List<Path> segments = listSegments(activeDirectory);
|
||||
for (Path path : segments) {
|
||||
String name = path.getFileName().toString();
|
||||
if (currentSegment == null || name.compareTo(currentSegment) > 0) {
|
||||
closeChannel();
|
||||
currentSegment = name;
|
||||
position = 0L;
|
||||
channel = FileChannel.open(path, StandardOpenOption.READ);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private byte[] readLine() throws IOException {
|
||||
long start = position;
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream(2048);
|
||||
ByteBuffer buffer = ByteBuffer.allocate(8192);
|
||||
while (true) {
|
||||
buffer.clear();
|
||||
channel.position(position);
|
||||
int count = channel.read(buffer);
|
||||
if (count < 0 || count == 0) {
|
||||
position = start;
|
||||
return null;
|
||||
}
|
||||
buffer.flip();
|
||||
int consumed = 0;
|
||||
while (buffer.hasRemaining()) {
|
||||
byte value = buffer.get();
|
||||
consumed++;
|
||||
if (value == (byte) '\n') {
|
||||
position += consumed;
|
||||
return output.toByteArray();
|
||||
}
|
||||
output.write(value);
|
||||
}
|
||||
position += consumed;
|
||||
}
|
||||
}
|
||||
|
||||
private void awaitDurable(long deadline) throws InterruptedException {
|
||||
long remainingNanos = deadline - System.nanoTime();
|
||||
if (remainingNanos <= 0) return;
|
||||
synchronized (durableMonitor) {
|
||||
TimeUnit.NANOSECONDS.timedWait(durableMonitor, remainingNanos);
|
||||
}
|
||||
}
|
||||
|
||||
private void closeChannel() throws IOException {
|
||||
if (channel != null) channel.close();
|
||||
channel = null;
|
||||
}
|
||||
|
||||
@Override public void close() throws IOException { closeChannel(); }
|
||||
}
|
||||
|
||||
public static class Checkpoint {
|
||||
public long sequence;
|
||||
public String segment;
|
||||
public long offset;
|
||||
|
||||
public Checkpoint() { this(0L, null, 0L); }
|
||||
public Checkpoint(long sequence, String segment, long offset) {
|
||||
this.sequence = sequence;
|
||||
this.segment = segment;
|
||||
this.offset = offset;
|
||||
}
|
||||
private Checkpoint copy() { return new Checkpoint(sequence, segment, offset); }
|
||||
}
|
||||
|
||||
private enum DiskState { NORMAL, HIGH, CRITICAL }
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import com.haobang.dispatch.DeviceContextResolver;
|
||||
import com.haobang.dispatch.SyslogWalDispatcher;
|
||||
import com.netty.SyslogMessageHandler;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/** 运维只读状态接口；不提供跳过、清空或人工推进游标等危险操作。 */
|
||||
@RestController
|
||||
@RequestMapping("/api/syslog/wal")
|
||||
public class SyslogWalStatusController {
|
||||
private final SyslogWalService wal;
|
||||
private final SyslogWalDispatcher dispatcher;
|
||||
private final DeviceContextResolver resolver;
|
||||
|
||||
public SyslogWalStatusController(SyslogWalService wal, SyslogWalDispatcher dispatcher,
|
||||
DeviceContextResolver resolver) {
|
||||
this.wal = wal;
|
||||
this.dispatcher = dispatcher;
|
||||
this.resolver = resolver;
|
||||
}
|
||||
|
||||
@GetMapping("/status")
|
||||
public Map<String, Object> status() {
|
||||
Map<String, Object> value = new LinkedHashMap<>();
|
||||
value.put("network_received", SyslogMessageHandler.getNetworkReceivedCount());
|
||||
value.put("wal_appended", wal.getAppendedCount());
|
||||
value.put("wal_queue_size", wal.getQueueSize());
|
||||
value.put("wal_queue_capacity", wal.getQueueCapacity());
|
||||
value.put("wal_durable_sequence", wal.getDurableSequence());
|
||||
value.put("wal_checkpoint_sequence", wal.getCheckpointSequence());
|
||||
value.put("wal_pending_approx", wal.getPendingRecordsApprox());
|
||||
value.put("wal_admission_backpressure", wal.getBackpressureCount());
|
||||
value.put("wal_terminal_recorded", wal.getTerminalRecordedCount());
|
||||
value.put("wal_quarantined", wal.getQuarantinedCount());
|
||||
Map<String, Long> pointTimeAssigned = new LinkedHashMap<>();
|
||||
pointTimeAssigned.put("ingress", wal.getIngressPointTimeAssignedCount());
|
||||
value.put("point_time_assigned_total", pointTimeAssigned);
|
||||
value.put("wal_disk_critical", wal.isDiskCritical());
|
||||
value.put("wal_fatal_error", wal.getFatalFailure() == null
|
||||
? null : wal.getFatalFailure().toString());
|
||||
value.put("dispatch_inflight", dispatcher.getInflight());
|
||||
value.put("kafka_acked", dispatcher.getKafkaAcked());
|
||||
value.put("dispatch_terminal_filtered", dispatcher.getTerminalFiltered());
|
||||
value.put("dispatch_retry", dispatcher.getRetryCount());
|
||||
value.put("tcp_pause_total", SyslogMessageHandler.getTcpPauseCount());
|
||||
value.put("udp_dropped_total", SyslogMessageHandler.getUdpDroppedCount());
|
||||
value.put("device_cache_size", resolver.getCacheSize());
|
||||
value.put("device_cache_hits", resolver.getHitCount());
|
||||
value.put("device_cache_misses", resolver.getMissCount());
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -1,178 +1,213 @@
|
||||
package com.kafka;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.util.Sm4Util;
|
||||
import org.apache.kafka.clients.producer.*;
|
||||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.haobang.config.AppConfig;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.util.Map;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import com.haobang.util.DeviceInfoUtil;
|
||||
import com.haobang.util.SpringContextUtil;
|
||||
import com.haobang.util.Sm4Util;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.apache.kafka.clients.producer.Producer;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.clients.producer.ProducerRecord;
|
||||
import org.apache.kafka.clients.producer.RecordMetadata;
|
||||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
import java.util.Properties;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 进程级单例 Kafka Producer。
|
||||
*
|
||||
* <p>保留历史类名以避免影响现有调用方，但实例由 Spring 托管。KafkaProducer 本身线程安全，
|
||||
* 长生命周期复用才能使用连接、RecordAccumulator、批量和压缩；逐条 new/close 会破坏这些机制。</p>
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class kafkaProducer {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(kafkaProducer.class);
|
||||
private static final ZoneId EVENT_ID_ZONE = ZoneId.of("Asia/Shanghai");
|
||||
private static final DateTimeFormatter EVENT_ID_DATE = DateTimeFormatter.ofPattern("yyyyMMdd");
|
||||
|
||||
private static AppProperties getProps() {
|
||||
return SpringContextUtil.getBean(AppProperties.class);
|
||||
}
|
||||
public static void main(String[] args) {
|
||||
AppProperties appProps = getProps();
|
||||
// 配置生产者属性
|
||||
private final AppProperties appProperties;
|
||||
private final Producer<String, String> producer;
|
||||
private final PartitionKeyStrategy partitionKeyStrategy;
|
||||
|
||||
@Autowired
|
||||
public kafkaProducer(AppProperties appProperties,
|
||||
@Value("${spring.kafka.producer.properties.retries:10}") int retries,
|
||||
@Value("${spring.kafka.producer.properties.delivery.timeout.ms:120000}") int deliveryTimeoutMs,
|
||||
@Value("${spring.kafka.producer.properties.request.timeout.ms:30000}") int requestTimeoutMs,
|
||||
@Value("${spring.kafka.producer.properties.linger.ms:5}") int lingerMs,
|
||||
@Value("${spring.kafka.producer.properties.batch.size:65536}") int batchSize,
|
||||
@Value("${spring.kafka.producer.compression-type:lz4}") String compressionType,
|
||||
@Value("${spring.kafka.producer.properties.max.block.ms:10000}") int maxBlockMs,
|
||||
@Value("${spring.kafka.producer.properties.buffer.memory:134217728}") long bufferMemory,
|
||||
@Value("${syslog.kafka.partition-key-strategy:event_id}") String partitionKeyStrategy) {
|
||||
this.appProperties = appProperties;
|
||||
this.partitionKeyStrategy = PartitionKeyStrategy.parse(partitionKeyStrategy);
|
||||
Properties props = new Properties();
|
||||
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProps.getKafkaProducerBootstrap());
|
||||
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProperties.getKafkaProducerBootstrap());
|
||||
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
||||
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
||||
|
||||
// 可选配置：提高可靠性
|
||||
// [SYSLOG-ALARM][KPROD][已修复] acks=all 与幂等发送共同避免 Leader 切换和客户端重试造成假成功或重复。
|
||||
props.put(ProducerConfig.ACKS_CONFIG, "all");
|
||||
props.put(ProducerConfig.RETRIES_CONFIG, 3);
|
||||
props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true);
|
||||
props.put(ProducerConfig.RETRIES_CONFIG, retries);
|
||||
props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 5);
|
||||
props.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, deliveryTimeoutMs);
|
||||
props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, requestTimeoutMs);
|
||||
props.put(ProducerConfig.LINGER_MS_CONFIG, lingerMs);
|
||||
props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
|
||||
props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, compressionType);
|
||||
// Dispatcher 已限制全局在途数；Producer 缓冲耗尽时快速返回失败，让事件留在 WAL 重试。
|
||||
props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, maxBlockMs);
|
||||
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);
|
||||
// [SYSLOG-ALARM][KPROD][已修复] linger、batch 与压缩恢复跨消息批量能力，吞吐优化不改变 Broker ACK 边界。
|
||||
// [SYSLOG-ALARM][KPROD][已修复] Producer 只在应用启动时构造一次，单条热路径不再创建网络线程和连接。
|
||||
this.producer = new KafkaProducer<>(props);
|
||||
log.info("Kafka Producer 已启用分区策略: strategy={}", this.partitionKeyStrategy.configValue);
|
||||
}
|
||||
|
||||
// 创建生产者实例
|
||||
Producer<String, String> producer = new KafkaProducer<>(props);
|
||||
|
||||
try {
|
||||
// 发送10条消息
|
||||
//for (int i = 0; i < 1; i++) {
|
||||
String key = "key-" + "1";
|
||||
String value = "message-" + "1" + " at " + System.currentTimeMillis();
|
||||
|
||||
// 创建生产者记录
|
||||
ProducerRecord<String, String> record =
|
||||
new ProducerRecord<>(appProps.getKafkaProducerTopic(), key, value);
|
||||
|
||||
// 发送消息（异步方式）
|
||||
producer.send(record, new Callback() {
|
||||
@Override
|
||||
public void onCompletion(RecordMetadata metadata, Exception exception) {
|
||||
if (exception == null) {
|
||||
System.out.println("消息发送成功: " +
|
||||
"主题=" + metadata.topic() +
|
||||
", 分区=" + metadata.partition() +
|
||||
", 偏移量=" + metadata.offset());
|
||||
} else {
|
||||
System.err.println("消息发送失败: " + exception.getMessage());
|
||||
}
|
||||
}
|
||||
});
|
||||
// 同步发送方式（如果需要）
|
||||
// RecordMetadata metadata = producer.send(record).get();
|
||||
// System.out.println("同步发送成功: " + metadata.offset());
|
||||
// }
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
// 关闭生产者
|
||||
producer.close();
|
||||
}
|
||||
/** 测试入口：复用真实的分区键选择和 ProducerRecord 构造逻辑，不建立 Kafka 网络连接。 */
|
||||
kafkaProducer(AppProperties appProperties, Producer<String, String> producer, String partitionKeyStrategy) {
|
||||
this.appProperties = appProperties;
|
||||
this.producer = producer;
|
||||
this.partitionKeyStrategy = PartitionKeyStrategy.parse(partitionKeyStrategy);
|
||||
}
|
||||
|
||||
/**
|
||||
* * syslog 文本消息
|
||||
* * @param strSyslog
|
||||
* [流程节点3] 为原始 Syslog 生成稳定 event_id，加密并异步投递 Kafka。
|
||||
* 返回 Future 只有在 Broker 按 acks=all 确认后成功完成，调用方必须据此更新接收审计。
|
||||
*/
|
||||
public CompletableFuture<DeliveryResult> messagePush(DeviceDevice deviceDevice,
|
||||
String syslog,
|
||||
String receiveTime) {
|
||||
return messagePush(deviceDevice, syslog, receiveTime, newEventId());
|
||||
}
|
||||
|
||||
public static void messagePush(DeviceDevice deviceDevice, String strSyslog,String strReceiveTime)
|
||||
{
|
||||
AppProperties appProps = getProps();
|
||||
// 配置生产者属性
|
||||
Properties props = new Properties();
|
||||
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProps.getKafkaProducerBootstrap());
|
||||
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
||||
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
||||
public CompletableFuture<DeliveryResult> messagePush(DeviceDevice deviceDevice,
|
||||
String syslog,
|
||||
String receiveTime,
|
||||
String eventId) {
|
||||
return messagePush(deviceDevice, syslog, receiveTime, eventId, null);
|
||||
}
|
||||
|
||||
// 可选配置：提高可靠性
|
||||
props.put(ProducerConfig.ACKS_CONFIG, "all");
|
||||
props.put(ProducerConfig.RETRIES_CONFIG, 3);
|
||||
props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true);
|
||||
public CompletableFuture<DeliveryResult> messagePush(DeviceDevice deviceDevice,
|
||||
String syslog,
|
||||
String receiveTime,
|
||||
String eventId,
|
||||
Long pointTimeNs) {
|
||||
// [流程节点3.1] event_id 在接入边界生成并写入加密载荷，Retry/Replay 不再重新生成。
|
||||
String fullMessage = DeviceInfoUtil.getFullLogString(
|
||||
deviceDevice, syslog, receiveTime, eventId, pointTimeNs, appProperties);
|
||||
// [流程节点3.2] Kafka 中保存携带 event_id 的完整密文，原始正文不写普通应用日志。
|
||||
String encrypted = Sm4Util.encryptCbc(fullMessage, appProperties.getSm4Key());
|
||||
ProducerRecord<String, String> record = buildRecord(deviceDevice.getId(), eventId, encrypted);
|
||||
|
||||
// 创建生产者实例
|
||||
Producer<String, String> producer = new KafkaProducer<>(props);
|
||||
CompletableFuture<DeliveryResult> result = new CompletableFuture<>();
|
||||
// [SYSLOG-ALARM][KPROD][已修复] 回调成功才表示 Broker 接管；send() 返回本身不再记录 PRODUCED。
|
||||
producer.send(record, (metadata, exception) -> {
|
||||
if (exception != null) {
|
||||
result.completeExceptionally(exception);
|
||||
return;
|
||||
}
|
||||
result.complete(new DeliveryResult(eventId, metadata));
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
ProducerRecord<String, String> buildRecord(Integer deviceId, String eventId, String encrypted) {
|
||||
String partitionKey = partitionKeyStrategy.partitionKey(deviceId, eventId);
|
||||
return new ProducerRecord<>(appProperties.getKafkaProducerTopic(), partitionKey, encrypted);
|
||||
}
|
||||
|
||||
public static String newEventId() {
|
||||
return newEventId(Instant.now());
|
||||
}
|
||||
|
||||
/** event_id 日期固定按接入业务时区计算，不依赖容器或 JVM 默认时区。 */
|
||||
public static String newEventId(Instant receivedAt) {
|
||||
return EVENT_ID_DATE.format(receivedAt.atZone(EVENT_ID_ZONE)) + "-" + UUID.randomUUID();
|
||||
}
|
||||
|
||||
/** 仅用于运维探针的同步健康验证，不用于逐条热路径。 */
|
||||
public DeliveryResult sendAndAwait(DeviceDevice deviceDevice, String syslog, String receiveTime,
|
||||
long timeoutMs) throws Exception {
|
||||
return messagePush(deviceDevice, syslog, receiveTime).get(timeoutMs, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void close() {
|
||||
// [SYSLOG-ALARM][KPROD][已修复] 只在优雅停机阶段 flush/close，确保缓冲中的已接收消息完成发送。
|
||||
try {
|
||||
|
||||
String key = "key-" + System.currentTimeMillis();
|
||||
//String value = DeviceInfoUtil.getFullLogString(strSyslog);
|
||||
//采用动态获取Syslog 请求的设备信息
|
||||
String value = DeviceInfoUtil.getFullLogString(deviceDevice,strSyslog,strReceiveTime);
|
||||
|
||||
//Message进行SM4加密写入kafka
|
||||
String Sm4message= Sm4Util.encryptCbc(value, appProps.getSm4Key());
|
||||
//System.out.println("Sm4message:"+Sm4message);
|
||||
|
||||
// 创建生产者记录
|
||||
ProducerRecord<String, String> record =
|
||||
new ProducerRecord<>(appProps.getKafkaProducerTopic(), key, Sm4message);
|
||||
|
||||
// 发送消息（异步方式）
|
||||
producer.send(record, new Callback() {
|
||||
@Override
|
||||
public void onCompletion(RecordMetadata metadata, Exception exception) {
|
||||
if (exception == null) {
|
||||
System.out.println("消息发送成功: " +
|
||||
"主题=" + metadata.topic() +
|
||||
", 分区=" + metadata.partition() +
|
||||
", 偏移量=" + metadata.offset());
|
||||
} else {
|
||||
System.err.println("消息发送失败: " + exception.getMessage());
|
||||
logger.error("消息发送失败: " + exception.getMessage());
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 同步发送方式（如果需要）
|
||||
// RecordMetadata metadata = producer.send(record).get();
|
||||
// System.out.println("同步发送成功: " + metadata.offset());
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
producer.flush();
|
||||
} finally {
|
||||
// 关闭生产者
|
||||
producer.close();
|
||||
producer.close(java.time.Duration.ofSeconds(10));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成json字符串，补充设备配置信息
|
||||
* @param strSyslog
|
||||
* @return
|
||||
*/
|
||||
private static String getLogJsonString(String strSyslog)
|
||||
{
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
// 创建设备配置信息MAP
|
||||
Map<String, Object> map = new LinkedHashMap<>();
|
||||
AppProperties appProps = getProps();
|
||||
map.put("device_id", appProps.getAppServiceDeviceId());
|
||||
map.put("device_name", appProps.getAppServiceDeviceName());
|
||||
map.put("vendor", appProps.getAppServiceVendor());
|
||||
map.put("data_type", appProps.getAppServiceDataType());
|
||||
// 添加syslogMessage
|
||||
map.put("syslogMessage", strSyslog);
|
||||
try {
|
||||
//生成json消息字符串
|
||||
String json = objectMapper.writeValueAsString(map);
|
||||
return json;
|
||||
enum PartitionKeyStrategy {
|
||||
EVENT_ID("event_id") {
|
||||
@Override
|
||||
String partitionKey(Integer deviceId, String eventId) {
|
||||
if (eventId == null || eventId.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("event_id 分区策略要求 eventId 非空");
|
||||
}
|
||||
return eventId;
|
||||
}
|
||||
},
|
||||
DEVICE_ID("device_id") {
|
||||
@Override
|
||||
String partitionKey(Integer deviceId, String eventId) {
|
||||
if (deviceId == null) {
|
||||
throw new IllegalArgumentException("device_id 分区策略要求 deviceId 非空");
|
||||
}
|
||||
return deviceId.toString();
|
||||
}
|
||||
};
|
||||
|
||||
private final String configValue;
|
||||
|
||||
PartitionKeyStrategy(String configValue) {
|
||||
this.configValue = configValue;
|
||||
}
|
||||
catch(Exception ex)
|
||||
{
|
||||
logger.error("kafkaProducer getLogJsonString(）生成json字符串 异常: syslog: "+ strSyslog);
|
||||
logger.error("Exception: : "+ ex.getMessage());
|
||||
|
||||
abstract String partitionKey(Integer deviceId, String eventId);
|
||||
|
||||
static PartitionKeyStrategy parse(String configuredValue) {
|
||||
String normalized = configuredValue == null
|
||||
? "" : configuredValue.trim().toLowerCase(Locale.ROOT);
|
||||
for (PartitionKeyStrategy strategy : values()) {
|
||||
if (strategy.configValue.equals(normalized)) return strategy;
|
||||
}
|
||||
throw new IllegalArgumentException(
|
||||
"不支持的 Kafka 分区键策略: " + configuredValue + "，仅支持 event_id 或 device_id");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Getter
|
||||
public static class DeliveryResult {
|
||||
private final String eventId;
|
||||
private final String topic;
|
||||
private final int partition;
|
||||
private final long offset;
|
||||
|
||||
|
||||
|
||||
DeliveryResult(String eventId, RecordMetadata metadata) {
|
||||
this.eventId = eventId;
|
||||
this.topic = metadata.topic();
|
||||
this.partition = metadata.partition();
|
||||
this.offset = metadata.offset();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+142
-120
@@ -1,5 +1,8 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.wal.SyslogIngressEvent;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import com.kafka.kafkaProducer;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.channel.SimpleChannelInboundHandler;
|
||||
import io.netty.channel.socket.DatagramPacket;
|
||||
@@ -7,147 +10,166 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.SocketAddress;
|
||||
import java.nio.charset.Charset;
|
||||
import com.kafka.kafkaProducer;
|
||||
import com.Modules.Device.DeviceProcess;
|
||||
import com.haobang.util.Sm4Util;
|
||||
import com.haobang.config.AppConfig;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Deque;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
|
||||
/**
|
||||
* Syslog 消息处理器
|
||||
* Syslog 网络接入处理器。
|
||||
*
|
||||
* <p>这里只解帧、生成首次 event_id/receive_time 并提交 WAL，不再查达梦、等待 Kafka ACK
|
||||
* 或写接收审计。TCP 在 WAL 达到高水位时关闭 AUTO_READ，并保留已经解出的当前帧，待 WAL
|
||||
* 恢复到低水位后按原顺序继续追加。</p>
|
||||
*/
|
||||
public class SyslogMessageHandler extends SimpleChannelInboundHandler<Object> {
|
||||
private static final Logger logger = LoggerFactory.getLogger(SyslogMessageHandler.class);
|
||||
private static final Charset SYSLOG_CHARSET = AppConfig.getSyslogCharset();
|
||||
private static final Logger log = LoggerFactory.getLogger(SyslogMessageHandler.class);
|
||||
private static final ZoneId RECEIVE_TIME_ZONE = ZoneId.of("Asia/Shanghai");
|
||||
private static final LongAdder NETWORK_RECEIVED = new LongAdder();
|
||||
private static final LongAdder TCP_PAUSE_COUNT = new LongAdder();
|
||||
private static final LongAdder UDP_DROPPED = new LongAdder();
|
||||
|
||||
/** 每个 TCP Channel 独立 Handler，pending 只保存关闭 AUTO_READ 前已进入 Pipeline 的帧。 */
|
||||
private final Deque<SyslogIngressEvent> pendingTcp = new ArrayDeque<>();
|
||||
private final Charset syslogCharset;
|
||||
private final SyslogWalService walService;
|
||||
private boolean retryScheduled;
|
||||
private boolean autoReadPaused;
|
||||
|
||||
SyslogMessageHandler(Charset syslogCharset, SyslogWalService walService) {
|
||||
this.syslogCharset = java.util.Objects.requireNonNull(syslogCharset, "syslogCharset");
|
||||
this.walService = java.util.Objects.requireNonNull(walService, "walService");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception {
|
||||
String message;
|
||||
String source;
|
||||
String source_ip;
|
||||
String networkProtocol;
|
||||
int source_port;
|
||||
protected void channelRead0(ChannelHandlerContext ctx, Object msg) {
|
||||
final String message;
|
||||
final String sourceIp;
|
||||
final int sourcePort;
|
||||
final String protocol;
|
||||
if (msg instanceof DatagramPacket) {
|
||||
// UDP 消息处理
|
||||
DatagramPacket packet = (DatagramPacket) msg;
|
||||
message = packet.content().toString(SYSLOG_CHARSET);
|
||||
source_ip=packet.sender().getAddress().getHostAddress();
|
||||
source_port=packet.sender().getPort();
|
||||
source = packet.sender().getAddress().getHostAddress() + ":" + packet.sender().getPort();
|
||||
networkProtocol="UDP";
|
||||
logger.info("Received syslog from {}: {}", source, message);
|
||||
message = packet.content().toString(syslogCharset);
|
||||
sourceIp = packet.sender().getAddress().getHostAddress();
|
||||
sourcePort = packet.sender().getPort();
|
||||
protocol = "UDP";
|
||||
} else if (msg instanceof String) {
|
||||
// TCP 消息处理
|
||||
message = (String) msg;
|
||||
InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress();
|
||||
source = remoteAddress.getAddress().getHostAddress() + ":" + remoteAddress.getPort();
|
||||
source_ip=remoteAddress.getAddress().getHostAddress();
|
||||
source_port=remoteAddress.getPort();
|
||||
networkProtocol="TCP";
|
||||
logger.info("Received syslog from {}: {}", source, message);
|
||||
SocketAddress address = ctx.channel().remoteAddress();
|
||||
if (address instanceof InetSocketAddress) {
|
||||
InetSocketAddress remote = (InetSocketAddress) address;
|
||||
sourceIp = remote.getAddress().getHostAddress();
|
||||
sourcePort = remote.getPort();
|
||||
} else {
|
||||
// 仅测试用 EmbeddedChannel 会进入该分支；真实 TCP Channel 必须是 InetSocketAddress。
|
||||
sourceIp = String.valueOf(address);
|
||||
sourcePort = 0;
|
||||
}
|
||||
protocol = "TCP";
|
||||
} else {
|
||||
logger.warn("Received unknown message type: {}", msg.getClass().getName());
|
||||
log.warn("收到不支持的 Syslog 消息类型: {}", msg == null ? "null" : msg.getClass().getName());
|
||||
return;
|
||||
}
|
||||
DeviceProcess deviceProcess = new DeviceProcess(source_ip);
|
||||
try {
|
||||
//日志推送的设备信息进行鉴权处理
|
||||
int deviceId = deviceProcess.getDeviceID(source_ip);
|
||||
if (deviceId < 0) {
|
||||
logger.info("syslog message 的请求设备IP:{}非系统注册，暂不做处理！",source_ip);
|
||||
//创建未知设备
|
||||
deviceProcess.saveDeviceUnknow(networkProtocol);
|
||||
return;
|
||||
}
|
||||
if (!deviceProcess.IsBelongDeviceCollectTask()) {
|
||||
logger.info("syslog message 的请求设备IP:{}不归属当前采集探针，暂不做处理！",source_ip);
|
||||
return;
|
||||
}
|
||||
//插入PG库接收记录
|
||||
deviceProcess.saveDeviceReceiveLog(message,true);
|
||||
kafkaProducer.messagePush(deviceProcess.getDeviceDevice(), message, deviceProcess.getDeviceReceiveLog().getReceiveTimeStr());
|
||||
|
||||
} catch (Exception e) {
|
||||
//记录日志推送失败
|
||||
deviceProcess.saveDeviceReceiveLog(message,false);
|
||||
logger.warn("Failed to Push syslog message", e);
|
||||
Instant receivedInstant = Instant.now();
|
||||
SyslogIngressEvent event = walService.prepareForWal(new SyslogIngressEvent(
|
||||
kafkaProducer.newEventId(receivedInstant),
|
||||
LocalDateTime.ofInstant(receivedInstant, RECEIVE_TIME_ZONE),
|
||||
sourceIp, sourcePort, protocol, message));
|
||||
NETWORK_RECEIVED.increment();
|
||||
|
||||
}
|
||||
// 处理 Syslog 消息
|
||||
//processSyslogMessage(message.trim(), source);
|
||||
}
|
||||
|
||||
private void processSyslogMessage(String message, String source) {
|
||||
// 这里可以添加 Syslog 消息解析逻辑
|
||||
// 根据 RFC 3164 或 RFC 5424 解析消息
|
||||
System.out.println("Received syslog from "+ source +", msg:"+ message);
|
||||
logger.info("Received syslog from {}: {}", source, message);
|
||||
|
||||
|
||||
try {
|
||||
SyslogMessage syslogMsg = parseSyslogMessage(message);
|
||||
logger.info("Parsed syslog - Facility: {}, Severity: {}, Timestamp: {}, Host: {}, Message: {}",
|
||||
syslogMsg.getFacility(),
|
||||
syslogMsg.getSeverity(),
|
||||
syslogMsg.getTimestamp(),
|
||||
syslogMsg.getHostname(),
|
||||
syslogMsg.getMessage());
|
||||
|
||||
// 这里可以添加消息存储或其他处理逻辑
|
||||
// storeToDatabase(syslogMsg);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.warn("Failed to parse syslog message: {}", message, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 简单的 Syslog 消息解析（RFC 3164 格式）
|
||||
*/
|
||||
private SyslogMessage parseSyslogMessage(String message) {
|
||||
SyslogMessage syslogMsg = new SyslogMessage();
|
||||
|
||||
// 尝试解析 PRI 部分
|
||||
if (message.startsWith("<")) {
|
||||
int priEnd = message.indexOf(">");
|
||||
if (priEnd > 0) {
|
||||
String priStr = message.substring(1, priEnd);
|
||||
try {
|
||||
int pri = Integer.parseInt(priStr);
|
||||
int facility = pri >> 3;
|
||||
int severity = pri & 0x07;
|
||||
|
||||
syslogMsg.setFacility(facility);
|
||||
syslogMsg.setSeverity(severity);
|
||||
|
||||
message = message.substring(priEnd + 1).trim();
|
||||
} catch (NumberFormatException e) {
|
||||
// 忽略 PRI 解析错误
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 尝试解析时间戳和主机名
|
||||
// 这是一个简化的解析，实际实现可能需要更复杂的逻辑
|
||||
String[] parts = message.split(" ", 5);
|
||||
if (parts.length >= 4) {
|
||||
// 假设前三个部分是时间戳，第四个部分是主机名
|
||||
String timestamp = parts[0] + " " + parts[1] + " " + parts[2];
|
||||
syslogMsg.setTimestamp(timestamp);
|
||||
syslogMsg.setHostname(parts[3]);
|
||||
|
||||
if (parts.length == 5) {
|
||||
syslogMsg.setMessage(parts[4]);
|
||||
}
|
||||
if ("UDP".equals(protocol)) {
|
||||
acceptUdp(walService, event);
|
||||
} else {
|
||||
syslogMsg.setMessage(message);
|
||||
acceptTcp(ctx, walService, event);
|
||||
}
|
||||
}
|
||||
|
||||
return syslogMsg;
|
||||
private void acceptUdp(SyslogWalService wal, SyslogIngressEvent event) {
|
||||
SyslogWalService.AdmissionResult result = wal.tryAppend(event);
|
||||
if (result == SyslogWalService.AdmissionResult.ACCEPTED) return;
|
||||
UDP_DROPPED.increment();
|
||||
long dropped = UDP_DROPPED.sum();
|
||||
if (dropped == 1 || dropped % 100 == 0) {
|
||||
log.error("UDP Syslog 无法进入 WAL；UDP 不支持背压，已发生明确丢弃: eventId={}, sourceIp={}, "
|
||||
+ "result={}, droppedTotal={}",
|
||||
event.getEventId(), event.getSourceIp(), result, dropped);
|
||||
}
|
||||
}
|
||||
|
||||
private void acceptTcp(ChannelHandlerContext ctx, SyslogWalService wal, SyslogIngressEvent event) {
|
||||
// pending 非空时必须维持同一 TCP Channel 的原始顺序，不能让新帧越过旧帧。
|
||||
if (!pendingTcp.isEmpty()) {
|
||||
pendingTcp.addLast(event);
|
||||
pauseAndSchedule(ctx, wal);
|
||||
return;
|
||||
}
|
||||
SyslogWalService.AdmissionResult result = wal.tryAppend(event);
|
||||
if (result != SyslogWalService.AdmissionResult.ACCEPTED) {
|
||||
pendingTcp.addLast(event);
|
||||
pauseAndSchedule(ctx, wal);
|
||||
return;
|
||||
}
|
||||
if (wal.shouldPauseTcp()) pauseAndSchedule(ctx, wal);
|
||||
}
|
||||
|
||||
private void pauseAndSchedule(ChannelHandlerContext ctx, SyslogWalService wal) {
|
||||
if (!autoReadPaused && ctx.channel().config().isAutoRead()) {
|
||||
ctx.channel().config().setAutoRead(false);
|
||||
autoReadPaused = true;
|
||||
TCP_PAUSE_COUNT.increment();
|
||||
log.warn("TCP Syslog 已按 WAL 高水位暂停读取: channel={}, pendingFrames={}, walQueue={}/{}, "
|
||||
+ "pauseTotal={}",
|
||||
ctx.channel().remoteAddress(), pendingTcp.size(), wal.getQueueSize(), wal.getQueueCapacity(),
|
||||
TCP_PAUSE_COUNT.sum());
|
||||
}
|
||||
scheduleDrain(ctx, wal);
|
||||
}
|
||||
|
||||
private void scheduleDrain(ChannelHandlerContext ctx, SyslogWalService wal) {
|
||||
if (retryScheduled) return;
|
||||
retryScheduled = true;
|
||||
ctx.executor().schedule(() -> {
|
||||
retryScheduled = false;
|
||||
drainPending(ctx, wal);
|
||||
}, wal.getRetryIntervalMs(), TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
private void drainPending(ChannelHandlerContext ctx, SyslogWalService wal) {
|
||||
while (!pendingTcp.isEmpty()) {
|
||||
SyslogIngressEvent event = pendingTcp.peekFirst();
|
||||
SyslogWalService.AdmissionResult result = wal.tryAppend(event);
|
||||
if (result != SyslogWalService.AdmissionResult.ACCEPTED) {
|
||||
scheduleDrain(ctx, wal);
|
||||
return;
|
||||
}
|
||||
pendingTcp.removeFirst();
|
||||
}
|
||||
if (wal.canResumeTcp()) {
|
||||
if (autoReadPaused && ctx.channel().isActive()) {
|
||||
ctx.channel().config().setAutoRead(true);
|
||||
ctx.read();
|
||||
log.info("TCP Syslog 已恢复读取: channel={}, walQueue={}/{}",
|
||||
ctx.channel().remoteAddress(), wal.getQueueSize(), wal.getQueueCapacity());
|
||||
}
|
||||
autoReadPaused = false;
|
||||
} else {
|
||||
scheduleDrain(ctx, wal);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
|
||||
logger.error("Exception in channel handler", cause);
|
||||
log.error("Syslog 网络接入异常: channel={}", ctx.channel().remoteAddress(), cause);
|
||||
ctx.close();
|
||||
}
|
||||
}
|
||||
|
||||
public static long getNetworkReceivedCount() { return NETWORK_RECEIVED.sum(); }
|
||||
public static long getTcpPauseCount() { return TCP_PAUSE_COUNT.sum(); }
|
||||
public static long getUdpDroppedCount() { return UDP_DROPPED.sum(); }
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.util.SpringContextUtil;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import io.netty.bootstrap.Bootstrap;
|
||||
import io.netty.bootstrap.ServerBootstrap;
|
||||
import io.netty.channel.ChannelFuture;
|
||||
@@ -11,6 +14,8 @@ import io.netty.channel.socket.nio.NioServerSocketChannel;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.nio.charset.Charset;
|
||||
|
||||
/**
|
||||
* Syslog 服务器主类，支持 TCP 和 UDP 协议
|
||||
*/
|
||||
@@ -34,6 +39,9 @@ public class SyslogServer {
|
||||
}
|
||||
|
||||
private void startTcpServer() throws InterruptedException {
|
||||
AppProperties properties = SpringContextUtil.getBean(AppProperties.class);
|
||||
SyslogWalService walService = SpringContextUtil.getBean(SyslogWalService.class);
|
||||
Charset charset = walService.getSyslogCharset();
|
||||
EventLoopGroup bossGroup = new NioEventLoopGroup(1);
|
||||
EventLoopGroup workerGroup = new NioEventLoopGroup();
|
||||
|
||||
@@ -41,7 +49,8 @@ public class SyslogServer {
|
||||
ServerBootstrap b = new ServerBootstrap();
|
||||
b.group(bossGroup, workerGroup)
|
||||
.channel(NioServerSocketChannel.class)
|
||||
.childHandler(new SyslogTcpChannelInitializer())
|
||||
.childHandler(new SyslogTcpChannelInitializer(
|
||||
properties.getSyslogMaxFrameLength(), charset, walService))
|
||||
.option(ChannelOption.SO_BACKLOG, 128)
|
||||
.childOption(ChannelOption.SO_KEEPALIVE, true);
|
||||
|
||||
@@ -56,13 +65,16 @@ public class SyslogServer {
|
||||
}
|
||||
|
||||
private void startUdpServer() throws InterruptedException {
|
||||
AppProperties properties = SpringContextUtil.getBean(AppProperties.class);
|
||||
SyslogWalService walService = SpringContextUtil.getBean(SyslogWalService.class);
|
||||
Charset charset = walService.getSyslogCharset();
|
||||
EventLoopGroup group = new NioEventLoopGroup();
|
||||
|
||||
try {
|
||||
Bootstrap b = new Bootstrap();
|
||||
b.group(group)
|
||||
.channel(NioDatagramChannel.class)
|
||||
.handler(new SyslogUdpChannelInitializer())
|
||||
.handler(new SyslogUdpChannelInitializer(charset, walService))
|
||||
.option(ChannelOption.SO_BROADCAST, true);
|
||||
|
||||
ChannelFuture f = b.bind(udpPort).sync();
|
||||
@@ -81,4 +93,4 @@ public class SyslogServer {
|
||||
SyslogServer server = new SyslogServer(tcpPort, udpPort);
|
||||
server.start();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,13 +14,20 @@ import org.slf4j.LoggerFactory;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import java.nio.charset.Charset;
|
||||
|
||||
/**
|
||||
* Syslog 服务器主类，支持 TCP 和 UDP 协议
|
||||
*/
|
||||
public class SyslogServerBoth {
|
||||
private static final Logger logger = LoggerFactory.getLogger(SyslogServer.class);
|
||||
private static final Logger logger = LoggerFactory.getLogger(SyslogServerBoth.class);
|
||||
|
||||
private final AppProperties properties;
|
||||
private final SyslogWalService walService;
|
||||
private final Charset syslogCharset;
|
||||
private final int tcpPort;
|
||||
private final int udpPort;
|
||||
|
||||
@@ -34,13 +41,17 @@ public class SyslogServerBoth {
|
||||
|
||||
private final ExecutorService serverExecutor = Executors.newFixedThreadPool(2);
|
||||
|
||||
public SyslogServerBoth(int tcpPort, int udpPort) {
|
||||
this.tcpPort = tcpPort;
|
||||
this.udpPort = udpPort;
|
||||
public SyslogServerBoth(AppProperties properties, SyslogWalService walService, Charset syslogCharset) {
|
||||
this.properties = java.util.Objects.requireNonNull(properties, "properties");
|
||||
this.walService = java.util.Objects.requireNonNull(walService, "walService");
|
||||
this.syslogCharset = java.util.Objects.requireNonNull(syslogCharset, "syslogCharset");
|
||||
this.tcpPort = properties.getSyslogTcpPort();
|
||||
this.udpPort = properties.getSyslogUdpPort();
|
||||
}
|
||||
|
||||
public void start() throws InterruptedException {
|
||||
logger.info("Starting Syslog server with TCP port {} and UDP port {}", tcpPort, udpPort);
|
||||
logger.info("Starting Syslog server with TCP port {}, UDP port {} and charset {}",
|
||||
tcpPort, udpPort, syslogCharset.name());
|
||||
|
||||
// 创建共享的EventLoopGroup
|
||||
bossGroup = new NioEventLoopGroup(1);
|
||||
@@ -48,6 +59,7 @@ public class SyslogServerBoth {
|
||||
udpGroup = new NioEventLoopGroup();
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(2);
|
||||
final AtomicReference<Throwable> startupFailure = new AtomicReference<>();
|
||||
|
||||
try {
|
||||
// 启动UDP服务器（使用独立的线程）
|
||||
@@ -57,8 +69,8 @@ public class SyslogServerBoth {
|
||||
latch.countDown();
|
||||
} catch (Exception e) {
|
||||
logger.error("UDP server failed to start", e);
|
||||
startupFailure.compareAndSet(null, e);
|
||||
latch.countDown();
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -69,24 +81,26 @@ public class SyslogServerBoth {
|
||||
latch.countDown();
|
||||
} catch (Exception e) {
|
||||
logger.error("TCP server failed to start", e);
|
||||
startupFailure.compareAndSet(null, e);
|
||||
latch.countDown();
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
|
||||
// 等待两个服务器都启动完成
|
||||
latch.await();
|
||||
if (startupFailure.get() != null) {
|
||||
throw new IllegalStateException("Syslog TCP/UDP 监听端口启动失败", startupFailure.get());
|
||||
}
|
||||
logger.info("Both TCP and UDP Syslog servers are running");
|
||||
|
||||
// 注册关闭钩子
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(this::shutdown));
|
||||
|
||||
// 保持主线程运行
|
||||
Thread.currentThread().join();
|
||||
// 端口生命周期由 Spring SmartLifecycle 管理；Netty EventLoop 自身保持服务运行。
|
||||
serverExecutor.shutdown();
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to start servers", e);
|
||||
shutdown();
|
||||
if (e instanceof InterruptedException) throw (InterruptedException) e;
|
||||
throw new IllegalStateException("Syslog server startup failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,9 +108,13 @@ public class SyslogServerBoth {
|
||||
ServerBootstrap b = new ServerBootstrap();
|
||||
b.group(bossGroup, workerGroup)
|
||||
.channel(NioServerSocketChannel.class)
|
||||
.childHandler(new SyslogTcpChannelInitializer())
|
||||
.option(ChannelOption.SO_BACKLOG, 128)
|
||||
.childOption(ChannelOption.SO_KEEPALIVE, true);
|
||||
.childHandler(new SyslogTcpChannelInitializer(
|
||||
properties.getSyslogMaxFrameLength(), syslogCharset, walService))
|
||||
// [SYSLOG-ALARM][NINGRESS][已修复] backlog/接收缓冲配置化；实际生效上限仍受操作系统 somaxconn/rmem_max 约束。
|
||||
.option(ChannelOption.SO_BACKLOG, properties.getSyslogTcpBacklog())
|
||||
.childOption(ChannelOption.SO_KEEPALIVE, true)
|
||||
.childOption(ChannelOption.AUTO_READ, true)
|
||||
.childOption(ChannelOption.SO_RCVBUF, properties.getSyslogTcpReceiveBufferBytes());
|
||||
|
||||
tcpChannelFuture = b.bind(tcpPort).sync();
|
||||
logger.info("TCP Syslog server started on port {}", tcpPort);
|
||||
@@ -113,8 +131,10 @@ public class SyslogServerBoth {
|
||||
Bootstrap b = new Bootstrap();
|
||||
b.group(udpGroup)
|
||||
.channel(NioDatagramChannel.class)
|
||||
.handler(new SyslogUdpChannelInitializer())
|
||||
.option(ChannelOption.SO_BROADCAST, true);
|
||||
.handler(new SyslogUdpChannelInitializer(syslogCharset, walService))
|
||||
// [SYSLOG-ALARM][NINGRESS][已修复] 显式配置 UDP 接收缓冲；UDP 仍无端到端确认，强可靠设备应改用 TCP/TLS 或 RELP。
|
||||
.option(ChannelOption.SO_BROADCAST, true)
|
||||
.option(ChannelOption.SO_RCVBUF, properties.getSyslogUdpReceiveBufferBytes());
|
||||
|
||||
udpChannelFuture = b.bind(udpPort).sync();
|
||||
logger.info("UDP Syslog server started on port {}", udpPort);
|
||||
@@ -172,31 +192,4 @@ public class SyslogServerBoth {
|
||||
(udpChannelFuture != null && udpChannelFuture.channel().isActive());
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
int tcpPort = 514;
|
||||
int udpPort = 514;
|
||||
|
||||
// 解析命令行参数
|
||||
if (args.length >= 1) {
|
||||
tcpPort = Integer.parseInt(args[0]);
|
||||
}
|
||||
if (args.length >= 2) {
|
||||
udpPort = Integer.parseInt(args[1]);
|
||||
}
|
||||
|
||||
// 如果两个端口相同，UDP端口自动+1避免冲突
|
||||
if (tcpPort == udpPort) {
|
||||
udpPort = tcpPort + 1;
|
||||
logger.warn("TCP and UDP ports cannot be the same. UDP port changed to {}", udpPort);
|
||||
}
|
||||
|
||||
SyslogServerBoth server = new SyslogServerBoth(tcpPort, udpPort);
|
||||
|
||||
// 注册全局异常处理器
|
||||
Thread.setDefaultUncaughtExceptionHandler((t, e) -> {
|
||||
logger.error("Uncaught exception in thread {}", t.getName(), e);
|
||||
});
|
||||
|
||||
server.start();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.dispatch.SyslogWalDispatcher;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.SmartLifecycle;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.nio.charset.Charset;
|
||||
|
||||
/**
|
||||
* 将 Netty 接入端口纳入 Spring 生命周期。
|
||||
*
|
||||
* <p>构造依赖确保 WAL 和 Dispatcher 初始化成功后才开放端口；SmartLifecycle.stop 会在 Bean
|
||||
* 销毁前先关闭 TCP/UDP，避免 WAL 停止后网络仍继续收包。</p>
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(name = "syslog.server.enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class SyslogServerLifecycle implements SmartLifecycle {
|
||||
private static final Logger log = LoggerFactory.getLogger(SyslogServerLifecycle.class);
|
||||
private final AppProperties properties;
|
||||
private final SyslogWalService walService;
|
||||
@SuppressWarnings("unused")
|
||||
private final SyslogWalDispatcher dispatcher;
|
||||
private final AtomicBoolean running = new AtomicBoolean();
|
||||
private SyslogServerBoth server;
|
||||
|
||||
public SyslogServerLifecycle(AppProperties properties, SyslogWalService walService,
|
||||
SyslogWalDispatcher dispatcher) {
|
||||
this.properties = properties;
|
||||
this.walService = walService;
|
||||
this.dispatcher = dispatcher;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
if (!running.compareAndSet(false, true)) return;
|
||||
try {
|
||||
Charset charset = walService.getSyslogCharset();
|
||||
server = new SyslogServerBoth(properties, walService, charset);
|
||||
server.start();
|
||||
log.info("Syslog TCP/UDP 端口已在 WAL 和 Dispatcher 就绪后开放: charset={}", charset.name());
|
||||
} catch (Exception failure) {
|
||||
running.set(false);
|
||||
throw new IllegalStateException("启动 Syslog TCP/UDP 端口失败", failure);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
if (!running.compareAndSet(true, false)) return;
|
||||
if (server != null) server.shutdown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop(Runnable callback) {
|
||||
try { stop(); }
|
||||
finally { callback.run(); }
|
||||
}
|
||||
|
||||
@Override public boolean isRunning() { return running.get(); }
|
||||
@Override public boolean isAutoStartup() { return true; }
|
||||
@Override public int getPhase() { return Integer.MAX_VALUE; }
|
||||
}
|
||||
+16
-8
@@ -1,7 +1,7 @@
|
||||
package com.netty;
|
||||
|
||||
|
||||
import com.haobang.config.AppConfig;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import io.netty.channel.ChannelInitializer;
|
||||
import io.netty.channel.ChannelPipeline;
|
||||
import io.netty.channel.socket.SocketChannel;
|
||||
@@ -15,8 +15,16 @@ import java.nio.charset.Charset;
|
||||
* TCP 通道初始化器
|
||||
*/
|
||||
public class SyslogTcpChannelInitializer extends ChannelInitializer<SocketChannel> {
|
||||
private static final int MAX_FRAME_LENGTH = AppConfig.getSyslogMaxFrameLength();
|
||||
private static final Charset SYSLOG_CHARSET = AppConfig.getSyslogCharset();
|
||||
private final int maxFrameLength;
|
||||
private final Charset syslogCharset;
|
||||
private final SyslogWalService walService;
|
||||
|
||||
SyslogTcpChannelInitializer(int maxFrameLength, Charset syslogCharset, SyslogWalService walService) {
|
||||
if (maxFrameLength <= 0) throw new IllegalArgumentException("maxFrameLength must be positive");
|
||||
this.maxFrameLength = maxFrameLength;
|
||||
this.syslogCharset = java.util.Objects.requireNonNull(syslogCharset, "syslogCharset");
|
||||
this.walService = java.util.Objects.requireNonNull(walService, "walService");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void initChannel(SocketChannel ch) throws Exception {
|
||||
@@ -24,14 +32,14 @@ public class SyslogTcpChannelInitializer extends ChannelInitializer<SocketChanne
|
||||
|
||||
// 添加基于分隔符的帧解码器，处理以换行符结尾的 Syslog 消息
|
||||
pipeline.addLast("framer", new DelimiterBasedFrameDecoder(
|
||||
MAX_FRAME_LENGTH,
|
||||
maxFrameLength,
|
||||
Delimiters.lineDelimiter()));
|
||||
|
||||
// 添加字符串解码器和编码器（使用可配置的字符集）
|
||||
pipeline.addLast("decoder", new StringDecoder(SYSLOG_CHARSET));
|
||||
pipeline.addLast("encoder", new StringEncoder(SYSLOG_CHARSET));
|
||||
pipeline.addLast("decoder", new StringDecoder(syslogCharset));
|
||||
pipeline.addLast("encoder", new StringEncoder(syslogCharset));
|
||||
|
||||
// 添加业务处理器
|
||||
pipeline.addLast("handler", new SyslogMessageHandler());
|
||||
pipeline.addLast("handler", new SyslogMessageHandler(syslogCharset, walService));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+12
-6
@@ -1,6 +1,6 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.config.AppConfig;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import io.netty.channel.ChannelInitializer;
|
||||
import io.netty.channel.ChannelPipeline;
|
||||
import io.netty.channel.socket.DatagramChannel;
|
||||
@@ -12,17 +12,23 @@ import java.nio.charset.Charset;
|
||||
* UDP 通道初始化器
|
||||
*/
|
||||
public class SyslogUdpChannelInitializer extends ChannelInitializer<DatagramChannel> {
|
||||
private static final Charset SYSLOG_CHARSET = AppConfig.getSyslogCharset();
|
||||
private final Charset syslogCharset;
|
||||
private final SyslogWalService walService;
|
||||
|
||||
SyslogUdpChannelInitializer(Charset syslogCharset, SyslogWalService walService) {
|
||||
this.syslogCharset = java.util.Objects.requireNonNull(syslogCharset, "syslogCharset");
|
||||
this.walService = java.util.Objects.requireNonNull(walService, "walService");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void initChannel(DatagramChannel ch) throws Exception {
|
||||
ChannelPipeline pipeline = ch.pipeline();
|
||||
|
||||
// 添加字符串解码器和编码器（使用可配置的字符集）
|
||||
pipeline.addLast("decoder", new StringDecoder(SYSLOG_CHARSET));
|
||||
pipeline.addLast("encoder", new StringEncoder(SYSLOG_CHARSET));
|
||||
pipeline.addLast("decoder", new StringDecoder(syslogCharset));
|
||||
pipeline.addLast("encoder", new StringEncoder(syslogCharset));
|
||||
|
||||
// 添加业务处理器
|
||||
pipeline.addLast("handler", new SyslogMessageHandler());
|
||||
pipeline.addLast("handler", new SyslogMessageHandler(syslogCharset, walService));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,14 +7,59 @@ server.error.include-message=always
|
||||
server.error.include-binding-errors=always
|
||||
|
||||
# Syslog Server Configuration
|
||||
syslog.server.enabled=${SYSLOG_SERVER_ENABLED:true}
|
||||
syslog.tcp.port=514
|
||||
syslog.udp.port=514
|
||||
syslog.tcp.backlog=${SYSLOG_TCP_BACKLOG:4096}
|
||||
syslog.tcp.receive-buffer-bytes=${SYSLOG_TCP_RCVBUF:4194304}
|
||||
syslog.udp.receive-buffer-bytes=${SYSLOG_UDP_RCVBUF:16777216}
|
||||
spring.kafka.producer.acks=all
|
||||
spring.kafka.producer.properties.enable.idempotence=true
|
||||
spring.kafka.producer.properties.linger.ms=5
|
||||
spring.kafka.producer.properties.batch.size=65536
|
||||
spring.kafka.producer.compression-type=lz4
|
||||
spring.kafka.producer.ack-timeout-ms=130000
|
||||
syslog.max.frame.length=262144
|
||||
syslog.buffer.size=1000
|
||||
# syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8
|
||||
syslog.charset=GBK
|
||||
syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a
|
||||
|
||||
# Kafka 接管前的本地持久化 WAL。必须挂载宿主机持久化磁盘，正式链路禁止关闭。
|
||||
# 兼容原 SYSLOG_RAW_ARCHIVE_DIRECTORY 环境变量，但新段使用 syslog-wal-* 前缀，不会自动重放旧 v1 归档。
|
||||
syslog.wal.enabled=${SYSLOG_WAL_ENABLED:true}
|
||||
syslog.wal.directory=${SYSLOG_WAL_DIRECTORY:${SYSLOG_RAW_ARCHIVE_DIRECTORY:/data/syslog-wal}}
|
||||
syslog.wal.queue-capacity=${SYSLOG_WAL_QUEUE_CAPACITY:10000}
|
||||
syslog.wal.high-watermark-percent=${SYSLOG_WAL_HIGH_WATERMARK_PERCENT:70}
|
||||
syslog.wal.low-watermark-percent=${SYSLOG_WAL_LOW_WATERMARK_PERCENT:40}
|
||||
syslog.wal.retry-interval-ms=${SYSLOG_WAL_RETRY_INTERVAL_MS:10}
|
||||
syslog.wal.disk-retry-interval-ms=${SYSLOG_WAL_DISK_RETRY_INTERVAL_MS:1000}
|
||||
syslog.wal.flush-records=${SYSLOG_WAL_FLUSH_RECORDS:200}
|
||||
syslog.wal.flush-interval-ms=${SYSLOG_WAL_FLUSH_INTERVAL_MS:200}
|
||||
syslog.wal.segment-max-bytes=${SYSLOG_WAL_SEGMENT_MAX_BYTES:268435456}
|
||||
syslog.wal.retention-days=${SYSLOG_WAL_RETENTION_DAYS:0}
|
||||
syslog.wal.shutdown-timeout-ms=${SYSLOG_WAL_SHUTDOWN_TIMEOUT_MS:30000}
|
||||
syslog.wal.checkpoint-flush-records=${SYSLOG_WAL_CHECKPOINT_FLUSH_RECORDS:200}
|
||||
syslog.wal.checkpoint-flush-interval-ms=${SYSLOG_WAL_CHECKPOINT_FLUSH_INTERVAL_MS:200}
|
||||
syslog.wal.disk-high-watermark-percent=${SYSLOG_WAL_DISK_HIGH_WATERMARK_PERCENT:80}
|
||||
syslog.wal.disk-critical-watermark-percent=${SYSLOG_WAL_DISK_CRITICAL_WATERMARK_PERCENT:90}
|
||||
syslog.wal.min-free-bytes=${SYSLOG_WAL_MIN_FREE_BYTES:10737418240}
|
||||
# 兼容阶段必须保持 false；全部实例具备 WAL v2/v3 reader 后再切换为 true。
|
||||
syslog.point-time.write-enabled=${SYSLOG_POINT_TIME_WRITE_ENABLED:true}
|
||||
|
||||
# WAL 后台投递 Kafka：内存只保留有限在途消息，失败记录留在 WAL 自动重试。
|
||||
syslog.dispatch.worker-threads=${SYSLOG_DISPATCH_WORKER_THREADS:4}
|
||||
syslog.dispatch.max-inflight=${SYSLOG_DISPATCH_MAX_INFLIGHT:5000}
|
||||
syslog.dispatch.retry-backoff-ms=${SYSLOG_DISPATCH_RETRY_BACKOFF_MS:1000}
|
||||
syslog.dispatch.shutdown-timeout-ms=${SYSLOG_DISPATCH_SHUTDOWN_TIMEOUT_MS:30000}
|
||||
syslog.device-context.cache-ttl-ms=${SYSLOG_DEVICE_CONTEXT_CACHE_TTL_MS:300000}
|
||||
syslog.status-log-interval-ms=${SYSLOG_STATUS_LOG_INTERVAL_MS:60000}
|
||||
|
||||
# 仅用于把旧 v1 归档中明确列出的拒绝 event_id 导入新 WAL；禁止自动导入整个历史归档。
|
||||
syslog.wal.legacy-import.enabled=${SYSLOG_WAL_LEGACY_IMPORT_ENABLED:false}
|
||||
syslog.wal.legacy-import.archive-directory=${SYSLOG_WAL_LEGACY_IMPORT_ARCHIVE_DIRECTORY:}
|
||||
syslog.wal.legacy-import.event-id-file=${SYSLOG_WAL_LEGACY_IMPORT_EVENT_ID_FILE:}
|
||||
|
||||
# APP Service Configuration
|
||||
app.service.device_id=1
|
||||
app.service.device_name=honeypot
|
||||
@@ -37,6 +82,10 @@ spring.kafka.producer.properties.connections.max.idle.ms=600000
|
||||
spring.kafka.producer.properties.socket.keepalive.enable=true
|
||||
spring.kafka.producer.properties.request.timeout.ms=30000
|
||||
spring.kafka.producer.properties.delivery.timeout.ms=120000
|
||||
spring.kafka.producer.properties.max.block.ms=10000
|
||||
spring.kafka.producer.properties.buffer.memory=134217728
|
||||
# event_id 使高流量设备的独立事件均匀分布到多个分区；device_id 仅用于兼容回滚。
|
||||
syslog.kafka.partition-key-strategy=${SYSLOG_KAFKA_PARTITION_KEY_STRATEGY:event_id}
|
||||
|
||||
#database Configuration
|
||||
spring.datasource.url=jdbc:dm://192.168.4.99:5237
|
||||
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package com.haobang.dispatch;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.common.service.DeviceUnknownService;
|
||||
import com.haobang.config.AppProperties;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Collections;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class DeviceContextResolverTest {
|
||||
@Test
|
||||
void repeatedSourceIpQueriesDatabaseOnlyOnceWithinTtl() {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
device.setIp("192.168.4.19");
|
||||
device.setDeviceCollectId(1);
|
||||
when(service.getByIpSafely("192.168.4.19")).thenReturn(Collections.singletonList(device));
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setAppServiceDeviceCollectId(1);
|
||||
properties.setSyslogDeviceContextCacheTtlMs(60000L);
|
||||
DeviceContextResolver resolver = new DeviceContextResolver(
|
||||
service, mock(DeviceUnknownService.class), properties);
|
||||
|
||||
assertTrue(resolver.resolve("192.168.4.19", "TCP").isRegistered());
|
||||
assertTrue(resolver.resolve("192.168.4.19", "TCP").isRegistered());
|
||||
|
||||
verify(service, times(1)).getByIpSafely("192.168.4.19");
|
||||
assertEquals(1L, resolver.getMissCount());
|
||||
assertEquals(1L, resolver.getHitCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deviceAssignedToOtherCollectorIsTerminal() {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
device.setDeviceCollectId(2);
|
||||
when(service.getByIpSafely("192.168.4.19")).thenReturn(Collections.singletonList(device));
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setAppServiceDeviceCollectId(1);
|
||||
properties.setSyslogDeviceContextCacheTtlMs(60000L);
|
||||
|
||||
DeviceContextResolver.Resolution result =
|
||||
new DeviceContextResolver(service, mock(DeviceUnknownService.class), properties)
|
||||
.resolve("192.168.4.19", "TCP");
|
||||
|
||||
assertEquals(DeviceContextResolver.Type.NOT_ASSIGNED, result.getType());
|
||||
}
|
||||
}
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
package com.haobang.dispatch;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.wal.SyslogIngressEvent;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import com.kafka.kafkaProducer;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Files;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class SyslogWalDispatcherTest {
|
||||
@TempDir Path tempDirectory;
|
||||
private SyslogWalService wal;
|
||||
private SyslogWalDispatcher dispatcher;
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
if (dispatcher != null) dispatcher.shutdown();
|
||||
if (wal != null) wal.shutdown();
|
||||
}
|
||||
|
||||
@Test
|
||||
void outOfOrderKafkaAckAdvancesOnlyContiguousCheckpoint() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setSyslogPointTimeWriteEnabled(true);
|
||||
wal = new SyslogWalService(properties, new ObjectMapper());
|
||||
wal.initialize();
|
||||
SyslogIngressEvent firstEvent = wal.prepareForWal(event("20260811-event-1", "first"));
|
||||
SyslogIngressEvent secondEvent = wal.prepareForWal(event("20260811-event-2", "second"));
|
||||
wal.tryAppend(firstEvent);
|
||||
wal.tryAppend(secondEvent);
|
||||
waitUntil(() -> wal.getAppendedCount() == 2L);
|
||||
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
device.setName("device");
|
||||
device.setVendor("vendor");
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
when(resolver.resolve(anyString(), anyString()))
|
||||
.thenReturn(DeviceContextResolver.Resolution.registered(device));
|
||||
|
||||
CompletableFuture<kafkaProducer.DeliveryResult> first = new CompletableFuture<>();
|
||||
CompletableFuture<kafkaProducer.DeliveryResult> second = new CompletableFuture<>();
|
||||
kafkaProducer producer = mock(kafkaProducer.class);
|
||||
when(producer.messagePush(any(DeviceDevice.class), anyString(), anyString(), anyString(), any()))
|
||||
.thenAnswer(invocation -> "20260811-event-1".equals(invocation.getArgument(3)) ? first : second);
|
||||
|
||||
dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
|
||||
dispatcher.start();
|
||||
waitUntil(() -> dispatcher.getInflight() == 2L);
|
||||
|
||||
second.complete(null);
|
||||
Thread.sleep(100L);
|
||||
assertEquals(0L, wal.getCheckpointSequence());
|
||||
|
||||
first.complete(null);
|
||||
waitUntil(() -> wal.getCheckpointSequence() == 2L);
|
||||
assertEquals(2L, dispatcher.getKafkaAcked());
|
||||
assertEquals(0L, dispatcher.getRetryCount());
|
||||
verify(producer).messagePush(any(DeviceDevice.class), eq("first"),
|
||||
eq("20260811200000000"), eq("20260811-event-1"), eq(firstEvent.getPointTimeNs()));
|
||||
verify(producer).messagePush(any(DeviceDevice.class), eq("second"),
|
||||
eq("20260811200000000"), eq("20260811-event-2"), eq(secondEvent.getPointTimeNs()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void terminalDeviceStateIsPersistedBeforeCheckpointAdvances() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
wal = new SyslogWalService(properties, new ObjectMapper());
|
||||
wal.initialize();
|
||||
wal.tryAppend(event("20260811-event-1", "first"));
|
||||
waitUntil(() -> wal.getAppendedCount() == 1L);
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
when(resolver.resolve(anyString(), anyString())).thenReturn(DeviceContextResolver.Resolution.terminal(
|
||||
DeviceContextResolver.Type.UNKNOWN_DEVICE, null, "not registered"));
|
||||
kafkaProducer producer = mock(kafkaProducer.class);
|
||||
|
||||
dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
|
||||
dispatcher.start();
|
||||
|
||||
waitUntil(() -> wal.getCheckpointSequence() == 1L);
|
||||
assertEquals(1L, dispatcher.getTerminalFiltered());
|
||||
assertTrue(Files.list(tempDirectory.resolve("terminal")).findAny().isPresent());
|
||||
verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
private SyslogIngressEvent event(String id, String raw) {
|
||||
return new SyslogIngressEvent(id, LocalDateTime.of(2026, 8, 11, 20, 0),
|
||||
"192.168.4.19", 514, "TCP", raw);
|
||||
}
|
||||
|
||||
private AppProperties properties(Path directory) {
|
||||
AppProperties value = new AppProperties();
|
||||
value.setSyslogWalEnabled(true);
|
||||
value.setSyslogWalDirectory(directory.toString());
|
||||
value.setSyslogWalQueueCapacity(16);
|
||||
value.setSyslogWalHighWatermarkPercent(70);
|
||||
value.setSyslogWalLowWatermarkPercent(40);
|
||||
value.setSyslogWalRetryIntervalMs(5);
|
||||
value.setSyslogWalDiskRetryIntervalMs(50);
|
||||
value.setSyslogWalFlushRecords(1);
|
||||
value.setSyslogWalFlushIntervalMs(5);
|
||||
value.setSyslogWalSegmentMaxBytes(1024 * 1024);
|
||||
value.setSyslogWalRetentionDays(0);
|
||||
value.setSyslogWalShutdownTimeoutMs(5000);
|
||||
value.setSyslogWalCheckpointFlushRecords(1);
|
||||
value.setSyslogWalCheckpointFlushIntervalMs(5);
|
||||
value.setSyslogWalDiskHighWatermarkPercent(98);
|
||||
value.setSyslogWalDiskCriticalWatermarkPercent(99);
|
||||
value.setSyslogWalMinFreeBytes(0);
|
||||
value.setSyslogCharset("UTF-8");
|
||||
value.setSyslogDispatchWorkerThreads(2);
|
||||
value.setSyslogDispatchMaxInflight(8);
|
||||
value.setSyslogDispatchRetryBackoffMs(10);
|
||||
value.setSyslogDispatchShutdownTimeoutMs(2000);
|
||||
value.setSyslogDeviceContextCacheTtlMs(60000);
|
||||
value.setSyslogStatusLogIntervalMs(60000);
|
||||
return value;
|
||||
}
|
||||
|
||||
private void waitUntil(Check condition) throws Exception {
|
||||
long deadline = System.currentTimeMillis() + 5000L;
|
||||
while (!condition.done() && System.currentTimeMillis() < deadline) Thread.sleep(10L);
|
||||
assertTrue(condition.done(), "condition not reached before timeout");
|
||||
}
|
||||
|
||||
private interface Check { boolean done() throws Exception; }
|
||||
}
|
||||
+7
-1
@@ -16,7 +16,13 @@ import java.util.List;
|
||||
/**
|
||||
* 达梦数据库连接测试 — 查询 device_device 表
|
||||
*/
|
||||
@SpringBootTest
|
||||
@SpringBootTest(properties = {
|
||||
"syslog.server.enabled=false",
|
||||
"syslog.wal.directory=${java.io.tmpdir}/syslog-serve-device-test-${random.uuid}",
|
||||
"syslog.wal.disk-high-watermark-percent=98",
|
||||
"syslog.wal.disk-critical-watermark-percent=99",
|
||||
"syslog.wal.min-free-bytes=0"
|
||||
})
|
||||
@ActiveProfiles("dev")
|
||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||
class DeviceDeviceTest {
|
||||
|
||||
+7
-1
@@ -3,7 +3,13 @@ package com.haobang.syslog;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
|
||||
@SpringBootTest
|
||||
@SpringBootTest(properties = {
|
||||
"syslog.server.enabled=false",
|
||||
"syslog.wal.directory=${java.io.tmpdir}/syslog-serve-context-${random.uuid}",
|
||||
"syslog.wal.disk-high-watermark-percent=98",
|
||||
"syslog.wal.disk-critical-watermark-percent=99",
|
||||
"syslog.wal.min-free-bytes=0"
|
||||
})
|
||||
class SysjavacollectApplicationTests {
|
||||
|
||||
@Test
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class PointTimeAllocatorTest {
|
||||
@Test void millionSuffixesAreUniqueAndCycleInsideReceiveMillisecond(){PointTimeAllocator a=new PointTimeAllocator();long ms=1_786_686_912_123L;Set<Long>s=new HashSet<>(1_333_334);for(int i=0;i<1_000_000;i++){long v=a.allocate(ms);assertEquals(ms,Math.floorDiv(v,1_000_000L));assertTrue(s.add(Math.floorMod(v,1_000_000L)));}assertEquals(0,Math.floorMod(a.allocate(ms),1_000_000L));}
|
||||
@Test void concurrentCallsAreAtomic()throws Exception{PointTimeAllocator a=new PointTimeAllocator();Set<Long>s=ConcurrentHashMap.newKeySet();ExecutorService p=Executors.newFixedThreadPool(8);for(int i=0;i<100_000;i++)p.submit(()->s.add(a.allocate(1_786_686_912_123L)));p.shutdown();assertTrue(p.awaitTermination(20,TimeUnit.SECONDS));assertEquals(100_000,s.size());}
|
||||
@Test void arithmeticOverflowFailsExplicitly(){assertThrows(ArithmeticException.class,()->new PointTimeAllocator().allocate(Long.MAX_VALUE));}
|
||||
}
|
||||
+315
@@ -0,0 +1,315 @@
|
||||
package com.haobang.wal;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.config.CacheConfig;
|
||||
import com.haobang.config.AppProperties;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class SyslogWalServiceTest {
|
||||
@TempDir Path tempDirectory;
|
||||
private SyslogWalService service;
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
if (service != null) service.shutdown();
|
||||
}
|
||||
|
||||
@Test
|
||||
void forceBeforeCursorAndPersistSchemaV2() throws Exception {
|
||||
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper());
|
||||
service.initialize();
|
||||
|
||||
SyslogIngressEvent event = event("20260811-event-1", "first");
|
||||
assertEquals(SyslogWalService.AdmissionResult.ACCEPTED, service.tryAppend(event));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord record = cursor.next(1000L);
|
||||
assertNotNull(record);
|
||||
assertEquals(1L, record.getSequence());
|
||||
assertEquals(event.getEventId(), record.getEventId());
|
||||
assertEquals(event.getRawMessage(), record.getRawMessage());
|
||||
assertTrue(record.getEndOffset() > record.getStartOffset());
|
||||
}
|
||||
|
||||
Path segment = Files.list(tempDirectory.resolve("active"))
|
||||
.filter(path -> path.getFileName().toString().startsWith("syslog-wal-"))
|
||||
.findFirst().orElseThrow(AssertionError::new);
|
||||
List<String> lines = Files.readAllLines(segment, StandardCharsets.UTF_8);
|
||||
JsonNode json = new ObjectMapper().readTree(lines.get(0));
|
||||
assertEquals(2, json.path("schema_version").asInt());
|
||||
assertEquals("20260811-event-1", json.path("event_id").asText());
|
||||
assertEquals(64, json.path("raw_sha256").asText().length());
|
||||
}
|
||||
|
||||
@Test
|
||||
void enabledPointTimeWritesSchemaV3AndRestoresSameIdentity() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setSyslogPointTimeWriteEnabled(true);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
SyslogIngressEvent prepared = service.prepareForWal(event("20260811-v3", "v3"));
|
||||
assertNotNull(prepared.getPointTimeNs());
|
||||
service.tryAppend(prepared);
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord restored = cursor.next(1000L);
|
||||
assertEquals(3, restored.getSchemaVersion());
|
||||
assertEquals(prepared.getPointTimeNs(), restored.getPointTimeNs());
|
||||
assertEquals(SyslogWalRecordValidator.receiveTimeMillis(restored.getReceiveTime()),
|
||||
Math.floorDiv(restored.getPointTimeNs(), PointTimeAllocator.NANOS_PER_MILLI));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void mixedV2V3RestartScanAndRecoveryRemainCompatible() throws Exception {
|
||||
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
|
||||
SyslogWalRecord v2 = walRecord(1L, "20260811-v2", "old");
|
||||
SyslogWalRecord v3 = walRecord(2L, "20260811-v3", "new"); v3.setSchemaVersion(3);
|
||||
v3.setPointTimeNs(Math.multiplyExact(SyslogWalRecordValidator.receiveTimeMillis(v3.getReceiveTime()), 1_000_000L));
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"),
|
||||
(mapper.writeValueAsString(v2)+"\n"+mapper.writeValueAsString(v3)+"\n").getBytes(StandardCharsets.UTF_8));
|
||||
service = new SyslogWalService(properties(tempDirectory), mapper); service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
assertEquals(2, cursor.next(1000L).getSchemaVersion());
|
||||
SyslogWalRecord restoredV3 = cursor.next(1000L); assertEquals(3, restoredV3.getSchemaVersion()); assertEquals(v3.getPointTimeNs(), restoredV3.getPointTimeNs());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void damagedV3IsQuarantinedAndNeverFallsBackAsV2() throws Exception {
|
||||
Path active = tempDirectory.resolve("active"); Files.createDirectories(active);
|
||||
String invalid = "{\"schema_version\":3,\"sequence\":1,\"event_id\":\"20260811-bad\","
|
||||
+ "\"receive_time\":\"2026-08-11T20:00:00.123\",\"point_time_ns\":-1}\n";
|
||||
Files.write(active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl"), invalid.getBytes(StandardCharsets.UTF_8));
|
||||
service = new SyslogWalService(properties(tempDirectory), new ObjectMapper()); service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord tombstone = cursor.next(1000L); assertTrue(tombstone.isQuarantinedInvalid()); assertEquals(3,tombstone.getSchemaVersion());
|
||||
}
|
||||
assertTrue(Files.list(tempDirectory.resolve("quarantine")).findAny().isPresent());
|
||||
assertEquals(1L, service.getQuarantinedCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatorRejectsEveryIllegalV3PointTimeShape() throws Exception {
|
||||
ObjectMapper mapper=new ObjectMapper();
|
||||
String prefix="{\"schema_version\":3,\"receive_time\":\"2026-08-11T20:00:00.123\"";
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+"}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":\"x\"}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":-1}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree(prefix+",\"point_time_ns\":1786636800124000000}")));
|
||||
assertThrows(IllegalArgumentException.class,()->SyslogWalRecordValidator.validateNode(mapper.readTree("{\"schema_version\":4}")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void productionRedisMapperIsSanitizedBeforeWalReadAndWrite() throws Exception {
|
||||
/*
|
||||
* 生产环境注入的是 CacheConfig 的 ObjectMapper，而不是测试常用的 new ObjectMapper()。
|
||||
* 这个回归测试必须使用真实配置，否则无法复现现场“写入 @class、读取又被白名单拒绝”的故障。
|
||||
*/
|
||||
ObjectMapper productionMapper = new CacheConfig().objectMapper();
|
||||
service = new SyslogWalService(properties(tempDirectory), productionMapper);
|
||||
service.initialize();
|
||||
service.tryAppend(event("20260811-production-mapper", "new-format"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
|
||||
Path segment = Files.list(tempDirectory.resolve("active"))
|
||||
.findFirst().orElseThrow(AssertionError::new);
|
||||
JsonNode persisted = new ObjectMapper().readTree(Files.readAllLines(
|
||||
segment, StandardCharsets.UTF_8).get(0));
|
||||
assertFalse(persisted.has("@class"), "WAL must never persist Java class metadata");
|
||||
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord restored = cursor.next(1000L);
|
||||
assertNotNull(restored);
|
||||
assertEquals("20260811-production-mapper", restored.getEventId());
|
||||
assertEquals("new-format", restored.getRawMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void repairedReaderAcceptsExistingSchemaV2RecordWithLegacyClassMetadata() throws Exception {
|
||||
Path active = tempDirectory.resolve("active");
|
||||
Files.createDirectories(active);
|
||||
Path segment = active.resolve("syslog-wal-00000000000000000001-20260811-233232601.jsonl");
|
||||
|
||||
// 使用发生现场故障的生产 Mapper 生成旧格式，确保样本确实包含 @class。
|
||||
ObjectMapper productionMapper = new CacheConfig().objectMapper();
|
||||
SyslogWalRecord legacy = walRecord(1L, "20260811-legacy-class", "existing-wal");
|
||||
String legacyJson = productionMapper.writeValueAsString(legacy);
|
||||
assertTrue(legacyJson.contains("\"@class\":\"com.haobang.wal.SyslogWalRecord\""));
|
||||
Files.write(segment, (legacyJson + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
service = new SyslogWalService(properties(tempDirectory), productionMapper);
|
||||
service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord restored = cursor.next(1000L);
|
||||
assertNotNull(restored);
|
||||
assertEquals(1L, restored.getSequence());
|
||||
assertEquals("20260811-legacy-class", restored.getEventId());
|
||||
assertEquals("existing-wal", restored.getRawMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void restartResumesAfterPersistedCheckpointWithoutChangingEventIdentity() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
service.tryAppend(event("20260811-event-1", "first"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord first = cursor.next(1000L);
|
||||
service.markCheckpoint(first);
|
||||
waitUntil(() -> service.getCheckpointSequence() == 1L);
|
||||
}
|
||||
service.shutdown();
|
||||
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
assertEquals(1L, service.getCheckpointSequence());
|
||||
service.tryAppend(event("20260811-event-2", "second"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord second = cursor.next(1000L);
|
||||
assertNotNull(second);
|
||||
assertEquals(2L, second.getSequence());
|
||||
assertEquals("20260811-event-2", second.getEventId());
|
||||
assertEquals("2026-08-11T20:00:00.123", second.getReceiveTime());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void restartTruncatesIncompleteTailAndKeepsCompleteRecord() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
service.tryAppend(event("20260811-event-1", "first"));
|
||||
waitUntil(() -> service.getAppendedCount() == 1L);
|
||||
service.shutdown();
|
||||
|
||||
Path segment = Files.list(tempDirectory.resolve("active")).findFirst().orElseThrow(AssertionError::new);
|
||||
Files.write(segment, "{\"schema_version\":2".getBytes(StandardCharsets.UTF_8),
|
||||
java.nio.file.StandardOpenOption.APPEND);
|
||||
service = new SyslogWalService(properties, new ObjectMapper());
|
||||
service.initialize();
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord record = cursor.next(1000L);
|
||||
assertNotNull(record);
|
||||
assertEquals("20260811-event-1", record.getEventId());
|
||||
}
|
||||
byte[] bytes = Files.readAllBytes(segment);
|
||||
assertEquals((byte) '\n', bytes[bytes.length - 1]);
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacyImporterImportsOnlyRequestedEventIdAndPreservesIdentity() throws Exception {
|
||||
Path archive = tempDirectory.resolve("legacy");
|
||||
Files.createDirectories(archive);
|
||||
String requestedId = "20260811-a3d874a0-6f50-4f0c-9e7a-9c20303360f5";
|
||||
String ignoredId = "20260811-53df7faa-7f78-4ce1-8eac-475965d9eeed";
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
Files.write(archive.resolve("syslog-raw-20260811-test.jsonl"), (
|
||||
legacyLine(mapper, requestedId, "requested") + "\n"
|
||||
+ legacyLine(mapper, ignoredId, "ignored") + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
Path ids = tempDirectory.resolve("rejected.txt");
|
||||
Files.write(ids, ("error eventId=" + requestedId).getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
AppProperties properties = properties(tempDirectory.resolve("wal"));
|
||||
properties.setSyslogWalLegacyImportEnabled(true);
|
||||
properties.setSyslogWalLegacyImportArchiveDirectory(archive.toString());
|
||||
properties.setSyslogWalLegacyImportEventIdFile(ids.toString());
|
||||
service = new SyslogWalService(properties, mapper);
|
||||
service.initialize();
|
||||
new SyslogLegacyArchiveImporter(properties, mapper, service).importIfRequested();
|
||||
|
||||
try (SyslogWalService.WalCursor cursor = service.openCursor()) {
|
||||
SyslogWalRecord imported = cursor.next(1000L);
|
||||
assertNotNull(imported);
|
||||
assertEquals(requestedId, imported.getEventId());
|
||||
assertEquals("2026-08-11T03:55:57.109", imported.getReceiveTime());
|
||||
assertEquals("requested", imported.getRawMessage());
|
||||
assertEquals(null, cursor.next(50L));
|
||||
}
|
||||
}
|
||||
|
||||
private String legacyLine(ObjectMapper mapper, String eventId, String raw) throws Exception {
|
||||
java.util.Map<String, Object> value = new java.util.LinkedHashMap<>();
|
||||
value.put("schema_version", 1);
|
||||
value.put("event_id", eventId);
|
||||
value.put("receive_time", "2026-08-11T03:55:57.109");
|
||||
value.put("source_ip", "192.168.4.19");
|
||||
value.put("source_port", 49768);
|
||||
value.put("protocol", "TCP");
|
||||
value.put("raw_message", raw);
|
||||
return mapper.writeValueAsString(value);
|
||||
}
|
||||
|
||||
private SyslogIngressEvent event(String eventId, String raw) {
|
||||
return new SyslogIngressEvent(eventId,
|
||||
LocalDateTime.of(2026, 8, 11, 20, 0, 0, 123_000_000),
|
||||
"192.168.4.19", 514, "TCP", raw);
|
||||
}
|
||||
|
||||
private SyslogWalRecord walRecord(long sequence, String eventId, String raw) {
|
||||
SyslogWalRecord record = new SyslogWalRecord();
|
||||
record.setSequence(sequence);
|
||||
record.setEventId(eventId);
|
||||
record.setReceiveTime("2026-08-11T23:32:32.531");
|
||||
record.setSourceIp("192.168.4.19");
|
||||
record.setSourcePort(45324);
|
||||
record.setProtocol("TCP");
|
||||
record.setCharset("UTF-8");
|
||||
record.setRawLength(raw.getBytes(StandardCharsets.UTF_8).length);
|
||||
record.setRawSha256("684b334d5690440f210e5778d304f778b5eaff5b30477667029968a711604400");
|
||||
record.setRawMessage(raw);
|
||||
return record;
|
||||
}
|
||||
|
||||
private AppProperties properties(Path directory) {
|
||||
AppProperties value = new AppProperties();
|
||||
value.setSyslogWalEnabled(true);
|
||||
value.setSyslogWalDirectory(directory.toString());
|
||||
value.setSyslogWalQueueCapacity(16);
|
||||
value.setSyslogWalHighWatermarkPercent(70);
|
||||
value.setSyslogWalLowWatermarkPercent(40);
|
||||
value.setSyslogWalRetryIntervalMs(5);
|
||||
value.setSyslogWalDiskRetryIntervalMs(50);
|
||||
value.setSyslogWalFlushRecords(1);
|
||||
value.setSyslogWalFlushIntervalMs(5);
|
||||
value.setSyslogWalSegmentMaxBytes(1024 * 1024);
|
||||
value.setSyslogWalRetentionDays(0);
|
||||
value.setSyslogWalShutdownTimeoutMs(5000);
|
||||
value.setSyslogWalCheckpointFlushRecords(1);
|
||||
value.setSyslogWalCheckpointFlushIntervalMs(5);
|
||||
value.setSyslogWalDiskHighWatermarkPercent(98);
|
||||
value.setSyslogWalDiskCriticalWatermarkPercent(99);
|
||||
value.setSyslogWalMinFreeBytes(0);
|
||||
value.setSyslogCharset("UTF-8");
|
||||
return value;
|
||||
}
|
||||
|
||||
private void waitUntil(Check condition) throws Exception {
|
||||
long deadline = System.currentTimeMillis() + 5000L;
|
||||
while (!condition.done() && System.currentTimeMillis() < deadline) Thread.sleep(10L);
|
||||
assertTrue(condition.done(), "condition not reached before timeout");
|
||||
}
|
||||
|
||||
private interface Check { boolean done() throws Exception; }
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
package com.kafka;
|
||||
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.haobang.util.DeviceInfoUtil;
|
||||
import org.apache.kafka.clients.producer.Producer;
|
||||
import org.apache.kafka.clients.producer.ProducerRecord;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
class KafkaProducerEventIdTest {
|
||||
|
||||
@Test
|
||||
void eventIdDateUsesShanghaiTimeInsteadOfJvmDefaultTimeZone() {
|
||||
Instant shanghaiMidnight = Instant.parse("2026-08-10T16:00:00Z");
|
||||
|
||||
String eventId = kafkaProducer.newEventId(shanghaiMidnight);
|
||||
|
||||
assertTrue(eventId.matches("20260811-[0-9a-f-]{36}"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void eventIdStrategyUsesTheSameEventIdAsProducerRecordKey() {
|
||||
kafkaProducer producer = producer("event_id");
|
||||
String eventId = "20260811-a3d874a0-6f50-4f0c-9e7a-9c20303360f5";
|
||||
|
||||
ProducerRecord<String, String> record = producer.buildRecord(126, eventId, "ciphertext");
|
||||
|
||||
assertEquals("agent-01-syslog-topic", record.topic());
|
||||
assertEquals(eventId, record.key());
|
||||
assertEquals("ciphertext", record.value());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deviceIdStrategyRemainsAvailableForRollback() {
|
||||
ProducerRecord<String, String> record =
|
||||
producer("device_id").buildRecord(126, "20260811-event", "ciphertext");
|
||||
|
||||
assertEquals("126", record.key());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidPartitionKeyStrategyFailsFast() {
|
||||
assertThrows(IllegalArgumentException.class, () -> producer("random"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void kafkaBusinessHeaderAddsOnlyPointTimeAndKeepsEventIdName() {
|
||||
DeviceDevice device = new DeviceDevice(); device.setId(126); device.setName("device"); device.setVendor("vendor");
|
||||
AppProperties properties = new AppProperties(); properties.setAppServiceDataType("json"); properties.setAppServiceDeviceCollectId(1);
|
||||
String payload = DeviceInfoUtil.getFullLogString(device, "raw", "20260814000000000",
|
||||
"20260814-event", 1_786_636_800_000_000_123L, properties);
|
||||
assertTrue(payload.contains("event_id=20260814-event"));
|
||||
assertTrue(payload.contains("point_time_ns=1786636800000000123"));
|
||||
assertTrue(payload.contains("receive_time=20260814000000000"));
|
||||
assertFalse(payload.contains("schema_version"));
|
||||
assertFalse(payload.contains("event_id_ns"));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private kafkaProducer producer(String strategy) {
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setKafkaProducerTopic("agent-01-syslog-topic");
|
||||
return new kafkaProducer(properties, mock(Producer.class), strategy);
|
||||
}
|
||||
}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.wal.SyslogIngressEvent;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.embedded.EmbeddedChannel;
|
||||
import io.netty.channel.socket.DatagramPacket;
|
||||
import io.netty.handler.codec.DelimiterBasedFrameDecoder;
|
||||
import io.netty.handler.codec.Delimiters;
|
||||
import io.netty.handler.codec.string.StringDecoder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class SyslogCharsetDecodingTest {
|
||||
private static final String CHINESE_MESSAGE =
|
||||
"<134>发现TCP传输层一句话小马行为 alarm_type=文件上传";
|
||||
|
||||
@Test
|
||||
void tcpUsesConfiguredUtf8CharsetBeforeAppendingToWal() {
|
||||
SyslogWalService wal = acceptingWal();
|
||||
EmbeddedChannel channel = new EmbeddedChannel(
|
||||
new DelimiterBasedFrameDecoder(4096, Delimiters.lineDelimiter()),
|
||||
new StringDecoder(StandardCharsets.UTF_8),
|
||||
new SyslogMessageHandler(StandardCharsets.UTF_8, wal));
|
||||
channel.connect(new InetSocketAddress("127.0.0.1", 5514));
|
||||
|
||||
channel.writeInbound(Unpooled.wrappedBuffer(
|
||||
(CHINESE_MESSAGE + "\n").getBytes(StandardCharsets.UTF_8)));
|
||||
|
||||
assertWalReceivedOriginalChinese(wal);
|
||||
channel.finishAndReleaseAll();
|
||||
}
|
||||
|
||||
@Test
|
||||
void udpUsesConfiguredUtf8CharsetBeforeAppendingToWal() {
|
||||
SyslogWalService wal = acceptingWal();
|
||||
EmbeddedChannel channel = new EmbeddedChannel(
|
||||
new SyslogMessageHandler(StandardCharsets.UTF_8, wal));
|
||||
DatagramPacket packet = new DatagramPacket(
|
||||
Unpooled.wrappedBuffer(CHINESE_MESSAGE.getBytes(StandardCharsets.UTF_8)),
|
||||
new InetSocketAddress("127.0.0.1", 514),
|
||||
new InetSocketAddress("192.168.4.19", 5514));
|
||||
|
||||
channel.writeInbound(packet);
|
||||
|
||||
assertWalReceivedOriginalChinese(wal);
|
||||
channel.finishAndReleaseAll();
|
||||
}
|
||||
|
||||
private SyslogWalService acceptingWal() {
|
||||
SyslogWalService wal = mock(SyslogWalService.class);
|
||||
when(wal.prepareForWal(any())).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
when(wal.tryAppend(any())).thenReturn(SyslogWalService.AdmissionResult.ACCEPTED);
|
||||
return wal;
|
||||
}
|
||||
|
||||
private void assertWalReceivedOriginalChinese(SyslogWalService wal) {
|
||||
ArgumentCaptor<SyslogIngressEvent> event = ArgumentCaptor.forClass(SyslogIngressEvent.class);
|
||||
verify(wal).tryAppend(event.capture());
|
||||
assertEquals(CHINESE_MESSAGE, event.getValue().getRawMessage());
|
||||
}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import io.netty.channel.embedded.EmbeddedChannel;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class SyslogMessageHandlerBackpressureTest {
|
||||
@Test
|
||||
void tcpWalBackpressureDisablesAutoReadInsteadOfDroppingCurrentFrame() {
|
||||
SyslogWalService wal = mock(SyslogWalService.class);
|
||||
when(wal.prepareForWal(any())).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
when(wal.tryAppend(any())).thenReturn(SyslogWalService.AdmissionResult.BACKPRESSURE);
|
||||
when(wal.getRetryIntervalMs()).thenReturn(60_000L);
|
||||
when(wal.getQueueCapacity()).thenReturn(10_000);
|
||||
EmbeddedChannel channel = new EmbeddedChannel(new SyslogMessageHandler(StandardCharsets.UTF_8, wal));
|
||||
channel.connect(new InetSocketAddress("127.0.0.1", 5514));
|
||||
|
||||
channel.writeInbound("<134>Aug 11 20:00:00 device event");
|
||||
|
||||
assertFalse(channel.config().isAutoRead());
|
||||
channel.close();
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.config.AppProperties;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class SyslogProfileCharsetConfigurationTest {
|
||||
@Test
|
||||
void prodProfileResolvesSyslogCharsetThroughSpringEnvironment() {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(TestConfiguration.class)
|
||||
.web(WebApplicationType.NONE)
|
||||
.profiles("prod")
|
||||
.properties("spring.main.banner-mode=off", "logging.level.root=OFF")
|
||||
.run()) {
|
||||
assertEquals("UTF-8", context.getBean(AppProperties.class).getSyslogCharset());
|
||||
}
|
||||
}
|
||||
|
||||
@Configuration
|
||||
@Import(AppProperties.class)
|
||||
static class TestConfiguration {
|
||||
}
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package com.netty;
|
||||
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
class SyslogServerLifecycleCharsetTest {
|
||||
@Test
|
||||
void invalidCharsetStopsWalConstructionBeforeOpeningSyslogPorts() {
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setSyslogCharset("NOT-A-REAL-CHARSET");
|
||||
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> new SyslogWalService(properties, new ObjectMapper()));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user