feat(syslog): 为三个生产模块增加启动版本标识
- syslog-serve 增加 SYSLOG_BUILD_READY 启动日志 - syslog-consumer 增加 SYSLOG_BUILD_READY 启动日志 - 更新 syslog-consumer-rule 代码修订标识至 v63 - 输出模块名、代码修订号、运行环境及关键非敏感配置 - 支持通过容器日志快速确认实际部署的 JAR 版本 - 补充 syslog-serve 和 syslog-consumer 启动日志测试 - 不输出数据库密码、API 凭据等敏感信息 - 不修改 Maven POM 配置
This commit is contained in:
+85
-12
@@ -1,26 +1,42 @@
|
||||
package com.common.controller;
|
||||
|
||||
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import com.fasterxml.jackson.annotation.JsonAnySetter;
|
||||
import com.haobang.dispatch.DeviceContextResolver;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import javax.validation.Valid;
|
||||
import javax.validation.constraints.NotNull;
|
||||
import java.util.List;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/devices")
|
||||
public class DeviceDeviceController {
|
||||
|
||||
@Autowired
|
||||
private DeviceDeviceService deviceDeviceService;
|
||||
private final DeviceDeviceService deviceDeviceService;
|
||||
private final DeviceContextResolver deviceContextResolver;
|
||||
|
||||
public DeviceDeviceController(DeviceDeviceService deviceDeviceService,
|
||||
DeviceContextResolver deviceContextResolver) {
|
||||
this.deviceDeviceService = deviceDeviceService;
|
||||
this.deviceContextResolver = deviceContextResolver;
|
||||
}
|
||||
|
||||
@GetMapping("/{id}")
|
||||
public DeviceDevice getById(@PathVariable Integer id) {
|
||||
System.out.println("device_id:"+id.toString());
|
||||
DeviceDevice deviceDevice=deviceDeviceService.getById(id);
|
||||
System.out.println("deviceDeviceService:"+deviceDevice.toString());
|
||||
return deviceDeviceService.getById(id);
|
||||
DeviceDevice device = deviceDeviceService.getById(id);
|
||||
if (device == null || device.getDeletedAt() != null
|
||||
|| (device.getDelFlag() != null && !"0".equals(device.getDelFlag()))) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "设备不存在或已删除");
|
||||
}
|
||||
return device;
|
||||
}
|
||||
|
||||
@GetMapping
|
||||
@@ -35,10 +51,6 @@ public class DeviceDeviceController {
|
||||
|
||||
@GetMapping("/ip/{ip}")
|
||||
public List<DeviceDevice> getByIp(@PathVariable String ip) {
|
||||
|
||||
List<DeviceDevice> deviceDevicelist =deviceDeviceService.getByIpSafely(ip);
|
||||
System.out.println("deviceDeviceService:"+deviceDevicelist.get(0).toString());
|
||||
|
||||
return deviceDeviceService.getByIpSafely(ip);
|
||||
}
|
||||
|
||||
@@ -46,4 +58,65 @@ public class DeviceDeviceController {
|
||||
public List<DeviceDevice> getMonitoringDevices() {
|
||||
return deviceDeviceService.getMonitoringDevices();
|
||||
}
|
||||
}
|
||||
|
||||
/** 清除所有设备的二级缓存、进程内解析缓存,并立即重载 Syslog 入站快照。 */
|
||||
@DeleteMapping("/cache")
|
||||
public Map<String, Object> evictAllDeviceCaches() {
|
||||
deviceDeviceService.evictAllDeviceCaches();
|
||||
deviceContextResolver.invalidateAll();
|
||||
deviceContextResolver.refreshIngressSnapshot();
|
||||
return cacheResponse("ALL", null, null);
|
||||
}
|
||||
|
||||
/** 只清除指定设备的 ID/IP 二级缓存及其进程内解析缓存。 */
|
||||
@DeleteMapping("/{id}/cache")
|
||||
public Map<String, Object> evictDeviceCaches(@PathVariable Integer id) {
|
||||
DeviceDevice device = deviceDeviceService.evictDeviceCaches(id);
|
||||
if (device == null) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "设备不存在");
|
||||
}
|
||||
deviceContextResolver.invalidateDevice(id, device.getIp());
|
||||
deviceContextResolver.refreshIngressSnapshot();
|
||||
return cacheResponse("DEVICE", id, device.getIp());
|
||||
}
|
||||
|
||||
/** 仅允许修改设备级采集策略,避免把完整设备实体直接绑定到更新接口。 */
|
||||
@PutMapping("/{id}/collection-mode")
|
||||
public DeviceDevice updateCollectionMode(@PathVariable Integer id,
|
||||
@Valid @RequestBody CollectionModeRequest request) {
|
||||
DeviceDevice updated = deviceDeviceService.updateCollectionMode(id, request.getCollectionMode());
|
||||
if (updated == null) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "设备不存在或已删除");
|
||||
}
|
||||
deviceContextResolver.invalidateAll();
|
||||
deviceContextResolver.refreshIngressSnapshot();
|
||||
return updated;
|
||||
}
|
||||
|
||||
public static final class CollectionModeRequest {
|
||||
@NotNull(message = "collectionMode 不能为空")
|
||||
private DeviceCollectionMode collectionMode;
|
||||
|
||||
public DeviceCollectionMode getCollectionMode() {
|
||||
return collectionMode;
|
||||
}
|
||||
|
||||
public void setCollectionMode(DeviceCollectionMode collectionMode) {
|
||||
this.collectionMode = collectionMode;
|
||||
}
|
||||
|
||||
@JsonAnySetter
|
||||
public void rejectUnknownField(String field, Object ignored) {
|
||||
throw new IllegalArgumentException("不允许修改字段: " + field);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> cacheResponse(String scope, Integer deviceId, String ip) {
|
||||
Map<String, Object> response = new LinkedHashMap<>();
|
||||
response.put("cleared", true);
|
||||
response.put("scope", scope);
|
||||
if (deviceId != null) response.put("deviceId", deviceId);
|
||||
if (ip != null) response.put("ip", ip);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.common.entity;
|
||||
|
||||
/** 设备级采集策略。任务 method 描述单个任务,本枚举决定设备允许的采集方向。 */
|
||||
public enum DeviceCollectionMode {
|
||||
SYSLOG_ONLY(true, false),
|
||||
API_ONLY(false, true),
|
||||
BOTH(true, true);
|
||||
|
||||
private final boolean syslogAllowed;
|
||||
private final boolean apiAllowed;
|
||||
|
||||
DeviceCollectionMode(boolean syslogAllowed, boolean apiAllowed) {
|
||||
this.syslogAllowed = syslogAllowed;
|
||||
this.apiAllowed = apiAllowed;
|
||||
}
|
||||
|
||||
public boolean allowsSyslog() {
|
||||
return syslogAllowed;
|
||||
}
|
||||
|
||||
public boolean allowsApi() {
|
||||
return apiAllowed;
|
||||
}
|
||||
|
||||
/** 兼容升级前写入 Redis 的设备对象;缺失字段必须保持原有双向采集能力。 */
|
||||
public static DeviceCollectionMode effective(DeviceCollectionMode value) {
|
||||
return value == null ? BOTH : value;
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,7 @@ public class DeviceDevice {
|
||||
public Integer todayNonLogCount;
|
||||
public Long createDept;
|
||||
public Integer deviceCollectId;
|
||||
public DeviceCollectionMode collectionMode;
|
||||
|
||||
|
||||
// Getter and Setter methods
|
||||
@@ -331,6 +332,14 @@ public class DeviceDevice {
|
||||
this.deviceCollectId = deviceCollectId;
|
||||
}
|
||||
|
||||
public DeviceCollectionMode getCollectionMode() {
|
||||
return DeviceCollectionMode.effective(collectionMode);
|
||||
}
|
||||
|
||||
public void setCollectionMode(DeviceCollectionMode collectionMode) {
|
||||
this.collectionMode = collectionMode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "DeviceDevice{" +
|
||||
@@ -370,7 +379,8 @@ public class DeviceDevice {
|
||||
", todayNonLogCount=" + todayNonLogCount +
|
||||
", createDept=" + createDept +
|
||||
", deviceCollectId=" + deviceCollectId +
|
||||
", collectionMode=" + getCollectionMode() +
|
||||
'}';
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+6
-1
@@ -2,6 +2,7 @@ package com.common.mapper;
|
||||
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
@@ -90,4 +91,8 @@ public interface DeviceDeviceMapper {
|
||||
* 根据厂商查询设备
|
||||
*/
|
||||
List<DeviceDevice> selectByVendor(String vendor);
|
||||
}
|
||||
|
||||
/** 只更新单台未删除设备的采集模式,禁止管理接口批量赋值其他字段。 */
|
||||
int updateCollectionMode(@Param("id") Integer id,
|
||||
@Param("collectionMode") DeviceCollectionMode collectionMode);
|
||||
}
|
||||
|
||||
+11
-1
@@ -1,6 +1,7 @@
|
||||
package com.common.service;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@@ -28,4 +29,13 @@ public interface DeviceDeviceService {
|
||||
List<DeviceDevice> getMonitoringDevices();
|
||||
|
||||
List<DeviceDevice> getActiveDevices();
|
||||
}
|
||||
|
||||
/** 返回更新后的设备;设备不存在或已删除时返回 null。 */
|
||||
DeviceDevice updateCollectionMode(Integer id, DeviceCollectionMode collectionMode);
|
||||
|
||||
/** 清除指定设备的 Spring Cache 和 SafeCacheUtil 原始 Redis Key;不存在时返回 null。 */
|
||||
DeviceDevice evictDeviceCaches(Integer id);
|
||||
|
||||
/** 清除全部设备的 Spring Cache 和 SafeCacheUtil 原始 Redis Key。 */
|
||||
void evictAllDeviceCaches();
|
||||
}
|
||||
|
||||
+70
-7
@@ -2,27 +2,35 @@ package com.common.service.impl;
|
||||
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import com.common.mapper.DeviceDeviceMapper;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.springframework.cache.annotation.CacheConfig;
|
||||
import org.springframework.cache.annotation.Cacheable;
|
||||
import org.springframework.cache.Cache;
|
||||
import org.springframework.cache.CacheManager;
|
||||
import com.haobang.util.SafeCacheUtil;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@CacheConfig(cacheNames = "device")
|
||||
@Service
|
||||
public class DeviceDeviceServiceImpl implements DeviceDeviceService {
|
||||
|
||||
@Autowired
|
||||
private DeviceDeviceMapper deviceDeviceMapper;
|
||||
private final DeviceDeviceMapper deviceDeviceMapper;
|
||||
private final SafeCacheUtil safeCacheUtil;
|
||||
private final CacheManager cacheManager;
|
||||
|
||||
|
||||
@Autowired
|
||||
private SafeCacheUtil safeCacheUtil;
|
||||
public DeviceDeviceServiceImpl(DeviceDeviceMapper deviceDeviceMapper,
|
||||
SafeCacheUtil safeCacheUtil,
|
||||
CacheManager cacheManager) {
|
||||
this.deviceDeviceMapper = deviceDeviceMapper;
|
||||
this.safeCacheUtil = safeCacheUtil;
|
||||
this.cacheManager = cacheManager;
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用安全的缓存方法
|
||||
@@ -100,4 +108,59 @@ public class DeviceDeviceServiceImpl implements DeviceDeviceService {
|
||||
public List<DeviceDevice> getActiveDevices() {
|
||||
return deviceDeviceMapper.selectActiveDevices();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public DeviceDevice updateCollectionMode(Integer id, DeviceCollectionMode collectionMode) {
|
||||
if (id == null || collectionMode == null) {
|
||||
throw new IllegalArgumentException("设备 ID 和采集模式不能为空");
|
||||
}
|
||||
DeviceDevice current = deviceDeviceMapper.selectById(id);
|
||||
if (current == null || current.getDeletedAt() != null
|
||||
|| (current.getDelFlag() != null && !"0".equals(current.getDelFlag()))) {
|
||||
return null;
|
||||
}
|
||||
if (deviceDeviceMapper.updateCollectionMode(id, collectionMode) != 1) return null;
|
||||
evictDeviceCacheEntries(id, current.getIp());
|
||||
// 采集模式更新属于策略变更,继续保持全量清空 Spring device Cache 的强一致行为。
|
||||
Cache springDeviceCache = cacheManager.getCache("device");
|
||||
if (springDeviceCache != null) springDeviceCache.clear();
|
||||
return deviceDeviceMapper.selectById(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DeviceDevice evictDeviceCaches(Integer id) {
|
||||
if (id == null) throw new IllegalArgumentException("设备 ID 不能为空");
|
||||
// 必须绕过二级缓存读取当前 IP,否则可能根据陈旧缓存删除错误的 IP Key。
|
||||
DeviceDevice device = deviceDeviceMapper.selectById(id);
|
||||
if (device == null) {
|
||||
// 即使数据库已无记录,也删除仍可能残留的 ID Key。
|
||||
safeCacheUtil.evict("device:id:" + id);
|
||||
Cache springDeviceCache = cacheManager.getCache("device");
|
||||
if (springDeviceCache != null) springDeviceCache.evict("device:id:" + id);
|
||||
return null;
|
||||
}
|
||||
evictDeviceCacheEntries(id, device.getIp());
|
||||
return device;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void evictAllDeviceCaches() {
|
||||
safeCacheUtil.evictMatching("device:id:*", "device:ip:*");
|
||||
Cache springDeviceCache = cacheManager.getCache("device");
|
||||
if (springDeviceCache != null) springDeviceCache.clear();
|
||||
}
|
||||
|
||||
private void evictDeviceCacheEntries(Integer id, String ip) {
|
||||
String idKey = "device:id:" + id;
|
||||
if (ip == null || ip.trim().isEmpty()) {
|
||||
safeCacheUtil.evict(idKey);
|
||||
} else {
|
||||
safeCacheUtil.evict(idKey, "device:ip:" + ip);
|
||||
}
|
||||
Cache springDeviceCache = cacheManager.getCache("device");
|
||||
if (springDeviceCache == null) return;
|
||||
springDeviceCache.evict(idKey);
|
||||
if (ip != null && !ip.trim().isEmpty()) springDeviceCache.evict("device:ip:" + ip);
|
||||
}
|
||||
}
|
||||
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package com.config;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** 在应用完全就绪后打印不可由现场配置覆盖的代码修订标识和关键开关。 */
|
||||
@Slf4j
|
||||
@Component
|
||||
public class SyslogBuildStartupLogger {
|
||||
|
||||
// 确认 JAR 包含设备级采集模式、API 拉取及全局 TLS 策略。
|
||||
static final String CODE_REVISION = "DEVICE_COLLECTION_API_TLS_V20260917_63";
|
||||
private final Environment environment;
|
||||
|
||||
public SyslogBuildStartupLogger(Environment environment) {
|
||||
this.environment = environment;
|
||||
}
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void onApplicationReady() {
|
||||
log.info("[SYSLOG_BUILD_READY] {}", startupSummary());
|
||||
}
|
||||
|
||||
String startupSummary() {
|
||||
return "codeRevision=" + CODE_REVISION
|
||||
+ ", application=syslog-serve"
|
||||
+ ", profiles=" + String.join(",", environment.getActiveProfiles())
|
||||
+ ", serverPort=" + value("server.port", "8189")
|
||||
+ ", contextPath=" + value("server.servlet.context-path", "/syslogserve")
|
||||
+ ", syslogEnabled=" + value("syslog.server.enabled", "true")
|
||||
+ ", apiPullEnabled=" + value("device.api.enabled", "true")
|
||||
+ ", apiTlsVerifyEnabled=" + value("device.api.tls-verify-enabled", "false")
|
||||
+ ", apiHostAllowlistEnabled=" + value("device.api.host-allowlist-enabled", "false")
|
||||
+ ", walEnabled=" + value("syslog.wal.enabled", "true")
|
||||
+ ", terminalRecordEnabled=" + value("syslog.dispatch.terminal-record-enabled", "false")
|
||||
+ ", deviceCollectId=" + value("app.service.device_collect_id", "7")
|
||||
+ ", kafkaTopic=" + value("spring.kafka.producer.topic", "test-topic");
|
||||
}
|
||||
|
||||
private String value(String key, String fallback) {
|
||||
return environment.getProperty(key, fallback);
|
||||
}
|
||||
}
|
||||
+17
-3
@@ -7,13 +7,14 @@ import org.springframework.stereotype.Component;
|
||||
/**
|
||||
* 进程级 API 拉取开关和资源上限。
|
||||
*
|
||||
* <p>这里只保存所有任务共享的运行保护参数;页大小、时间窗口、厂商适配器等任务级参数 固化在达梦数据库中。总开关默认关闭,避免系统升级后未经验证就访问真实设备。
|
||||
* <p>这里只保存所有任务共享的运行保护参数;页大小、时间窗口、厂商适配器等任务级参数固化在达梦数据库中。
|
||||
* 总开关默认开启,但新建任务保持暂停,必须完成校验并 resume 后才会访问真实设备。
|
||||
*/
|
||||
@Data
|
||||
@Component
|
||||
public class ApiPullProperties {
|
||||
@Value("${device.api.enabled:false}")
|
||||
private boolean enabled;
|
||||
@Value("${device.api.enabled:true}")
|
||||
private boolean enabled = true;
|
||||
|
||||
@Value("${device.api.scheduler.instance-id:${HOSTNAME:local}-${random.uuid}}")
|
||||
private String instanceId;
|
||||
@@ -48,6 +49,19 @@ public class ApiPullProperties {
|
||||
@Value("${device.api.allowed-hosts:}")
|
||||
private String allowedHosts;
|
||||
|
||||
/** 现场设备经常增减;默认关闭静态主机白名单,启用后才检查 allowedHosts。 */
|
||||
@Value("${device.api.host-allowlist-enabled:false}")
|
||||
private boolean hostAllowlistEnabled = false;
|
||||
|
||||
/**
|
||||
* 所有设备 API HTTPS 是否校验证书链和主机名。
|
||||
*
|
||||
* <p>现场默认为内网自签名环境,因此默认关闭。该开关是进程级全局策略:关闭后微步、NDR、全悉等
|
||||
* 所有 API 拉取适配器都不校验证书链和主机名;HTTPS 协议限制和禁止重定向仍然生效。
|
||||
*/
|
||||
@Value("${device.api.tls-verify-enabled:false}")
|
||||
private boolean tlsVerifyEnabled = false;
|
||||
|
||||
/** 在调度器创建线程池前快速失败,防止以危险的并发或租约参数启动。 */
|
||||
public void validate() {
|
||||
if (scanIntervalMs < 100
|
||||
|
||||
+36
-2
@@ -1,6 +1,7 @@
|
||||
package com.haobang.apipull;
|
||||
|
||||
import com.common.mapper.DeviceCollectApiMapper;
|
||||
import com.haobang.dispatch.DeviceContextResolver;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiRun;
|
||||
@@ -31,6 +32,7 @@ public class ApiPullScheduler {
|
||||
private final DeviceCollectApiMapper mapper;
|
||||
private final ApiPullRunner runner;
|
||||
private final SyslogWalService wal;
|
||||
private final DeviceContextResolver deviceResolver;
|
||||
private ExecutorService executor;
|
||||
private Semaphore global;
|
||||
private final ConcurrentHashMap<String, Semaphore> hosts = new ConcurrentHashMap<>();
|
||||
@@ -40,11 +42,13 @@ public class ApiPullScheduler {
|
||||
ApiPullProperties properties,
|
||||
DeviceCollectApiMapper mapper,
|
||||
ApiPullRunner runner,
|
||||
SyslogWalService wal) {
|
||||
SyslogWalService wal,
|
||||
DeviceContextResolver deviceResolver) {
|
||||
this.properties = properties;
|
||||
this.mapper = mapper;
|
||||
this.runner = runner;
|
||||
this.wal = wal;
|
||||
this.deviceResolver = deviceResolver;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
@@ -110,6 +114,7 @@ public class ApiPullScheduler {
|
||||
|
||||
private DeviceCollectApiRun trySubmit(DeviceCollectApiConfig c, String trigger) {
|
||||
if (!c.isEnabled()) return null;
|
||||
if (!requireApiCollectionMode(c, "MANUAL".equals(trigger))) return null;
|
||||
if (wal.getFatalFailure() != null || wal.isDiskCritical() || wal.shouldPauseTcp()) {
|
||||
mapper.scheduleNext(c.getCollectTaskId(), c.getConfigVersion(), c.getPollIntervalSeconds());
|
||||
return null;
|
||||
@@ -167,7 +172,18 @@ public class ApiPullScheduler {
|
||||
executor.execute(
|
||||
() -> {
|
||||
try {
|
||||
runner.execute(c, leasedCheckpoint, run);
|
||||
// 模式可能在排队期间变化;执行 HTTP 前再次校验,避免已提交线程绕过新策略。
|
||||
if (requireApiCollectionMode(c, false)) {
|
||||
runner.execute(c, leasedCheckpoint, run);
|
||||
} else {
|
||||
run.setStatus("FAILED");
|
||||
run.setErrorCode("COLLECTION_MODE_DENIED");
|
||||
run.setErrorSummary("设备采集模式不允许 API_PULL");
|
||||
run.setFinishedAt(LocalDateTime.now(ApiPullTime.DATABASE_ZONE));
|
||||
mapper.updateRun(run);
|
||||
mapper.releaseLease(
|
||||
c.getCollectTaskId(), properties.getInstanceId(), leasedCheckpoint.getFencingToken());
|
||||
}
|
||||
} finally {
|
||||
releaseLocal(c.getCollectTaskId(), hostPermit);
|
||||
}
|
||||
@@ -188,6 +204,24 @@ public class ApiPullScheduler {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 调度扫描遇到禁止 API 的设备时推进 next_run_at,避免同一到期任务按 5 秒扫描周期空转。
|
||||
*/
|
||||
private boolean requireApiCollectionMode(DeviceCollectApiConfig c, boolean failRequest) {
|
||||
DeviceContextResolver.Resolution resolution = deviceResolver.resolveDeviceId(c.getDeviceId());
|
||||
if (resolution.isRegistered()) return true;
|
||||
mapper.scheduleNext(c.getCollectTaskId(), c.getConfigVersion(), c.getPollIntervalSeconds());
|
||||
String reason = resolution.getReason() == null ? "设备不允许 API_PULL" : resolution.getReason();
|
||||
log.warn("设备 API 拉取已被设备策略跳过: taskId={}, deviceId={}, type={}, reason={}",
|
||||
c.getCollectTaskId(), c.getDeviceId(), resolution.getType(), reason);
|
||||
if (failRequest) {
|
||||
String code = resolution.getType() == DeviceContextResolver.Type.COLLECTION_MODE_DENIED
|
||||
? "COLLECTION_MODE_DENIED" : "DEVICE_CONTEXT_DENIED";
|
||||
throw new ApiPullException(code, reason, org.springframework.http.HttpStatus.CONFLICT);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void releaseLocal(Integer taskId, Semaphore host) {
|
||||
if (host != null) host.release();
|
||||
global.release();
|
||||
|
||||
+33
-7
@@ -1,6 +1,7 @@
|
||||
package com.haobang.apipull;
|
||||
|
||||
import com.common.entity.DeviceCollectTask;
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.mapper.DeviceCollectApiMapper;
|
||||
import com.common.mapper.DeviceCollectTaskMapper;
|
||||
@@ -81,11 +82,7 @@ public class ApiPullTaskService {
|
||||
@Transactional
|
||||
public ApiPullTaskResponse create(CreateApiPullTaskRequest request) {
|
||||
// 设备必须归属当前探针,避免一个采集实例替其他探针访问设备内网地址。
|
||||
DeviceDevice device = devices.getByIdSafely(request.getDeviceId());
|
||||
if (device == null) throw error("INVALID_REQUEST", "设备不存在", HttpStatus.BAD_REQUEST);
|
||||
if (device.getDeviceCollectId() == null
|
||||
|| device.getDeviceCollectId() != app.getAppServiceDeviceCollectId())
|
||||
throw error("FORBIDDEN", "设备不属于当前采集探针", HttpStatus.FORBIDDEN);
|
||||
DeviceDevice device = requireApiAllowedDevice(request.getDeviceId());
|
||||
DeviceCollectApiConfig config = fromCreate(request);
|
||||
ApiPullAdapter adapter = adapters.require(config.getAdapterType());
|
||||
adapter.validateConfig(config);
|
||||
@@ -237,6 +234,7 @@ public class ApiPullTaskService {
|
||||
DeviceCollectApiConfig c = requireConfig(taskId);
|
||||
if (c.getConfigVersion() != version)
|
||||
throw error("CONFIG_VERSION_CONFLICT", "配置版本冲突", HttpStatus.CONFLICT);
|
||||
requireApiAllowedDevice(c.getDeviceId());
|
||||
return scheduler.trigger(taskId);
|
||||
}
|
||||
|
||||
@@ -245,6 +243,8 @@ public class ApiPullTaskService {
|
||||
DeviceCollectApiConfig c = requireConfig(taskId);
|
||||
if (c.getConfigVersion() != version)
|
||||
throw error("CONFIG_VERSION_CONFLICT", "配置版本冲突", HttpStatus.CONFLICT);
|
||||
// 必须在创建验证审计和发出任何 HTTP 请求前执行设备级策略。
|
||||
requireApiAllowedDevice(c.getDeviceId());
|
||||
DeviceCollectApiRun audit = new DeviceCollectApiRun();
|
||||
audit.setRunId(UUID.randomUUID().toString());
|
||||
audit.setCollectTaskId(taskId);
|
||||
@@ -301,7 +301,7 @@ public class ApiPullTaskService {
|
||||
Instant start = end.minusSeconds(1);
|
||||
// 验证仅做 limit=1 的闭合窗口查询,不推进 checkpoint,也不写入 WAL/Kafka。
|
||||
adapter.pull(new PullRequest(c, start, end, 1, 0));
|
||||
checks.add(check("TLS_HANDSHAKE", "PASS", null, "TLS 和主机名校验通过", pullStarted));
|
||||
checks.add(check("TLS_HANDSHAKE", "PASS", null, tlsValidationSummary(), pullStarted));
|
||||
checks.add(check("AUTHENTICATION", "PASS", null, "厂商认证及查询通过", pullStarted));
|
||||
checks.add(check("API_PERMISSION", "PASS", null, "业务接口可访问", pullStarted));
|
||||
checks.add(
|
||||
@@ -323,7 +323,7 @@ public class ApiPullTaskService {
|
||||
checks.add(skipped("API_PERMISSION"));
|
||||
} else if (failureCode.startsWith("QUERY_")) {
|
||||
// QUERY_ 表示已完成 TLS、Challenge/Verify 并收到业务接口的 2xx 响应,不能误报为认证失败。
|
||||
checks.add(check("TLS_HANDSHAKE", "PASS", null, "TLS 和主机名校验通过", pullStarted));
|
||||
checks.add(check("TLS_HANDSHAKE", "PASS", null, tlsValidationSummary(), pullStarted));
|
||||
checks.add(check("AUTHENTICATION", "PASS", null, "厂商认证及查询通过", pullStarted));
|
||||
checks.add(check("API_PERMISSION", "PASS", null, "业务接口可访问", pullStarted));
|
||||
checks.add(check(queryCheck, "FAIL", failureCode, safe(e), pullStarted));
|
||||
@@ -362,6 +362,12 @@ public class ApiPullTaskService {
|
||||
return r;
|
||||
}
|
||||
|
||||
private String tlsValidationSummary() {
|
||||
return apiProperties.isTlsVerifyEnabled()
|
||||
? "TLS 证书链和主机名校验通过"
|
||||
: "TLS 连接通过(全局已关闭证书链和主机名校验)";
|
||||
}
|
||||
|
||||
public ApiPullRuntimeResponse runtime(Integer taskId) {
|
||||
DeviceCollectApiConfig c = requireConfig(taskId);
|
||||
DeviceCollectApiCheckpoint cp = mapper.selectCheckpoint(taskId);
|
||||
@@ -529,6 +535,26 @@ public class ApiPullTaskService {
|
||||
return c;
|
||||
}
|
||||
|
||||
private DeviceDevice requireApiAllowedDevice(Integer deviceId) {
|
||||
DeviceDevice device = devices.getByIdSafely(deviceId);
|
||||
if (device == null || device.getDeletedAt() != null
|
||||
|| (device.getDelFlag() != null && !"0".equals(device.getDelFlag()))) {
|
||||
throw error("INVALID_REQUEST", "设备不存在或已删除", HttpStatus.BAD_REQUEST);
|
||||
}
|
||||
if (device.getDeviceCollectId() == null
|
||||
|| device.getDeviceCollectId() != app.getAppServiceDeviceCollectId()) {
|
||||
throw error("FORBIDDEN", "设备不属于当前采集探针", HttpStatus.FORBIDDEN);
|
||||
}
|
||||
DeviceCollectionMode mode = device.getCollectionMode();
|
||||
if (!mode.allowsApi()) {
|
||||
throw error(
|
||||
"COLLECTION_MODE_DENIED",
|
||||
"设备采集模式不允许 API_PULL: deviceId=" + deviceId + ", collectionMode=" + mode,
|
||||
HttpStatus.CONFLICT);
|
||||
}
|
||||
return device;
|
||||
}
|
||||
|
||||
private DeviceCollectTask requireTask(Integer id) {
|
||||
DeviceCollectTask t = tasks.selectById(id);
|
||||
if (t == null || t.getMethod() == null || t.getMethod() != API_PULL_METHOD)
|
||||
|
||||
+28
-2
@@ -3,6 +3,7 @@ package com.haobang.apipull.adapter.ndr;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.haobang.apipull.ApiPullProperties;
|
||||
import com.haobang.apipull.adapter.AdapterException;
|
||||
import com.haobang.apipull.adapter.ApiPullAdapter;
|
||||
import com.haobang.apipull.adapter.PullPage;
|
||||
@@ -42,8 +43,12 @@ import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.entity.ContentType;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.conn.ssl.NoopHostnameVerifier;
|
||||
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.http.ssl.SSLContextBuilder;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -104,8 +109,11 @@ public class NdrServerApiAdapter implements ApiPullAdapter {
|
||||
*/
|
||||
@Autowired
|
||||
public NdrServerApiAdapter(
|
||||
CredentialProvider credentials, SafeBaseUrlValidator urlValidator, ObjectMapper mapper) {
|
||||
this(credentials, urlValidator, mapper, HttpClients.custom().disableAutomaticRetries().build());
|
||||
CredentialProvider credentials,
|
||||
SafeBaseUrlValidator urlValidator,
|
||||
ObjectMapper mapper,
|
||||
ApiPullProperties properties) {
|
||||
this(credentials, urlValidator, mapper, buildHttpClient(properties.isTlsVerifyEnabled()));
|
||||
}
|
||||
|
||||
NdrServerApiAdapter(
|
||||
@@ -119,6 +127,24 @@ public class NdrServerApiAdapter implements ApiPullAdapter {
|
||||
this.http = http;
|
||||
}
|
||||
|
||||
/**
|
||||
* 遵循设备 API 全局 TLS 校验开关,但始终禁止自动重试和重定向。
|
||||
*/
|
||||
static CloseableHttpClient buildHttpClient(boolean tlsVerifyEnabled) {
|
||||
HttpClientBuilder builder =
|
||||
HttpClients.custom().disableAutomaticRetries().disableRedirectHandling();
|
||||
if (tlsVerifyEnabled) return builder.build();
|
||||
try {
|
||||
javax.net.ssl.SSLContext sslContext =
|
||||
SSLContextBuilder.create().loadTrustMaterial((chain, authType) -> true).build();
|
||||
SSLConnectionSocketFactory socketFactory =
|
||||
new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
|
||||
return builder.setSSLSocketFactory(socketFactory).build();
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException("初始化 NDR 内网 TLS 客户端失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return TYPE;
|
||||
|
||||
+27
-3
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonToken;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.haobang.apipull.ApiPullProperties;
|
||||
import com.haobang.apipull.adapter.*;
|
||||
import com.haobang.apipull.credential.*;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
|
||||
@@ -26,8 +27,12 @@ import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.entity.ContentType;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.conn.ssl.NoopHostnameVerifier;
|
||||
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.http.ssl.SSLContextBuilder;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -49,9 +54,12 @@ public class QuanxiApiAdapter implements ApiPullAdapter {
|
||||
private final CloseableHttpClient http;
|
||||
|
||||
@Autowired
|
||||
public QuanxiApiAdapter(CredentialProvider credentials, SafeBaseUrlValidator urls, ObjectMapper json) {
|
||||
this(credentials, urls, json,
|
||||
HttpClients.custom().disableAutomaticRetries().disableRedirectHandling().build());
|
||||
public QuanxiApiAdapter(
|
||||
CredentialProvider credentials,
|
||||
SafeBaseUrlValidator urls,
|
||||
ObjectMapper json,
|
||||
ApiPullProperties properties) {
|
||||
this(credentials, urls, json, buildHttpClient(properties.isTlsVerifyEnabled()));
|
||||
}
|
||||
|
||||
QuanxiApiAdapter(CredentialProvider credentials, SafeBaseUrlValidator urls,
|
||||
@@ -63,6 +71,22 @@ public class QuanxiApiAdapter implements ApiPullAdapter {
|
||||
this.http = http;
|
||||
}
|
||||
|
||||
/** 遵循设备 API 全局 TLS 校验开关,但始终禁止自动重试和重定向。 */
|
||||
static CloseableHttpClient buildHttpClient(boolean tlsVerifyEnabled) {
|
||||
HttpClientBuilder builder =
|
||||
HttpClients.custom().disableAutomaticRetries().disableRedirectHandling();
|
||||
if (tlsVerifyEnabled) return builder.build();
|
||||
try {
|
||||
javax.net.ssl.SSLContext sslContext =
|
||||
SSLContextBuilder.create().loadTrustMaterial((chain, authType) -> true).build();
|
||||
SSLConnectionSocketFactory socketFactory =
|
||||
new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
|
||||
return builder.setSSLSocketFactory(socketFactory).build();
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException("初始化全悉内网 TLS 客户端失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override public String type() { return TYPE; }
|
||||
@Override public int version() { return 1; }
|
||||
@Override public Set<String> streams() { return Collections.singleton(STREAM); }
|
||||
|
||||
+481
@@ -0,0 +1,481 @@
|
||||
package com.haobang.apipull.adapter.weibu;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonParser;
|
||||
import com.fasterxml.jackson.core.JsonToken;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.haobang.apipull.ApiPullProperties;
|
||||
import com.haobang.apipull.adapter.AdapterException;
|
||||
import com.haobang.apipull.adapter.ApiPullAdapter;
|
||||
import com.haobang.apipull.adapter.PullPage;
|
||||
import com.haobang.apipull.adapter.PullRequest;
|
||||
import com.haobang.apipull.adapter.PulledEvent;
|
||||
import com.haobang.apipull.credential.CredentialException;
|
||||
import com.haobang.apipull.credential.CredentialProvider;
|
||||
import com.haobang.apipull.credential.CredentialSchemas;
|
||||
import com.haobang.apipull.credential.CredentialSnapshot;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
|
||||
import com.haobang.apipull.security.SafeBaseUrlValidator;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.net.URI;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Base64;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import javax.annotation.PreDestroy;
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.utils.URIBuilder;
|
||||
import org.apache.http.entity.ContentType;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.conn.ssl.NoopHostnameVerifier;
|
||||
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.http.ssl.SSLContextBuilder;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 微步 API 配置文档中的逐条日志查询协议。
|
||||
*
|
||||
* <p>固定使用 {@code /api/v1/log/searchBySql},响应 {@code data.data[]} 中每个对象作为一条原始
|
||||
* 日志进入 WAL/Kafka。该接口没有 offset/page 参数,因此任务强制 {@code maxPagesPerRun=1};通用
|
||||
* Runner 先读取响应 {@code total},当窗口内总数超过 {@code pageSize} 时只丢弃探测响应并继续缩小
|
||||
* 时间窗口,直到单次响应能够完整返回。最小一秒窗口仍超限时失败且不推进 checkpoint。
|
||||
*
|
||||
* <p>externalEventId 使用厂商日志 {@code id},事件时间使用 {@code time}。事件 ID 只由任务命名空间
|
||||
* 和厂商稳定 ID 生成,lookback 重叠查询不会制造新事件版本。适配器不把 API JSON 伪装成 Syslog,
|
||||
* 业务字段标准化由下游针对 API 的泛化规则完成。
|
||||
*/
|
||||
@Component
|
||||
public class WeibuApiAdapter implements ApiPullAdapter {
|
||||
public static final String TYPE = "WEIBU_V1";
|
||||
public static final String LOG = "WEIBU_LOG";
|
||||
static final String PATH = "/api/v1/log/searchBySql";
|
||||
private static final ZoneId DEVICE_ZONE = ZoneId.of("Asia/Shanghai");
|
||||
private static final DateTimeFormatter EVENT_ID_DATE =
|
||||
DateTimeFormatter.ofPattern("yyyyMMdd").withZone(DEVICE_ZONE);
|
||||
private static final UUID EVENT_NAMESPACE =
|
||||
UUID.fromString("89a53d59-bb84-5d41-924f-38319448cf46");
|
||||
private static final long MIN_EVENT_EPOCH_SECOND = 946684800L;
|
||||
private static final long MAX_EVENT_EPOCH_SECOND = 4102444800L;
|
||||
|
||||
private final CredentialProvider credentials;
|
||||
private final SafeBaseUrlValidator urls;
|
||||
private final ObjectMapper json;
|
||||
private final CloseableHttpClient http;
|
||||
|
||||
@Autowired
|
||||
public WeibuApiAdapter(
|
||||
CredentialProvider credentials,
|
||||
SafeBaseUrlValidator urls,
|
||||
ObjectMapper json,
|
||||
ApiPullProperties properties) {
|
||||
this(
|
||||
credentials,
|
||||
urls,
|
||||
json,
|
||||
buildHttpClient(properties.isTlsVerifyEnabled()));
|
||||
}
|
||||
|
||||
WeibuApiAdapter(
|
||||
CredentialProvider credentials,
|
||||
SafeBaseUrlValidator urls,
|
||||
ObjectMapper json,
|
||||
CloseableHttpClient http) {
|
||||
this.credentials = credentials;
|
||||
this.urls = urls;
|
||||
this.json =
|
||||
json.copy()
|
||||
.deactivateDefaultTyping()
|
||||
.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION);
|
||||
this.http = http;
|
||||
}
|
||||
|
||||
/**
|
||||
* 进程级全局开关可为使用自签名证书的内网设备关闭证书链和主机名校验。
|
||||
* 客户端仍拒绝重定向和非 HTTPS URL;静态主机白名单是否启用由部署配置决定。
|
||||
*/
|
||||
static CloseableHttpClient buildHttpClient(boolean tlsVerifyEnabled) {
|
||||
HttpClientBuilder builder =
|
||||
HttpClients.custom().disableAutomaticRetries().disableRedirectHandling();
|
||||
if (tlsVerifyEnabled) return builder.build();
|
||||
try {
|
||||
javax.net.ssl.SSLContext sslContext =
|
||||
SSLContextBuilder.create().loadTrustMaterial((chain, authType) -> true).build();
|
||||
SSLConnectionSocketFactory socketFactory =
|
||||
new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
|
||||
return builder.setSSLSocketFactory(socketFactory).build();
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException("初始化微步内网 TLS 客户端失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int version() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> streams() {
|
||||
return Collections.singleton(LOG);
|
||||
}
|
||||
|
||||
/** 厂商时间字段为 Unix 秒,窗口不能安全拆到亚秒。 */
|
||||
@Override
|
||||
public int minimumWindowSeconds() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean requiresDistinctExternalIds() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validateConfig(DeviceCollectApiConfig c) {
|
||||
if (c == null || !TYPE.equals(c.getAdapterType())) fail("CONFIG_SCHEMA_ERROR");
|
||||
if (!LOG.equals(c.getStreamKey())) fail("STREAM_NOT_SUPPORTED");
|
||||
range(c.getPageSize(), 1, 1000);
|
||||
if (c.getMaxPagesPerRun() == null || c.getMaxPagesPerRun() != 1)
|
||||
fail("CONFIG_SCHEMA_ERROR");
|
||||
range(c.getPollIntervalSeconds(), 1, 86400);
|
||||
range(c.getWindowSizeSeconds(), 1, 86400);
|
||||
range(c.getLookbackSeconds(), 0, 86400);
|
||||
range(c.getSafetyDelaySeconds(), 0, 86400);
|
||||
range(c.getConnectTimeoutMs(), 100, 30000);
|
||||
range(c.getRequestTimeoutMs(), 1000, 120000);
|
||||
if (c.getMaxResponseBytes() == null
|
||||
|| c.getMaxResponseBytes() < 1024
|
||||
|| c.getMaxResponseBytes() > 67108864L) fail("CONFIG_SCHEMA_ERROR");
|
||||
URI base = urls.validate(c.getBaseUrl());
|
||||
String path = base.getPath();
|
||||
if (!(path == null || path.isEmpty() || "/".equals(path))) fail("CONFIG_SCHEMA_ERROR");
|
||||
if (c.getExtraConfigJson() == null || !"{}".equals(c.getExtraConfigJson().trim()))
|
||||
fail("CONFIG_SCHEMA_ERROR");
|
||||
}
|
||||
|
||||
@Override
|
||||
public PullPage pull(PullRequest request) {
|
||||
DeviceCollectApiConfig config = request.getConfig();
|
||||
validateConfig(config);
|
||||
validateRequest(request);
|
||||
|
||||
Instant vendorStart = request.getQueryStart().truncatedTo(ChronoUnit.SECONDS);
|
||||
Instant vendorEnd = ceilToSecond(request.getQueryEnd()).minusSeconds(1L);
|
||||
if (vendorEnd.isBefore(vendorStart)) fail("REQUEST_RANGE_INVALID");
|
||||
|
||||
ObjectNode body = json.createObjectNode();
|
||||
body.putArray("assets_group");
|
||||
body.put("search_connection", false);
|
||||
body.putArray("net_data_type").add("attack").add("risk");
|
||||
body.put("time_from", vendorStart.getEpochSecond());
|
||||
body.put("time_to", vendorEnd.getEpochSecond());
|
||||
body.put("sql", "");
|
||||
body.putArray("columns");
|
||||
body.put("asc", false);
|
||||
// 厂商接口只有 size,没有 offset/page。Runner 通过 total 自动缩小窗口,确保该次
|
||||
// size 足以取回窗口内全部日志,而不是把 size 当作伪分页游标。
|
||||
body.put("size", request.getLimit());
|
||||
|
||||
byte[] responseBytes = null;
|
||||
HttpPost post = null;
|
||||
long httpStarted = System.nanoTime();
|
||||
try (CredentialSnapshot snapshot =
|
||||
credentials.load(config.getCredentialRef(), TYPE, config.getDeviceId())) {
|
||||
char[] keyChars = snapshot.require("app_key").copy();
|
||||
char[] secretChars = snapshot.require("app_secret").copy();
|
||||
try {
|
||||
String apiKey = new String(keyChars);
|
||||
String secret = new String(secretChars);
|
||||
CredentialSchemas.validate(TYPE, apiKey, secret, null);
|
||||
long timestamp = Instant.now().getEpochSecond();
|
||||
String signature = sign(apiKey, secretChars, timestamp);
|
||||
URI endpoint =
|
||||
new URIBuilder(endpoint(urls.validate(config.getBaseUrl())))
|
||||
.addParameter("api_key", apiKey)
|
||||
.addParameter("auth_timestamp", Long.toString(timestamp))
|
||||
.addParameter("sign", signature)
|
||||
.build();
|
||||
post = new HttpPost(endpoint);
|
||||
post.setConfig(requestConfig(config));
|
||||
post.setHeader("Accept", "application/json");
|
||||
post.setEntity(
|
||||
new StringEntity(json.writeValueAsString(body), ContentType.APPLICATION_JSON));
|
||||
try (CloseableHttpResponse response = http.execute(post)) {
|
||||
int status = response.getStatusLine().getStatusCode();
|
||||
if (status < 200 || status >= 300) {
|
||||
String code =
|
||||
status == 401
|
||||
? "AUTH_HTTP_401"
|
||||
: status == 403 ? "AUTH_HTTP_403" : "HTTP_ERROR";
|
||||
throw new AdapterException(code, "微步 API HTTP 请求失败: " + status, status);
|
||||
}
|
||||
responseBytes = readBounded(response.getEntity(), config.getMaxResponseBytes());
|
||||
long httpMs = elapsed(httpStarted);
|
||||
long parseStarted = System.nanoTime();
|
||||
return parsePage(responseBytes, status, httpMs, parseStarted, request);
|
||||
}
|
||||
} finally {
|
||||
Arrays.fill(keyChars, '\0');
|
||||
Arrays.fill(secretChars, '\0');
|
||||
}
|
||||
} catch (AdapterException | CredentialException e) {
|
||||
throw e;
|
||||
} catch (SocketTimeoutException | org.apache.http.conn.ConnectTimeoutException e) {
|
||||
fail("HTTP_TIMEOUT");
|
||||
return null;
|
||||
} catch (IOException e) {
|
||||
fail("DEVICE_API_UNAVAILABLE");
|
||||
return null;
|
||||
} catch (Exception e) {
|
||||
fail("REQUEST_BUILD_ERROR");
|
||||
return null;
|
||||
} finally {
|
||||
if (post != null) post.releaseConnection();
|
||||
if (responseBytes != null) Arrays.fill(responseBytes, (byte) 0);
|
||||
}
|
||||
}
|
||||
|
||||
private PullPage parsePage(
|
||||
byte[] bytes,
|
||||
int status,
|
||||
long httpMs,
|
||||
long parseStarted,
|
||||
PullRequest request) {
|
||||
try {
|
||||
JsonNode root = parseBounded(bytes);
|
||||
if (!root.isObject()) fail("RESPONSE_SCHEMA_ERROR");
|
||||
JsonNode responseCode = root.get("response_code");
|
||||
if (responseCode == null || !responseCode.isIntegralNumber())
|
||||
fail("RESPONSE_SCHEMA_ERROR");
|
||||
if (responseCode.intValue() != 0) fail("VENDOR_RESPONSE_ERROR");
|
||||
JsonNode data = root.get("data");
|
||||
if (data == null || !data.isObject()) fail("RESPONSE_SCHEMA_ERROR");
|
||||
JsonNode items = data.get("data");
|
||||
if (items == null || !items.isArray())
|
||||
fail("RESPONSE_SCHEMA_ERROR");
|
||||
long total = nonNegativeLong(data.get("total"));
|
||||
long expectedItems = Math.min(total, request.getLimit());
|
||||
if (items.size() != expectedItems) fail("PAGINATION_INCONSISTENT");
|
||||
|
||||
ArrayList<PulledEvent> events = new ArrayList<>();
|
||||
Set<String> pageIds = new HashSet<>();
|
||||
UUID taskNamespace =
|
||||
uuidV5(
|
||||
EVENT_NAMESPACE,
|
||||
configIdentity(request.getConfig()));
|
||||
for (JsonNode item : items) {
|
||||
if (!item.isObject()) fail("RESPONSE_SCHEMA_ERROR");
|
||||
String external = requiredText(item.get("id"), 512);
|
||||
if (!pageIds.add(external)) fail("PAGINATION_DUPLICATE_DOCUMENT");
|
||||
long eventSecond = integral(item.get("time"));
|
||||
if (eventSecond < MIN_EVENT_EPOCH_SECOND || eventSecond >= MAX_EVENT_EPOCH_SECOND)
|
||||
fail("EVENT_TIME_INVALID");
|
||||
Instant eventTime = Instant.ofEpochSecond(eventSecond);
|
||||
// 请求使用外扩后的整秒闭区间,最终仍严格服从平台的半开窗口。厂商若返回
|
||||
// 外扩边界记录,只按设备返回数完成完整性校验,不让该记录进入 WAL。
|
||||
if (eventTime.isBefore(request.getQueryStart())
|
||||
|| !eventTime.isBefore(request.getQueryEnd())) continue;
|
||||
String raw = json.writeValueAsString(item);
|
||||
if (raw.getBytes(StandardCharsets.UTF_8).length > 1048576) fail("EVENT_TOO_LARGE");
|
||||
String eventId =
|
||||
EVENT_ID_DATE.format(eventTime) + "-" + uuidV5(taskNamespace, external);
|
||||
events.add(new PulledEvent(eventId, external, eventTime, raw));
|
||||
}
|
||||
return new PullPage(
|
||||
events,
|
||||
total,
|
||||
status,
|
||||
bytes.length,
|
||||
httpMs,
|
||||
elapsed(parseStarted),
|
||||
Long.toString(total),
|
||||
items.size());
|
||||
} catch (AdapterException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
fail("RESPONSE_SCHEMA_ERROR");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void validateRequest(PullRequest r) {
|
||||
if (r.getConfig().getDeviceId() == null
|
||||
|| r.getConfig().getCollectTaskId() == null
|
||||
|| r.getQueryStart() == null
|
||||
|| r.getQueryEnd() == null
|
||||
|| !r.getQueryEnd().isAfter(r.getQueryStart())
|
||||
|| r.getLimit() < 1
|
||||
|| r.getLimit() > r.getConfig().getPageSize()
|
||||
|| r.getOffset() != 0L) fail("REQUEST_RANGE_INVALID");
|
||||
}
|
||||
|
||||
private URI endpoint(URI base) throws Exception {
|
||||
return new URI(base.getScheme(), null, base.getHost(), base.getPort(), PATH, null, null);
|
||||
}
|
||||
|
||||
private RequestConfig requestConfig(DeviceCollectApiConfig c) {
|
||||
return RequestConfig.custom()
|
||||
.setConnectTimeout(c.getConnectTimeoutMs())
|
||||
.setConnectionRequestTimeout(c.getConnectTimeoutMs())
|
||||
.setSocketTimeout(c.getRequestTimeoutMs())
|
||||
.setRedirectsEnabled(false)
|
||||
.build();
|
||||
}
|
||||
|
||||
static String sign(String apiKey, char[] secret, long timestamp) throws Exception {
|
||||
byte[] secretBytes = new String(secret).getBytes(StandardCharsets.UTF_8);
|
||||
try {
|
||||
Mac mac = Mac.getInstance("HmacSHA256");
|
||||
mac.init(new SecretKeySpec(secretBytes, "HmacSHA256"));
|
||||
byte[] digest =
|
||||
mac.doFinal((apiKey + timestamp).getBytes(StandardCharsets.UTF_8));
|
||||
try {
|
||||
return Base64.getUrlEncoder().withoutPadding().encodeToString(digest);
|
||||
} finally {
|
||||
Arrays.fill(digest, (byte) 0);
|
||||
}
|
||||
} finally {
|
||||
Arrays.fill(secretBytes, (byte) 0);
|
||||
}
|
||||
}
|
||||
|
||||
static Instant ceilToSecond(Instant value) {
|
||||
Instant floor = value.truncatedTo(ChronoUnit.SECONDS);
|
||||
return floor.equals(value) ? floor : floor.plusSeconds(1L);
|
||||
}
|
||||
|
||||
private byte[] readBounded(HttpEntity entity, long maxBytes) throws IOException {
|
||||
if (entity == null) fail("RESPONSE_SCHEMA_ERROR");
|
||||
if (entity.getContentLength() > maxBytes) fail("RESPONSE_TOO_LARGE");
|
||||
try (InputStream in = entity.getContent();
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||
byte[] buffer = new byte[8192];
|
||||
long size = 0;
|
||||
int n;
|
||||
while ((n = in.read(buffer)) != -1) {
|
||||
size += n;
|
||||
if (size > maxBytes) fail("RESPONSE_TOO_LARGE");
|
||||
out.write(buffer, 0, n);
|
||||
}
|
||||
return out.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
private JsonNode parseBounded(byte[] bytes) {
|
||||
try {
|
||||
try (JsonParser parser = json.getFactory().createParser(bytes)) {
|
||||
int depth = 0;
|
||||
int tokens = 0;
|
||||
JsonToken token;
|
||||
while ((token = parser.nextToken()) != null) {
|
||||
if (++tokens > 2000000) fail("RESPONSE_SCHEMA_ERROR");
|
||||
if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) {
|
||||
if (++depth > 64) fail("RESPONSE_SCHEMA_ERROR");
|
||||
} else if (token == JsonToken.END_OBJECT || token == JsonToken.END_ARRAY) {
|
||||
depth--;
|
||||
} else if ((token == JsonToken.VALUE_STRING && parser.getTextLength() > 1048576)
|
||||
|| (token == JsonToken.FIELD_NAME && parser.getCurrentName().length() > 512)) {
|
||||
fail("RESPONSE_SCHEMA_ERROR");
|
||||
}
|
||||
}
|
||||
}
|
||||
try (JsonParser parser = json.getFactory().createParser(bytes)) {
|
||||
JsonNode value = json.readTree(parser);
|
||||
if (value == null || parser.nextToken() != null) fail("RESPONSE_SCHEMA_ERROR");
|
||||
return value;
|
||||
}
|
||||
} catch (AdapterException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
fail("RESPONSE_SCHEMA_ERROR");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String configIdentity(DeviceCollectApiConfig c) {
|
||||
return c.getDeviceId() + "|" + c.getCollectTaskId() + "|" + TYPE + "|" + LOG;
|
||||
}
|
||||
|
||||
private long nonNegativeLong(JsonNode value) {
|
||||
long result = integral(value);
|
||||
if (result < 0) fail("RESPONSE_SCHEMA_ERROR");
|
||||
return result;
|
||||
}
|
||||
|
||||
private long integral(JsonNode value) {
|
||||
if (value == null || !value.isIntegralNumber() || !value.canConvertToLong())
|
||||
fail("RESPONSE_SCHEMA_ERROR");
|
||||
return value.longValue();
|
||||
}
|
||||
|
||||
private String requiredText(JsonNode value, int max) {
|
||||
if (value == null
|
||||
|| !value.isTextual()
|
||||
|| value.textValue().trim().isEmpty()
|
||||
|| value.textValue().length() > max) fail("RESPONSE_SCHEMA_ERROR");
|
||||
return value.textValue();
|
||||
}
|
||||
|
||||
private void range(Integer value, int min, int max) {
|
||||
if (value == null || value < min || value > max) fail("CONFIG_SCHEMA_ERROR");
|
||||
}
|
||||
|
||||
static UUID uuidV5(UUID namespace, String value) throws Exception {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-1");
|
||||
digest.update(
|
||||
ByteBuffer.allocate(16)
|
||||
.putLong(namespace.getMostSignificantBits())
|
||||
.putLong(namespace.getLeastSignificantBits())
|
||||
.array());
|
||||
byte[] hash = digest.digest(value.getBytes(StandardCharsets.UTF_8));
|
||||
hash[6] = (byte) ((hash[6] & 15) | 80);
|
||||
hash[8] = (byte) ((hash[8] & 63) | 128);
|
||||
ByteBuffer buffer = ByteBuffer.wrap(hash);
|
||||
return new UUID(buffer.getLong(), buffer.getLong());
|
||||
}
|
||||
|
||||
private long elapsed(long started) {
|
||||
return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started);
|
||||
}
|
||||
|
||||
private void fail(String code) {
|
||||
throw new AdapterException(code, "微步 API: " + code);
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void close() {
|
||||
try {
|
||||
http.close();
|
||||
} catch (IOException ignored) {
|
||||
// Spring shutdown best effort.
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -5,7 +5,7 @@ public final class CredentialSchemas {
|
||||
private CredentialSchemas() {}
|
||||
|
||||
public static void validate(String type, String key, String secret, String token) {
|
||||
if ("NDRSERVER_V1".equals(type)) {
|
||||
if ("NDRSERVER_V1".equals(type) || "WEIBU_V1".equals(type)) {
|
||||
require(key, 512);
|
||||
require(secret, 2048);
|
||||
if (token != null) fail();
|
||||
|
||||
+5
-3
@@ -14,14 +14,16 @@ import org.springframework.stereotype.Component;
|
||||
/**
|
||||
* 设备基础地址的 SSRF 防护边界。
|
||||
*
|
||||
* <p>只有显式加入白名单的 HTTPS 主机才能访问;拒绝 user-info、query、fragment 和路径穿越。 主机采用精确匹配,不允许后缀或通配符匹配,避免相似域名绕过。TLS
|
||||
* 证书链和主机名继续由 JVM 默认校验器验证。
|
||||
* <p>始终拒绝非 HTTPS、user-info、query、fragment 和路径穿越。现场默认关闭静态主机白名单,
|
||||
* 避免设备增减必须重启;显式启用后采用精确主机匹配,不允许后缀或通配符匹配。
|
||||
*/
|
||||
@Component
|
||||
public class SafeBaseUrlValidator {
|
||||
private final boolean hostAllowlistEnabled;
|
||||
private final Set<String> allowedHosts;
|
||||
|
||||
public SafeBaseUrlValidator(ApiPullProperties properties) {
|
||||
hostAllowlistEnabled = properties.isHostAllowlistEnabled();
|
||||
String configured = properties.getAllowedHosts();
|
||||
allowedHosts =
|
||||
configured == null || configured.trim().isEmpty()
|
||||
@@ -45,7 +47,7 @@ public class SafeBaseUrlValidator {
|
||||
throw new AdapterException(
|
||||
"SSRF_REJECTED", "baseUrl 必须是无 user-info/query/fragment 的 HTTPS URL");
|
||||
String host = uri.getHost().toLowerCase();
|
||||
if (allowedHosts.isEmpty() || !allowedHosts.contains(host))
|
||||
if (hostAllowlistEnabled && (allowedHosts.isEmpty() || !allowedHosts.contains(host)))
|
||||
throw new AdapterException("SSRF_REJECTED", "设备主机不在 device.api.allowed-hosts 中: " + host);
|
||||
InetAddress[] addresses = InetAddress.getAllByName(host);
|
||||
if (addresses.length == 0)
|
||||
|
||||
+49
-13
@@ -1,6 +1,7 @@
|
||||
package com.haobang.dispatch;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import com.common.entity.DeviceUnknown;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.common.service.DeviceUnknownService;
|
||||
@@ -80,7 +81,7 @@ public class DeviceContextResolver {
|
||||
for (Map.Entry<String, List<DeviceDevice>> entry : grouped.entrySet()) {
|
||||
List<DeviceDevice> sameIp = entry.getValue();
|
||||
Resolution resolution = sameIp.size() == 1
|
||||
? validateAssignment(sameIp.get(0))
|
||||
? validateSyslogDevice(sameIp.get(0))
|
||||
: Resolution.terminal(Type.DUPLICATE_DEVICE, null,
|
||||
"来源 IP 存在多条设备记录: " + entry.getKey());
|
||||
resolutions.put(entry.getKey(), resolution);
|
||||
@@ -147,15 +148,7 @@ public class DeviceContextResolver {
|
||||
resolution = Resolution.terminal(Type.DUPLICATE_DEVICE, null,
|
||||
"来源 IP 存在多条设备记录: " + sourceIp);
|
||||
} else {
|
||||
DeviceDevice device = devices.get(0);
|
||||
Integer assignedCollector = device.getDeviceCollectId();
|
||||
if (assignedCollector == null
|
||||
|| assignedCollector != properties.getAppServiceDeviceCollectId()) {
|
||||
resolution = Resolution.terminal(Type.NOT_ASSIGNED, device,
|
||||
"设备不属于当前采集探针: deviceId=" + device.getId());
|
||||
} else {
|
||||
resolution = Resolution.registered(device);
|
||||
}
|
||||
resolution = validateSyslogDevice(devices.get(0));
|
||||
}
|
||||
cache.put(sourceIp, new CacheEntry(resolution,
|
||||
now + properties.getSyslogDeviceContextCacheTtlMs()));
|
||||
@@ -196,9 +189,10 @@ public class DeviceContextResolver {
|
||||
}
|
||||
misses.increment();
|
||||
DeviceDevice device = deviceService.getByIdSafely(deviceId);
|
||||
Resolution resolution = device == null
|
||||
Resolution resolution = device == null || device.getDeletedAt() != null
|
||||
|| (device.getDelFlag() != null && !"0".equals(device.getDelFlag()))
|
||||
? Resolution.terminal(Type.UNKNOWN_DEVICE, null, "设备不存在: deviceId=" + deviceId)
|
||||
: validateAssignment(device);
|
||||
: validateApiDevice(device);
|
||||
deviceIdCache.put(deviceId, new CacheEntry(resolution,
|
||||
now + properties.getSyslogDeviceContextCacheTtlMs()));
|
||||
return resolution;
|
||||
@@ -214,10 +208,46 @@ public class DeviceContextResolver {
|
||||
return Resolution.registered(device);
|
||||
}
|
||||
|
||||
private Resolution validateSyslogDevice(DeviceDevice device) {
|
||||
Resolution assignment = validateAssignment(device);
|
||||
if (!assignment.isRegistered()) return assignment;
|
||||
DeviceCollectionMode mode = device.getCollectionMode();
|
||||
if (!mode.allowsSyslog()) {
|
||||
return Resolution.terminal(Type.COLLECTION_MODE_DENIED, device,
|
||||
"设备采集模式不允许 SYSLOG_PUSH: deviceId=" + device.getId()
|
||||
+ ", collectionMode=" + mode);
|
||||
}
|
||||
return assignment;
|
||||
}
|
||||
|
||||
private Resolution validateApiDevice(DeviceDevice device) {
|
||||
Resolution assignment = validateAssignment(device);
|
||||
if (!assignment.isRegistered()) return assignment;
|
||||
DeviceCollectionMode mode = device.getCollectionMode();
|
||||
if (!mode.allowsApi()) {
|
||||
return Resolution.terminal(Type.COLLECTION_MODE_DENIED, device,
|
||||
"设备采集模式不允许 API_PULL: deviceId=" + device.getId()
|
||||
+ ", collectionMode=" + mode);
|
||||
}
|
||||
return assignment;
|
||||
}
|
||||
|
||||
public void invalidate(String sourceIp) {
|
||||
if (sourceIp != null) cache.remove(sourceIp);
|
||||
}
|
||||
|
||||
/** 清除指定设备的 device_id 和所有关联 source_ip 本地缓存。 */
|
||||
public void invalidateDevice(Integer deviceId, String currentIp) {
|
||||
if (deviceId != null) deviceIdCache.remove(deviceId);
|
||||
cache.forEach((sourceIp, entry) -> {
|
||||
DeviceDevice cachedDevice = entry.resolution.getDevice();
|
||||
if ((currentIp != null && currentIp.equals(sourceIp))
|
||||
|| (deviceId != null && cachedDevice != null && deviceId.equals(cachedDevice.getId()))) {
|
||||
cache.remove(sourceIp, entry);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public void invalidateAll() { cache.clear(); deviceIdCache.clear(); }
|
||||
public long getHitCount() { return hits.sum(); }
|
||||
public long getMissCount() { return misses.sum(); }
|
||||
@@ -255,7 +285,13 @@ public class DeviceContextResolver {
|
||||
}
|
||||
}
|
||||
|
||||
public enum Type { REGISTERED, UNKNOWN_DEVICE, DUPLICATE_DEVICE, NOT_ASSIGNED }
|
||||
public enum Type {
|
||||
REGISTERED,
|
||||
UNKNOWN_DEVICE,
|
||||
DUPLICATE_DEVICE,
|
||||
NOT_ASSIGNED,
|
||||
COLLECTION_MODE_DENIED
|
||||
}
|
||||
public enum IngressStatus { REGISTERED, INVALID, UNAVAILABLE }
|
||||
|
||||
private static final class IngressSnapshot {
|
||||
|
||||
+8
-5
@@ -2,6 +2,7 @@ package com.haobang.dispatch;
|
||||
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.diagnostics.SyslogIpMetricsRegistry;
|
||||
import com.haobang.wal.IngressType;
|
||||
import com.haobang.wal.SyslogWalRecord;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
import com.kafka.kafkaProducer;
|
||||
@@ -30,9 +31,9 @@ import java.util.concurrent.atomic.LongAdder;
|
||||
/**
|
||||
* 从 WAL 有界并发异步投递 Kafka,并只按连续 ACK 推进 WAL 游标。
|
||||
*
|
||||
* <p>v2/v3 保持原有按 source_ip 识别设备和单时间参数的生产路径;v4 改为按可信 device_id
|
||||
* 识别设备,并分别传递 receive_time/event_time。兼容分支留在 Dispatcher,Kafka 消息外层结构
|
||||
* 不因接入方式改变。
|
||||
* <p>v2/v3 保持原有按 source_ip 识别设备和单时间参数的生产路径;v4 API_PULL 按可信
|
||||
* device_id 识别设备,并分别传递 receive_time/event_time。Dispatcher 按 ingress_type 选择
|
||||
* 设备策略,确保模式变化后已经进入 WAL 的记录仍会被最终校验。
|
||||
*/
|
||||
@Component
|
||||
public class SyslogWalDispatcher {
|
||||
@@ -148,8 +149,10 @@ public class SyslogWalDispatcher {
|
||||
}
|
||||
final DeviceContextResolver.Resolution resolution;
|
||||
try {
|
||||
// API 的 HTTP 对端可能是代理或网关;v4 必须使用任务写入 WAL 的 device_id。
|
||||
if (record.getSchemaVersion() >= 4 && record.getDeviceId() != null) {
|
||||
// API 的 HTTP 对端可能是代理或网关;API_PULL 必须使用任务写入 WAL 的 device_id。
|
||||
// 不能仅凭 schema/device_id 判断,否则未来携带 device_id 的 SYSLOG_PUSH 会误用 API 权限。
|
||||
if (record.getSchemaVersion() >= 4
|
||||
&& IngressType.API_PULL.name().equals(record.getIngressType())) {
|
||||
resolution = deviceResolver.resolveDeviceId(record.getDeviceId());
|
||||
} else {
|
||||
resolution = deviceResolver.resolve(record.getSourceIp(), record.getProtocol());
|
||||
|
||||
@@ -9,6 +9,11 @@ import java.util.function.Supplier;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List ;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.Arrays;
|
||||
import java.util.ArrayList;
|
||||
import org.springframework.data.redis.core.Cursor;
|
||||
import org.springframework.data.redis.core.RedisCallback;
|
||||
import org.springframework.data.redis.core.ScanOptions;
|
||||
|
||||
@Component
|
||||
public class SafeCacheUtil {
|
||||
@@ -105,4 +110,39 @@ public class SafeCacheUtil {
|
||||
return loader.get();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 严格删除 SafeCacheUtil 使用的原始 Redis Key。
|
||||
*
|
||||
* <p>设备策略更新不能像读缓存一样静默降级,否则接口成功后仍可能读取旧策略。</p>
|
||||
*/
|
||||
public void evict(String... keys) {
|
||||
if (keys == null || keys.length == 0) return;
|
||||
redisTemplate.delete(Arrays.asList(keys));
|
||||
}
|
||||
|
||||
/** 使用 Redis SCAN 分批删除匹配的原始 Key,避免生产环境执行阻塞式 KEYS。 */
|
||||
public void evictMatching(String... patterns) {
|
||||
if (patterns == null || patterns.length == 0) return;
|
||||
redisTemplate.execute((RedisCallback<Void>) connection -> {
|
||||
for (String pattern : patterns) {
|
||||
if (pattern == null || pattern.trim().isEmpty()) continue;
|
||||
ScanOptions options = ScanOptions.scanOptions().match(pattern).count(500).build();
|
||||
try (Cursor<byte[]> cursor = connection.scan(options)) {
|
||||
List<byte[]> batch = new ArrayList<>(500);
|
||||
while (cursor.hasNext()) {
|
||||
batch.add(cursor.next());
|
||||
if (batch.size() == 500) {
|
||||
connection.del(batch.toArray(new byte[batch.size()][]));
|
||||
batch.clear();
|
||||
}
|
||||
}
|
||||
if (!batch.isEmpty()) {
|
||||
connection.del(batch.toArray(new byte[batch.size()][]));
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,8 +7,8 @@ server.error.include-message=always
|
||||
server.error.include-binding-errors=always
|
||||
|
||||
# Syslog Server Configuration
|
||||
# 当前测试只验证设备 API 拉取,关闭 TCP/UDP Netty 监听;可用环境变量重新开启。
|
||||
syslog.server.enabled=${SYSLOG_SERVER_ENABLED:false}
|
||||
# 默认同时开启 TCP/UDP Syslog 与设备 API 拉取;设备级 collectionMode 再决定具体接入方式。
|
||||
syslog.server.enabled=${SYSLOG_SERVER_ENABLED:true}
|
||||
syslog.tcp.port=514
|
||||
syslog.udp.port=514
|
||||
syslog.tcp.backlog=${SYSLOG_TCP_BACKLOG:4096}
|
||||
@@ -127,7 +127,7 @@ mybatis.mapper-locations=classpath:mapper/*.xml
|
||||
mybatis.type-aliases-package=com.common.entity
|
||||
mybatis.configuration.map-underscore-to-camel-case=true
|
||||
|
||||
# 当前测试环境开启设备 API 拉取调度。
|
||||
# 默认开启设备 API 拉取调度;单个任务仍须通过 resume 启用。
|
||||
device.api.enabled=${DEVICE_API_ENABLED:true}
|
||||
# 设备 AppKey/AppSecret 由 device_collect_api_credential 表管理,配置文件不再保存厂商凭据。
|
||||
device.api.scheduler.instance-id=${DEVICE_API_INSTANCE_ID:${HOSTNAME:local}}
|
||||
@@ -141,9 +141,12 @@ device.api.scheduler.max-pages-per-slice=${DEVICE_API_MAX_PAGES_PER_SLICE:20}
|
||||
# HTTP + 解析 + 本页 WAL durable 总耗时达到 1 秒即打印一条结构化 WARN。
|
||||
device.api.page.slow-threshold-ms=${DEVICE_API_SLOW_PAGE_THRESHOLD_MS:1000}
|
||||
device.api.page.backpressure-timeout-ms=${DEVICE_API_BACKPRESSURE_TIMEOUT_MS:120000}
|
||||
# 逗号分隔的设备 DNS 名或 IP;必须显式配置,防止管理接口被用作 SSRF 跳板。
|
||||
# NDR 测试设备证书仅声明 CN=wlza.com;通过宿主机/容器 hosts 将该域名固定映射到 192.168.4.19。
|
||||
# 现场默认关闭静态主机白名单,新增 API 设备无需修改配置和重启;需要收紧时可显式开启。
|
||||
device.api.host-allowlist-enabled=${DEVICE_API_HOST_ALLOWLIST_ENABLED:false}
|
||||
# 白名单开启时使用的逗号分隔 DNS 名或 IP;主机精确匹配,不支持通配符。
|
||||
device.api.allowed-hosts=${DEVICE_API_ALLOWED_HOSTS:192.168.4.19,wlza.com}
|
||||
# 所有设备 API 默认关闭证书链和主机名校验;仍只允许 HTTPS,并禁止重定向。
|
||||
device.api.tls-verify-enabled=${DEVICE_API_TLS_VERIFY_ENABLED:false}
|
||||
|
||||
# 开发环境缓存配置
|
||||
spring.redis.host=localhost
|
||||
|
||||
@@ -42,6 +42,9 @@
|
||||
<result column="today_non_log_count" property="todayNonLogCount" />
|
||||
<result column="create_dept" property="createDept" />
|
||||
<result column="device_collect_id" property="deviceCollectId" />
|
||||
<result column="collection_mode" property="collectionMode"
|
||||
javaType="com.common.entity.DeviceCollectionMode"
|
||||
typeHandler="org.apache.ibatis.type.EnumTypeHandler" />
|
||||
</resultMap>
|
||||
|
||||
<!-- 基础查询列 -->
|
||||
@@ -51,7 +54,8 @@
|
||||
control_agent_id, license_start_time, license_end_time, is_monitoring,
|
||||
security_scope_id, owner_id, ssh_config_id, status, created_by_id, decode_type,
|
||||
miss_policy, tenant_id, create_time, update_time, create_by, update_by, del_flag,
|
||||
manager_name, today_parse_count, today_non_log_count, create_dept, device_collect_id
|
||||
manager_name, today_parse_count, today_non_log_count, create_dept, device_collect_id,
|
||||
collection_mode
|
||||
</sql>
|
||||
|
||||
<!-- 根据ID查询 -->
|
||||
@@ -235,4 +239,14 @@
|
||||
WHERE vendor = #{vendor}
|
||||
</select>
|
||||
|
||||
</mapper>
|
||||
<update id="updateCollectionMode">
|
||||
UPDATE device_device
|
||||
SET collection_mode = #{collectionMode,javaType=com.common.entity.DeviceCollectionMode,typeHandler=org.apache.ibatis.type.EnumTypeHandler},
|
||||
updated_at = SYSDATE,
|
||||
update_time = SYSDATE
|
||||
WHERE id = #{id}
|
||||
AND deleted_at IS NULL
|
||||
AND (del_flag IS NULL OR del_flag = '0')
|
||||
</update>
|
||||
|
||||
</mapper>
|
||||
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
package com.common.controller;
|
||||
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.haobang.dispatch.DeviceContextResolver;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
class DeviceDeviceControllerTest {
|
||||
@Test
|
||||
void rejectsInvalidCollectionMode() throws Exception {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
MockMvc mvc = MockMvcBuilders.standaloneSetup(
|
||||
new DeviceDeviceController(service, resolver)).build();
|
||||
|
||||
mvc.perform(put("/api/devices/40/collection-mode")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("{\"collectionMode\":\"INVALID\"}"))
|
||||
.andExpect(status().isBadRequest());
|
||||
|
||||
verify(service, never()).updateCollectionMode(
|
||||
org.mockito.ArgumentMatchers.anyInt(), org.mockito.ArgumentMatchers.any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsAttemptsToUpdateOtherDeviceFields() throws Exception {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
MockMvc mvc = MockMvcBuilders.standaloneSetup(
|
||||
new DeviceDeviceController(service, resolver)).build();
|
||||
|
||||
mvc.perform(put("/api/devices/40/collection-mode")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("{\"collectionMode\":\"API_ONLY\",\"name\":\"tampered\"}"))
|
||||
.andExpect(status().isBadRequest());
|
||||
|
||||
verify(service, never()).updateCollectionMode(
|
||||
org.mockito.ArgumentMatchers.anyInt(), org.mockito.ArgumentMatchers.any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void updateInvalidatesLocalCachesAndRefreshesIngressSnapshot() throws Exception {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
DeviceDevice updated = new DeviceDevice();
|
||||
updated.setId(40);
|
||||
updated.setIp("10.56.171.164");
|
||||
updated.setCollectionMode(DeviceCollectionMode.API_ONLY);
|
||||
when(service.updateCollectionMode(40, DeviceCollectionMode.API_ONLY)).thenReturn(updated);
|
||||
MockMvc mvc = MockMvcBuilders.standaloneSetup(
|
||||
new DeviceDeviceController(service, resolver)).build();
|
||||
|
||||
mvc.perform(put("/api/devices/40/collection-mode")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("{\"collectionMode\":\"API_ONLY\"}"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.id").value(40))
|
||||
.andExpect(jsonPath("$.collectionMode").value("API_ONLY"));
|
||||
|
||||
verify(resolver).invalidateAll();
|
||||
verify(resolver).refreshIngressSnapshot();
|
||||
}
|
||||
|
||||
@Test
|
||||
void returnsNotFoundForMissingOrDeletedDevice() throws Exception {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
MockMvc mvc = MockMvcBuilders.standaloneSetup(
|
||||
new DeviceDeviceController(service, resolver)).build();
|
||||
|
||||
mvc.perform(put("/api/devices/404/collection-mode")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("{\"collectionMode\":\"API_ONLY\"}"))
|
||||
.andExpect(status().isNotFound());
|
||||
|
||||
verify(resolver, never()).refreshIngressSnapshot();
|
||||
}
|
||||
|
||||
@Test
|
||||
void clearsAllDeviceCacheLayersAndRefreshesSnapshot() throws Exception {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
MockMvc mvc = MockMvcBuilders.standaloneSetup(
|
||||
new DeviceDeviceController(service, resolver)).build();
|
||||
|
||||
mvc.perform(delete("/api/devices/cache"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.cleared").value(true))
|
||||
.andExpect(jsonPath("$.scope").value("ALL"));
|
||||
|
||||
verify(service).evictAllDeviceCaches();
|
||||
verify(resolver).invalidateAll();
|
||||
verify(resolver).refreshIngressSnapshot();
|
||||
}
|
||||
|
||||
@Test
|
||||
void clearsSpecifiedDeviceCacheLayersAndRefreshesSnapshot() throws Exception {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(40);
|
||||
device.setIp("10.56.171.164");
|
||||
when(service.evictDeviceCaches(40)).thenReturn(device);
|
||||
MockMvc mvc = MockMvcBuilders.standaloneSetup(
|
||||
new DeviceDeviceController(service, resolver)).build();
|
||||
|
||||
mvc.perform(delete("/api/devices/40/cache"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.cleared").value(true))
|
||||
.andExpect(jsonPath("$.scope").value("DEVICE"))
|
||||
.andExpect(jsonPath("$.deviceId").value(40))
|
||||
.andExpect(jsonPath("$.ip").value("10.56.171.164"));
|
||||
|
||||
verify(service).evictDeviceCaches(40);
|
||||
verify(resolver).invalidateDevice(40, "10.56.171.164");
|
||||
verify(resolver).refreshIngressSnapshot();
|
||||
}
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package com.common.mapper;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import org.apache.ibatis.builder.xml.XMLMapperBuilder;
|
||||
import org.apache.ibatis.io.Resources;
|
||||
import org.apache.ibatis.mapping.ResultMapping;
|
||||
import org.apache.ibatis.session.Configuration;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DeviceDeviceMapperXmlTest {
|
||||
@Test
|
||||
void parsesCollectionModeMappingAndDedicatedUpdateStatement() throws Exception {
|
||||
Configuration configuration = new Configuration();
|
||||
String resource = "mapper/DeviceDeviceMapper.xml";
|
||||
try (InputStream input = Resources.getResourceAsStream(resource)) {
|
||||
new XMLMapperBuilder(input, configuration, resource, configuration.getSqlFragments()).parse();
|
||||
}
|
||||
|
||||
ResultMapping collectionMode = configuration
|
||||
.getResultMap("com.common.mapper.DeviceDeviceMapper.BaseResultMap")
|
||||
.getResultMappings().stream()
|
||||
.filter(mapping -> "collectionMode".equals(mapping.getProperty()))
|
||||
.findFirst().orElseThrow(() -> new AssertionError("collectionMode mapping missing"));
|
||||
assertTrue("collection_mode".equalsIgnoreCase(collectionMode.getColumn()));
|
||||
|
||||
Map<String, Object> parameters = new HashMap<>();
|
||||
parameters.put("id", 40);
|
||||
parameters.put("collectionMode", "API_ONLY");
|
||||
String sql = configuration
|
||||
.getMappedStatement("com.common.mapper.DeviceDeviceMapper.updateCollectionMode")
|
||||
.getBoundSql(parameters).getSql().replaceAll("\\s+", " ").toLowerCase();
|
||||
assertTrue(sql.contains("set collection_mode = ?"));
|
||||
assertTrue(sql.contains("where id = ?"));
|
||||
assertTrue(sql.contains("deleted_at is null"));
|
||||
}
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package com.common.service;
|
||||
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.mapper.DeviceDeviceMapper;
|
||||
import com.common.service.impl.DeviceDeviceServiceImpl;
|
||||
import com.haobang.util.SafeCacheUtil;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.cache.Cache;
|
||||
import org.springframework.cache.CacheManager;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class DeviceDeviceServiceImplTest {
|
||||
@Test
|
||||
void updateEvictsRawRedisKeysAndEntireSpringDeviceCache() {
|
||||
DeviceDeviceMapper mapper = mock(DeviceDeviceMapper.class);
|
||||
SafeCacheUtil safeCache = mock(SafeCacheUtil.class);
|
||||
CacheManager cacheManager = mock(CacheManager.class);
|
||||
Cache springCache = mock(Cache.class);
|
||||
DeviceDevice current = new DeviceDevice();
|
||||
current.setId(40);
|
||||
current.setIp("10.56.171.164");
|
||||
current.setDelFlag("0");
|
||||
DeviceDevice updated = new DeviceDevice();
|
||||
updated.setId(40);
|
||||
updated.setIp(current.getIp());
|
||||
updated.setCollectionMode(DeviceCollectionMode.API_ONLY);
|
||||
when(mapper.selectById(40)).thenReturn(current, updated);
|
||||
when(mapper.updateCollectionMode(40, DeviceCollectionMode.API_ONLY)).thenReturn(1);
|
||||
when(cacheManager.getCache("device")).thenReturn(springCache);
|
||||
DeviceDeviceServiceImpl service = new DeviceDeviceServiceImpl(mapper, safeCache, cacheManager);
|
||||
|
||||
DeviceDevice result = service.updateCollectionMode(40, DeviceCollectionMode.API_ONLY);
|
||||
|
||||
assertEquals(DeviceCollectionMode.API_ONLY, result.getCollectionMode());
|
||||
verify(safeCache).evict("device:id:40", "device:ip:10.56.171.164");
|
||||
verify(springCache).clear();
|
||||
}
|
||||
|
||||
@Test
|
||||
void evictsOnlySpecifiedDeviceIdAndIpEntries() {
|
||||
DeviceDeviceMapper mapper = mock(DeviceDeviceMapper.class);
|
||||
SafeCacheUtil safeCache = mock(SafeCacheUtil.class);
|
||||
CacheManager cacheManager = mock(CacheManager.class);
|
||||
Cache springCache = mock(Cache.class);
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(40);
|
||||
device.setIp("10.56.171.164");
|
||||
when(mapper.selectById(40)).thenReturn(device);
|
||||
when(cacheManager.getCache("device")).thenReturn(springCache);
|
||||
DeviceDeviceServiceImpl service = new DeviceDeviceServiceImpl(mapper, safeCache, cacheManager);
|
||||
|
||||
DeviceDevice result = service.evictDeviceCaches(40);
|
||||
|
||||
assertEquals(40, result.getId());
|
||||
verify(safeCache).evict("device:id:40", "device:ip:10.56.171.164");
|
||||
verify(springCache).evict("device:id:40");
|
||||
verify(springCache).evict("device:ip:10.56.171.164");
|
||||
}
|
||||
|
||||
@Test
|
||||
void evictsAllRawRedisAndSpringDeviceCaches() {
|
||||
DeviceDeviceMapper mapper = mock(DeviceDeviceMapper.class);
|
||||
SafeCacheUtil safeCache = mock(SafeCacheUtil.class);
|
||||
CacheManager cacheManager = mock(CacheManager.class);
|
||||
Cache springCache = mock(Cache.class);
|
||||
when(cacheManager.getCache("device")).thenReturn(springCache);
|
||||
DeviceDeviceServiceImpl service = new DeviceDeviceServiceImpl(mapper, safeCache, cacheManager);
|
||||
|
||||
service.evictAllDeviceCaches();
|
||||
|
||||
verify(safeCache).evictMatching("device:id:*", "device:ip:*");
|
||||
verify(springCache).clear();
|
||||
}
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package com.config;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import ch.qos.logback.classic.Logger;
|
||||
import ch.qos.logback.classic.spi.ILoggingEvent;
|
||||
import ch.qos.logback.core.read.ListAppender;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.mock.env.MockEnvironment;
|
||||
|
||||
class SyslogBuildStartupLoggerTest {
|
||||
@Test
|
||||
void logsCodeRevisionAndEffectiveOverrides() {
|
||||
MockEnvironment environment = new MockEnvironment()
|
||||
.withProperty("device.api.enabled", "false")
|
||||
.withProperty("device.api.tls-verify-enabled", "true")
|
||||
.withProperty("app.service.device_collect_id", "40")
|
||||
.withProperty("spring.kafka.producer.topic", "prod-syslog");
|
||||
environment.setActiveProfiles("prod");
|
||||
SyslogBuildStartupLogger reporter = new SyslogBuildStartupLogger(environment);
|
||||
Logger logger = (Logger) LoggerFactory.getLogger(SyslogBuildStartupLogger.class);
|
||||
ListAppender<ILoggingEvent> appender = new ListAppender<>();
|
||||
appender.start();
|
||||
logger.addAppender(appender);
|
||||
try {
|
||||
reporter.onApplicationReady();
|
||||
assertEquals(1, appender.list.size());
|
||||
String message = appender.list.get(0).getFormattedMessage();
|
||||
assertTrue(message.contains("[SYSLOG_BUILD_READY] codeRevision="
|
||||
+ SyslogBuildStartupLogger.CODE_REVISION));
|
||||
assertTrue(message.contains("application=syslog-serve"));
|
||||
assertTrue(message.contains("profiles=prod"));
|
||||
assertTrue(message.contains("apiPullEnabled=false"));
|
||||
assertTrue(message.contains("apiTlsVerifyEnabled=true"));
|
||||
assertTrue(message.contains("deviceCollectId=40"));
|
||||
assertTrue(message.contains("kafkaTopic=prod-syslog"));
|
||||
} finally {
|
||||
logger.detachAppender(appender);
|
||||
appender.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void usesSafeDefaultsAndIgnoresConfiguredRevision() {
|
||||
String summary = new SyslogBuildStartupLogger(new MockEnvironment()
|
||||
.withProperty("codeRevision", "fake")).startupSummary();
|
||||
assertTrue(summary.contains("codeRevision=" + SyslogBuildStartupLogger.CODE_REVISION));
|
||||
assertFalse(summary.contains("codeRevision=fake"));
|
||||
assertTrue(summary.contains("syslogEnabled=true"));
|
||||
assertTrue(summary.contains("apiPullEnabled=true"));
|
||||
assertTrue(summary.contains("apiTlsVerifyEnabled=false"));
|
||||
assertTrue(summary.contains("walEnabled=true"));
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package com.haobang.apipull;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class ApiPullPropertiesTest {
|
||||
@Test
|
||||
void apiPullIsEnabledByDefault() {
|
||||
ApiPullProperties properties = new ApiPullProperties();
|
||||
assertTrue(properties.isEnabled());
|
||||
assertFalse(properties.isTlsVerifyEnabled());
|
||||
}
|
||||
}
|
||||
+38
@@ -154,6 +154,44 @@ class ApiPullRunnerTest {
|
||||
verify(fixture.ingress, times(3)).tryAppend(any(IngressEvent.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void unpagedAdapterSplitsUntilSingleResponseContainsWholeWindow() {
|
||||
RunnerFixture fixture = new RunnerFixture();
|
||||
fixture.config.setPageSize(2);
|
||||
fixture.config.setMaxPagesPerRun(1);
|
||||
|
||||
when(fixture.adapter.pull(any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
PullRequest request = invocation.getArgument(0);
|
||||
long windowSeconds =
|
||||
Duration.between(WINDOW_START, request.getQueryEnd()).getSeconds();
|
||||
return windowSeconds == 60L ? page(3L, 2, 0L) : page(2L, 2, 0L);
|
||||
});
|
||||
|
||||
AtomicReference<LocalDateTime> committedEnd = new AtomicReference<>();
|
||||
when(fixture.mapper.commitCheckpoint(
|
||||
anyInt(), anyLong(), anyString(), anyLong(), anyLong(), anyString(), any(), any(), any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
committedEnd.set(invocation.getArgument(6));
|
||||
return 1;
|
||||
});
|
||||
|
||||
fixture.runner.execute(fixture.config, fixture.checkpoint, fixture.run);
|
||||
|
||||
assertEquals("SUCCESS", fixture.run.getStatus());
|
||||
assertEquals(Integer.valueOf(1), fixture.run.getPagesFetched());
|
||||
assertEquals(Long.valueOf(2L), fixture.run.getRecordsDurable());
|
||||
assertEquals(ApiPullTime.toDatabase(WINDOW_START.plusSeconds(30L)), committedEnd.get());
|
||||
org.mockito.ArgumentCaptor<PullRequest> requests =
|
||||
org.mockito.ArgumentCaptor.forClass(PullRequest.class);
|
||||
verify(fixture.adapter, times(2)).pull(requests.capture());
|
||||
assertEquals(0L, requests.getAllValues().get(0).getOffset());
|
||||
assertEquals(0L, requests.getAllValues().get(1).getOffset());
|
||||
verify(fixture.ingress, times(2)).tryAppend(any(IngressEvent.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void schedulingSliceCanSplitBelowTheHardRunCapacity() {
|
||||
RunnerFixture fixture = new RunnerFixture();
|
||||
|
||||
+28
-1
@@ -1,6 +1,12 @@
|
||||
package com.haobang.apipull;
|
||||
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.common.service.DeviceUnknownService;
|
||||
import com.common.mapper.DeviceCollectApiMapper;
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.dispatch.DeviceContextResolver;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
|
||||
import com.haobang.wal.SyslogWalService;
|
||||
@@ -19,12 +25,33 @@ class ApiPullSchedulerTest {
|
||||
DeviceCollectApiCheckpoint cp=new DeviceCollectApiCheckpoint();cp.setFencingToken(1L);when(mapper.selectCheckpoint(1)).thenReturn(cp);
|
||||
CountDownLatch entered=new CountDownLatch(1);CountDownLatch release=new CountDownLatch(1);
|
||||
doAnswer(i->{entered.countDown();release.await(2,TimeUnit.SECONDS);return null;}).when(runner).execute(eq(first),eq(cp),any());
|
||||
ApiPullScheduler scheduler=new ApiPullScheduler(p,mapper,runner,wal);scheduler.init();
|
||||
ApiPullScheduler scheduler=new ApiPullScheduler(p,mapper,runner,wal,resolver(DeviceCollectionMode.BOTH));scheduler.init();
|
||||
try{assertNotNull(scheduler.trigger(1));assertTrue(entered.await(1,TimeUnit.SECONDS));
|
||||
ApiPullException conflict=assertThrows(ApiPullException.class,()->scheduler.trigger(2));assertEquals("LEASE_CONFLICT",conflict.getCode());
|
||||
verify(mapper,never()).acquireLease(eq(2),anyLong(),anyString(),anyInt());}
|
||||
finally{release.countDown();scheduler.shutdown();}
|
||||
}
|
||||
|
||||
@Test void syslogOnlyScheduledTaskAdvancesNextRunWithoutLeaseOrHttp(){
|
||||
ApiPullProperties p=new ApiPullProperties();p.setEnabled(true);p.setInstanceId("i1");p.setScanIntervalMs(5000);p.setLeaseSeconds(120);
|
||||
p.setGlobalConcurrency(1);p.setPerHostConcurrency(1);p.setSlowPageThresholdMs(1000);p.setBackpressureTimeoutMs(1000);
|
||||
DeviceCollectApiMapper mapper=mock(DeviceCollectApiMapper.class);ApiPullRunner runner=mock(ApiPullRunner.class);SyslogWalService wal=mock(SyslogWalService.class);
|
||||
DeviceCollectApiConfig config=config(1,"https://localhost/ndr/api");
|
||||
when(mapper.selectDueConfigs()).thenReturn(java.util.Collections.singletonList(config));
|
||||
ApiPullScheduler scheduler=new ApiPullScheduler(p,mapper,runner,wal,resolver(DeviceCollectionMode.SYSLOG_ONLY));scheduler.init();
|
||||
try{scheduler.scan();
|
||||
verify(mapper).scheduleNext(1,1L,10);
|
||||
verify(mapper,never()).acquireLease(anyInt(),anyLong(),anyString(),anyInt());
|
||||
verifyNoInteractions(runner);
|
||||
}finally{scheduler.shutdown();}
|
||||
}
|
||||
|
||||
private DeviceContextResolver resolver(DeviceCollectionMode mode){
|
||||
DeviceDeviceService devices=mock(DeviceDeviceService.class);
|
||||
when(devices.getByIdSafely(anyInt())).thenAnswer(i->{DeviceDevice d=new DeviceDevice();d.setId(i.getArgument(0));d.setDeviceCollectId(1);d.setCollectionMode(mode);return d;});
|
||||
AppProperties app=new AppProperties();app.setAppServiceDeviceCollectId(1);app.setSyslogDeviceContextCacheTtlMs(60000L);
|
||||
return new DeviceContextResolver(devices,mock(DeviceUnknownService.class),app);
|
||||
}
|
||||
private DeviceCollectApiConfig config(int id,String base){DeviceCollectApiConfig c=new DeviceCollectApiConfig();c.setCollectTaskId(id);c.setEnabled(1);c.setConfigVersion(1L);
|
||||
c.setBaseUrl(base);c.setPollIntervalSeconds(10);c.setDeviceId(id);c.setAdapterType("NDRSERVER_V1");c.setAdapterVersion(1);c.setStreamKey("NDR_ALERT");return c;}
|
||||
}
|
||||
|
||||
+71
@@ -2,13 +2,16 @@ package com.haobang.apipull;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.common.entity.DeviceCollectTask;
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.mapper.DeviceCollectApiMapper;
|
||||
import com.common.mapper.DeviceCollectTaskMapper;
|
||||
@@ -157,6 +160,11 @@ class ApiPullTaskServiceTest {
|
||||
(credentialRef, adapterType, deviceId) ->
|
||||
new CredentialSnapshot(1, "v1", adapterType, Collections.emptyMap());
|
||||
AppProperties app = new AppProperties();
|
||||
app.setAppServiceDeviceCollectId(7);
|
||||
DeviceDevice allowed = new DeviceDevice();
|
||||
allowed.setId(126);
|
||||
allowed.setDeviceCollectId(7);
|
||||
when(devices.getByIdSafely(126)).thenReturn(allowed);
|
||||
ApiPullProperties apiProperties = new ApiPullProperties();
|
||||
apiProperties.setAllowedHosts("localhost");
|
||||
ApiPullTaskService service =
|
||||
@@ -178,12 +186,75 @@ class ApiPullTaskServiceTest {
|
||||
.collect(Collectors.toMap(ValidationCheckResponse::getName, value -> value));
|
||||
|
||||
assertEquals("PASS", checks.get("TLS_HANDSHAKE").getStatus());
|
||||
assertEquals(
|
||||
"TLS 连接通过(全局已关闭证书链和主机名校验)",
|
||||
checks.get("TLS_HANDSHAKE").getSummary());
|
||||
assertEquals("PASS", checks.get("AUTHENTICATION").getStatus());
|
||||
assertEquals("PASS", checks.get("API_PERMISSION").getStatus());
|
||||
assertEquals("FAIL", checks.get("MINIMAL_ALERT_QUERY").getStatus());
|
||||
assertEquals("QUERY_RESPONSE_SCHEMA_ERROR", checks.get("MINIMAL_ALERT_QUERY").getErrorCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void syslogOnlyDeviceCannotCreateApiPullTask() {
|
||||
DeviceCollectTaskMapper tasks = mock(DeviceCollectTaskMapper.class);
|
||||
DeviceCollectApiMapper apiMapper = mock(DeviceCollectApiMapper.class);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
device.setDeviceCollectId(7);
|
||||
device.setCollectionMode(DeviceCollectionMode.SYSLOG_ONLY);
|
||||
when(devices.getByIdSafely(126)).thenReturn(device);
|
||||
AppProperties app = new AppProperties();
|
||||
app.setAppServiceDeviceCollectId(7);
|
||||
ApiPullProperties properties = new ApiPullProperties();
|
||||
properties.setAllowedHosts("localhost");
|
||||
ApiPullTaskService service = new ApiPullTaskService(
|
||||
tasks, apiMapper, devices, mock(AdapterRegistry.class),
|
||||
mock(CredentialProvider.class), new SafeBaseUrlValidator(properties),
|
||||
mock(ApiPullScheduler.class), app, properties, new ObjectMapper());
|
||||
|
||||
ApiPullException failure = assertThrows(ApiPullException.class, () -> service.create(request()));
|
||||
|
||||
assertEquals("COLLECTION_MODE_DENIED", failure.getCode());
|
||||
verify(tasks, never()).insertApiPullTask(any());
|
||||
verify(apiMapper, never()).insertConfig(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void syslogOnlyDeviceCannotValidateResumeOrManuallyTriggerApiTask() {
|
||||
DeviceCollectApiMapper apiMapper = mock(DeviceCollectApiMapper.class);
|
||||
DeviceCollectApiConfig config = new DeviceCollectApiConfig();
|
||||
config.setCollectTaskId(12);
|
||||
config.setDeviceId(126);
|
||||
config.setConfigVersion(3L);
|
||||
when(apiMapper.selectConfig(12)).thenReturn(config);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
device.setDeviceCollectId(7);
|
||||
device.setCollectionMode(DeviceCollectionMode.SYSLOG_ONLY);
|
||||
when(devices.getByIdSafely(126)).thenReturn(device);
|
||||
AppProperties app = new AppProperties();
|
||||
app.setAppServiceDeviceCollectId(7);
|
||||
ApiPullProperties properties = new ApiPullProperties();
|
||||
AdapterRegistry adapters = mock(AdapterRegistry.class);
|
||||
ApiPullScheduler scheduler = mock(ApiPullScheduler.class);
|
||||
ApiPullTaskService service = new ApiPullTaskService(
|
||||
mock(DeviceCollectTaskMapper.class), apiMapper, devices, adapters,
|
||||
mock(CredentialProvider.class), new SafeBaseUrlValidator(properties), scheduler,
|
||||
app, properties, new ObjectMapper());
|
||||
|
||||
assertEquals("COLLECTION_MODE_DENIED",
|
||||
assertThrows(ApiPullException.class, () -> service.validate(12, 3L)).getCode());
|
||||
assertEquals("COLLECTION_MODE_DENIED",
|
||||
assertThrows(ApiPullException.class, () -> service.resume(12, 3L)).getCode());
|
||||
assertEquals("COLLECTION_MODE_DENIED",
|
||||
assertThrows(ApiPullException.class, () -> service.trigger(12, 3L)).getCode());
|
||||
verifyNoInteractions(adapters, scheduler);
|
||||
verify(apiMapper, never()).insertRun(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void updateFromOldClientKeepsExistingSchedulerPriority() {
|
||||
DeviceCollectTaskMapper tasks = mock(DeviceCollectTaskMapper.class);
|
||||
|
||||
+9
@@ -29,6 +29,15 @@ import org.apache.http.util.EntityUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class NdrServerApiAdapterTest {
|
||||
@Test
|
||||
void httpClientCanExplicitlyDisableOrKeepTlsVerification() throws Exception {
|
||||
try (CloseableHttpClient verified = NdrServerApiAdapter.buildHttpClient(true);
|
||||
CloseableHttpClient internalInsecure = NdrServerApiAdapter.buildHttpClient(false)) {
|
||||
assertNotNull(verified);
|
||||
assertNotNull(internalInsecure);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void challengeAnswerUsesExactUtf8AndLowercaseMd5() throws Exception {
|
||||
assertEquals(
|
||||
|
||||
+1
@@ -31,6 +31,7 @@ class NdrServerSpringWiringTest {
|
||||
};
|
||||
|
||||
context.registerBean(CredentialProvider.class, () -> credentialProvider);
|
||||
context.registerBean(ApiPullProperties.class, () -> properties);
|
||||
context.registerBean(SafeBaseUrlValidator.class, () -> new SafeBaseUrlValidator(properties));
|
||||
context.registerBean(ObjectMapper.class, () -> new ObjectMapper());
|
||||
context.register(NdrServerApiAdapter.class);
|
||||
|
||||
+8
@@ -30,6 +30,14 @@ class QuanxiApiAdapterTest {
|
||||
static final Instant END = START.plusSeconds(86400);
|
||||
static final String TOKEN = "synthetic-unit-test-token";
|
||||
|
||||
@Test void supportsVerifiedAndGlobalInsecureTlsClients() throws Exception {
|
||||
try (CloseableHttpClient verified = QuanxiApiAdapter.buildHttpClient(true);
|
||||
CloseableHttpClient internalInsecure = QuanxiApiAdapter.buildHttpClient(false)) {
|
||||
assertNotNull(verified);
|
||||
assertNotNull(internalInsecure);
|
||||
}
|
||||
}
|
||||
|
||||
@Test void realFixtureBecomesTwentyIndependentOriginalDocumentsAndGoldenIds() throws Exception {
|
||||
Fixture f = new Fixture();
|
||||
PullPage page = f.pull();
|
||||
|
||||
+17
-4
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.apipull.*;
|
||||
import com.haobang.apipull.adapter.*;
|
||||
import com.haobang.apipull.adapter.ndr.NdrServerApiAdapter;
|
||||
import com.haobang.apipull.adapter.weibu.WeibuApiAdapter;
|
||||
import com.haobang.apipull.api.ApiPullDtos.*;
|
||||
import com.haobang.apipull.credential.*;
|
||||
import com.haobang.apipull.persistence.*;
|
||||
@@ -22,17 +23,25 @@ import static org.mockito.Mockito.*;
|
||||
import static org.mockito.ArgumentMatchers.*;
|
||||
|
||||
class QuanxiManagementTest {
|
||||
@Test void productionConstructorsRegisterBothVendorsWithoutExternalSystems() {
|
||||
@Test void productionConstructorsRegisterAllVendorsWithoutExternalSystems() {
|
||||
try (AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext()) {
|
||||
ApiPullProperties props = new ApiPullProperties(); props.setAllowedHosts("192.0.2.10");
|
||||
context.registerBean(ApiPullProperties.class, () -> props);
|
||||
context.registerBean(CredentialProvider.class, () -> mock(CredentialProvider.class));
|
||||
context.registerBean(ObjectMapper.class, () -> new ObjectMapper());
|
||||
context.registerBean(SafeBaseUrlValidator.class, () -> new SafeBaseUrlValidator(props));
|
||||
context.register(QuanxiApiAdapter.class, NdrServerApiAdapter.class, AdapterRegistry.class);
|
||||
context.register(
|
||||
QuanxiApiAdapter.class,
|
||||
NdrServerApiAdapter.class,
|
||||
WeibuApiAdapter.class,
|
||||
AdapterRegistry.class);
|
||||
context.refresh();
|
||||
AdapterRegistry registry = context.getBean(AdapterRegistry.class);
|
||||
assertEquals(Collections.singleton(QuanxiApiAdapter.STREAM), registry.require("QUANXI_V1").streams());
|
||||
assertTrue(registry.require("NDRSERVER_V1").streams().contains("NDR_ALERT"));
|
||||
assertEquals(
|
||||
Collections.singleton(WeibuApiAdapter.LOG),
|
||||
registry.require(WeibuApiAdapter.TYPE).streams());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,11 +84,15 @@ class QuanxiManagementTest {
|
||||
f.config.setConfigVersion(1L); f.config.setAdapterVersion(1);
|
||||
DeviceCollectApiMapper mapper = mock(DeviceCollectApiMapper.class);
|
||||
when(mapper.selectConfig(11)).thenReturn(f.config);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = new DeviceDevice(); device.setId(f.config.getDeviceId()); device.setDeviceCollectId(3);
|
||||
when(devices.getByIdSafely(f.config.getDeviceId())).thenReturn(device);
|
||||
AppProperties app = new AppProperties(); app.setAppServiceDeviceCollectId(3);
|
||||
ApiPullProperties props = new ApiPullProperties(); props.setAllowedHosts("192.0.2.10");
|
||||
CredentialProvider credentials = (ref, type, id) -> new CredentialSnapshot(1, "fixture", type, Collections.emptyMap());
|
||||
ApiPullTaskService service = new ApiPullTaskService(mock(DeviceCollectTaskMapper.class), mapper,
|
||||
mock(DeviceDeviceService.class), new AdapterRegistry(Collections.singletonList(f.adapter)),
|
||||
credentials, new SafeBaseUrlValidator(props), mock(ApiPullScheduler.class), new AppProperties(), props, new ObjectMapper());
|
||||
devices, new AdapterRegistry(Collections.singletonList(f.adapter)),
|
||||
credentials, new SafeBaseUrlValidator(props), mock(ApiPullScheduler.class), app, props, new ObjectMapper());
|
||||
ValidateApiPullTaskResponse response = service.validate(11, 1L);
|
||||
assertFalse(response.isValid());
|
||||
Map<String, ValidationCheckResponse> checks = new HashMap<>();
|
||||
|
||||
+358
@@ -0,0 +1,358 @@
|
||||
package com.haobang.apipull.adapter.weibu;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.haobang.apipull.ApiPullProperties;
|
||||
import com.haobang.apipull.adapter.AdapterException;
|
||||
import com.haobang.apipull.adapter.PullPage;
|
||||
import com.haobang.apipull.adapter.PullRequest;
|
||||
import com.haobang.apipull.credential.CredentialException;
|
||||
import com.haobang.apipull.credential.CredentialProvider;
|
||||
import com.haobang.apipull.credential.CredentialSchemas;
|
||||
import com.haobang.apipull.credential.CredentialSnapshot;
|
||||
import com.haobang.apipull.credential.SecretValue;
|
||||
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
|
||||
import com.haobang.apipull.security.SafeBaseUrlValidator;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Instant;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.StatusLine;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.methods.HttpUriRequest;
|
||||
import org.apache.http.client.utils.URLEncodedUtils;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** 离线协议测试:不访问真实设备、数据库、Kafka 或真实凭据。 */
|
||||
class WeibuApiAdapterTest {
|
||||
private static final ObjectMapper JSON = new ObjectMapper();
|
||||
private static final String API_KEY = "synthetic-api-key";
|
||||
private static final String SECRET = "synthetic-secret";
|
||||
private static final Instant START = Instant.parse("2026-09-16T00:00:00Z");
|
||||
private static final Instant END = START.plusSeconds(3600);
|
||||
|
||||
@Test
|
||||
void documentedHmacSha256UrlSafeSignatureIsExact() throws Exception {
|
||||
assertEquals(
|
||||
"giV2UIgRTNvcaVczQ2ifRtqZrLKZoFcoinMbWlTsCzQ",
|
||||
WeibuApiAdapter.sign("api-key", "secret".toCharArray(), 1700000000L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void httpClientCanExplicitlyDisableOrKeepTlsVerification() throws Exception {
|
||||
try (CloseableHttpClient verified = WeibuApiAdapter.buildHttpClient(true);
|
||||
CloseableHttpClient internalInsecure = WeibuApiAdapter.buildHttpClient(false)) {
|
||||
assertNotNull(verified);
|
||||
assertNotNull(internalInsecure);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void requestUsesSignedQueryAndDocumentedLogSearchShape() throws Exception {
|
||||
Fixture f = new Fixture();
|
||||
PullPage page = f.pull(0L);
|
||||
|
||||
HttpPost request = f.request.get();
|
||||
assertEquals(WeibuApiAdapter.PATH, request.getURI().getPath());
|
||||
Map<String, String> query = query(request);
|
||||
assertEquals(API_KEY, query.get("api_key"));
|
||||
long timestamp = Long.parseLong(query.get("auth_timestamp"));
|
||||
assertEquals(
|
||||
WeibuApiAdapter.sign(API_KEY, SECRET.toCharArray(), timestamp), query.get("sign"));
|
||||
assertEquals(3, query.size());
|
||||
assertFalse(request.getConfig().isRedirectsEnabled());
|
||||
|
||||
JsonNode body = f.body.get();
|
||||
assertEquals(START.getEpochSecond(), body.path("time_from").asLong());
|
||||
assertEquals(END.getEpochSecond() - 1L, body.path("time_to").asLong());
|
||||
assertEquals(2, body.path("size").asInt());
|
||||
assertTrue(body.path("assets_group").isArray());
|
||||
assertFalse(body.path("search_connection").asBoolean());
|
||||
assertEquals("attack", body.path("net_data_type").get(0).asText());
|
||||
assertEquals("risk", body.path("net_data_type").get(1).asText());
|
||||
assertEquals("", body.path("sql").asText());
|
||||
assertTrue(body.path("columns").isArray());
|
||||
assertFalse(body.path("asc").asBoolean());
|
||||
assertFalse(body.has("condition"));
|
||||
assertFalse(body.has("page"));
|
||||
|
||||
assertEquals(2, page.getEvents().size());
|
||||
assertEquals(2L, page.getTotal());
|
||||
assertEquals("2", page.getDatasetFingerprint());
|
||||
assertEquals("log-1", page.getEvents().get(0).getExternalEventId());
|
||||
assertEquals(START.plusSeconds(100), page.getEvents().get(0).getEventTime());
|
||||
assertEquals(f.items().get(0).toString(), page.getEvents().get(0).getRawMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void stableLogIdIsIdempotentAndOffsetPaginationIsRejected() throws Exception {
|
||||
Fixture f = new Fixture();
|
||||
String stable = f.pull(0L).getEvents().get(0).getEventId();
|
||||
assertEquals(stable, f.pull(0L).getEvents().get(0).getEventId());
|
||||
((ObjectNode) f.items().get(0)).put("machine", "192.0.2.99");
|
||||
assertEquals(stable, f.pull(0L).getEvents().get(0).getEventId());
|
||||
|
||||
assertEquals(
|
||||
"REQUEST_RANGE_INVALID",
|
||||
assertThrows(AdapterException.class, () -> f.pull(2L)).getCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fractionalPlatformWindowIsExpandedThenStrictlyFiltered() throws Exception {
|
||||
Fixture f = new Fixture();
|
||||
Instant start = START.plusMillis(500);
|
||||
Instant end = START.plusSeconds(3600).plusMillis(250);
|
||||
f.pull(start, end, 0L);
|
||||
assertEquals(START.getEpochSecond(), f.body.get().path("time_from").asLong());
|
||||
assertEquals(
|
||||
START.plusSeconds(3600).getEpochSecond(),
|
||||
f.body.get().path("time_to").asLong());
|
||||
|
||||
((ObjectNode) f.items().get(0)).put("time", START.getEpochSecond());
|
||||
PullPage filtered = f.pull(start, end, 0L);
|
||||
assertEquals(1, filtered.getEvents().size());
|
||||
assertEquals(2, filtered.getSourceRecordCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void vendorAndPaginationSchemaFailuresFailClosed() throws Exception {
|
||||
Fixture vendor = new Fixture();
|
||||
vendor.root.put("response_code", -1).put("verbose_msg", SECRET);
|
||||
AdapterException vendorFailure = assertThrows(AdapterException.class, () -> vendor.pull(0L));
|
||||
assertEquals("VENDOR_RESPONSE_ERROR", vendorFailure.getCode());
|
||||
assertFalse(vendorFailure.toString().contains(SECRET));
|
||||
|
||||
Fixture total = new Fixture();
|
||||
total.root.with("data").put("total", 3);
|
||||
PullPage capacityProbe = total.pull(0L);
|
||||
assertEquals(3L, capacityProbe.getTotal());
|
||||
assertEquals(2, capacityProbe.getSourceRecordCount());
|
||||
|
||||
Fixture inconsistent = new Fixture();
|
||||
inconsistent.root.with("data").put("total", 1);
|
||||
assertEquals(
|
||||
"PAGINATION_INCONSISTENT",
|
||||
assertThrows(AdapterException.class, () -> inconsistent.pull(0L)).getCode());
|
||||
|
||||
Fixture duplicate = new Fixture();
|
||||
((ObjectNode) duplicate.items().get(1)).put("id", "log-1");
|
||||
assertEquals(
|
||||
"PAGINATION_DUPLICATE_DOCUMENT",
|
||||
assertThrows(AdapterException.class, () -> duplicate.pull(0L)).getCode());
|
||||
|
||||
Fixture missing = new Fixture();
|
||||
missing.root.with("data").remove("total");
|
||||
assertEquals(
|
||||
"RESPONSE_SCHEMA_ERROR",
|
||||
assertThrows(AdapterException.class, () -> missing.pull(0L)).getCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void responseBoundsTimeoutsAndHttpErrorsAreClassifiedWithoutBodyLeaks() throws Exception {
|
||||
Fixture oversized = new Fixture();
|
||||
oversized.config.setMaxResponseBytes(1024L);
|
||||
((ObjectNode) oversized.items().get(0)).put("large", repeat('x', 2000));
|
||||
assertEquals(
|
||||
"RESPONSE_TOO_LARGE",
|
||||
assertThrows(AdapterException.class, () -> oversized.pull(0L)).getCode());
|
||||
|
||||
Fixture timeout = new Fixture();
|
||||
when(timeout.http.execute(any(HttpUriRequest.class)))
|
||||
.thenThrow(new SocketTimeoutException(SECRET));
|
||||
AdapterException timeoutFailure =
|
||||
assertThrows(AdapterException.class, () -> timeout.pull(0L));
|
||||
assertEquals("HTTP_TIMEOUT", timeoutFailure.getCode());
|
||||
assertFalse(timeoutFailure.toString().contains(SECRET));
|
||||
|
||||
Fixture forbidden = new Fixture();
|
||||
forbidden.status = 403;
|
||||
forbidden.rawOverride = SECRET;
|
||||
AdapterException httpFailure =
|
||||
assertThrows(AdapterException.class, () -> forbidden.pull(0L));
|
||||
assertEquals("AUTH_HTTP_403", httpFailure.getCode());
|
||||
assertFalse(httpFailure.toString().contains(SECRET));
|
||||
}
|
||||
|
||||
@Test
|
||||
void configAndCredentialSchemasRejectCrossVendorOrUnsafeShapesBeforeHttp() throws Exception {
|
||||
Fixture f = new Fixture();
|
||||
f.config.setBaseUrl("https://192.0.2.10/api");
|
||||
assertEquals(
|
||||
"CONFIG_SCHEMA_ERROR",
|
||||
assertThrows(AdapterException.class, () -> f.pull(0L)).getCode());
|
||||
verifyNoInteractions(f.http);
|
||||
|
||||
Fixture unpaged = new Fixture();
|
||||
unpaged.config.setMaxPagesPerRun(2);
|
||||
assertEquals(
|
||||
"CONFIG_SCHEMA_ERROR",
|
||||
assertThrows(AdapterException.class, () -> unpaged.pull(0L)).getCode());
|
||||
verifyNoInteractions(unpaged.http);
|
||||
|
||||
Fixture incident = new Fixture();
|
||||
incident.config.setStreamKey("WEIBU_INCIDENT");
|
||||
assertEquals(
|
||||
"STREAM_NOT_SUPPORTED",
|
||||
assertThrows(AdapterException.class, () -> incident.pull(0L)).getCode());
|
||||
verifyNoInteractions(incident.http);
|
||||
|
||||
CredentialSchemas.validate(WeibuApiAdapter.TYPE, API_KEY, SECRET, null);
|
||||
assertThrows(
|
||||
CredentialException.class,
|
||||
() -> CredentialSchemas.validate(WeibuApiAdapter.TYPE, API_KEY, SECRET, "token"));
|
||||
assertThrows(
|
||||
CredentialException.class,
|
||||
() -> CredentialSchemas.validate(WeibuApiAdapter.TYPE, null, SECRET, null));
|
||||
}
|
||||
|
||||
private static String repeat(char value, int count) {
|
||||
char[] chars = new char[count];
|
||||
java.util.Arrays.fill(chars, value);
|
||||
return new String(chars);
|
||||
}
|
||||
|
||||
private static Map<String, String> query(HttpPost request) {
|
||||
Map<String, String> result = new LinkedHashMap<>();
|
||||
List<NameValuePair> pairs =
|
||||
URLEncodedUtils.parse(request.getURI(), StandardCharsets.UTF_8);
|
||||
for (NameValuePair pair : pairs) result.put(pair.getName(), pair.getValue());
|
||||
return result;
|
||||
}
|
||||
|
||||
private static DeviceCollectApiConfig config() {
|
||||
DeviceCollectApiConfig c = new DeviceCollectApiConfig();
|
||||
c.setCollectTaskId(22);
|
||||
c.setDeviceId(9);
|
||||
c.setAdapterType(WeibuApiAdapter.TYPE);
|
||||
c.setAdapterVersion(1);
|
||||
c.setStreamKey(WeibuApiAdapter.LOG);
|
||||
c.setBaseUrl("https://192.0.2.10");
|
||||
c.setCredentialRef("weibu-fixture");
|
||||
c.setPageSize(2);
|
||||
c.setPollIntervalSeconds(30);
|
||||
c.setMaxPagesPerRun(1);
|
||||
c.setConnectTimeoutMs(1000);
|
||||
c.setRequestTimeoutMs(3000);
|
||||
c.setWindowSizeSeconds(3600);
|
||||
c.setSafetyDelaySeconds(120);
|
||||
c.setLookbackSeconds(300);
|
||||
c.setMaxResponseBytes(1024 * 1024L);
|
||||
c.setExtraConfigJson("{}");
|
||||
return c;
|
||||
}
|
||||
|
||||
private static ObjectNode response() {
|
||||
ObjectNode root = JSON.createObjectNode();
|
||||
root.put("response_code", 0);
|
||||
ObjectNode data = root.putObject("data");
|
||||
data.put("total", 2);
|
||||
ArrayNode items = data.putArray("data");
|
||||
items.addObject()
|
||||
.put("id", "log-1")
|
||||
.put("machine", "192.0.2.10")
|
||||
.put("time", START.plusSeconds(100).getEpochSecond())
|
||||
.put("direction", "in")
|
||||
.putObject("threat")
|
||||
.put("level", "attack")
|
||||
.put("name", "synthetic attack one");
|
||||
items.addObject()
|
||||
.put("id", "log-2")
|
||||
.put("machine", "192.0.2.11")
|
||||
.put("time", START.plusSeconds(200).getEpochSecond())
|
||||
.put("direction", "lateral")
|
||||
.putObject("threat")
|
||||
.put("level", "risk")
|
||||
.put("name", "synthetic risk two");
|
||||
return root;
|
||||
}
|
||||
|
||||
private static CloseableHttpResponse response(int status, HttpEntity body) {
|
||||
CloseableHttpResponse result = mock(CloseableHttpResponse.class);
|
||||
StatusLine statusLine = mock(StatusLine.class);
|
||||
when(statusLine.getStatusCode()).thenReturn(status);
|
||||
when(result.getStatusLine()).thenReturn(statusLine);
|
||||
when(result.getEntity()).thenReturn(body);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static final class Fixture {
|
||||
private final CloseableHttpClient http = mock(CloseableHttpClient.class);
|
||||
private final AtomicReference<HttpPost> request = new AtomicReference<>();
|
||||
private final AtomicReference<JsonNode> body = new AtomicReference<>();
|
||||
private final ObjectNode root = response();
|
||||
private DeviceCollectApiConfig config = config();
|
||||
private int status = 200;
|
||||
private boolean unknownLength;
|
||||
private String rawOverride;
|
||||
private final WeibuApiAdapter adapter;
|
||||
|
||||
private Fixture() throws Exception {
|
||||
ApiPullProperties properties = new ApiPullProperties();
|
||||
properties.setAllowedHosts("192.0.2.10");
|
||||
CredentialProvider provider =
|
||||
(ref, type, device) -> {
|
||||
Map<String, SecretValue> values = new LinkedHashMap<>();
|
||||
values.put("app_key", new SecretValue(API_KEY.toCharArray()));
|
||||
values.put("app_secret", new SecretValue(SECRET.toCharArray()));
|
||||
return new CredentialSnapshot(1, "fixture-v1", type, values);
|
||||
};
|
||||
adapter =
|
||||
new WeibuApiAdapter(
|
||||
provider, new SafeBaseUrlValidator(properties), JSON, http);
|
||||
when(http.execute(any(HttpUriRequest.class)))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
HttpPost post = invocation.getArgument(0);
|
||||
request.set(post);
|
||||
body.set(JSON.readTree(EntityUtils.toString(post.getEntity(), "UTF-8")));
|
||||
String text = rawOverride == null ? root.toString() : rawOverride;
|
||||
HttpEntity entity;
|
||||
if (unknownLength) {
|
||||
entity = mock(HttpEntity.class);
|
||||
when(entity.getContentLength()).thenReturn(-1L);
|
||||
when(entity.getContent())
|
||||
.thenReturn(
|
||||
new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)));
|
||||
} else {
|
||||
entity = new StringEntity(text, "UTF-8");
|
||||
}
|
||||
return response(status, entity);
|
||||
});
|
||||
}
|
||||
|
||||
private ArrayNode items() {
|
||||
return (ArrayNode) root.path("data").path("data");
|
||||
}
|
||||
|
||||
private PullPage pull(long offset) {
|
||||
return pull(START, END, offset);
|
||||
}
|
||||
|
||||
private PullPage pull(Instant start, Instant end, long offset) {
|
||||
return adapter.pull(new PullRequest(config, start, end, 2, offset));
|
||||
}
|
||||
}
|
||||
}
|
||||
+17
@@ -39,6 +39,23 @@ class DatabaseCredentialProviderTest {
|
||||
assertEquals("CREDENTIAL_NOT_FOUND", error.getCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void weibuUsesTheExistingAppKeyAndSecretCredentialColumns() {
|
||||
DeviceApiCredentialMapper mapper = mock(DeviceApiCredentialMapper.class);
|
||||
DeviceApiCredential record = credential();
|
||||
record.setCredentialRef("weibu-v1");
|
||||
record.setDeviceId(9);
|
||||
record.setAdapterType("WEIBU_V1");
|
||||
when(mapper.selectForLoad("weibu-v1", 9, "WEIBU_V1")).thenReturn(record);
|
||||
|
||||
DatabaseCredentialProvider provider = new DatabaseCredentialProvider(mapper);
|
||||
try (CredentialSnapshot snapshot = provider.load("weibu-v1", "WEIBU_V1", 9)) {
|
||||
assertArrayEquals("app-key".toCharArray(), snapshot.require("app_key").copy());
|
||||
assertEquals("[REDACTED]", snapshot.require("app_secret").toString());
|
||||
assertThrows(CredentialException.class, () -> snapshot.require("api_token"));
|
||||
}
|
||||
}
|
||||
|
||||
private DeviceApiCredential credential() {
|
||||
DeviceApiCredential record = new DeviceApiCredential();
|
||||
record.setCredentialRef("ndr-v1");
|
||||
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
package com.haobang.apipull.security;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.haobang.apipull.ApiPullProperties;
|
||||
import com.haobang.apipull.adapter.AdapterException;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class SafeBaseUrlValidatorTest {
|
||||
@Test
|
||||
void disabledAllowlistAcceptsNewHttpsDeviceWithoutConfigurationChange() {
|
||||
ApiPullProperties properties = new ApiPullProperties();
|
||||
properties.setHostAllowlistEnabled(false);
|
||||
properties.setAllowedHosts("");
|
||||
|
||||
java.net.URI accepted =
|
||||
new SafeBaseUrlValidator(properties).validate("https://192.0.2.20/");
|
||||
assertEquals("https", accepted.getScheme());
|
||||
assertEquals("192.0.2.20", accepted.getHost());
|
||||
}
|
||||
|
||||
@Test
|
||||
void enabledAllowlistStillUsesExactHostMatching() {
|
||||
ApiPullProperties properties = new ApiPullProperties();
|
||||
properties.setHostAllowlistEnabled(true);
|
||||
properties.setAllowedHosts("192.0.2.10");
|
||||
SafeBaseUrlValidator validator = new SafeBaseUrlValidator(properties);
|
||||
|
||||
assertEquals(
|
||||
"https://192.0.2.10", validator.validate("https://192.0.2.10").toString());
|
||||
AdapterException failure =
|
||||
assertThrows(
|
||||
AdapterException.class, () -> validator.validate("https://192.0.2.20"));
|
||||
assertEquals("SSRF_REJECTED", failure.getCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void disabledAllowlistDoesNotRelaxUrlShapeValidation() {
|
||||
ApiPullProperties properties = new ApiPullProperties();
|
||||
properties.setHostAllowlistEnabled(false);
|
||||
SafeBaseUrlValidator validator = new SafeBaseUrlValidator(properties);
|
||||
|
||||
assertEquals(
|
||||
"SSRF_REJECTED",
|
||||
assertThrows(
|
||||
AdapterException.class,
|
||||
() -> validator.validate("http://192.0.2.20/path"))
|
||||
.getCode());
|
||||
assertEquals(
|
||||
"SSRF_REJECTED",
|
||||
assertThrows(
|
||||
AdapterException.class,
|
||||
() -> validator.validate("https://user@192.0.2.20/path"))
|
||||
.getCode());
|
||||
assertEquals(
|
||||
"SSRF_REJECTED",
|
||||
assertThrows(
|
||||
AdapterException.class,
|
||||
() -> validator.validate("https://192.0.2.20/../admin"))
|
||||
.getCode());
|
||||
}
|
||||
}
|
||||
+89
@@ -1,6 +1,7 @@
|
||||
package com.haobang.dispatch;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.common.service.DeviceUnknownService;
|
||||
import com.haobang.config.AppProperties;
|
||||
@@ -16,6 +17,8 @@ import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class DeviceContextResolverTest {
|
||||
private static final String IP = "192.168.4.19";
|
||||
|
||||
@Test
|
||||
void repeatedSourceIpQueriesDatabaseOnlyOnceWithinTtl() {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
@@ -83,4 +86,90 @@ class DeviceContextResolverTest {
|
||||
assertTrue(resolver.isIngressSnapshotReady());
|
||||
assertEquals(1, resolver.getIngressSnapshotSize());
|
||||
}
|
||||
|
||||
@Test
|
||||
void apiOnlyIsInvalidForIngressButAllowedByDeviceId() {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = device(DeviceCollectionMode.API_ONLY);
|
||||
when(service.getActiveDevices()).thenReturn(Collections.singletonList(device));
|
||||
when(service.getByIdSafely(126)).thenReturn(device);
|
||||
DeviceContextResolver resolver = resolver(service);
|
||||
|
||||
resolver.refreshIngressSnapshot();
|
||||
|
||||
assertEquals(DeviceContextResolver.IngressStatus.INVALID,
|
||||
resolver.resolveIngress(IP, "UDP"));
|
||||
assertTrue(resolver.resolveDeviceId(126).isRegistered());
|
||||
}
|
||||
|
||||
@Test
|
||||
void syslogOnlyDeniesApiWithStableResolutionType() {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
when(service.getByIdSafely(126)).thenReturn(device(DeviceCollectionMode.SYSLOG_ONLY));
|
||||
|
||||
DeviceContextResolver.Resolution result = resolver(service).resolveDeviceId(126);
|
||||
|
||||
assertEquals(DeviceContextResolver.Type.COLLECTION_MODE_DENIED, result.getType());
|
||||
assertTrue(result.getReason().contains("API_PULL"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void bothAllowsSyslogAndApi() {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = device(DeviceCollectionMode.BOTH);
|
||||
when(service.getByIpSafely(IP)).thenReturn(Collections.singletonList(device));
|
||||
when(service.getByIdSafely(126)).thenReturn(device);
|
||||
DeviceContextResolver resolver = resolver(service);
|
||||
|
||||
assertTrue(resolver.resolve(IP, "TCP").isRegistered());
|
||||
assertTrue(resolver.resolveDeviceId(126).isRegistered());
|
||||
}
|
||||
|
||||
@Test
|
||||
void nullCollectionModeFromLegacyCacheBehavesAsBoth() {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = device(null);
|
||||
when(service.getByIpSafely(IP)).thenReturn(Collections.singletonList(device));
|
||||
when(service.getByIdSafely(126)).thenReturn(device);
|
||||
DeviceContextResolver resolver = resolver(service);
|
||||
|
||||
assertEquals(DeviceCollectionMode.BOTH, device.getCollectionMode());
|
||||
assertTrue(resolver.resolve(IP, "TCP").isRegistered());
|
||||
assertTrue(resolver.resolveDeviceId(126).isRegistered());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidateDeviceClearsBothIpAndIdLocalEntries() {
|
||||
DeviceDeviceService service = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = device(DeviceCollectionMode.BOTH);
|
||||
when(service.getByIpSafely(IP)).thenReturn(Collections.singletonList(device));
|
||||
when(service.getByIdSafely(126)).thenReturn(device);
|
||||
DeviceContextResolver resolver = resolver(service);
|
||||
resolver.resolve(IP, "TCP");
|
||||
resolver.resolveDeviceId(126);
|
||||
|
||||
resolver.invalidateDevice(126, IP);
|
||||
resolver.resolve(IP, "TCP");
|
||||
resolver.resolveDeviceId(126);
|
||||
|
||||
verify(service, times(2)).getByIpSafely(IP);
|
||||
verify(service, times(2)).getByIdSafely(126);
|
||||
}
|
||||
|
||||
private DeviceContextResolver resolver(DeviceDeviceService service) {
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setAppServiceDeviceCollectId(1);
|
||||
properties.setSyslogDispatchTerminalRecordEnabled(false);
|
||||
properties.setSyslogDeviceContextCacheTtlMs(60000L);
|
||||
return new DeviceContextResolver(service, mock(DeviceUnknownService.class), properties);
|
||||
}
|
||||
|
||||
private DeviceDevice device(DeviceCollectionMode mode) {
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(126);
|
||||
device.setIp(IP);
|
||||
device.setDeviceCollectId(1);
|
||||
device.setCollectionMode(mode);
|
||||
return device;
|
||||
}
|
||||
}
|
||||
|
||||
+67
@@ -1,6 +1,9 @@
|
||||
package com.haobang.dispatch;
|
||||
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.common.service.DeviceUnknownService;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.diagnostics.SyslogIpMetricsRegistry;
|
||||
@@ -19,6 +22,7 @@ import java.time.LocalDateTime;
|
||||
import java.time.Instant;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.Collections;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
@@ -58,6 +62,39 @@ class SyslogWalDispatcherTest {
|
||||
verify(producer).messagePush(eq(device),eq("{}"),eq("20260831100102123"),eq("20260831100058127"),eq("20260831-api"),eq(event.getPointTimeNs()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void syslogOnlyApiRecordAlreadyInWalIsStoppedByDispatcher() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setAppServiceDeviceCollectId(1);
|
||||
properties.setSyslogDispatchTerminalRecordEnabled(true);
|
||||
wal = new SyslogWalService(properties, new ObjectMapper());
|
||||
wal.initialize();
|
||||
IngressEvent event = new IngressEvent(
|
||||
"20260831-denied-api", Instant.parse("2026-08-31T02:01:02.123Z"),
|
||||
Instant.parse("2026-08-31T02:00:58.127Z"), IngressType.API_PULL, "HTTPS", 40, 2001,
|
||||
"WEIBU_V1", "incident:1", "weibu.example", 443, "JSON",
|
||||
StandardCharsets.UTF_8, "{}", null);
|
||||
wal.tryAppendUnified(wal.prepareIngressEvent(event));
|
||||
waitUntil(() -> wal.getAppendedCount() == 1L);
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(40);
|
||||
device.setDeviceCollectId(1);
|
||||
device.setCollectionMode(DeviceCollectionMode.SYSLOG_ONLY);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
when(devices.getByIdSafely(40)).thenReturn(device);
|
||||
DeviceContextResolver resolver = new DeviceContextResolver(
|
||||
devices, mock(DeviceUnknownService.class), properties);
|
||||
kafkaProducer producer = mock(kafkaProducer.class);
|
||||
|
||||
dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
|
||||
dispatcher.start();
|
||||
|
||||
waitUntil(() -> wal.getCheckpointSequence() == 1L);
|
||||
assertEquals(1L, dispatcher.getTerminalFiltered());
|
||||
assertEquals(0L, dispatcher.getKafkaAcked());
|
||||
verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void outOfOrderKafkaAckAdvancesOnlyContiguousCheckpoint() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
@@ -128,6 +165,36 @@ class SyslogWalDispatcherTest {
|
||||
verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void apiOnlySyslogAlreadyInWalIsStoppedByDispatcher() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
properties.setAppServiceDeviceCollectId(1);
|
||||
properties.setSyslogDispatchTerminalRecordEnabled(true);
|
||||
wal = new SyslogWalService(properties, new ObjectMapper());
|
||||
wal.initialize();
|
||||
wal.tryAppend(event("20260811-api-only-syslog", "must-not-reach-kafka"));
|
||||
waitUntil(() -> wal.getAppendedCount() == 1L);
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(40);
|
||||
device.setIp("192.168.4.19");
|
||||
device.setDeviceCollectId(1);
|
||||
device.setCollectionMode(DeviceCollectionMode.API_ONLY);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
when(devices.getByIpSafely("192.168.4.19"))
|
||||
.thenReturn(Collections.singletonList(device));
|
||||
DeviceContextResolver resolver = new DeviceContextResolver(
|
||||
devices, mock(DeviceUnknownService.class), properties);
|
||||
kafkaProducer producer = mock(kafkaProducer.class);
|
||||
|
||||
dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
|
||||
dispatcher.start();
|
||||
|
||||
waitUntil(() -> wal.getCheckpointSequence() == 1L);
|
||||
assertEquals(1L, dispatcher.getTerminalFiltered());
|
||||
assertEquals(0L, dispatcher.getKafkaAcked());
|
||||
verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void terminalDeviceStateIsDroppedWithoutFileWhenRecordingDisabled() throws Exception {
|
||||
AppProperties properties = properties(tempDirectory);
|
||||
|
||||
+23
-5
@@ -1,5 +1,9 @@
|
||||
package com.netty;
|
||||
|
||||
import com.common.entity.DeviceCollectionMode;
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.common.service.DeviceUnknownService;
|
||||
import com.haobang.dispatch.DeviceContextResolver;
|
||||
import com.haobang.config.AppProperties;
|
||||
import com.haobang.diagnostics.SyslogIpMetricsRegistry;
|
||||
@@ -10,6 +14,7 @@ import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Collections;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
@@ -73,18 +78,31 @@ class SyslogMessageHandlerBackpressureTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidDeviceIsDroppedBeforeWalInPerformanceMode() {
|
||||
void apiOnlyDeviceIsCountedAndDroppedBeforeWalInPerformanceMode() {
|
||||
SyslogWalService wal = mock(SyslogWalService.class);
|
||||
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
|
||||
when(resolver.resolveIngress(anyString(), anyString()))
|
||||
.thenReturn(DeviceContextResolver.IngressStatus.INVALID);
|
||||
DeviceDeviceService devices = mock(DeviceDeviceService.class);
|
||||
DeviceDevice device = new DeviceDevice();
|
||||
device.setId(40);
|
||||
device.setIp("embedded");
|
||||
device.setDeviceCollectId(1);
|
||||
device.setCollectionMode(DeviceCollectionMode.API_ONLY);
|
||||
when(devices.getActiveDevices()).thenReturn(Collections.singletonList(device));
|
||||
AppProperties properties = new AppProperties();
|
||||
properties.setAppServiceDeviceCollectId(1);
|
||||
properties.setSyslogDispatchTerminalRecordEnabled(false);
|
||||
DeviceContextResolver resolver = new DeviceContextResolver(
|
||||
devices, mock(DeviceUnknownService.class), properties);
|
||||
resolver.refreshIngressSnapshot();
|
||||
long networkBefore = SyslogMessageHandler.getNetworkReceivedCount();
|
||||
long invalidBefore = SyslogMessageHandler.getInvalidDeviceDroppedCount();
|
||||
EmbeddedChannel channel = new EmbeddedChannel(
|
||||
new SyslogMessageHandler(StandardCharsets.UTF_8, wal, resolver, true));
|
||||
channel.connect(new InetSocketAddress("192.168.4.19", 5514));
|
||||
|
||||
channel.writeInbound("<134>Aug 11 20:00:00 invalid device event");
|
||||
|
||||
verify(resolver).resolveIngress("embedded", "TCP");
|
||||
assertEquals(networkBefore + 1L, SyslogMessageHandler.getNetworkReceivedCount());
|
||||
assertEquals(invalidBefore + 1L, SyslogMessageHandler.getInvalidDeviceDroppedCount());
|
||||
verify(wal, never()).prepareForWal(any());
|
||||
verify(wal, never()).tryAppend(any());
|
||||
channel.close();
|
||||
|
||||
Reference in New Issue
Block a user