feat(syslog-serve): 完善微步全量拉取、探针归属控制及诊断接口

- 按 DEVICE_DEVICE.DEVICE_COLLECT_ID 过滤 API 拉取任务
- 仅允许与 APP_SERVICE_DEVICE_COLLECT_ID 匹配的节点执行任务
- 在任务扫描和数据库租约获取阶段双重校验设备归属
- 设备迁移后原节点不执行 HTTP 请求且不推进 NEXT_RUN_AT
- 保留数据库租约,避免同一探针的多实例重复拉取

- 微步 searchBySql 改为全量逐条日志查询
- 移除 net_data_type=attack/risk 固定过滤
- 支持采集 attack、action、risk 并原样写入 WAL/Kafka
- 保留空 sql、完整字段和时间窗口查询

- 新增微步原始查询诊断接口
- 支持自定义 size、page、pageSize、cur_page 等查询字段
- 复用任务绑定的地址、凭据、TLS 和超时配置
- 诊断查询不写 WAL/Kafka、不推进 checkpoint
- 响应隐藏 API Key、Secret、签名和认证字段

- 更新启动版本标识为 WEIBU_DIAGNOSTIC_V20260917_66
- 更新 v6 部署、验证和 curl 调用文档
- Docker 基础镜像调整为 eclipse-temurin:8-jre
- 补充探针归属、全量查询、诊断透传和状态隔离测试

测试:
- JDK 8
- Tests run: 165
- Failures: 0
- Errors: 0
- Skipped: 2
- git diff --check 通过
This commit is contained in:
王绍宏
2026-09-17 22:25:40 +08:00
parent 2d297eb996
commit 261f1d38e9
15 changed files with 439 additions and 36 deletions
@@ -1,6 +1,7 @@
package com.haobang.apipull;
import com.common.mapper.DeviceCollectApiMapper;
import com.haobang.config.AppProperties;
import com.haobang.dispatch.DeviceContextResolver;
import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
@@ -33,6 +34,7 @@ public class ApiPullScheduler {
private final ApiPullRunner runner;
private final SyslogWalService wal;
private final DeviceContextResolver deviceResolver;
private final AppProperties appProperties;
private ExecutorService executor;
private Semaphore global;
private final ConcurrentHashMap<String, Semaphore> hosts = new ConcurrentHashMap<>();
@@ -43,12 +45,14 @@ public class ApiPullScheduler {
DeviceCollectApiMapper mapper,
ApiPullRunner runner,
SyslogWalService wal,
DeviceContextResolver deviceResolver) {
DeviceContextResolver deviceResolver,
AppProperties appProperties) {
this.properties = properties;
this.mapper = mapper;
this.runner = runner;
this.wal = wal;
this.deviceResolver = deviceResolver;
this.appProperties = appProperties;
}
@PostConstruct
@@ -78,7 +82,8 @@ public class ApiPullScheduler {
@Scheduled(fixedDelayString = "${device.api.scheduler.scan-interval-ms:5000}")
public void scan() {
if (!properties.isEnabled()) return;
for (DeviceCollectApiConfig c : mapper.selectDueConfigs())
for (DeviceCollectApiConfig c
: mapper.selectDueConfigs(appProperties.getAppServiceDeviceCollectId()))
try {
trySubmit(c, "SCHEDULED");
} catch (Throwable failure) {
@@ -146,7 +151,8 @@ public class ApiPullScheduler {
c.getCollectTaskId(),
c.getConfigVersion(),
properties.getInstanceId(),
properties.getLeaseSeconds())
properties.getLeaseSeconds(),
appProperties.getAppServiceDeviceCollectId())
!= 1) {
releaseLocal(c.getCollectTaskId(), hostPermit);
return null;
@@ -210,7 +216,11 @@ public class ApiPullScheduler {
private boolean requireApiCollectionMode(DeviceCollectApiConfig c, boolean failRequest) {
DeviceContextResolver.Resolution resolution = deviceResolver.resolveDeviceId(c.getDeviceId());
if (resolution.isRegistered()) return true;
mapper.scheduleNext(c.getCollectTaskId(), c.getConfigVersion(), c.getPollIntervalSeconds());
// 非归属节点不能修改任务调度时间;设备归属可能在扫描后、执行前发生变化。
// 手工触发失败同样不应改变正常调度计划。
if (!failRequest && resolution.getType() != DeviceContextResolver.Type.NOT_ASSIGNED) {
mapper.scheduleNext(c.getCollectTaskId(), c.getConfigVersion(), c.getPollIntervalSeconds());
}
String reason = resolution.getReason() == null ? "设备不允许 API_PULL" : resolution.getReason();
log.warn("设备 API 拉取已被设备策略跳过: taskId={}, deviceId={}, type={}, reason={}",
c.getCollectTaskId(), c.getDeviceId(), resolution.getType(), reason);