210 lines
8.1 KiB
Java
210 lines
8.1 KiB
Java
|
package com.haobang.apipull;
|
||
|
|
|
||
|
|
import com.common.mapper.DeviceCollectApiMapper;
|
||
|
|
import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
|
||
|
|
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
|
||
|
|
import com.haobang.apipull.persistence.DeviceCollectApiRun;
|
||
|
|
import com.haobang.wal.SyslogWalService;
|
||
|
|
import java.net.URI;
|
||
|
|
import java.time.LocalDateTime;
|
||
|
|
import java.util.Set;
|
||
|
|
import java.util.UUID;
|
||
|
|
import java.util.concurrent.*;
|
||
|
|
import java.util.concurrent.ConcurrentHashMap;
|
||
|
|
import javax.annotation.PostConstruct;
|
||
|
|
import javax.annotation.PreDestroy;
|
||
|
|
import org.slf4j.Logger;
|
||
|
|
import org.slf4j.LoggerFactory;
|
||
|
|
import org.springframework.scheduling.annotation.Scheduled;
|
||
|
|
import org.springframework.stereotype.Component;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* API 拉取任务调度器和并发闸门。
|
||
|
|
*
|
||
|
|
* <p>并发限制分三层：同一 JVM 内同一任务只能运行一次、全局并发不超过线程池大小、同一设备 主机不超过 per-host 上限。跨实例互斥由达梦租约和单调递增的 fencing token
|
||
|
|
* 保证。
|
||
|
|
*/
|
||
|
|
@Component
|
||
|
|
public class ApiPullScheduler {
|
||
|
|
private static final Logger log = LoggerFactory.getLogger(ApiPullScheduler.class);
|
||
|
|
private final ApiPullProperties properties;
|
||
|
|
private final DeviceCollectApiMapper mapper;
|
||
|
|
private final ApiPullRunner runner;
|
||
|
|
private final SyslogWalService wal;
|
||
|
|
private ExecutorService executor;
|
||
|
|
private Semaphore global;
|
||
|
|
private final ConcurrentHashMap<String, Semaphore> hosts = new ConcurrentHashMap<>();
|
||
|
|
private final Set<Integer> localRunning = ConcurrentHashMap.newKeySet();
|
||
|
|
|
||
|
|
public ApiPullScheduler(
|
||
|
|
ApiPullProperties properties,
|
||
|
|
DeviceCollectApiMapper mapper,
|
||
|
|
ApiPullRunner runner,
|
||
|
|
SyslogWalService wal) {
|
||
|
|
this.properties = properties;
|
||
|
|
this.mapper = mapper;
|
||
|
|
this.runner = runner;
|
||
|
|
this.wal = wal;
|
||
|
|
}
|
||
|
|
|
||
|
|
@PostConstruct
|
||
|
|
public void init() {
|
||
|
|
properties.validate();
|
||
|
|
global = new Semaphore(properties.getGlobalConcurrency());
|
||
|
|
executor =
|
||
|
|
Executors.newFixedThreadPool(
|
||
|
|
properties.getGlobalConcurrency(),
|
||
|
|
r -> {
|
||
|
|
Thread t = new Thread(r, "device-api-pull");
|
||
|
|
t.setDaemon(true);
|
||
|
|
return t;
|
||
|
|
});
|
||
|
|
log.info(
|
||
|
|
"设备 API 拉取调度器配置: enabled={}, globalConcurrency={}, perHostConcurrency={}, leaseSeconds={},"
|
||
|
|
+ " maxPagesPerSlice={}, slowPageThresholdMs={}",
|
||
|
|
properties.isEnabled(),
|
||
|
|
properties.getGlobalConcurrency(),
|
||
|
|
properties.getPerHostConcurrency(),
|
||
|
|
properties.getLeaseSeconds(),
|
||
|
|
properties.getMaxPagesPerSlice(),
|
||
|
|
properties.getSlowPageThresholdMs());
|
||
|
|
}
|
||
|
|
|
||
|
|
/** 扫描到期任务；总开关关闭时不查询数据库，也不创建任何设备请求。 */
|
||
|
|
@Scheduled(fixedDelayString = "${device.api.scheduler.scan-interval-ms:5000}")
|
||
|
|
public void scan() {
|
||
|
|
if (!properties.isEnabled()) return;
|
||
|
|
for (DeviceCollectApiConfig c : mapper.selectDueConfigs())
|
||
|
|
try {
|
||
|
|
trySubmit(c, "SCHEDULED");
|
||
|
|
} catch (Throwable failure) {
|
||
|
|
log.warn("设备 API 调度提交失败: taskId={}, error={}", c.getCollectTaskId(), failure.toString());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/** 提交一次人工运行；手工操作不会绕过任务启用状态或资源保护。 */
|
||
|
|
public DeviceCollectApiRun trigger(Integer taskId) {
|
||
|
|
if (!properties.isEnabled())
|
||
|
|
throw new ApiPullException(
|
||
|
|
"API_PULL_DISABLED",
|
||
|
|
"设备 API 拉取总开关未开启",
|
||
|
|
org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE);
|
||
|
|
if (wal.getFatalFailure() != null || wal.isDiskCritical() || wal.shouldPauseTcp())
|
||
|
|
throw new ApiPullException(
|
||
|
|
"WAL_UNAVAILABLE",
|
||
|
|
"WAL 不可用或处于接入保护水位",
|
||
|
|
org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE);
|
||
|
|
DeviceCollectApiConfig c = mapper.selectConfig(taskId);
|
||
|
|
if (c == null)
|
||
|
|
throw new ApiPullException(
|
||
|
|
"TASK_NOT_FOUND", "API 拉取任务不存在", org.springframework.http.HttpStatus.NOT_FOUND);
|
||
|
|
if (!c.isEnabled())
|
||
|
|
throw new ApiPullException(
|
||
|
|
"TASK_NOT_PAUSED", "任务处于暂停状态，不能手工触发", org.springframework.http.HttpStatus.CONFLICT);
|
||
|
|
DeviceCollectApiRun run = trySubmit(c, "MANUAL");
|
||
|
|
if (run == null)
|
||
|
|
throw new ApiPullException(
|
||
|
|
"LEASE_CONFLICT", "任务正在运行或无可用并发槽", org.springframework.http.HttpStatus.CONFLICT);
|
||
|
|
return run;
|
||
|
|
}
|
||
|
|
|
||
|
|
private DeviceCollectApiRun trySubmit(DeviceCollectApiConfig c, String trigger) {
|
||
|
|
if (!c.isEnabled()) return null;
|
||
|
|
if (wal.getFatalFailure() != null || wal.isDiskCritical() || wal.shouldPauseTcp()) {
|
||
|
|
mapper.scheduleNext(c.getCollectTaskId(), c.getConfigVersion(), c.getPollIntervalSeconds());
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
// 获取顺序固定为任务互斥 -> 全局许可 -> 主机许可 -> DB 租约，失败时按持有情况释放。
|
||
|
|
if (!localRunning.add(c.getCollectTaskId())) return null;
|
||
|
|
if (!global.tryAcquire()) {
|
||
|
|
localRunning.remove(c.getCollectTaskId());
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
String host;
|
||
|
|
try {
|
||
|
|
host = new URI(c.getBaseUrl()).getHost();
|
||
|
|
} catch (Exception e) {
|
||
|
|
releaseLocal(c.getCollectTaskId(), null);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
Semaphore hostPermit =
|
||
|
|
hosts.computeIfAbsent(host, h -> new Semaphore(properties.getPerHostConcurrency()));
|
||
|
|
if (!hostPermit.tryAcquire()) {
|
||
|
|
releaseLocal(c.getCollectTaskId(), null);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
DeviceCollectApiCheckpoint cp = null;
|
||
|
|
DeviceCollectApiRun submittedRun = null;
|
||
|
|
try {
|
||
|
|
// DB 租约是跨实例最终互斥；本地 semaphore 只用于资源控制，不能代替 fencing。
|
||
|
|
if (mapper.acquireLease(
|
||
|
|
c.getCollectTaskId(),
|
||
|
|
c.getConfigVersion(),
|
||
|
|
properties.getInstanceId(),
|
||
|
|
properties.getLeaseSeconds())
|
||
|
|
!= 1) {
|
||
|
|
releaseLocal(c.getCollectTaskId(), hostPermit);
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
cp = mapper.selectCheckpoint(c.getCollectTaskId());
|
||
|
|
DeviceCollectApiRun run = new DeviceCollectApiRun();
|
||
|
|
run.setRunId(UUID.randomUUID().toString());
|
||
|
|
run.setCollectTaskId(c.getCollectTaskId());
|
||
|
|
run.setDeviceId(c.getDeviceId());
|
||
|
|
run.setAdapterType(c.getAdapterType());
|
||
|
|
run.setAdapterVersion(c.getAdapterVersion());
|
||
|
|
run.setStreamKey(c.getStreamKey());
|
||
|
|
run.setInstanceId(properties.getInstanceId());
|
||
|
|
run.setConfigVersion(c.getConfigVersion());
|
||
|
|
run.setFencingToken(cp.getFencingToken());
|
||
|
|
run.setTriggerType(trigger);
|
||
|
|
run.setStatus("RUNNING");
|
||
|
|
run.setScheduledAt(LocalDateTime.now(ApiPullTime.DATABASE_ZONE));
|
||
|
|
mapper.insertRun(run);
|
||
|
|
submittedRun = run;
|
||
|
|
final DeviceCollectApiCheckpoint leasedCheckpoint = cp;
|
||
|
|
// 成功提交后，许可的所有权交给工作线程并由其 finally 统一释放。
|
||
|
|
executor.execute(
|
||
|
|
() -> {
|
||
|
|
try {
|
||
|
|
runner.execute(c, leasedCheckpoint, run);
|
||
|
|
} finally {
|
||
|
|
releaseLocal(c.getCollectTaskId(), hostPermit);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
return run;
|
||
|
|
} catch (Throwable e) {
|
||
|
|
if (submittedRun != null) {
|
||
|
|
submittedRun.setStatus("FAILED");
|
||
|
|
submittedRun.setErrorCode("SCHEDULER_REJECTED");
|
||
|
|
submittedRun.setErrorSummary("任务未能提交到本地执行器");
|
||
|
|
submittedRun.setFinishedAt(LocalDateTime.now(ApiPullTime.DATABASE_ZONE));
|
||
|
|
mapper.updateRun(submittedRun);
|
||
|
|
}
|
||
|
|
if (cp != null && cp.getFencingToken() != null)
|
||
|
|
mapper.releaseLease(c.getCollectTaskId(), properties.getInstanceId(), cp.getFencingToken());
|
||
|
|
releaseLocal(c.getCollectTaskId(), hostPermit);
|
||
|
|
throw e;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private void releaseLocal(Integer taskId, Semaphore host) {
|
||
|
|
if (host != null) host.release();
|
||
|
|
global.release();
|
||
|
|
localRunning.remove(taskId);
|
||
|
|
}
|
||
|
|
|
||
|
|
@PreDestroy
|
||
|
|
public void shutdown() {
|
||
|
|
// 等待在途窗口正常收尾；超时中断时，未提交的 checkpoint 会使该窗口下次重新拉取。
|
||
|
|
if (executor == null) return;
|
||
|
|
executor.shutdown();
|
||
|
|
try {
|
||
|
|
if (!executor.awaitTermination(30, TimeUnit.SECONDS)) executor.shutdownNow();
|
||
|
|
} catch (InterruptedException e) {
|
||
|
|
Thread.currentThread().interrupt();
|
||
|
|
executor.shutdownNow();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|