ai-security-xdr/haobang-security-dm/upgrade-doc/v2/database/02-device-api-pull-upgrade.sql

-- =============================================================================
-- 脚本：02-device-api-pull-upgrade.sql
-- 用途：创建设备 API 拉取需要的四张基础表、约束和索引。
-- 性质：数据库结构变更脚本，会创建对象，不是只读脚本。
-- 执行时机：01 预检通过后，部署新版 syslog-serve 之前。
--
-- 前置条件：
-- 1. 已完成数据库备份；
-- 2. API 拉取任务和旧版服务已经停止；
-- 3. 已确认 device_collect_task.id 与 device_device.id 的真实类型；
-- 4. 四张 DEVICE_COLLECT_API_* 表均不存在。
--
-- 注意：本脚本不是重复执行脚本。执行成功后不要再次执行。
-- 如果中途报错，应保留现场并执行 03 验证实际创建情况，不要直接重跑。
-- =============================================================================

-- 凭据表：保存设备、适配器和 AppKey/AppSecret 的对应关系。
-- 第一阶段为明文保存，因此禁止把 app_secret 输出到日志或普通查询结果中。
CREATE TABLE device_collect_api_credential (
  credential_ref VARCHAR2(128) NOT NULL,
  device_id BIGINT NOT NULL,
  adapter_type VARCHAR2(64) NOT NULL,
  credential_version VARCHAR2(64) NOT NULL,
  app_key VARCHAR2(512) NOT NULL,
  app_secret VARCHAR2(2048) NOT NULL,
  config_version BIGINT DEFAULT 1 NOT NULL,
  created_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
  updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
  CONSTRAINT pk_dc_api_credential PRIMARY KEY(credential_ref),
  CONSTRAINT fk_dc_api_credential_device FOREIGN KEY(device_id) REFERENCES device_device(id),
  CONSTRAINT ck_dc_api_credential_ver CHECK(config_version >= 1),
  CONSTRAINT uk_dc_api_credential_device UNIQUE(credential_ref, device_id, adapter_type)
);

-- 任务配置表：每个 API 拉取任务一条记录。
-- collect_task_id 关联现有 device_collect_task.id；任务默认关闭，验证通过后再开启。
CREATE TABLE device_collect_api_config (
  collect_task_id INTEGER NOT NULL,
  device_id BIGINT NOT NULL,
  adapter_type VARCHAR2(64) NOT NULL,
  adapter_version INTEGER DEFAULT 1 NOT NULL,
  base_url VARCHAR2(1024) NOT NULL,
  stream_key VARCHAR2(64) NOT NULL,
  credential_ref VARCHAR2(128) NOT NULL,
  enabled SMALLINT DEFAULT 0 NOT NULL,
  next_run_at TIMESTAMP,
  poll_interval_seconds INTEGER DEFAULT 10 NOT NULL,
  page_size INTEGER DEFAULT 100 NOT NULL,
  max_pages_per_run INTEGER DEFAULT 1000 NOT NULL,
  connect_timeout_ms INTEGER DEFAULT 5000 NOT NULL,
  request_timeout_ms INTEGER DEFAULT 30000 NOT NULL,
  window_size_seconds INTEGER DEFAULT 60 NOT NULL,
  safety_delay_seconds INTEGER DEFAULT 120 NOT NULL,
  lookback_seconds INTEGER DEFAULT 0 NOT NULL,
  max_response_bytes BIGINT DEFAULT 16777216 NOT NULL,
  initial_start_time TIMESTAMP NOT NULL,
  extra_config_json CLOB NOT NULL,
  config_version BIGINT DEFAULT 1 NOT NULL,
  created_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
  updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
  CONSTRAINT pk_dc_api_cfg PRIMARY KEY(collect_task_id),
  CONSTRAINT fk_dc_api_cfg_task FOREIGN KEY(collect_task_id) REFERENCES device_collect_task(id),
  CONSTRAINT fk_dc_api_cfg_credential FOREIGN KEY(credential_ref,device_id,adapter_type)
    REFERENCES device_collect_api_credential(credential_ref,device_id,adapter_type),
  CONSTRAINT uk_dc_api_cfg_stream UNIQUE(device_id,adapter_type,stream_key),
  CONSTRAINT ck_dc_api_cfg_enabled CHECK(enabled IN (0,1)),
  CONSTRAINT ck_dc_api_cfg_adapter_ver CHECK(adapter_version>=1),
  CONSTRAINT ck_dc_api_cfg_poll CHECK(poll_interval_seconds BETWEEN 1 AND 86400),
  CONSTRAINT ck_dc_api_cfg_page CHECK(page_size BETWEEN 1 AND 50000),
  CONSTRAINT ck_dc_api_cfg_pages CHECK(max_pages_per_run BETWEEN 1 AND 100000),
  CONSTRAINT ck_dc_api_cfg_conn_to CHECK(connect_timeout_ms BETWEEN 100 AND 120000),
  CONSTRAINT ck_dc_api_cfg_req_to CHECK(request_timeout_ms BETWEEN 100 AND 600000),
  CONSTRAINT ck_dc_api_cfg_window CHECK(window_size_seconds BETWEEN 1 AND 1209600),
  CONSTRAINT ck_dc_api_cfg_delay CHECK(safety_delay_seconds BETWEEN 0 AND 604800),
  CONSTRAINT ck_dc_api_cfg_lookback CHECK(lookback_seconds BETWEEN 0 AND 1209600),
  CONSTRAINT ck_dc_api_cfg_resp CHECK(max_response_bytes BETWEEN 1024 AND 536870912),
  CONSTRAINT ck_dc_api_cfg_ver CHECK(config_version>=1),
  CONSTRAINT ck_dc_api_cfg_https CHECK(base_url LIKE 'https://%')
);
CREATE INDEX idx_dc_api_cfg_due ON device_collect_api_config(enabled,next_run_at,collect_task_id);

-- 进度表：记录任务已经安全写入 WAL 的时间位置，并保存跨实例租约信息。
-- committed_window_end 只有在当前窗口完整处理成功后才向后推进。
CREATE TABLE device_collect_api_checkpoint (
  collect_task_id INTEGER NOT NULL,
  checkpoint_type VARCHAR2(32) NOT NULL,
  checkpoint_value CLOB NOT NULL,
  committed_window_end TIMESTAMP,
  adapter_type VARCHAR2(64) NOT NULL,
  adapter_version INTEGER NOT NULL,
  version BIGINT DEFAULT 1 NOT NULL,
  lease_owner VARCHAR2(128),
  lease_expires_at TIMESTAMP,
  fencing_token BIGINT DEFAULT 0 NOT NULL,
  last_event_time TIMESTAMP,
  last_external_event_id VARCHAR2(512),
  created_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
  updated_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
  CONSTRAINT pk_dc_api_cp PRIMARY KEY(collect_task_id),
  CONSTRAINT fk_dc_api_cp_cfg FOREIGN KEY(collect_task_id) REFERENCES device_collect_api_config(collect_task_id),
  CONSTRAINT ck_dc_api_cp_type CHECK(checkpoint_type IN ('TIME_WINDOW','TIME_ID','OPAQUE_CURSOR','EXTERNAL_ID')),
  CONSTRAINT ck_dc_api_cp_adapter_ver CHECK(adapter_version>=1),
  CONSTRAINT ck_dc_api_cp_ver CHECK(version>=1), CONSTRAINT ck_dc_api_cp_fence CHECK(fencing_token>=0),
  CONSTRAINT ck_dc_api_cp_lease CHECK((lease_owner IS NULL AND lease_expires_at IS NULL)
    OR (lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL))
);
CREATE INDEX idx_dc_api_cp_lease ON device_collect_api_checkpoint(lease_expires_at,collect_task_id);

-- 运行记录表：每次调度、手工执行或验证都会留下运行结果，便于排查进度和错误。
-- 本表只保存脱敏错误摘要，不保存完整设备响应和认证信息。
CREATE TABLE device_collect_api_run (
  run_id VARCHAR2(36) NOT NULL,
  collect_task_id INTEGER NOT NULL,
  device_id BIGINT NOT NULL,
  adapter_type VARCHAR2(64) NOT NULL,
  adapter_version INTEGER NOT NULL,
  stream_key VARCHAR2(64) NOT NULL,
  instance_id VARCHAR2(128) NOT NULL,
  config_version BIGINT NOT NULL,
  fencing_token BIGINT NOT NULL,
  trigger_type VARCHAR2(16) NOT NULL,
  status VARCHAR2(16) NOT NULL,
  logical_window_start TIMESTAMP,
  logical_window_end TIMESTAMP,
  query_start TIMESTAMP,
  query_end TIMESTAMP,
  pages_fetched INTEGER DEFAULT 0 NOT NULL,
  records_fetched BIGINT DEFAULT 0 NOT NULL,
  records_durable BIGINT DEFAULT 0 NOT NULL,
  records_quarantined BIGINT DEFAULT 0 NOT NULL,
  duplicate_candidates BIGINT DEFAULT 0 NOT NULL,
  last_http_status INTEGER,
  error_code VARCHAR2(128),
  error_summary VARCHAR2(2000),
  scheduled_at TIMESTAMP,
  started_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
  finished_at TIMESTAMP,
  created_at TIMESTAMP DEFAULT SYSDATE NOT NULL,
  CONSTRAINT pk_dc_api_run PRIMARY KEY(run_id),
  CONSTRAINT fk_dc_api_run_cfg FOREIGN KEY(collect_task_id) REFERENCES device_collect_api_config(collect_task_id),
  CONSTRAINT ck_dc_api_run_trigger CHECK(trigger_type IN ('SCHEDULED','MANUAL','VALIDATE')),
  CONSTRAINT ck_dc_api_run_status CHECK(status IN ('RUNNING','SUCCESS','FAILED','SKIPPED')),
  CONSTRAINT ck_dc_api_run_counts CHECK(pages_fetched>=0 AND records_fetched>=0 AND records_durable>=0
    AND records_quarantined>=0 AND duplicate_candidates>=0),
  CONSTRAINT ck_dc_api_run_fence CHECK(fencing_token>=0)
);
CREATE INDEX idx_dc_api_run_task_time ON device_collect_api_run(collect_task_id,started_at);
CREATE INDEX idx_dc_api_run_status ON device_collect_api_run(status,started_at);

-- 数据字典注释：方便 DBA 在数据库管理工具中直接查看各表用途。
COMMENT ON TABLE device_collect_api_config IS '设备 API 拉取任务非敏感配置，一任务一条';
COMMENT ON TABLE device_collect_api_credential IS '设备 API 厂商凭据；第一阶段 AppKey/AppSecret 明文保存，禁止日志和接口响应回显凭据正文';
COMMENT ON COLUMN device_collect_api_credential.credential_ref IS '任务引用的全局唯一凭据名称';
COMMENT ON COLUMN device_collect_api_credential.app_secret IS '第一阶段明文 AppSecret，后续迁移为密文字段';
COMMENT ON TABLE device_collect_api_checkpoint IS '设备 API 已安全进入 WAL 的业务水位及跨实例租约';
COMMENT ON TABLE device_collect_api_run IS '设备 API 每次调度、手工触发或验证的脱敏运行审计';
COMMENT ON COLUMN device_collect_api_run.error_summary IS '最长 2000 字符的脱敏摘要，禁止保存原始响应和认证信息';

-- 创建完成后，请立即执行 03-device-api-pull-verify.sql。