diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/config/SyslogBuildStartupLogger.java b/haobang-security-dm/syslog-serve/src/main/java/com/config/SyslogBuildStartupLogger.java index af15311..cd789ef 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/config/SyslogBuildStartupLogger.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/config/SyslogBuildStartupLogger.java @@ -12,7 +12,7 @@ import org.springframework.stereotype.Component; public class SyslogBuildStartupLogger { // 确认 JAR 包含设备级采集模式、API 拉取、探针归属过滤、微步全量查询及诊断接口。 - static final String CODE_REVISION = "WEIBU_5S_50000_SCAN1S_V20260917_70"; + static final String CODE_REVISION = "WEIBU_5S_50000_512M_V20260917_71"; private final Environment environment; public SyslogBuildStartupLogger(Environment environment) { diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapter.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapter.java index 057449f..c08ac3d 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapter.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapter.java @@ -81,6 +81,8 @@ public class WeibuApiAdapter implements ApiPullAdapter { UUID.fromString("89a53d59-bb84-5d41-924f-38319448cf46"); private static final long MIN_EVENT_EPOCH_SECOND = 946684800L; private static final long MAX_EVENT_EPOCH_SECOND = 4102444800L; + private static final long MAX_RESPONSE_BYTES = 536870912L; + private static final int MAX_JSON_TOKENS = 25000000; private final CredentialProvider credentials; private final SafeBaseUrlValidator urls; @@ -176,7 +178,7 @@ public class WeibuApiAdapter implements ApiPullAdapter { range(c.getRequestTimeoutMs(), 1000, 120000); if (c.getMaxResponseBytes() == null || c.getMaxResponseBytes() < 1024 - || c.getMaxResponseBytes() > 67108864L) fail("CONFIG_SCHEMA_ERROR"); + || c.getMaxResponseBytes() > MAX_RESPONSE_BYTES) fail("CONFIG_SCHEMA_ERROR"); URI base = urls.validate(c.getBaseUrl()); String path = base.getPath(); if (!(path == null || path.isEmpty() || "/".equals(path))) fail("CONFIG_SCHEMA_ERROR"); @@ -541,7 +543,7 @@ public class WeibuApiAdapter implements ApiPullAdapter { int tokens = 0; JsonToken token; while ((token = parser.nextToken()) != null) { - if (++tokens > 2000000) fail("RESPONSE_SCHEMA_ERROR"); + if (++tokens > MAX_JSON_TOKENS) fail("RESPONSE_SCHEMA_ERROR"); if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { if (++depth > 64) fail("RESPONSE_SCHEMA_ERROR"); } else if (token == JsonToken.END_OBJECT || token == JsonToken.END_ARRAY) { diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullDtos.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullDtos.java index 2474044..bde17b6 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullDtos.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullDtos.java @@ -83,7 +83,7 @@ public final class ApiPullDtos { private Integer lookbackSeconds = 0; @Min(1024) - @Max(268435456) + @Max(536870912) private Long maxResponseBytes = 16777216L; @NotNull private OffsetDateTime initialStartTime; @@ -145,7 +145,7 @@ public final class ApiPullDtos { private Integer lookbackSeconds; @Min(1024) - @Max(268435456) + @Max(536870912) private Long maxResponseBytes; @NotNull private Map extraConfig; diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullContractTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullContractTest.java index 351ac85..15fe16f 100644 --- a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullContractTest.java +++ b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullContractTest.java @@ -45,9 +45,16 @@ class ApiPullContractTest { @Test void managementDtoAllowsWeibuFiftyThousandRecordWindow() throws Exception { - Max max = CreateApiPullTaskRequest.class.getDeclaredField("pageSize").getAnnotation(Max.class); - assertNotNull(max); - assertEquals(50000L, max.value()); + Max pageMax = + CreateApiPullTaskRequest.class.getDeclaredField("pageSize").getAnnotation(Max.class); + assertNotNull(pageMax); + assertEquals(50000L, pageMax.value()); + Max responseMax = + CreateApiPullTaskRequest.class + .getDeclaredField("maxResponseBytes") + .getAnnotation(Max.class); + assertNotNull(responseMax); + assertEquals(536870912L, responseMax.value()); } @Test diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapterTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapterTest.java index 7aa8533..d263838 100644 --- a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapterTest.java +++ b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapterTest.java @@ -261,6 +261,18 @@ class WeibuApiAdapterTest { .getCode()); verifyNoInteractions(capacity.http); + Fixture responseCapacity = new Fixture(); + responseCapacity.config.setMaxResponseBytes(536870912L); + responseCapacity.adapter.validateConfig(responseCapacity.config); + responseCapacity.config.setMaxResponseBytes(536870913L); + assertEquals( + "CONFIG_SCHEMA_ERROR", + assertThrows( + AdapterException.class, + () -> responseCapacity.adapter.validateConfig(responseCapacity.config)) + .getCode()); + verifyNoInteractions(responseCapacity.http); + Fixture incident = new Fixture(); incident.config.setStreamKey("WEIBU_INCIDENT"); assertEquals( diff --git a/haobang-security-dm/upgrade-doc/v2/database/02-device-api-pull-upgrade.sql b/haobang-security-dm/upgrade-doc/v2/database/02-device-api-pull-upgrade.sql index 1e0aa73..c287564 100644 --- a/haobang-security-dm/upgrade-doc/v2/database/02-device-api-pull-upgrade.sql +++ b/haobang-security-dm/upgrade-doc/v2/database/02-device-api-pull-upgrade.sql @@ -73,7 +73,7 @@ CREATE TABLE device_collect_api_config ( CONSTRAINT ck_dc_api_cfg_window CHECK(window_size_seconds BETWEEN 1 AND 1209600), CONSTRAINT ck_dc_api_cfg_delay CHECK(safety_delay_seconds BETWEEN 0 AND 604800), CONSTRAINT ck_dc_api_cfg_lookback CHECK(lookback_seconds BETWEEN 0 AND 1209600), - CONSTRAINT ck_dc_api_cfg_resp CHECK(max_response_bytes BETWEEN 1024 AND 268435456), + CONSTRAINT ck_dc_api_cfg_resp CHECK(max_response_bytes BETWEEN 1024 AND 536870912), CONSTRAINT ck_dc_api_cfg_ver CHECK(config_version>=1), CONSTRAINT ck_dc_api_cfg_https CHECK(base_url LIKE 'https://%') ); diff --git a/haobang-security-dm/upgrade-doc/v6/README.md b/haobang-security-dm/upgrade-doc/v6/README.md index 88da89c..8d77e96 100644 --- a/haobang-security-dm/upgrade-doc/v6/README.md +++ b/haobang-security-dm/upgrade-doc/v6/README.md @@ -36,6 +36,8 @@ API 拉取任务的通用 `LOOKBACK_SECONDS` 默认值同时由 `300` 调整为 不会批量更新已有任务;确实需要迟到补偿的任务继续保留或显式配置正数。 数据库 `PAGE_SIZE` 检查约束上限由 `10000` 放宽为 `50000`,用于微步 5 秒窗口;其他适配器仍执行 各自更严格的 Java 配置校验。已有任务值不会被升级脚本修改。 +数据库 `MAX_RESPONSE_BYTES` 检查约束上限同时放宽为 `536870912`(512 MiB),用于容纳 +50000 条 × 每条最大 10 KiB 的微步响应;生产部署必须使用文档指定的 24 GiB JVM 堆配置。 ## API 拉取任务的探针归属 diff --git a/haobang-security-dm/upgrade-doc/v6/api/README.md b/haobang-security-dm/upgrade-doc/v6/api/README.md index 93ff6e1..69b2849 100644 --- a/haobang-security-dm/upgrade-doc/v6/api/README.md +++ b/haobang-security-dm/upgrade-doc/v6/api/README.md @@ -26,7 +26,7 @@ API 相关升级只看本页。示例设备:`deviceId=40`、`IP=10.56.171.164` docker logs syslog-serve-dm 2>&1 | grep '\[SYSLOG_BUILD_READY\]' | tail -n 1 ``` - 必须看到 `codeRevision=WEIBU_5S_50000_SCAN1S_V20260917_70`、`apiPullEnabled=true`、 + 必须看到 `codeRevision=WEIBU_5S_50000_512M_V20260917_71`、`apiPullEnabled=true`、 `apiSchedulerScanIntervalMs=1000`,并确认 `deviceCollectId` 与设备 40 的 `DEVICE_DEVICE.DEVICE_COLLECT_ID` 一致。内网默认 `apiTlsVerifyEnabled=false`。 @@ -64,8 +64,9 @@ cat /tmp/weibu-40-task.json 同时确认本次常规任务使用默认的 `lookbackSeconds=0`。只有现场确有迟到日志补偿需求并完成容量评估后, 才显式配置正数;微步接口没有已确认的分页游标,正数回看可能增加单个时间窗口的数据量。 还必须确认 `windowSizeSeconds=5`、`pageSize=50000`、`maxPagesPerRun=1`、 -`pollIntervalSeconds=1` 和 `maxResponseBytes=67108864`。条数上限与响应字节上限同时生效;响应超过 -64 MiB 时任务失败且不推进 checkpoint,不会截断后伪装成功。 +`pollIntervalSeconds=1` 和 `maxResponseBytes=536870912`。50000 条按每条最大 10 KiB 估算约为 +488 MiB,响应上限取整为 512 MiB;超限时任务失败且不推进 checkpoint,不会截断后伪装成功。 +生产容器必须使用 `JAVA_TOOL_OPTIONS=-Xms8g -Xmx24g ...`,不得使用只有 `-Xmx1g` 的开发配置。 ```bash docker exec -i syslog-serve-dm curl -fsS -X POST \ diff --git a/haobang-security-dm/upgrade-doc/v6/api/weibu-40-task.json b/haobang-security-dm/upgrade-doc/v6/api/weibu-40-task.json index 2b532b7..150974c 100644 --- a/haobang-security-dm/upgrade-doc/v6/api/weibu-40-task.json +++ b/haobang-security-dm/upgrade-doc/v6/api/weibu-40-task.json @@ -14,7 +14,7 @@ "windowSizeSeconds": 5, "safetyDelaySeconds": 120, "lookbackSeconds": 0, - "maxResponseBytes": 67108864, + "maxResponseBytes": 536870912, "initialStartTime": "__INITIAL_START_TIME__", "extraConfig": {} } diff --git a/haobang-security-dm/upgrade-doc/v6/database/01-device-collection-mode-precheck.sql b/haobang-security-dm/upgrade-doc/v6/database/01-device-collection-mode-precheck.sql index 152d6b8..95641d5 100644 --- a/haobang-security-dm/upgrade-doc/v6/database/01-device-collection-mode-precheck.sql +++ b/haobang-security-dm/upgrade-doc/v6/database/01-device-collection-mode-precheck.sql @@ -49,3 +49,10 @@ SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION FROM USER_CONSTRAINTS WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG' AND CONSTRAINT_NAME = 'CK_DC_API_CFG_PAGE'; + +-- 8. 记录响应字节约束。旧版上限通常为 268435456;新版微步按 50000 条 × 10 KiB +-- 取整使用 512 MiB,需要将上限放宽为 536870912。 +SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION +FROM USER_CONSTRAINTS +WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG' + AND CONSTRAINT_NAME = 'CK_DC_API_CFG_RESP'; diff --git a/haobang-security-dm/upgrade-doc/v6/database/02-device-collection-mode-upgrade.sql b/haobang-security-dm/upgrade-doc/v6/database/02-device-collection-mode-upgrade.sql index 993de83..f39984e 100644 --- a/haobang-security-dm/upgrade-doc/v6/database/02-device-collection-mode-upgrade.sql +++ b/haobang-security-dm/upgrade-doc/v6/database/02-device-collection-mode-upgrade.sql @@ -44,3 +44,14 @@ ALTER TABLE DEVICE_COLLECT_API_CONFIG CHECK (PAGE_SIZE BETWEEN 1 AND 50000); COMMIT; + +-- 50000 条 × 每条 10 KiB 约为 488 MiB,响应上限取整放宽到 512 MiB。 +-- 微步适配器仍执行自身上限;其他适配器继续保持各自更小的 Java Schema 上限。 +ALTER TABLE DEVICE_COLLECT_API_CONFIG + DROP CONSTRAINT CK_DC_API_CFG_RESP; + +ALTER TABLE DEVICE_COLLECT_API_CONFIG + ADD CONSTRAINT CK_DC_API_CFG_RESP + CHECK (MAX_RESPONSE_BYTES BETWEEN 1024 AND 536870912); + +COMMIT; diff --git a/haobang-security-dm/upgrade-doc/v6/database/03-device-collection-mode-verify.sql b/haobang-security-dm/upgrade-doc/v6/database/03-device-collection-mode-verify.sql index f2b3e8f..f296b90 100644 --- a/haobang-security-dm/upgrade-doc/v6/database/03-device-collection-mode-verify.sql +++ b/haobang-security-dm/upgrade-doc/v6/database/03-device-collection-mode-verify.sql @@ -44,3 +44,9 @@ SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION FROM USER_CONSTRAINTS WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG' AND CONSTRAINT_NAME = 'CK_DC_API_CFG_PAGE'; + +-- 8. 预期 1 行且 SEARCH_CONDITION 包含 MAX_RESPONSE_BYTES BETWEEN 1024 AND 536870912。 +SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION +FROM USER_CONSTRAINTS +WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG' + AND CONSTRAINT_NAME = 'CK_DC_API_CFG_RESP';