From 3acd90ecc930aad3c267fa541fc9c1b575d020c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E7=BB=8D=E5=AE=8F?= <994233303@qq.com> Date: Thu, 17 Sep 2026 23:40:07 +0800 Subject: [PATCH] =?UTF-8?q?fix(syslog-serve):=20=E6=89=A9=E5=A4=A7?= =?UTF-8?q?=E5=BE=AE=E6=AD=A55=E4=B8=87=E6=9D=A1=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=93=8D=E5=BA=94=E5=AE=B9=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 按50000条、每条最大10KiB将响应上限调整为512MiB - 将微步适配器最大响应字节数提高至536870912 - 将JSON解析token上限由200万提高至2500万 - 同步放宽管理接口和达梦数据库响应大小约束 - 更新微步任务模板、v6升级脚本和现场部署文档 - 增加50000条及512MiB配置边界测试 - 更新启动版本标识为WEIBU_5S_50000_512M_V20260917_71 验证： - JDK 8测试通过：167个，失败0，错误0，跳过2 - Maven BUILD SUCCESS - git diff --check通过 --- .../java/com/config/SyslogBuildStartupLogger.java | 2 +- .../apipull/adapter/weibu/WeibuApiAdapter.java | 6 ++++-- .../java/com/haobang/apipull/api/ApiPullDtos.java | 4 ++-- .../com/haobang/apipull/ApiPullContractTest.java | 13 ++++++++++--- .../apipull/adapter/weibu/WeibuApiAdapterTest.java | 12 ++++++++++++ .../v2/database/02-device-api-pull-upgrade.sql | 2 +- haobang-security-dm/upgrade-doc/v6/README.md | 2 ++ haobang-security-dm/upgrade-doc/v6/api/README.md | 7 ++++--- .../upgrade-doc/v6/api/weibu-40-task.json | 2 +- .../database/01-device-collection-mode-precheck.sql | 7 +++++++ .../database/02-device-collection-mode-upgrade.sql | 11 +++++++++++ .../database/03-device-collection-mode-verify.sql | 6 ++++++ 12 files changed, 61 insertions(+), 13 deletions(-) diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/config/SyslogBuildStartupLogger.java b/haobang-security-dm/syslog-serve/src/main/java/com/config/SyslogBuildStartupLogger.java index af15311..cd789ef 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/config/SyslogBuildStartupLogger.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/config/SyslogBuildStartupLogger.java @@ -12,7 +12,7 @@ import org.springframework.stereotype.Component; public class SyslogBuildStartupLogger { // 确认 JAR 包含设备级采集模式、API 拉取、探针归属过滤、微步全量查询及诊断接口。 - static final String CODE_REVISION = "WEIBU_5S_50000_SCAN1S_V20260917_70"; + static final String CODE_REVISION = "WEIBU_5S_50000_512M_V20260917_71"; private final Environment environment; public SyslogBuildStartupLogger(Environment environment) { diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapter.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapter.java index 057449f..c08ac3d 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapter.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapter.java @@ -81,6 +81,8 @@ public class WeibuApiAdapter implements ApiPullAdapter { UUID.fromString("89a53d59-bb84-5d41-924f-38319448cf46"); private static final long MIN_EVENT_EPOCH_SECOND = 946684800L; private static final long MAX_EVENT_EPOCH_SECOND = 4102444800L; + private static final long MAX_RESPONSE_BYTES = 536870912L; + private static final int MAX_JSON_TOKENS = 25000000; private final CredentialProvider credentials; private final SafeBaseUrlValidator urls; @@ -176,7 +178,7 @@ public class WeibuApiAdapter implements ApiPullAdapter { range(c.getRequestTimeoutMs(), 1000, 120000); if (c.getMaxResponseBytes() == null || c.getMaxResponseBytes() < 1024 - || c.getMaxResponseBytes() > 67108864L) fail("CONFIG_SCHEMA_ERROR"); + || c.getMaxResponseBytes() > MAX_RESPONSE_BYTES) fail("CONFIG_SCHEMA_ERROR"); URI base = urls.validate(c.getBaseUrl()); String path = base.getPath(); if (!(path == null || path.isEmpty() || "/".equals(path))) fail("CONFIG_SCHEMA_ERROR"); @@ -541,7 +543,7 @@ public class WeibuApiAdapter implements ApiPullAdapter { int tokens = 0; JsonToken token; while ((token = parser.nextToken()) != null) { - if (++tokens > 2000000) fail("RESPONSE_SCHEMA_ERROR"); + if (++tokens > MAX_JSON_TOKENS) fail("RESPONSE_SCHEMA_ERROR"); if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { if (++depth > 64) fail("RESPONSE_SCHEMA_ERROR"); } else if (token == JsonToken.END_OBJECT || token == JsonToken.END_ARRAY) { diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullDtos.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullDtos.java index 2474044..bde17b6 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullDtos.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/apipull/api/ApiPullDtos.java @@ -83,7 +83,7 @@ public final class ApiPullDtos { private Integer lookbackSeconds = 0; @Min(1024) - @Max(268435456) + @Max(536870912) private Long maxResponseBytes = 16777216L; @NotNull private OffsetDateTime initialStartTime; @@ -145,7 +145,7 @@ public final class ApiPullDtos { private Integer lookbackSeconds; @Min(1024) - @Max(268435456) + @Max(536870912) private Long maxResponseBytes; @NotNull private Map extraConfig; diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullContractTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullContractTest.java index 351ac85..15fe16f 100644 --- a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullContractTest.java +++ b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/ApiPullContractTest.java @@ -45,9 +45,16 @@ class ApiPullContractTest { @Test void managementDtoAllowsWeibuFiftyThousandRecordWindow() throws Exception { - Max max = CreateApiPullTaskRequest.class.getDeclaredField("pageSize").getAnnotation(Max.class); - assertNotNull(max); - assertEquals(50000L, max.value()); + Max pageMax = + CreateApiPullTaskRequest.class.getDeclaredField("pageSize").getAnnotation(Max.class); + assertNotNull(pageMax); + assertEquals(50000L, pageMax.value()); + Max responseMax = + CreateApiPullTaskRequest.class + .getDeclaredField("maxResponseBytes") + .getAnnotation(Max.class); + assertNotNull(responseMax); + assertEquals(536870912L, responseMax.value()); } @Test diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapterTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapterTest.java index 7aa8533..d263838 100644 --- a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapterTest.java +++ b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/apipull/adapter/weibu/WeibuApiAdapterTest.java @@ -261,6 +261,18 @@ class WeibuApiAdapterTest { .getCode()); verifyNoInteractions(capacity.http); + Fixture responseCapacity = new Fixture(); + responseCapacity.config.setMaxResponseBytes(536870912L); + responseCapacity.adapter.validateConfig(responseCapacity.config); + responseCapacity.config.setMaxResponseBytes(536870913L); + assertEquals( + "CONFIG_SCHEMA_ERROR", + assertThrows( + AdapterException.class, + () -> responseCapacity.adapter.validateConfig(responseCapacity.config)) + .getCode()); + verifyNoInteractions(responseCapacity.http); + Fixture incident = new Fixture(); incident.config.setStreamKey("WEIBU_INCIDENT"); assertEquals( diff --git a/haobang-security-dm/upgrade-doc/v2/database/02-device-api-pull-upgrade.sql b/haobang-security-dm/upgrade-doc/v2/database/02-device-api-pull-upgrade.sql index 1e0aa73..c287564 100644 --- a/haobang-security-dm/upgrade-doc/v2/database/02-device-api-pull-upgrade.sql +++ b/haobang-security-dm/upgrade-doc/v2/database/02-device-api-pull-upgrade.sql @@ -73,7 +73,7 @@ CREATE TABLE device_collect_api_config ( CONSTRAINT ck_dc_api_cfg_window CHECK(window_size_seconds BETWEEN 1 AND 1209600), CONSTRAINT ck_dc_api_cfg_delay CHECK(safety_delay_seconds BETWEEN 0 AND 604800), CONSTRAINT ck_dc_api_cfg_lookback CHECK(lookback_seconds BETWEEN 0 AND 1209600), - CONSTRAINT ck_dc_api_cfg_resp CHECK(max_response_bytes BETWEEN 1024 AND 268435456), + CONSTRAINT ck_dc_api_cfg_resp CHECK(max_response_bytes BETWEEN 1024 AND 536870912), CONSTRAINT ck_dc_api_cfg_ver CHECK(config_version>=1), CONSTRAINT ck_dc_api_cfg_https CHECK(base_url LIKE 'https://%') ); diff --git a/haobang-security-dm/upgrade-doc/v6/README.md b/haobang-security-dm/upgrade-doc/v6/README.md index 88da89c..8d77e96 100644 --- a/haobang-security-dm/upgrade-doc/v6/README.md +++ b/haobang-security-dm/upgrade-doc/v6/README.md @@ -36,6 +36,8 @@ API 拉取任务的通用 `LOOKBACK_SECONDS` 默认值同时由 `300` 调整为 不会批量更新已有任务；确实需要迟到补偿的任务继续保留或显式配置正数。 数据库 `PAGE_SIZE` 检查约束上限由 `10000` 放宽为 `50000`，用于微步 5 秒窗口；其他适配器仍执行 各自更严格的 Java 配置校验。已有任务值不会被升级脚本修改。 +数据库 `MAX_RESPONSE_BYTES` 检查约束上限同时放宽为 `536870912`（512 MiB），用于容纳 +50000 条 × 每条最大 10 KiB 的微步响应；生产部署必须使用文档指定的 24 GiB JVM 堆配置。 ## API 拉取任务的探针归属 diff --git a/haobang-security-dm/upgrade-doc/v6/api/README.md b/haobang-security-dm/upgrade-doc/v6/api/README.md index 93ff6e1..69b2849 100644 --- a/haobang-security-dm/upgrade-doc/v6/api/README.md +++ b/haobang-security-dm/upgrade-doc/v6/api/README.md @@ -26,7 +26,7 @@ API 相关升级只看本页。示例设备：`deviceId=40`、`IP=10.56.171.164` docker logs syslog-serve-dm 2>&1 | grep '\[SYSLOG_BUILD_READY\]' | tail -n 1 ``` - 必须看到 `codeRevision=WEIBU_5S_50000_SCAN1S_V20260917_70`、`apiPullEnabled=true`、 + 必须看到 `codeRevision=WEIBU_5S_50000_512M_V20260917_71`、`apiPullEnabled=true`、 `apiSchedulerScanIntervalMs=1000`，并确认 `deviceCollectId` 与设备 40 的 `DEVICE_DEVICE.DEVICE_COLLECT_ID` 一致。内网默认 `apiTlsVerifyEnabled=false`。 @@ -64,8 +64,9 @@ cat /tmp/weibu-40-task.json 同时确认本次常规任务使用默认的 `lookbackSeconds=0`。只有现场确有迟到日志补偿需求并完成容量评估后， 才显式配置正数；微步接口没有已确认的分页游标，正数回看可能增加单个时间窗口的数据量。 还必须确认 `windowSizeSeconds=5`、`pageSize=50000`、`maxPagesPerRun=1`、 -`pollIntervalSeconds=1` 和 `maxResponseBytes=67108864`。条数上限与响应字节上限同时生效；响应超过 -64 MiB 时任务失败且不推进 checkpoint，不会截断后伪装成功。 +`pollIntervalSeconds=1` 和 `maxResponseBytes=536870912`。50000 条按每条最大 10 KiB 估算约为 +488 MiB，响应上限取整为 512 MiB；超限时任务失败且不推进 checkpoint，不会截断后伪装成功。 +生产容器必须使用 `JAVA_TOOL_OPTIONS=-Xms8g -Xmx24g ...`，不得使用只有 `-Xmx1g` 的开发配置。 ```bash docker exec -i syslog-serve-dm curl -fsS -X POST \ diff --git a/haobang-security-dm/upgrade-doc/v6/api/weibu-40-task.json b/haobang-security-dm/upgrade-doc/v6/api/weibu-40-task.json index 2b532b7..150974c 100644 --- a/haobang-security-dm/upgrade-doc/v6/api/weibu-40-task.json +++ b/haobang-security-dm/upgrade-doc/v6/api/weibu-40-task.json @@ -14,7 +14,7 @@ "windowSizeSeconds": 5, "safetyDelaySeconds": 120, "lookbackSeconds": 0, - "maxResponseBytes": 67108864, + "maxResponseBytes": 536870912, "initialStartTime": "__INITIAL_START_TIME__", "extraConfig": {} } diff --git a/haobang-security-dm/upgrade-doc/v6/database/01-device-collection-mode-precheck.sql b/haobang-security-dm/upgrade-doc/v6/database/01-device-collection-mode-precheck.sql index 152d6b8..95641d5 100644 --- a/haobang-security-dm/upgrade-doc/v6/database/01-device-collection-mode-precheck.sql +++ b/haobang-security-dm/upgrade-doc/v6/database/01-device-collection-mode-precheck.sql @@ -49,3 +49,10 @@ SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION FROM USER_CONSTRAINTS WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG' AND CONSTRAINT_NAME = 'CK_DC_API_CFG_PAGE'; + +-- 8. 记录响应字节约束。旧版上限通常为 268435456；新版微步按 50000 条 × 10 KiB +-- 取整使用 512 MiB，需要将上限放宽为 536870912。 +SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION +FROM USER_CONSTRAINTS +WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG' + AND CONSTRAINT_NAME = 'CK_DC_API_CFG_RESP'; diff --git a/haobang-security-dm/upgrade-doc/v6/database/02-device-collection-mode-upgrade.sql b/haobang-security-dm/upgrade-doc/v6/database/02-device-collection-mode-upgrade.sql index 993de83..f39984e 100644 --- a/haobang-security-dm/upgrade-doc/v6/database/02-device-collection-mode-upgrade.sql +++ b/haobang-security-dm/upgrade-doc/v6/database/02-device-collection-mode-upgrade.sql @@ -44,3 +44,14 @@ ALTER TABLE DEVICE_COLLECT_API_CONFIG CHECK (PAGE_SIZE BETWEEN 1 AND 50000); COMMIT; + +-- 50000 条 × 每条 10 KiB 约为 488 MiB，响应上限取整放宽到 512 MiB。 +-- 微步适配器仍执行自身上限；其他适配器继续保持各自更小的 Java Schema 上限。 +ALTER TABLE DEVICE_COLLECT_API_CONFIG + DROP CONSTRAINT CK_DC_API_CFG_RESP; + +ALTER TABLE DEVICE_COLLECT_API_CONFIG + ADD CONSTRAINT CK_DC_API_CFG_RESP + CHECK (MAX_RESPONSE_BYTES BETWEEN 1024 AND 536870912); + +COMMIT; diff --git a/haobang-security-dm/upgrade-doc/v6/database/03-device-collection-mode-verify.sql b/haobang-security-dm/upgrade-doc/v6/database/03-device-collection-mode-verify.sql index f2b3e8f..f296b90 100644 --- a/haobang-security-dm/upgrade-doc/v6/database/03-device-collection-mode-verify.sql +++ b/haobang-security-dm/upgrade-doc/v6/database/03-device-collection-mode-verify.sql @@ -44,3 +44,9 @@ SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION FROM USER_CONSTRAINTS WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG' AND CONSTRAINT_NAME = 'CK_DC_API_CFG_PAGE'; + +-- 8. 预期 1 行且 SEARCH_CONDITION 包含 MAX_RESPONSE_BYTES BETWEEN 1024 AND 536870912。 +SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION +FROM USER_CONSTRAINTS +WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG' + AND CONSTRAINT_NAME = 'CK_DC_API_CFG_RESP';