fix(syslog-serve): 扩大微步5万条日志响应容量

- 按50000条、每条最大10KiB将响应上限调整为512MiB - 将微步适配器最大响应字节数提高至536870912 - 将JSON解析token上限由200万提高至2500万 - 同步放宽管理接口和达梦数据库响应大小约束 - 更新微步任务模板、v6升级脚本和现场部署文档 - 增加50000条及512MiB配置边界测试 - 更新启动版本标识为WEIBU_5S_50000_512M_V20260917_71 验证： - JDK 8测试通过：167个，失败0，错误0，跳过2 - Maven BUILD SUCCESS - git diff --check通过
2026-09-17 23:40:07 +08:00
parent d7fcbdde92
commit 3acd90ecc9
12 changed files with 61 additions and 13 deletions
@@ -12,7 +12,7 @@ import org.springframework.stereotype.Component;
 public class SyslogBuildStartupLogger {

    // 确认 JAR 包含设备级采集模式、API 拉取、探针归属过滤、微步全量查询及诊断接口。
-    static final String CODE_REVISION = "WEIBU_5S_50000_SCAN1S_V20260917_70";
+    static final String CODE_REVISION = "WEIBU_5S_50000_512M_V20260917_71";
    private final Environment environment;

    public SyslogBuildStartupLogger(Environment environment) {
@@ -81,6 +81,8 @@ public class WeibuApiAdapter implements ApiPullAdapter {
      UUID.fromString("89a53d59-bb84-5d41-924f-38319448cf46");
  private static final long MIN_EVENT_EPOCH_SECOND = 946684800L;
  private static final long MAX_EVENT_EPOCH_SECOND = 4102444800L;
+  private static final long MAX_RESPONSE_BYTES = 536870912L;
+  private static final int MAX_JSON_TOKENS = 25000000;

  private final CredentialProvider credentials;
  private final SafeBaseUrlValidator urls;
@@ -176,7 +178,7 @@ public class WeibuApiAdapter implements ApiPullAdapter {
    range(c.getRequestTimeoutMs(), 1000, 120000);
    if (c.getMaxResponseBytes() == null
        || c.getMaxResponseBytes() < 1024
-        || c.getMaxResponseBytes() > 67108864L) fail("CONFIG_SCHEMA_ERROR");
+        || c.getMaxResponseBytes() > MAX_RESPONSE_BYTES) fail("CONFIG_SCHEMA_ERROR");
    URI base = urls.validate(c.getBaseUrl());
    String path = base.getPath();
    if (!(path == null || path.isEmpty() || "/".equals(path))) fail("CONFIG_SCHEMA_ERROR");
@@ -541,7 +543,7 @@ public class WeibuApiAdapter implements ApiPullAdapter {
        int tokens = 0;
        JsonToken token;
        while ((token = parser.nextToken()) != null) {
-          if (++tokens > 2000000) fail("RESPONSE_SCHEMA_ERROR");
+          if (++tokens > MAX_JSON_TOKENS) fail("RESPONSE_SCHEMA_ERROR");
          if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) {
            if (++depth > 64) fail("RESPONSE_SCHEMA_ERROR");
          } else if (token == JsonToken.END_OBJECT || token == JsonToken.END_ARRAY) {
@@ -83,7 +83,7 @@ public final class ApiPullDtos {
    private Integer lookbackSeconds = 0;

    @Min(1024)
-    @Max(268435456)
+    @Max(536870912)
    private Long maxResponseBytes = 16777216L;

    @NotNull private OffsetDateTime initialStartTime;
@@ -145,7 +145,7 @@ public final class ApiPullDtos {
    private Integer lookbackSeconds;

    @Min(1024)
-    @Max(268435456)
+    @Max(536870912)
    private Long maxResponseBytes;

    @NotNull private Map<String, Object> extraConfig;
@@ -45,9 +45,16 @@ class ApiPullContractTest {

  @Test
  void managementDtoAllowsWeibuFiftyThousandRecordWindow() throws Exception {
-    Max max = CreateApiPullTaskRequest.class.getDeclaredField("pageSize").getAnnotation(Max.class);
-    assertNotNull(max);
-    assertEquals(50000L, max.value());
+    Max pageMax =
+        CreateApiPullTaskRequest.class.getDeclaredField("pageSize").getAnnotation(Max.class);
+    assertNotNull(pageMax);
+    assertEquals(50000L, pageMax.value());
+    Max responseMax =
+        CreateApiPullTaskRequest.class
+            .getDeclaredField("maxResponseBytes")
+            .getAnnotation(Max.class);
+    assertNotNull(responseMax);
+    assertEquals(536870912L, responseMax.value());
  }

  @Test
@@ -261,6 +261,18 @@ class WeibuApiAdapterTest {
            .getCode());
    verifyNoInteractions(capacity.http);

+    Fixture responseCapacity = new Fixture();
+    responseCapacity.config.setMaxResponseBytes(536870912L);
+    responseCapacity.adapter.validateConfig(responseCapacity.config);
+    responseCapacity.config.setMaxResponseBytes(536870913L);
+    assertEquals(
+        "CONFIG_SCHEMA_ERROR",
+        assertThrows(
+                AdapterException.class,
+                () -> responseCapacity.adapter.validateConfig(responseCapacity.config))
+            .getCode());
+    verifyNoInteractions(responseCapacity.http);
+
    Fixture incident = new Fixture();
    incident.config.setStreamKey("WEIBU_INCIDENT");
    assertEquals(
@@ -73,7 +73,7 @@ CREATE TABLE device_collect_api_config (
  CONSTRAINT ck_dc_api_cfg_window CHECK(window_size_seconds BETWEEN 1 AND 1209600),
  CONSTRAINT ck_dc_api_cfg_delay CHECK(safety_delay_seconds BETWEEN 0 AND 604800),
  CONSTRAINT ck_dc_api_cfg_lookback CHECK(lookback_seconds BETWEEN 0 AND 1209600),
-  CONSTRAINT ck_dc_api_cfg_resp CHECK(max_response_bytes BETWEEN 1024 AND 268435456),
+  CONSTRAINT ck_dc_api_cfg_resp CHECK(max_response_bytes BETWEEN 1024 AND 536870912),
  CONSTRAINT ck_dc_api_cfg_ver CHECK(config_version>=1),
  CONSTRAINT ck_dc_api_cfg_https CHECK(base_url LIKE 'https://%')
 );
@@ -36,6 +36,8 @@ API 拉取任务的通用 `LOOKBACK_SECONDS` 默认值同时由 `300` 调整为
 不会批量更新已有任务；确实需要迟到补偿的任务继续保留或显式配置正数。
 数据库 `PAGE_SIZE` 检查约束上限由 `10000` 放宽为 `50000`，用于微步 5 秒窗口；其他适配器仍执行
 各自更严格的 Java 配置校验。已有任务值不会被升级脚本修改。
+数据库 `MAX_RESPONSE_BYTES` 检查约束上限同时放宽为 `536870912`（512 MiB），用于容纳
+50000 条 × 每条最大 10 KiB 的微步响应；生产部署必须使用文档指定的 24 GiB JVM 堆配置。

 ## API 拉取任务的探针归属

@@ -26,7 +26,7 @@ API 相关升级只看本页。示例设备：`deviceId=40`、`IP=10.56.171.164`
   docker logs syslog-serve-dm 2>&1 | grep '\[SYSLOG_BUILD_READY\]' | tail -n 1
   ```

-   必须看到 `codeRevision=WEIBU_5S_50000_SCAN1S_V20260917_70`、`apiPullEnabled=true`、
+   必须看到 `codeRevision=WEIBU_5S_50000_512M_V20260917_71`、`apiPullEnabled=true`、
   `apiSchedulerScanIntervalMs=1000`，并确认
   `deviceCollectId` 与设备 40 的 `DEVICE_DEVICE.DEVICE_COLLECT_ID` 一致。内网默认
   `apiTlsVerifyEnabled=false`。
@@ -64,8 +64,9 @@ cat /tmp/weibu-40-task.json
 同时确认本次常规任务使用默认的 `lookbackSeconds=0`。只有现场确有迟到日志补偿需求并完成容量评估后，
 才显式配置正数；微步接口没有已确认的分页游标，正数回看可能增加单个时间窗口的数据量。
 还必须确认 `windowSizeSeconds=5`、`pageSize=50000`、`maxPagesPerRun=1`、
-`pollIntervalSeconds=1` 和 `maxResponseBytes=67108864`。条数上限与响应字节上限同时生效；响应超过
-64 MiB 时任务失败且不推进 checkpoint，不会截断后伪装成功。
+`pollIntervalSeconds=1` 和 `maxResponseBytes=536870912`。50000 条按每条最大 10 KiB 估算约为
+488 MiB，响应上限取整为 512 MiB；超限时任务失败且不推进 checkpoint，不会截断后伪装成功。
+生产容器必须使用 `JAVA_TOOL_OPTIONS=-Xms8g -Xmx24g ...`，不得使用只有 `-Xmx1g` 的开发配置。

 ```bash
 docker exec -i syslog-serve-dm curl -fsS -X POST \
@@ -14,7 +14,7 @@
  "windowSizeSeconds": 5,
  "safetyDelaySeconds": 120,
  "lookbackSeconds": 0,
-  "maxResponseBytes": 67108864,
+  "maxResponseBytes": 536870912,
  "initialStartTime": "__INITIAL_START_TIME__",
  "extraConfig": {}
 }
@@ -49,3 +49,10 @@ SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION
 FROM USER_CONSTRAINTS
 WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG'
  AND CONSTRAINT_NAME = 'CK_DC_API_CFG_PAGE';
+
+-- 8. 记录响应字节约束。旧版上限通常为 268435456；新版微步按 50000 条 × 10 KiB
+-- 取整使用 512 MiB，需要将上限放宽为 536870912。
+SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION
+FROM USER_CONSTRAINTS
+WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG'
+  AND CONSTRAINT_NAME = 'CK_DC_API_CFG_RESP';
@@ -44,3 +44,14 @@ ALTER TABLE DEVICE_COLLECT_API_CONFIG
  CHECK (PAGE_SIZE BETWEEN 1 AND 50000);

 COMMIT;
+
+-- 50000 条 × 每条 10 KiB 约为 488 MiB，响应上限取整放宽到 512 MiB。
+-- 微步适配器仍执行自身上限；其他适配器继续保持各自更小的 Java Schema 上限。
+ALTER TABLE DEVICE_COLLECT_API_CONFIG
+  DROP CONSTRAINT CK_DC_API_CFG_RESP;
+
+ALTER TABLE DEVICE_COLLECT_API_CONFIG
+  ADD CONSTRAINT CK_DC_API_CFG_RESP
+  CHECK (MAX_RESPONSE_BYTES BETWEEN 1024 AND 536870912);
+
+COMMIT;
@@ -44,3 +44,9 @@ SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION
 FROM USER_CONSTRAINTS
 WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG'
  AND CONSTRAINT_NAME = 'CK_DC_API_CFG_PAGE';
+
+-- 8. 预期 1 行且 SEARCH_CONDITION 包含 MAX_RESPONSE_BYTES BETWEEN 1024 AND 536870912。
+SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION
+FROM USER_CONSTRAINTS
+WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG'
+  AND CONSTRAINT_NAME = 'CK_DC_API_CFG_RESP';