fix(syslog-serve): 扩大微步5万条日志响应容量
- 按50000条、每条最大10KiB将响应上限调整为512MiB - 将微步适配器最大响应字节数提高至536870912 - 将JSON解析token上限由200万提高至2500万 - 同步放宽管理接口和达梦数据库响应大小约束 - 更新微步任务模板、v6升级脚本和现场部署文档 - 增加50000条及512MiB配置边界测试 - 更新启动版本标识为WEIBU_5S_50000_512M_V20260917_71 验证: - JDK 8测试通过:167个,失败0,错误0,跳过2 - Maven BUILD SUCCESS - git diff --check通过
This commit is contained in:
+1
-1
@@ -12,7 +12,7 @@ import org.springframework.stereotype.Component;
|
|||||||
public class SyslogBuildStartupLogger {
|
public class SyslogBuildStartupLogger {
|
||||||
|
|
||||||
// 确认 JAR 包含设备级采集模式、API 拉取、探针归属过滤、微步全量查询及诊断接口。
|
// 确认 JAR 包含设备级采集模式、API 拉取、探针归属过滤、微步全量查询及诊断接口。
|
||||||
static final String CODE_REVISION = "WEIBU_5S_50000_SCAN1S_V20260917_70";
|
static final String CODE_REVISION = "WEIBU_5S_50000_512M_V20260917_71";
|
||||||
private final Environment environment;
|
private final Environment environment;
|
||||||
|
|
||||||
public SyslogBuildStartupLogger(Environment environment) {
|
public SyslogBuildStartupLogger(Environment environment) {
|
||||||
|
|||||||
+4
-2
@@ -81,6 +81,8 @@ public class WeibuApiAdapter implements ApiPullAdapter {
|
|||||||
UUID.fromString("89a53d59-bb84-5d41-924f-38319448cf46");
|
UUID.fromString("89a53d59-bb84-5d41-924f-38319448cf46");
|
||||||
private static final long MIN_EVENT_EPOCH_SECOND = 946684800L;
|
private static final long MIN_EVENT_EPOCH_SECOND = 946684800L;
|
||||||
private static final long MAX_EVENT_EPOCH_SECOND = 4102444800L;
|
private static final long MAX_EVENT_EPOCH_SECOND = 4102444800L;
|
||||||
|
private static final long MAX_RESPONSE_BYTES = 536870912L;
|
||||||
|
private static final int MAX_JSON_TOKENS = 25000000;
|
||||||
|
|
||||||
private final CredentialProvider credentials;
|
private final CredentialProvider credentials;
|
||||||
private final SafeBaseUrlValidator urls;
|
private final SafeBaseUrlValidator urls;
|
||||||
@@ -176,7 +178,7 @@ public class WeibuApiAdapter implements ApiPullAdapter {
|
|||||||
range(c.getRequestTimeoutMs(), 1000, 120000);
|
range(c.getRequestTimeoutMs(), 1000, 120000);
|
||||||
if (c.getMaxResponseBytes() == null
|
if (c.getMaxResponseBytes() == null
|
||||||
|| c.getMaxResponseBytes() < 1024
|
|| c.getMaxResponseBytes() < 1024
|
||||||
|| c.getMaxResponseBytes() > 67108864L) fail("CONFIG_SCHEMA_ERROR");
|
|| c.getMaxResponseBytes() > MAX_RESPONSE_BYTES) fail("CONFIG_SCHEMA_ERROR");
|
||||||
URI base = urls.validate(c.getBaseUrl());
|
URI base = urls.validate(c.getBaseUrl());
|
||||||
String path = base.getPath();
|
String path = base.getPath();
|
||||||
if (!(path == null || path.isEmpty() || "/".equals(path))) fail("CONFIG_SCHEMA_ERROR");
|
if (!(path == null || path.isEmpty() || "/".equals(path))) fail("CONFIG_SCHEMA_ERROR");
|
||||||
@@ -541,7 +543,7 @@ public class WeibuApiAdapter implements ApiPullAdapter {
|
|||||||
int tokens = 0;
|
int tokens = 0;
|
||||||
JsonToken token;
|
JsonToken token;
|
||||||
while ((token = parser.nextToken()) != null) {
|
while ((token = parser.nextToken()) != null) {
|
||||||
if (++tokens > 2000000) fail("RESPONSE_SCHEMA_ERROR");
|
if (++tokens > MAX_JSON_TOKENS) fail("RESPONSE_SCHEMA_ERROR");
|
||||||
if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) {
|
if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) {
|
||||||
if (++depth > 64) fail("RESPONSE_SCHEMA_ERROR");
|
if (++depth > 64) fail("RESPONSE_SCHEMA_ERROR");
|
||||||
} else if (token == JsonToken.END_OBJECT || token == JsonToken.END_ARRAY) {
|
} else if (token == JsonToken.END_OBJECT || token == JsonToken.END_ARRAY) {
|
||||||
|
|||||||
+2
-2
@@ -83,7 +83,7 @@ public final class ApiPullDtos {
|
|||||||
private Integer lookbackSeconds = 0;
|
private Integer lookbackSeconds = 0;
|
||||||
|
|
||||||
@Min(1024)
|
@Min(1024)
|
||||||
@Max(268435456)
|
@Max(536870912)
|
||||||
private Long maxResponseBytes = 16777216L;
|
private Long maxResponseBytes = 16777216L;
|
||||||
|
|
||||||
@NotNull private OffsetDateTime initialStartTime;
|
@NotNull private OffsetDateTime initialStartTime;
|
||||||
@@ -145,7 +145,7 @@ public final class ApiPullDtos {
|
|||||||
private Integer lookbackSeconds;
|
private Integer lookbackSeconds;
|
||||||
|
|
||||||
@Min(1024)
|
@Min(1024)
|
||||||
@Max(268435456)
|
@Max(536870912)
|
||||||
private Long maxResponseBytes;
|
private Long maxResponseBytes;
|
||||||
|
|
||||||
@NotNull private Map<String, Object> extraConfig;
|
@NotNull private Map<String, Object> extraConfig;
|
||||||
|
|||||||
+10
-3
@@ -45,9 +45,16 @@ class ApiPullContractTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
void managementDtoAllowsWeibuFiftyThousandRecordWindow() throws Exception {
|
void managementDtoAllowsWeibuFiftyThousandRecordWindow() throws Exception {
|
||||||
Max max = CreateApiPullTaskRequest.class.getDeclaredField("pageSize").getAnnotation(Max.class);
|
Max pageMax =
|
||||||
assertNotNull(max);
|
CreateApiPullTaskRequest.class.getDeclaredField("pageSize").getAnnotation(Max.class);
|
||||||
assertEquals(50000L, max.value());
|
assertNotNull(pageMax);
|
||||||
|
assertEquals(50000L, pageMax.value());
|
||||||
|
Max responseMax =
|
||||||
|
CreateApiPullTaskRequest.class
|
||||||
|
.getDeclaredField("maxResponseBytes")
|
||||||
|
.getAnnotation(Max.class);
|
||||||
|
assertNotNull(responseMax);
|
||||||
|
assertEquals(536870912L, responseMax.value());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
+12
@@ -261,6 +261,18 @@ class WeibuApiAdapterTest {
|
|||||||
.getCode());
|
.getCode());
|
||||||
verifyNoInteractions(capacity.http);
|
verifyNoInteractions(capacity.http);
|
||||||
|
|
||||||
|
Fixture responseCapacity = new Fixture();
|
||||||
|
responseCapacity.config.setMaxResponseBytes(536870912L);
|
||||||
|
responseCapacity.adapter.validateConfig(responseCapacity.config);
|
||||||
|
responseCapacity.config.setMaxResponseBytes(536870913L);
|
||||||
|
assertEquals(
|
||||||
|
"CONFIG_SCHEMA_ERROR",
|
||||||
|
assertThrows(
|
||||||
|
AdapterException.class,
|
||||||
|
() -> responseCapacity.adapter.validateConfig(responseCapacity.config))
|
||||||
|
.getCode());
|
||||||
|
verifyNoInteractions(responseCapacity.http);
|
||||||
|
|
||||||
Fixture incident = new Fixture();
|
Fixture incident = new Fixture();
|
||||||
incident.config.setStreamKey("WEIBU_INCIDENT");
|
incident.config.setStreamKey("WEIBU_INCIDENT");
|
||||||
assertEquals(
|
assertEquals(
|
||||||
|
|||||||
@@ -73,7 +73,7 @@ CREATE TABLE device_collect_api_config (
|
|||||||
CONSTRAINT ck_dc_api_cfg_window CHECK(window_size_seconds BETWEEN 1 AND 1209600),
|
CONSTRAINT ck_dc_api_cfg_window CHECK(window_size_seconds BETWEEN 1 AND 1209600),
|
||||||
CONSTRAINT ck_dc_api_cfg_delay CHECK(safety_delay_seconds BETWEEN 0 AND 604800),
|
CONSTRAINT ck_dc_api_cfg_delay CHECK(safety_delay_seconds BETWEEN 0 AND 604800),
|
||||||
CONSTRAINT ck_dc_api_cfg_lookback CHECK(lookback_seconds BETWEEN 0 AND 1209600),
|
CONSTRAINT ck_dc_api_cfg_lookback CHECK(lookback_seconds BETWEEN 0 AND 1209600),
|
||||||
CONSTRAINT ck_dc_api_cfg_resp CHECK(max_response_bytes BETWEEN 1024 AND 268435456),
|
CONSTRAINT ck_dc_api_cfg_resp CHECK(max_response_bytes BETWEEN 1024 AND 536870912),
|
||||||
CONSTRAINT ck_dc_api_cfg_ver CHECK(config_version>=1),
|
CONSTRAINT ck_dc_api_cfg_ver CHECK(config_version>=1),
|
||||||
CONSTRAINT ck_dc_api_cfg_https CHECK(base_url LIKE 'https://%')
|
CONSTRAINT ck_dc_api_cfg_https CHECK(base_url LIKE 'https://%')
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -36,6 +36,8 @@ API 拉取任务的通用 `LOOKBACK_SECONDS` 默认值同时由 `300` 调整为
|
|||||||
不会批量更新已有任务;确实需要迟到补偿的任务继续保留或显式配置正数。
|
不会批量更新已有任务;确实需要迟到补偿的任务继续保留或显式配置正数。
|
||||||
数据库 `PAGE_SIZE` 检查约束上限由 `10000` 放宽为 `50000`,用于微步 5 秒窗口;其他适配器仍执行
|
数据库 `PAGE_SIZE` 检查约束上限由 `10000` 放宽为 `50000`,用于微步 5 秒窗口;其他适配器仍执行
|
||||||
各自更严格的 Java 配置校验。已有任务值不会被升级脚本修改。
|
各自更严格的 Java 配置校验。已有任务值不会被升级脚本修改。
|
||||||
|
数据库 `MAX_RESPONSE_BYTES` 检查约束上限同时放宽为 `536870912`(512 MiB),用于容纳
|
||||||
|
50000 条 × 每条最大 10 KiB 的微步响应;生产部署必须使用文档指定的 24 GiB JVM 堆配置。
|
||||||
|
|
||||||
## API 拉取任务的探针归属
|
## API 拉取任务的探针归属
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ API 相关升级只看本页。示例设备:`deviceId=40`、`IP=10.56.171.164`
|
|||||||
docker logs syslog-serve-dm 2>&1 | grep '\[SYSLOG_BUILD_READY\]' | tail -n 1
|
docker logs syslog-serve-dm 2>&1 | grep '\[SYSLOG_BUILD_READY\]' | tail -n 1
|
||||||
```
|
```
|
||||||
|
|
||||||
必须看到 `codeRevision=WEIBU_5S_50000_SCAN1S_V20260917_70`、`apiPullEnabled=true`、
|
必须看到 `codeRevision=WEIBU_5S_50000_512M_V20260917_71`、`apiPullEnabled=true`、
|
||||||
`apiSchedulerScanIntervalMs=1000`,并确认
|
`apiSchedulerScanIntervalMs=1000`,并确认
|
||||||
`deviceCollectId` 与设备 40 的 `DEVICE_DEVICE.DEVICE_COLLECT_ID` 一致。内网默认
|
`deviceCollectId` 与设备 40 的 `DEVICE_DEVICE.DEVICE_COLLECT_ID` 一致。内网默认
|
||||||
`apiTlsVerifyEnabled=false`。
|
`apiTlsVerifyEnabled=false`。
|
||||||
@@ -64,8 +64,9 @@ cat /tmp/weibu-40-task.json
|
|||||||
同时确认本次常规任务使用默认的 `lookbackSeconds=0`。只有现场确有迟到日志补偿需求并完成容量评估后,
|
同时确认本次常规任务使用默认的 `lookbackSeconds=0`。只有现场确有迟到日志补偿需求并完成容量评估后,
|
||||||
才显式配置正数;微步接口没有已确认的分页游标,正数回看可能增加单个时间窗口的数据量。
|
才显式配置正数;微步接口没有已确认的分页游标,正数回看可能增加单个时间窗口的数据量。
|
||||||
还必须确认 `windowSizeSeconds=5`、`pageSize=50000`、`maxPagesPerRun=1`、
|
还必须确认 `windowSizeSeconds=5`、`pageSize=50000`、`maxPagesPerRun=1`、
|
||||||
`pollIntervalSeconds=1` 和 `maxResponseBytes=67108864`。条数上限与响应字节上限同时生效;响应超过
|
`pollIntervalSeconds=1` 和 `maxResponseBytes=536870912`。50000 条按每条最大 10 KiB 估算约为
|
||||||
64 MiB 时任务失败且不推进 checkpoint,不会截断后伪装成功。
|
488 MiB,响应上限取整为 512 MiB;超限时任务失败且不推进 checkpoint,不会截断后伪装成功。
|
||||||
|
生产容器必须使用 `JAVA_TOOL_OPTIONS=-Xms8g -Xmx24g ...`,不得使用只有 `-Xmx1g` 的开发配置。
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker exec -i syslog-serve-dm curl -fsS -X POST \
|
docker exec -i syslog-serve-dm curl -fsS -X POST \
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
"windowSizeSeconds": 5,
|
"windowSizeSeconds": 5,
|
||||||
"safetyDelaySeconds": 120,
|
"safetyDelaySeconds": 120,
|
||||||
"lookbackSeconds": 0,
|
"lookbackSeconds": 0,
|
||||||
"maxResponseBytes": 67108864,
|
"maxResponseBytes": 536870912,
|
||||||
"initialStartTime": "__INITIAL_START_TIME__",
|
"initialStartTime": "__INITIAL_START_TIME__",
|
||||||
"extraConfig": {}
|
"extraConfig": {}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,3 +49,10 @@ SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION
|
|||||||
FROM USER_CONSTRAINTS
|
FROM USER_CONSTRAINTS
|
||||||
WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG'
|
WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG'
|
||||||
AND CONSTRAINT_NAME = 'CK_DC_API_CFG_PAGE';
|
AND CONSTRAINT_NAME = 'CK_DC_API_CFG_PAGE';
|
||||||
|
|
||||||
|
-- 8. 记录响应字节约束。旧版上限通常为 268435456;新版微步按 50000 条 × 10 KiB
|
||||||
|
-- 取整使用 512 MiB,需要将上限放宽为 536870912。
|
||||||
|
SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION
|
||||||
|
FROM USER_CONSTRAINTS
|
||||||
|
WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG'
|
||||||
|
AND CONSTRAINT_NAME = 'CK_DC_API_CFG_RESP';
|
||||||
|
|||||||
@@ -44,3 +44,14 @@ ALTER TABLE DEVICE_COLLECT_API_CONFIG
|
|||||||
CHECK (PAGE_SIZE BETWEEN 1 AND 50000);
|
CHECK (PAGE_SIZE BETWEEN 1 AND 50000);
|
||||||
|
|
||||||
COMMIT;
|
COMMIT;
|
||||||
|
|
||||||
|
-- 50000 条 × 每条 10 KiB 约为 488 MiB,响应上限取整放宽到 512 MiB。
|
||||||
|
-- 微步适配器仍执行自身上限;其他适配器继续保持各自更小的 Java Schema 上限。
|
||||||
|
ALTER TABLE DEVICE_COLLECT_API_CONFIG
|
||||||
|
DROP CONSTRAINT CK_DC_API_CFG_RESP;
|
||||||
|
|
||||||
|
ALTER TABLE DEVICE_COLLECT_API_CONFIG
|
||||||
|
ADD CONSTRAINT CK_DC_API_CFG_RESP
|
||||||
|
CHECK (MAX_RESPONSE_BYTES BETWEEN 1024 AND 536870912);
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
|||||||
@@ -44,3 +44,9 @@ SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION
|
|||||||
FROM USER_CONSTRAINTS
|
FROM USER_CONSTRAINTS
|
||||||
WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG'
|
WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG'
|
||||||
AND CONSTRAINT_NAME = 'CK_DC_API_CFG_PAGE';
|
AND CONSTRAINT_NAME = 'CK_DC_API_CFG_PAGE';
|
||||||
|
|
||||||
|
-- 8. 预期 1 行且 SEARCH_CONDITION 包含 MAX_RESPONSE_BYTES BETWEEN 1024 AND 536870912。
|
||||||
|
SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS, SEARCH_CONDITION
|
||||||
|
FROM USER_CONSTRAINTS
|
||||||
|
WHERE TABLE_NAME = 'DEVICE_COLLECT_API_CONFIG'
|
||||||
|
AND CONSTRAINT_NAME = 'CK_DC_API_CFG_RESP';
|
||||||
|
|||||||
Reference in New Issue
Block a user