feat(syslog): 为三个生产模块增加启动版本标识

- syslog-serve 增加 SYSLOG_BUILD_READY 启动日志
- syslog-consumer 增加 SYSLOG_BUILD_READY 启动日志
- 更新 syslog-consumer-rule 代码修订标识至 v63
- 输出模块名、代码修订号、运行环境及关键非敏感配置
- 支持通过容器日志快速确认实际部署的 JAR 版本
- 补充 syslog-serve 和 syslog-consumer 启动日志测试
- 不输出数据库密码、API 凭据等敏感信息
- 不修改 Maven POM 配置
This commit is contained in:
王绍宏
2026-09-17 18:47:25 +08:00
parent 8f1a8b6283
commit 2d297eb996
67 changed files with 3132 additions and 88 deletions
@@ -63,8 +63,30 @@ public class NormalizationCacheInvalidationService {
}
}
/**
* 清空规则专属 Spring Cache,并按设备精确广播 JetCache BOTH 失效。
*
* <p>JetCache 没有安全的全库 clear;调用方传入数据库中的全部设备 ID,既能删除共享 L2,
* 又能通过 syncLocal 让其他实例立即移除对应 L1,且不会误删同 Redis 中的其他业务数据。</p>
*/
public void invalidateAllRules(Collection<Long> deviceIds) {
clearSpring("normalizeRule");
metrics.invalidation("local");
if (deviceIds == null) return;
for (Long deviceId : deviceIds) {
if (deviceId == null) continue;
jetCacheInvalidator.rulesForDevice(deviceId);
metrics.invalidation("remote");
}
}
private void evictSpring(String cacheName, Object key) {
Cache cache = springCacheManager.getCache(cacheName);
if (cache != null) cache.evict(key);
}
private void clearSpring(String cacheName) {
Cache cache = springCacheManager.getCache(cacheName);
if (cache != null) cache.clear();
}
}
@@ -0,0 +1,85 @@
package com.Modules.NormalData.cache;
import com.Modules.NormalData.compiled.CompiledRuleCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.redis.connection.Message;
import org.springframework.data.redis.connection.MessageListener;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.StringJoiner;
/** 通过 Redis Pub/Sub 让所有消费节点同步清理 JVM 本地编译规则缓存。 */
@Component
public class NormalizationCompiledCacheInvalidationBus implements MessageListener {
public static final String TOPIC = "xdr:norm:v1:compiled-rule:invalidate";
private static final String ALL = "ALL";
private static final String RULES_PREFIX = "RULES:";
private static final Logger log = LoggerFactory.getLogger(
NormalizationCompiledCacheInvalidationBus.class);
private final CompiledRuleCache compiledRuleCache;
private final StringRedisTemplate redisTemplate;
public NormalizationCompiledCacheInvalidationBus(CompiledRuleCache compiledRuleCache,
StringRedisTemplate redisTemplate) {
this.compiledRuleCache = compiledRuleCache;
this.redisTemplate = redisTemplate;
}
public void invalidateRulesEverywhere(Collection<Long> ruleIds) {
Set<Long> distinct = positiveIds(ruleIds);
if (distinct.isEmpty()) return;
for (Long ruleId : distinct) compiledRuleCache.invalidateRule(ruleId);
redisTemplate.convertAndSend(TOPIC, ruleMessage(distinct));
}
public void invalidateAllEverywhere() {
compiledRuleCache.invalidateAll();
redisTemplate.convertAndSend(TOPIC, ALL);
}
@Override
public void onMessage(Message message, byte[] pattern) {
if (message == null || message.getBody() == null) return;
String command = new String(message.getBody(), StandardCharsets.UTF_8).trim();
if (ALL.equals(command)) {
compiledRuleCache.invalidateAll();
return;
}
if (!command.startsWith(RULES_PREFIX)) {
log.warn("忽略未知的规则缓存失效广播: {}", command);
return;
}
String ids = command.substring(RULES_PREFIX.length());
if (ids.isEmpty()) return;
for (String value : ids.split(",")) {
try {
long ruleId = Long.parseLong(value);
if (ruleId > 0L) compiledRuleCache.invalidateRule(ruleId);
} catch (NumberFormatException invalidId) {
log.warn("忽略非法的规则缓存失效广播参数: {}", value);
}
}
}
private static Set<Long> positiveIds(Collection<Long> ruleIds) {
Set<Long> distinct = new LinkedHashSet<>();
if (ruleIds == null) return distinct;
for (Long ruleId : ruleIds) {
if (ruleId != null && ruleId > 0L) distinct.add(ruleId);
}
return distinct;
}
private static String ruleMessage(Collection<Long> ruleIds) {
StringJoiner values = new StringJoiner(",", RULES_PREFIX, "");
for (Long ruleId : ruleIds) values.add(ruleId.toString());
return values.toString();
}
}
@@ -0,0 +1,83 @@
package com.Modules.NormalData.cache;
import com.common.mapper.DeviceDeviceMapper;
import com.common.mapper.DmNormalizeRuleMapper;
import org.springframework.stereotype.Service;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
/** 管理接口使用的规则缓存清理编排;不会清除设备、任务或其他业务 Redis 数据。 */
@Service
public class NormalizationRuleCacheManagementService {
private final NormalizationCacheInvalidationService invalidationService;
private final NormalizationCompiledCacheInvalidationBus compiledCacheInvalidationBus;
private final DeviceDeviceMapper deviceMapper;
private final DmNormalizeRuleMapper ruleMapper;
public NormalizationRuleCacheManagementService(
NormalizationCacheInvalidationService invalidationService,
NormalizationCompiledCacheInvalidationBus compiledCacheInvalidationBus,
DeviceDeviceMapper deviceMapper,
DmNormalizeRuleMapper ruleMapper) {
this.invalidationService = invalidationService;
this.compiledCacheInvalidationBus = compiledCacheInvalidationBus;
this.deviceMapper = deviceMapper;
this.ruleMapper = ruleMapper;
}
/** 清理一个设备的规则快照,并移除该设备当前关联规则的实体和编译缓存。 */
public ClearResult clearDevice(long deviceId) {
if (deviceId <= 0L) throw new IllegalArgumentException("deviceId 必须大于 0");
Set<Long> ruleIds = new LinkedHashSet<>();
addAllNonNull(ruleIds, nullSafe(ruleMapper.selectRuleIdsByDeviceId(deviceId)));
invalidationService.invalidateRulesForDevice(deviceId);
for (Long ruleId : ruleIds) {
invalidationService.invalidateRuleEntity(ruleId);
}
compiledCacheInvalidationBus.invalidateRulesEverywhere(ruleIds);
return new ClearResult("DEVICE", deviceId, 1, ruleIds.size());
}
/** 清理所有设备规则快照、规则实体和当前 JVM 的全部编译缓存。 */
public ClearResult clearAll() {
Set<Long> deviceIds = new LinkedHashSet<>();
addAllNonNull(deviceIds, deviceMapper.selectAllIds());
// 即使存在历史孤立关联,也应删除其规则快照键。
addAllNonNull(deviceIds, ruleMapper.selectAllAssociatedDeviceIds());
invalidationService.invalidateAllRules(deviceIds);
compiledCacheInvalidationBus.invalidateAllEverywhere();
return new ClearResult("ALL", null, deviceIds.size(), 0);
}
private static List<Long> nullSafe(List<Long> values) {
return values == null ? Collections.emptyList() : values;
}
private static void addAllNonNull(Set<Long> target, Collection<Long> values) {
if (values == null) return;
for (Long value : values) if (value != null) target.add(value);
}
public static final class ClearResult {
private final String scope;
private final Long deviceId;
private final int deviceCount;
private final int associatedRuleCount;
public ClearResult(String scope, Long deviceId, int deviceCount, int associatedRuleCount) {
this.scope = scope;
this.deviceId = deviceId;
this.deviceCount = deviceCount;
this.associatedRuleCount = associatedRuleCount;
}
public String getScope() { return scope; }
public Long getDeviceId() { return deviceId; }
public int getDeviceCount() { return deviceCount; }
public int getAssociatedRuleCount() { return associatedRuleCount; }
}
}
@@ -1,6 +1,5 @@
package com.Modules.NormalData.cache;
import com.Modules.NormalData.compiled.CompiledRuleCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@@ -14,21 +13,21 @@ public class NormalizeRuleCacheInvalidationListener {
private static final Logger log = LoggerFactory.getLogger(NormalizeRuleCacheInvalidationListener.class);
private final NormalizationCacheInvalidationService invalidationService;
private final CompiledRuleCache compiledRuleCache;
private final NormalizationCompiledCacheInvalidationBus compiledCacheInvalidationBus;
private final NormalizationRuleMetrics metrics;
public NormalizeRuleCacheInvalidationListener(NormalizationCacheInvalidationService invalidationService,
CompiledRuleCache compiledRuleCache,
NormalizationCompiledCacheInvalidationBus compiledCacheInvalidationBus,
NormalizationRuleMetrics metrics) {
this.invalidationService = invalidationService;
this.compiledRuleCache = compiledRuleCache;
this.compiledCacheInvalidationBus = compiledCacheInvalidationBus;
this.metrics = metrics;
}
@TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT)
public void afterCommit(NormalizeRuleChangedEvent event) {
metrics.invalidation("after_commit");
// 先移除跨 JVM 的规则事实,再清除本 JVM 所有历史 fingerprint 编译对象。
// 先移除跨 JVM 的规则事实,再广播清除所有 JVM 的历史 fingerprint 对象。
for (Long deviceId : event.getDeviceIds()) {
try {
invalidationService.invalidateRulesForDevice(deviceId);
@@ -39,7 +38,8 @@ public class NormalizeRuleCacheInvalidationListener {
}
}
try {
compiledRuleCache.invalidateRule(event.getRuleId());
compiledCacheInvalidationBus.invalidateRulesEverywhere(
java.util.Collections.singleton(event.getRuleId()));
} catch (RuntimeException cacheFailure) {
metrics.invalidation("failure");
// 数据库事实已提交;raw rules 的短 TTL 是最终一致性兜底。
@@ -192,6 +192,18 @@ public class CompiledRuleCache {
metrics.compiledCacheGauge(compiled.estimatedSize(), evictions.sum());
}
/** 清空当前 JVM 中所有规则编译结果、匹配画像及短失败缓存。 */
public void invalidateAll() {
compiled.invalidateAll();
matchProfiles.invalidateAll();
failures.invalidateAll();
matchFailures.invalidateAll();
loggedFailures.invalidateAll();
keysByRule.invalidateAll();
matchKeysByRule.invalidateAll();
metrics.compiledCacheGauge(0L, evictions.sum());
}
public long estimatedSize() { return compiled.estimatedSize(); }
public long compileSuccessCount() { return compileSuccess.sum(); }
public long compileFailureCount() { return compileFailure.sum(); }
@@ -21,6 +21,9 @@ public interface DeviceDeviceMapper {
*/
List<DeviceDevice> selectAll();
/** 规则缓存全量失效只需要主键,避免加载整行设备信息。 */
List<Long> selectAllIds();
/**
* 根据IP地址查询设备
*/
@@ -24,6 +24,10 @@ public interface DmNormalizeRuleMapper {
List<Long> selectDeviceIdsByRuleId(@Param("ruleId") Long ruleId);
List<Long> selectRuleIdsByDeviceId(@Param("deviceId") Long deviceId);
List<Long> selectAllAssociatedDeviceIds();
/**
* 根据ID更新数据标准化规则
* @param rule 数据标准化规则对象
@@ -0,0 +1,24 @@
package com.config;
import com.Modules.NormalData.cache.NormalizationCompiledCacheInvalidationBus;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.listener.ChannelTopic;
import org.springframework.data.redis.listener.RedisMessageListenerContainer;
/** 规则编译缓存的跨实例失效广播通道。 */
@Configuration
public class NormalizationCacheInvalidationRedisConfig {
@Bean
public RedisMessageListenerContainer normalizationCacheInvalidationListenerContainer(
RedisConnectionFactory connectionFactory,
NormalizationCompiledCacheInvalidationBus listener) {
RedisMessageListenerContainer container = new RedisMessageListenerContainer();
container.setConnectionFactory(connectionFactory);
container.addMessageListener(listener,
new ChannelTopic(NormalizationCompiledCacheInvalidationBus.TOPIC));
return container;
}
}
@@ -12,7 +12,7 @@ import org.springframework.stereotype.Component;
public class SyslogBuildStartupLogger {
// 确认包含此功能修订,不代表唯一构建;后续功能修订时同步更新此标识。
static final String CODE_REVISION = "ASYNC_STATS_ADAPTIVE_RAW_V20260913_57";
static final String CODE_REVISION = "ASYNC_STATS_ADAPTIVE_RAW_V20260917_63";
private final Environment environment;
public SyslogBuildStartupLogger(Environment environment) {
@@ -0,0 +1,52 @@
package com.controllers;
import com.Modules.NormalData.cache.NormalizationRuleCacheManagementService;
import com.Modules.NormalData.cache.NormalizationRuleCacheManagementService.ClearResult;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.LinkedHashMap;
import java.util.Map;
/** 只清理标准化规则相关缓存,不执行现有 /api/cache/all 的全 Redis 删除。 */
@RestController
@RequestMapping("/api/syslog/normalization/cache")
public class NormalizationRuleCacheController {
private final NormalizationRuleCacheManagementService service;
public NormalizationRuleCacheController(NormalizationRuleCacheManagementService service) {
this.service = service;
}
@DeleteMapping("/devices/{deviceId}")
public ResponseEntity<Map<String, Object>> clearDevice(@PathVariable Long deviceId) {
if (deviceId == null || deviceId <= 0L) {
Map<String, Object> error = new LinkedHashMap<>();
error.put("cleared", false);
error.put("errorCode", "INVALID_DEVICE_ID");
error.put("message", "deviceId 必须大于 0");
return ResponseEntity.badRequest().body(error);
}
return ResponseEntity.ok(response(service.clearDevice(deviceId)));
}
@DeleteMapping
public Map<String, Object> clearAll() {
return response(service.clearAll());
}
private Map<String, Object> response(ClearResult result) {
Map<String, Object> body = new LinkedHashMap<>();
body.put("cleared", true);
body.put("scope", result.getScope());
if (result.getDeviceId() != null) body.put("deviceId", result.getDeviceId());
body.put("deviceCount", result.getDeviceCount());
if (result.getDeviceId() != null) {
body.put("associatedRuleCount", result.getAssociatedRuleCount());
}
return body;
}
}
@@ -70,6 +70,10 @@
FROM device_device
</select>
<select id="selectAllIds" resultType="java.lang.Long">
SELECT id FROM device_device
</select>
<!-- 根据IP查询 -->
<select id="selectByIp" parameterType="java.lang.String" resultMap="BaseResultMap">
SELECT
@@ -96,6 +96,17 @@
WHERE normalize_rule_id = #{ruleId}
</select>
<select id="selectRuleIdsByDeviceId" parameterType="java.lang.Long" resultType="java.lang.Long">
SELECT normalize_rule_id
FROM dm_normalize_rule_device
WHERE device_id = #{deviceId}
</select>
<select id="selectAllAssociatedDeviceIds" resultType="java.lang.Long">
SELECT DISTINCT device_id
FROM dm_normalize_rule_device
</select>
<!-- 根据ID和租户ID查询 -->
<select id="selectByIdAndTenant" resultMap="BaseResultMap">
@@ -10,6 +10,8 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.Arrays;
class NormalizationCacheInvalidationServiceTest {
@Test
void evictsMatchingLegacyAndJetCacheKeys() {
@@ -35,4 +37,20 @@ class NormalizationCacheInvalidationServiceTest {
verify(jet).collectTask(8);
verify(jet).rulesForDevice(9L);
}
@Test
void clearsOnlyRuleSpringCacheAndBroadcastsEveryDevice() {
CacheManager manager = mock(CacheManager.class);
Cache rules = mock(Cache.class);
when(manager.getCache("normalizeRule")).thenReturn(rules);
NormalizationCacheInvalidator jet = mock(NormalizationCacheInvalidator.class);
NormalizationCacheInvalidationService service = new NormalizationCacheInvalidationService(
manager, jet, new NormalizationRuleMetrics());
service.invalidateAllRules(Arrays.asList(9L, null, 10L));
verify(rules).clear();
verify(jet).rulesForDevice(9L);
verify(jet).rulesForDevice(10L);
}
}
@@ -0,0 +1,65 @@
package com.Modules.NormalData.cache;
import com.Modules.NormalData.compiled.CompiledRuleCache;
import org.junit.jupiter.api.Test;
import org.springframework.data.redis.connection.DefaultMessage;
import org.springframework.data.redis.core.StringRedisTemplate;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
class NormalizationCompiledCacheInvalidationBusTest {
@Test
void clearsLocalRulesAndPublishesOneClusterCommand() {
CompiledRuleCache cache = mock(CompiledRuleCache.class);
StringRedisTemplate redis = mock(StringRedisTemplate.class);
NormalizationCompiledCacheInvalidationBus bus =
new NormalizationCompiledCacheInvalidationBus(cache, redis);
bus.invalidateRulesEverywhere(Arrays.asList(7L, 8L, 8L, null));
verify(cache).invalidateRule(7L);
verify(cache).invalidateRule(8L);
verify(redis).convertAndSend(
NormalizationCompiledCacheInvalidationBus.TOPIC, "RULES:7,8");
}
@Test
void clearsAllLocallyAndPublishesClusterCommand() {
CompiledRuleCache cache = mock(CompiledRuleCache.class);
StringRedisTemplate redis = mock(StringRedisTemplate.class);
NormalizationCompiledCacheInvalidationBus bus =
new NormalizationCompiledCacheInvalidationBus(cache, redis);
bus.invalidateAllEverywhere();
verify(cache).invalidateAll();
verify(redis).convertAndSend(
NormalizationCompiledCacheInvalidationBus.TOPIC, "ALL");
}
@Test
void subscriberClearsCompiledCacheOnEveryInstance() {
CompiledRuleCache cache = mock(CompiledRuleCache.class);
NormalizationCompiledCacheInvalidationBus bus =
new NormalizationCompiledCacheInvalidationBus(
cache, mock(StringRedisTemplate.class));
bus.onMessage(message("RULES:7,8"), null);
bus.onMessage(message("ALL"), null);
verify(cache).invalidateRule(7L);
verify(cache).invalidateRule(8L);
verify(cache).invalidateAll();
}
private static DefaultMessage message(String body) {
return new DefaultMessage(
NormalizationCompiledCacheInvalidationBus.TOPIC.getBytes(StandardCharsets.UTF_8),
body.getBytes(StandardCharsets.UTF_8));
}
}
@@ -0,0 +1,79 @@
package com.Modules.NormalData.cache;
import com.common.mapper.DeviceDeviceMapper;
import com.common.mapper.DmNormalizeRuleMapper;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.Collections;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class NormalizationRuleCacheManagementServiceTest {
@Test
void clearsDeviceRuleViewsEntitiesAndCompiledRules() {
NormalizationCacheInvalidationService invalidation =
mock(NormalizationCacheInvalidationService.class);
NormalizationCompiledCacheInvalidationBus compiled =
mock(NormalizationCompiledCacheInvalidationBus.class);
DeviceDeviceMapper devices = mock(DeviceDeviceMapper.class);
DmNormalizeRuleMapper rules = mock(DmNormalizeRuleMapper.class);
when(rules.selectRuleIdsByDeviceId(40L)).thenReturn(Arrays.asList(7L, 8L, 8L));
NormalizationRuleCacheManagementService service =
new NormalizationRuleCacheManagementService(invalidation, compiled, devices, rules);
NormalizationRuleCacheManagementService.ClearResult result = service.clearDevice(40L);
assertEquals("DEVICE", result.getScope());
assertEquals(40L, result.getDeviceId());
assertEquals(2, result.getAssociatedRuleCount());
verify(invalidation).invalidateRulesForDevice(40L);
verify(invalidation).invalidateRuleEntity(7L);
verify(invalidation).invalidateRuleEntity(8L);
verify(compiled).invalidateRulesEverywhere(
new java.util.LinkedHashSet<>(Arrays.asList(7L, 8L)));
}
@Test
void clearsAllDistinctDeviceViewsAndCompiledCache() {
NormalizationCacheInvalidationService invalidation =
mock(NormalizationCacheInvalidationService.class);
NormalizationCompiledCacheInvalidationBus compiled =
mock(NormalizationCompiledCacheInvalidationBus.class);
DeviceDeviceMapper devices = mock(DeviceDeviceMapper.class);
DmNormalizeRuleMapper rules = mock(DmNormalizeRuleMapper.class);
when(devices.selectAllIds()).thenReturn(Arrays.asList(40L, 41L));
when(rules.selectAllAssociatedDeviceIds()).thenReturn(Arrays.asList(41L, 42L));
NormalizationRuleCacheManagementService service =
new NormalizationRuleCacheManagementService(invalidation, compiled, devices, rules);
NormalizationRuleCacheManagementService.ClearResult result = service.clearAll();
assertEquals("ALL", result.getScope());
assertEquals(3, result.getDeviceCount());
verify(invalidation).invalidateAllRules(
org.mockito.ArgumentMatchers.argThat(ids -> ids.equals(
new java.util.LinkedHashSet<>(Arrays.asList(40L, 41L, 42L)))));
verify(compiled).invalidateAllEverywhere();
}
@Test
void acceptsDeviceWithoutAssociatedRules() {
NormalizationCacheInvalidationService invalidation =
mock(NormalizationCacheInvalidationService.class);
NormalizationCompiledCacheInvalidationBus compiled =
mock(NormalizationCompiledCacheInvalidationBus.class);
DeviceDeviceMapper devices = mock(DeviceDeviceMapper.class);
DmNormalizeRuleMapper rules = mock(DmNormalizeRuleMapper.class);
when(rules.selectRuleIdsByDeviceId(40L)).thenReturn(Collections.emptyList());
NormalizationRuleCacheManagementService service =
new NormalizationRuleCacheManagementService(invalidation, compiled, devices, rules);
assertEquals(0, service.clearDevice(40L).getAssociatedRuleCount());
verify(invalidation).invalidateRulesForDevice(40L);
}
}
@@ -101,6 +101,24 @@ class CompiledRuleCacheTest {
assertEquals(4, compiler.compiles.get());
}
@Test
void invalidatingAllRemovesEveryCompiledRule() {
CountingCompiler compiler = new CountingCompiler();
CompiledRuleCache cache = new CompiledRuleCache(compiler, properties());
RuleSource first = source("v1", "{\"mappers\":[]}");
RuleSource second = new RuleSource(9L, "rule-9", "rule-9", "json",
"{\"mappers\":[]}", true, "0");
cache.get(first, null);
cache.get(second, null);
assertEquals(2, compiler.compiles.get());
cache.invalidateAll();
assertEquals(0L, cache.estimatedSize());
cache.get(first, null);
cache.get(second, null);
assertEquals(4, compiler.compiles.get());
}
@Test
void diagnosticRuleTextKeepsExecutionFieldsAndRemovesEmbeddedSamples() {
String diagnostic = CompiledRuleCache.diagnosticRuleText("{"
@@ -0,0 +1,36 @@
package com.common.mapper;
import org.apache.ibatis.builder.xml.XMLMapperBuilder;
import org.apache.ibatis.session.Configuration;
import org.junit.jupiter.api.Test;
import java.io.InputStream;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
class NormalizationRuleCacheMapperXmlTest {
@Test
void mapperXmlContainsCacheManagementQueries() throws Exception {
Configuration configuration = new Configuration();
parse(configuration, "mapper/DeviceDeviceMapper.xml");
parse(configuration, "mapper/DmNormalizeRuleMapper.xml");
assertTrue(configuration.hasStatement(
"com.common.mapper.DeviceDeviceMapper.selectAllIds"));
assertTrue(configuration.hasStatement(
"com.common.mapper.DmNormalizeRuleMapper.selectRuleIdsByDeviceId"));
assertTrue(configuration.hasStatement(
"com.common.mapper.DmNormalizeRuleMapper.selectAllAssociatedDeviceIds"));
}
private static void parse(Configuration configuration, String resource) throws Exception {
try (InputStream stream = Thread.currentThread().getContextClassLoader()
.getResourceAsStream(resource)) {
assertNotNull(stream, resource);
new XMLMapperBuilder(stream, configuration, resource,
configuration.getSqlFragments()).parse();
}
}
}
@@ -0,0 +1,58 @@
package com.controllers;
import com.Modules.NormalData.cache.NormalizationRuleCacheManagementService;
import org.junit.jupiter.api.Test;
import org.springframework.http.ResponseEntity;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class NormalizationRuleCacheControllerTest {
@Test
void exposesDeviceAndAllRuleCacheOperations() {
NormalizationRuleCacheManagementService service =
mock(NormalizationRuleCacheManagementService.class);
when(service.clearDevice(40L)).thenReturn(result("DEVICE", 40L, 1, 2));
when(service.clearAll()).thenReturn(result("ALL", null, 8, 0));
NormalizationRuleCacheController controller =
new NormalizationRuleCacheController(service);
ResponseEntity<Map<String, Object>> device = controller.clearDevice(40L);
Map<String, Object> all = controller.clearAll();
assertEquals(200, device.getStatusCodeValue());
assertEquals(true, device.getBody().get("cleared"));
assertEquals("DEVICE", device.getBody().get("scope"));
assertEquals(40L, device.getBody().get("deviceId"));
assertEquals("ALL", all.get("scope"));
assertEquals(8, all.get("deviceCount"));
assertFalse(all.containsKey("associatedRuleCount"));
verify(service).clearDevice(40L);
verify(service).clearAll();
}
@Test
void rejectsNonPositiveDeviceIdWithoutCallingService() {
NormalizationRuleCacheManagementService service =
mock(NormalizationRuleCacheManagementService.class);
NormalizationRuleCacheController controller =
new NormalizationRuleCacheController(service);
ResponseEntity<Map<String, Object>> response = controller.clearDevice(0L);
assertEquals(400, response.getStatusCodeValue());
assertEquals("INVALID_DEVICE_ID", response.getBody().get("errorCode"));
}
private static NormalizationRuleCacheManagementService.ClearResult result(
String scope, Long deviceId, int deviceCount, int ruleCount) {
return new NormalizationRuleCacheManagementService.ClearResult(
scope, deviceId, deviceCount, ruleCount);
}
}
@@ -0,0 +1,44 @@
package com.config;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
/** 在应用完全就绪后打印不可由现场配置覆盖的代码修订标识和关键开关。 */
@Slf4j
@Component
public class SyslogBuildStartupLogger {
// 用于确认当前运行的原始日志存储模块 JAR,后续功能修订时同步更新。
static final String CODE_REVISION = "RAW_STORAGE_PIPELINE_V20260917_63";
private final Environment environment;
public SyslogBuildStartupLogger(Environment environment) {
this.environment = environment;
}
@EventListener(ApplicationReadyEvent.class)
public void onApplicationReady() {
log.info("[SYSLOG_BUILD_READY] {}", startupSummary());
}
String startupSummary() {
return "codeRevision=" + CODE_REVISION
+ ", application=syslog-consumer"
+ ", profiles=" + String.join(",", environment.getActiveProfiles())
+ ", serverPort=" + value("server.port", "8089")
+ ", contextPath=" + value("server.servlet.context-path", "/xdrservice")
+ ", rawConsumerEnabled=" + value("syslog.raw-consumer.enabled", "false")
+ ", kafkaTopic=" + value("spring.kafka.consumer.topic", "agent-syslog-topic")
+ ", kafkaGroupId=" + value("spring.kafka.consumer.group-id", "agent-syslog-group")
+ ", rawWriteSchema=" + value("influxdb.raw.write-schema", "V2")
+ ", rawReadMode=" + value("influxdb.raw.read-mode", "V1_V2")
+ ", reliabilityVersion=" + value("syslog.reliability.consumer-version", "1.0.0");
}
private String value(String key, String fallback) {
return environment.getProperty(key, fallback);
}
}
@@ -169,11 +169,17 @@ analysis.realtime.check-interval-seconds: 10
analysis.realtime.allowed-lateness-seconds=${ANALYSIS_REALTIME_ALLOWED_LATENESS_SECONDS:120}
# 迟到标准化告警使用独立状态表重放已完成规则窗口,不回退正常实时分析 checkpoint。
# 是否启用迟到告警规则重放;关闭后不再扫描和重放历史窗口,不影响正常实时分析。
analysis.late-replay.enabled=${ANALYSIS_LATE_REPLAY_ENABLED:true}
# 两轮扫描之间的固定间隔,单位毫秒;5000 表示上一轮结束 5 秒后再执行下一轮。
analysis.late-replay.scan-interval-ms=${ANALYSIS_LATE_REPLAY_SCAN_INTERVAL_MS:5000}
# 通知桶和重放窗口被实例领取后的租约时长,单位秒;超时未完成时可由其他扫描重新领取。
analysis.late-replay.lease-seconds=${ANALYSIS_LATE_REPLAY_LEASE_SECONDS:1800}
# 每轮最多展开的迟到数据分钟桶数;将分钟桶转换为受影响的历史规则窗口。
analysis.late-replay.max-buckets-per-scan=${ANALYSIS_LATE_REPLAY_MAX_BUCKETS_PER_SCAN:100}
# 每轮最多执行的历史规则窗口数;用于限制单轮重放对数据库和规则引擎的压力。
analysis.late-replay.max-windows-per-scan=${ANALYSIS_LATE_REPLAY_MAX_WINDOWS_PER_SCAN:10}
# 重放失败后的再次尝试等待时间,单位秒。
analysis.late-replay.retry-delay-seconds=${ANALYSIS_LATE_REPLAY_RETRY_DELAY_SECONDS:30}
# 关联分析规则达梦安全版子查询模式开关(默认false,生产环境达梦安全版设为true)
@@ -0,0 +1,56 @@
package com.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.read.ListAppender;
import org.junit.jupiter.api.Test;
import org.slf4j.LoggerFactory;
import org.springframework.mock.env.MockEnvironment;
class SyslogBuildStartupLoggerTest {
@Test
void logsCodeRevisionAndEffectiveOverrides() {
MockEnvironment environment = new MockEnvironment()
.withProperty("syslog.raw-consumer.enabled", "true")
.withProperty("spring.kafka.consumer.topic", "prod-syslog")
.withProperty("spring.kafka.consumer.group-id", "raw-store")
.withProperty("influxdb.raw.write-schema", "V2_ONLY");
environment.setActiveProfiles("prod");
SyslogBuildStartupLogger reporter = new SyslogBuildStartupLogger(environment);
Logger logger = (Logger) LoggerFactory.getLogger(SyslogBuildStartupLogger.class);
ListAppender<ILoggingEvent> appender = new ListAppender<>();
appender.start();
logger.addAppender(appender);
try {
reporter.onApplicationReady();
assertEquals(1, appender.list.size());
String message = appender.list.get(0).getFormattedMessage();
assertTrue(message.contains("[SYSLOG_BUILD_READY] codeRevision="
+ SyslogBuildStartupLogger.CODE_REVISION));
assertTrue(message.contains("application=syslog-consumer"));
assertTrue(message.contains("profiles=prod"));
assertTrue(message.contains("rawConsumerEnabled=true"));
assertTrue(message.contains("kafkaTopic=prod-syslog"));
assertTrue(message.contains("kafkaGroupId=raw-store"));
assertTrue(message.contains("rawWriteSchema=V2_ONLY"));
} finally {
logger.detachAppender(appender);
appender.stop();
}
}
@Test
void usesSafeDefaultsAndIgnoresConfiguredRevision() {
String summary = new SyslogBuildStartupLogger(new MockEnvironment()
.withProperty("codeRevision", "fake")).startupSummary();
assertTrue(summary.contains("codeRevision=" + SyslogBuildStartupLogger.CODE_REVISION));
assertFalse(summary.contains("codeRevision=fake"));
assertTrue(summary.contains("rawConsumerEnabled=false"));
assertTrue(summary.contains("rawWriteSchema=V2"));
assertTrue(summary.contains("rawReadMode=V1_V2"));
}
}
@@ -1,26 +1,42 @@
package com.common.controller;
import com.common.entity.DeviceCollectionMode;
import com.common.entity.DeviceDevice;
import com.common.service.DeviceDeviceService;
import org.springframework.beans.factory.annotation.Autowired;
import com.fasterxml.jackson.annotation.JsonAnySetter;
import com.haobang.dispatch.DeviceContextResolver;
import org.springframework.http.HttpStatus;
import org.springframework.web.server.ResponseStatusException;
import org.springframework.web.bind.annotation.*;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import java.util.List;
import java.util.LinkedHashMap;
import java.util.Map;
@RestController
@RequestMapping("/api/devices")
public class DeviceDeviceController {
@Autowired
private DeviceDeviceService deviceDeviceService;
private final DeviceDeviceService deviceDeviceService;
private final DeviceContextResolver deviceContextResolver;
public DeviceDeviceController(DeviceDeviceService deviceDeviceService,
DeviceContextResolver deviceContextResolver) {
this.deviceDeviceService = deviceDeviceService;
this.deviceContextResolver = deviceContextResolver;
}
@GetMapping("/{id}")
public DeviceDevice getById(@PathVariable Integer id) {
System.out.println("device_id:"+id.toString());
DeviceDevice deviceDevice=deviceDeviceService.getById(id);
System.out.println("deviceDeviceService:"+deviceDevice.toString());
return deviceDeviceService.getById(id);
DeviceDevice device = deviceDeviceService.getById(id);
if (device == null || device.getDeletedAt() != null
|| (device.getDelFlag() != null && !"0".equals(device.getDelFlag()))) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "设备不存在或已删除");
}
return device;
}
@GetMapping
@@ -35,10 +51,6 @@ public class DeviceDeviceController {
@GetMapping("/ip/{ip}")
public List<DeviceDevice> getByIp(@PathVariable String ip) {
List<DeviceDevice> deviceDevicelist =deviceDeviceService.getByIpSafely(ip);
System.out.println("deviceDeviceService:"+deviceDevicelist.get(0).toString());
return deviceDeviceService.getByIpSafely(ip);
}
@@ -46,4 +58,65 @@ public class DeviceDeviceController {
public List<DeviceDevice> getMonitoringDevices() {
return deviceDeviceService.getMonitoringDevices();
}
/** 清除所有设备的二级缓存、进程内解析缓存,并立即重载 Syslog 入站快照。 */
@DeleteMapping("/cache")
public Map<String, Object> evictAllDeviceCaches() {
deviceDeviceService.evictAllDeviceCaches();
deviceContextResolver.invalidateAll();
deviceContextResolver.refreshIngressSnapshot();
return cacheResponse("ALL", null, null);
}
/** 只清除指定设备的 ID/IP 二级缓存及其进程内解析缓存。 */
@DeleteMapping("/{id}/cache")
public Map<String, Object> evictDeviceCaches(@PathVariable Integer id) {
DeviceDevice device = deviceDeviceService.evictDeviceCaches(id);
if (device == null) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "设备不存在");
}
deviceContextResolver.invalidateDevice(id, device.getIp());
deviceContextResolver.refreshIngressSnapshot();
return cacheResponse("DEVICE", id, device.getIp());
}
/** 仅允许修改设备级采集策略,避免把完整设备实体直接绑定到更新接口。 */
@PutMapping("/{id}/collection-mode")
public DeviceDevice updateCollectionMode(@PathVariable Integer id,
@Valid @RequestBody CollectionModeRequest request) {
DeviceDevice updated = deviceDeviceService.updateCollectionMode(id, request.getCollectionMode());
if (updated == null) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "设备不存在或已删除");
}
deviceContextResolver.invalidateAll();
deviceContextResolver.refreshIngressSnapshot();
return updated;
}
public static final class CollectionModeRequest {
@NotNull(message = "collectionMode 不能为空")
private DeviceCollectionMode collectionMode;
public DeviceCollectionMode getCollectionMode() {
return collectionMode;
}
public void setCollectionMode(DeviceCollectionMode collectionMode) {
this.collectionMode = collectionMode;
}
@JsonAnySetter
public void rejectUnknownField(String field, Object ignored) {
throw new IllegalArgumentException("不允许修改字段: " + field);
}
}
private Map<String, Object> cacheResponse(String scope, Integer deviceId, String ip) {
Map<String, Object> response = new LinkedHashMap<>();
response.put("cleared", true);
response.put("scope", scope);
if (deviceId != null) response.put("deviceId", deviceId);
if (ip != null) response.put("ip", ip);
return response;
}
}
@@ -0,0 +1,29 @@
package com.common.entity;
/** 设备级采集策略。任务 method 描述单个任务,本枚举决定设备允许的采集方向。 */
public enum DeviceCollectionMode {
SYSLOG_ONLY(true, false),
API_ONLY(false, true),
BOTH(true, true);
private final boolean syslogAllowed;
private final boolean apiAllowed;
DeviceCollectionMode(boolean syslogAllowed, boolean apiAllowed) {
this.syslogAllowed = syslogAllowed;
this.apiAllowed = apiAllowed;
}
public boolean allowsSyslog() {
return syslogAllowed;
}
public boolean allowsApi() {
return apiAllowed;
}
/** 兼容升级前写入 Redis 的设备对象;缺失字段必须保持原有双向采集能力。 */
public static DeviceCollectionMode effective(DeviceCollectionMode value) {
return value == null ? BOTH : value;
}
}
@@ -40,6 +40,7 @@ public class DeviceDevice {
public Integer todayNonLogCount;
public Long createDept;
public Integer deviceCollectId;
public DeviceCollectionMode collectionMode;
// Getter and Setter methods
@@ -331,6 +332,14 @@ public class DeviceDevice {
this.deviceCollectId = deviceCollectId;
}
public DeviceCollectionMode getCollectionMode() {
return DeviceCollectionMode.effective(collectionMode);
}
public void setCollectionMode(DeviceCollectionMode collectionMode) {
this.collectionMode = collectionMode;
}
@Override
public String toString() {
return "DeviceDevice{" +
@@ -370,6 +379,7 @@ public class DeviceDevice {
", todayNonLogCount=" + todayNonLogCount +
", createDept=" + createDept +
", deviceCollectId=" + deviceCollectId +
", collectionMode=" + getCollectionMode() +
'}';
}
@@ -2,6 +2,7 @@ package com.common.mapper;
import com.common.entity.DeviceDevice;
import com.common.entity.DeviceCollectionMode;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
@@ -90,4 +91,8 @@ public interface DeviceDeviceMapper {
* 根据厂商查询设备
*/
List<DeviceDevice> selectByVendor(String vendor);
/** 只更新单台未删除设备的采集模式,禁止管理接口批量赋值其他字段。 */
int updateCollectionMode(@Param("id") Integer id,
@Param("collectionMode") DeviceCollectionMode collectionMode);
}
@@ -1,6 +1,7 @@
package com.common.service;
import com.common.entity.DeviceDevice;
import com.common.entity.DeviceCollectionMode;
import java.util.List;
import java.util.Map;
@@ -28,4 +29,13 @@ public interface DeviceDeviceService {
List<DeviceDevice> getMonitoringDevices();
List<DeviceDevice> getActiveDevices();
/** 返回更新后的设备;设备不存在或已删除时返回 null。 */
DeviceDevice updateCollectionMode(Integer id, DeviceCollectionMode collectionMode);
/** 清除指定设备的 Spring Cache 和 SafeCacheUtil 原始 Redis Key;不存在时返回 null。 */
DeviceDevice evictDeviceCaches(Integer id);
/** 清除全部设备的 Spring Cache 和 SafeCacheUtil 原始 Redis Key。 */
void evictAllDeviceCaches();
}
@@ -2,27 +2,35 @@ package com.common.service.impl;
import com.common.entity.DeviceDevice;
import com.common.entity.DeviceCollectionMode;
import com.common.mapper.DeviceDeviceMapper;
import com.common.service.DeviceDeviceService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import com.haobang.util.SafeCacheUtil;
import org.springframework.transaction.annotation.Transactional;
@CacheConfig(cacheNames = "device")
@Service
public class DeviceDeviceServiceImpl implements DeviceDeviceService {
@Autowired
private DeviceDeviceMapper deviceDeviceMapper;
private final DeviceDeviceMapper deviceDeviceMapper;
private final SafeCacheUtil safeCacheUtil;
private final CacheManager cacheManager;
@Autowired
private SafeCacheUtil safeCacheUtil;
public DeviceDeviceServiceImpl(DeviceDeviceMapper deviceDeviceMapper,
SafeCacheUtil safeCacheUtil,
CacheManager cacheManager) {
this.deviceDeviceMapper = deviceDeviceMapper;
this.safeCacheUtil = safeCacheUtil;
this.cacheManager = cacheManager;
}
/**
* 使用安全的缓存方法
@@ -100,4 +108,59 @@ public class DeviceDeviceServiceImpl implements DeviceDeviceService {
public List<DeviceDevice> getActiveDevices() {
return deviceDeviceMapper.selectActiveDevices();
}
@Override
@Transactional
public DeviceDevice updateCollectionMode(Integer id, DeviceCollectionMode collectionMode) {
if (id == null || collectionMode == null) {
throw new IllegalArgumentException("设备 ID 和采集模式不能为空");
}
DeviceDevice current = deviceDeviceMapper.selectById(id);
if (current == null || current.getDeletedAt() != null
|| (current.getDelFlag() != null && !"0".equals(current.getDelFlag()))) {
return null;
}
if (deviceDeviceMapper.updateCollectionMode(id, collectionMode) != 1) return null;
evictDeviceCacheEntries(id, current.getIp());
// 采集模式更新属于策略变更,继续保持全量清空 Spring device Cache 的强一致行为。
Cache springDeviceCache = cacheManager.getCache("device");
if (springDeviceCache != null) springDeviceCache.clear();
return deviceDeviceMapper.selectById(id);
}
@Override
public DeviceDevice evictDeviceCaches(Integer id) {
if (id == null) throw new IllegalArgumentException("设备 ID 不能为空");
// 必须绕过二级缓存读取当前 IP,否则可能根据陈旧缓存删除错误的 IP Key。
DeviceDevice device = deviceDeviceMapper.selectById(id);
if (device == null) {
// 即使数据库已无记录,也删除仍可能残留的 ID Key。
safeCacheUtil.evict("device:id:" + id);
Cache springDeviceCache = cacheManager.getCache("device");
if (springDeviceCache != null) springDeviceCache.evict("device:id:" + id);
return null;
}
evictDeviceCacheEntries(id, device.getIp());
return device;
}
@Override
public void evictAllDeviceCaches() {
safeCacheUtil.evictMatching("device:id:*", "device:ip:*");
Cache springDeviceCache = cacheManager.getCache("device");
if (springDeviceCache != null) springDeviceCache.clear();
}
private void evictDeviceCacheEntries(Integer id, String ip) {
String idKey = "device:id:" + id;
if (ip == null || ip.trim().isEmpty()) {
safeCacheUtil.evict(idKey);
} else {
safeCacheUtil.evict(idKey, "device:ip:" + ip);
}
Cache springDeviceCache = cacheManager.getCache("device");
if (springDeviceCache == null) return;
springDeviceCache.evict(idKey);
if (ip != null && !ip.trim().isEmpty()) springDeviceCache.evict("device:ip:" + ip);
}
}
@@ -0,0 +1,46 @@
package com.config;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
/** 在应用完全就绪后打印不可由现场配置覆盖的代码修订标识和关键开关。 */
@Slf4j
@Component
public class SyslogBuildStartupLogger {
// 确认 JAR 包含设备级采集模式、API 拉取及全局 TLS 策略。
static final String CODE_REVISION = "DEVICE_COLLECTION_API_TLS_V20260917_63";
private final Environment environment;
public SyslogBuildStartupLogger(Environment environment) {
this.environment = environment;
}
@EventListener(ApplicationReadyEvent.class)
public void onApplicationReady() {
log.info("[SYSLOG_BUILD_READY] {}", startupSummary());
}
String startupSummary() {
return "codeRevision=" + CODE_REVISION
+ ", application=syslog-serve"
+ ", profiles=" + String.join(",", environment.getActiveProfiles())
+ ", serverPort=" + value("server.port", "8189")
+ ", contextPath=" + value("server.servlet.context-path", "/syslogserve")
+ ", syslogEnabled=" + value("syslog.server.enabled", "true")
+ ", apiPullEnabled=" + value("device.api.enabled", "true")
+ ", apiTlsVerifyEnabled=" + value("device.api.tls-verify-enabled", "false")
+ ", apiHostAllowlistEnabled=" + value("device.api.host-allowlist-enabled", "false")
+ ", walEnabled=" + value("syslog.wal.enabled", "true")
+ ", terminalRecordEnabled=" + value("syslog.dispatch.terminal-record-enabled", "false")
+ ", deviceCollectId=" + value("app.service.device_collect_id", "7")
+ ", kafkaTopic=" + value("spring.kafka.producer.topic", "test-topic");
}
private String value(String key, String fallback) {
return environment.getProperty(key, fallback);
}
}
@@ -7,13 +7,14 @@ import org.springframework.stereotype.Component;
/**
* 进程级 API 拉取开关和资源上限。
*
* <p>这里只保存所有任务共享的运行保护参数;页大小、时间窗口、厂商适配器等任务级参数 固化在达梦数据库中。总开关默认关闭,避免系统升级后未经验证就访问真实设备。
* <p>这里只保存所有任务共享的运行保护参数;页大小、时间窗口、厂商适配器等任务级参数固化在达梦数据库中。
* 总开关默认开启,但新建任务保持暂停,必须完成校验并 resume 后才会访问真实设备。
*/
@Data
@Component
public class ApiPullProperties {
@Value("${device.api.enabled:false}")
private boolean enabled;
@Value("${device.api.enabled:true}")
private boolean enabled = true;
@Value("${device.api.scheduler.instance-id:${HOSTNAME:local}-${random.uuid}}")
private String instanceId;
@@ -48,6 +49,19 @@ public class ApiPullProperties {
@Value("${device.api.allowed-hosts:}")
private String allowedHosts;
/** 现场设备经常增减;默认关闭静态主机白名单,启用后才检查 allowedHosts。 */
@Value("${device.api.host-allowlist-enabled:false}")
private boolean hostAllowlistEnabled = false;
/**
* 所有设备 API HTTPS 是否校验证书链和主机名。
*
* <p>现场默认为内网自签名环境,因此默认关闭。该开关是进程级全局策略:关闭后微步、NDR、全悉等
* 所有 API 拉取适配器都不校验证书链和主机名;HTTPS 协议限制和禁止重定向仍然生效。
*/
@Value("${device.api.tls-verify-enabled:false}")
private boolean tlsVerifyEnabled = false;
/** 在调度器创建线程池前快速失败,防止以危险的并发或租约参数启动。 */
public void validate() {
if (scanIntervalMs < 100
@@ -1,6 +1,7 @@
package com.haobang.apipull;
import com.common.mapper.DeviceCollectApiMapper;
import com.haobang.dispatch.DeviceContextResolver;
import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
import com.haobang.apipull.persistence.DeviceCollectApiRun;
@@ -31,6 +32,7 @@ public class ApiPullScheduler {
private final DeviceCollectApiMapper mapper;
private final ApiPullRunner runner;
private final SyslogWalService wal;
private final DeviceContextResolver deviceResolver;
private ExecutorService executor;
private Semaphore global;
private final ConcurrentHashMap<String, Semaphore> hosts = new ConcurrentHashMap<>();
@@ -40,11 +42,13 @@ public class ApiPullScheduler {
ApiPullProperties properties,
DeviceCollectApiMapper mapper,
ApiPullRunner runner,
SyslogWalService wal) {
SyslogWalService wal,
DeviceContextResolver deviceResolver) {
this.properties = properties;
this.mapper = mapper;
this.runner = runner;
this.wal = wal;
this.deviceResolver = deviceResolver;
}
@PostConstruct
@@ -110,6 +114,7 @@ public class ApiPullScheduler {
private DeviceCollectApiRun trySubmit(DeviceCollectApiConfig c, String trigger) {
if (!c.isEnabled()) return null;
if (!requireApiCollectionMode(c, "MANUAL".equals(trigger))) return null;
if (wal.getFatalFailure() != null || wal.isDiskCritical() || wal.shouldPauseTcp()) {
mapper.scheduleNext(c.getCollectTaskId(), c.getConfigVersion(), c.getPollIntervalSeconds());
return null;
@@ -167,7 +172,18 @@ public class ApiPullScheduler {
executor.execute(
() -> {
try {
// 模式可能在排队期间变化;执行 HTTP 前再次校验,避免已提交线程绕过新策略。
if (requireApiCollectionMode(c, false)) {
runner.execute(c, leasedCheckpoint, run);
} else {
run.setStatus("FAILED");
run.setErrorCode("COLLECTION_MODE_DENIED");
run.setErrorSummary("设备采集模式不允许 API_PULL");
run.setFinishedAt(LocalDateTime.now(ApiPullTime.DATABASE_ZONE));
mapper.updateRun(run);
mapper.releaseLease(
c.getCollectTaskId(), properties.getInstanceId(), leasedCheckpoint.getFencingToken());
}
} finally {
releaseLocal(c.getCollectTaskId(), hostPermit);
}
@@ -188,6 +204,24 @@ public class ApiPullScheduler {
}
}
/**
* 调度扫描遇到禁止 API 的设备时推进 next_run_at,避免同一到期任务按 5 秒扫描周期空转。
*/
private boolean requireApiCollectionMode(DeviceCollectApiConfig c, boolean failRequest) {
DeviceContextResolver.Resolution resolution = deviceResolver.resolveDeviceId(c.getDeviceId());
if (resolution.isRegistered()) return true;
mapper.scheduleNext(c.getCollectTaskId(), c.getConfigVersion(), c.getPollIntervalSeconds());
String reason = resolution.getReason() == null ? "设备不允许 API_PULL" : resolution.getReason();
log.warn("设备 API 拉取已被设备策略跳过: taskId={}, deviceId={}, type={}, reason={}",
c.getCollectTaskId(), c.getDeviceId(), resolution.getType(), reason);
if (failRequest) {
String code = resolution.getType() == DeviceContextResolver.Type.COLLECTION_MODE_DENIED
? "COLLECTION_MODE_DENIED" : "DEVICE_CONTEXT_DENIED";
throw new ApiPullException(code, reason, org.springframework.http.HttpStatus.CONFLICT);
}
return false;
}
private void releaseLocal(Integer taskId, Semaphore host) {
if (host != null) host.release();
global.release();
@@ -1,6 +1,7 @@
package com.haobang.apipull;
import com.common.entity.DeviceCollectTask;
import com.common.entity.DeviceCollectionMode;
import com.common.entity.DeviceDevice;
import com.common.mapper.DeviceCollectApiMapper;
import com.common.mapper.DeviceCollectTaskMapper;
@@ -81,11 +82,7 @@ public class ApiPullTaskService {
@Transactional
public ApiPullTaskResponse create(CreateApiPullTaskRequest request) {
// 设备必须归属当前探针,避免一个采集实例替其他探针访问设备内网地址。
DeviceDevice device = devices.getByIdSafely(request.getDeviceId());
if (device == null) throw error("INVALID_REQUEST", "设备不存在", HttpStatus.BAD_REQUEST);
if (device.getDeviceCollectId() == null
|| device.getDeviceCollectId() != app.getAppServiceDeviceCollectId())
throw error("FORBIDDEN", "设备不属于当前采集探针", HttpStatus.FORBIDDEN);
DeviceDevice device = requireApiAllowedDevice(request.getDeviceId());
DeviceCollectApiConfig config = fromCreate(request);
ApiPullAdapter adapter = adapters.require(config.getAdapterType());
adapter.validateConfig(config);
@@ -237,6 +234,7 @@ public class ApiPullTaskService {
DeviceCollectApiConfig c = requireConfig(taskId);
if (c.getConfigVersion() != version)
throw error("CONFIG_VERSION_CONFLICT", "配置版本冲突", HttpStatus.CONFLICT);
requireApiAllowedDevice(c.getDeviceId());
return scheduler.trigger(taskId);
}
@@ -245,6 +243,8 @@ public class ApiPullTaskService {
DeviceCollectApiConfig c = requireConfig(taskId);
if (c.getConfigVersion() != version)
throw error("CONFIG_VERSION_CONFLICT", "配置版本冲突", HttpStatus.CONFLICT);
// 必须在创建验证审计和发出任何 HTTP 请求前执行设备级策略。
requireApiAllowedDevice(c.getDeviceId());
DeviceCollectApiRun audit = new DeviceCollectApiRun();
audit.setRunId(UUID.randomUUID().toString());
audit.setCollectTaskId(taskId);
@@ -301,7 +301,7 @@ public class ApiPullTaskService {
Instant start = end.minusSeconds(1);
// 验证仅做 limit=1 的闭合窗口查询,不推进 checkpoint,也不写入 WAL/Kafka。
adapter.pull(new PullRequest(c, start, end, 1, 0));
checks.add(check("TLS_HANDSHAKE", "PASS", null, "TLS 和主机名校验通过", pullStarted));
checks.add(check("TLS_HANDSHAKE", "PASS", null, tlsValidationSummary(), pullStarted));
checks.add(check("AUTHENTICATION", "PASS", null, "厂商认证及查询通过", pullStarted));
checks.add(check("API_PERMISSION", "PASS", null, "业务接口可访问", pullStarted));
checks.add(
@@ -323,7 +323,7 @@ public class ApiPullTaskService {
checks.add(skipped("API_PERMISSION"));
} else if (failureCode.startsWith("QUERY_")) {
// QUERY_ 表示已完成 TLS、Challenge/Verify 并收到业务接口的 2xx 响应,不能误报为认证失败。
checks.add(check("TLS_HANDSHAKE", "PASS", null, "TLS 和主机名校验通过", pullStarted));
checks.add(check("TLS_HANDSHAKE", "PASS", null, tlsValidationSummary(), pullStarted));
checks.add(check("AUTHENTICATION", "PASS", null, "厂商认证及查询通过", pullStarted));
checks.add(check("API_PERMISSION", "PASS", null, "业务接口可访问", pullStarted));
checks.add(check(queryCheck, "FAIL", failureCode, safe(e), pullStarted));
@@ -362,6 +362,12 @@ public class ApiPullTaskService {
return r;
}
private String tlsValidationSummary() {
return apiProperties.isTlsVerifyEnabled()
? "TLS 证书链和主机名校验通过"
: "TLS 连接通过(全局已关闭证书链和主机名校验)";
}
public ApiPullRuntimeResponse runtime(Integer taskId) {
DeviceCollectApiConfig c = requireConfig(taskId);
DeviceCollectApiCheckpoint cp = mapper.selectCheckpoint(taskId);
@@ -529,6 +535,26 @@ public class ApiPullTaskService {
return c;
}
private DeviceDevice requireApiAllowedDevice(Integer deviceId) {
DeviceDevice device = devices.getByIdSafely(deviceId);
if (device == null || device.getDeletedAt() != null
|| (device.getDelFlag() != null && !"0".equals(device.getDelFlag()))) {
throw error("INVALID_REQUEST", "设备不存在或已删除", HttpStatus.BAD_REQUEST);
}
if (device.getDeviceCollectId() == null
|| device.getDeviceCollectId() != app.getAppServiceDeviceCollectId()) {
throw error("FORBIDDEN", "设备不属于当前采集探针", HttpStatus.FORBIDDEN);
}
DeviceCollectionMode mode = device.getCollectionMode();
if (!mode.allowsApi()) {
throw error(
"COLLECTION_MODE_DENIED",
"设备采集模式不允许 API_PULL: deviceId=" + deviceId + ", collectionMode=" + mode,
HttpStatus.CONFLICT);
}
return device;
}
private DeviceCollectTask requireTask(Integer id) {
DeviceCollectTask t = tasks.selectById(id);
if (t == null || t.getMethod() == null || t.getMethod() != API_PULL_METHOD)
@@ -3,6 +3,7 @@ package com.haobang.apipull.adapter.ndr;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.haobang.apipull.ApiPullProperties;
import com.haobang.apipull.adapter.AdapterException;
import com.haobang.apipull.adapter.ApiPullAdapter;
import com.haobang.apipull.adapter.PullPage;
@@ -42,8 +43,12 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.ssl.SSLContextBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -104,8 +109,11 @@ public class NdrServerApiAdapter implements ApiPullAdapter {
*/
@Autowired
public NdrServerApiAdapter(
CredentialProvider credentials, SafeBaseUrlValidator urlValidator, ObjectMapper mapper) {
this(credentials, urlValidator, mapper, HttpClients.custom().disableAutomaticRetries().build());
CredentialProvider credentials,
SafeBaseUrlValidator urlValidator,
ObjectMapper mapper,
ApiPullProperties properties) {
this(credentials, urlValidator, mapper, buildHttpClient(properties.isTlsVerifyEnabled()));
}
NdrServerApiAdapter(
@@ -119,6 +127,24 @@ public class NdrServerApiAdapter implements ApiPullAdapter {
this.http = http;
}
/**
* 遵循设备 API 全局 TLS 校验开关,但始终禁止自动重试和重定向。
*/
static CloseableHttpClient buildHttpClient(boolean tlsVerifyEnabled) {
HttpClientBuilder builder =
HttpClients.custom().disableAutomaticRetries().disableRedirectHandling();
if (tlsVerifyEnabled) return builder.build();
try {
javax.net.ssl.SSLContext sslContext =
SSLContextBuilder.create().loadTrustMaterial((chain, authType) -> true).build();
SSLConnectionSocketFactory socketFactory =
new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
return builder.setSSLSocketFactory(socketFactory).build();
} catch (Exception e) {
throw new IllegalStateException("初始化 NDR 内网 TLS 客户端失败", e);
}
}
@Override
public String type() {
return TYPE;
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.haobang.apipull.ApiPullProperties;
import com.haobang.apipull.adapter.*;
import com.haobang.apipull.credential.*;
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
@@ -26,8 +27,12 @@ import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.ssl.SSLContextBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -49,9 +54,12 @@ public class QuanxiApiAdapter implements ApiPullAdapter {
private final CloseableHttpClient http;
@Autowired
public QuanxiApiAdapter(CredentialProvider credentials, SafeBaseUrlValidator urls, ObjectMapper json) {
this(credentials, urls, json,
HttpClients.custom().disableAutomaticRetries().disableRedirectHandling().build());
public QuanxiApiAdapter(
CredentialProvider credentials,
SafeBaseUrlValidator urls,
ObjectMapper json,
ApiPullProperties properties) {
this(credentials, urls, json, buildHttpClient(properties.isTlsVerifyEnabled()));
}
QuanxiApiAdapter(CredentialProvider credentials, SafeBaseUrlValidator urls,
@@ -63,6 +71,22 @@ public class QuanxiApiAdapter implements ApiPullAdapter {
this.http = http;
}
/** 遵循设备 API 全局 TLS 校验开关,但始终禁止自动重试和重定向。 */
static CloseableHttpClient buildHttpClient(boolean tlsVerifyEnabled) {
HttpClientBuilder builder =
HttpClients.custom().disableAutomaticRetries().disableRedirectHandling();
if (tlsVerifyEnabled) return builder.build();
try {
javax.net.ssl.SSLContext sslContext =
SSLContextBuilder.create().loadTrustMaterial((chain, authType) -> true).build();
SSLConnectionSocketFactory socketFactory =
new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
return builder.setSSLSocketFactory(socketFactory).build();
} catch (Exception e) {
throw new IllegalStateException("初始化全悉内网 TLS 客户端失败", e);
}
}
@Override public String type() { return TYPE; }
@Override public int version() { return 1; }
@Override public Set<String> streams() { return Collections.singleton(STREAM); }
@@ -0,0 +1,481 @@
package com.haobang.apipull.adapter.weibu;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.haobang.apipull.ApiPullProperties;
import com.haobang.apipull.adapter.AdapterException;
import com.haobang.apipull.adapter.ApiPullAdapter;
import com.haobang.apipull.adapter.PullPage;
import com.haobang.apipull.adapter.PullRequest;
import com.haobang.apipull.adapter.PulledEvent;
import com.haobang.apipull.credential.CredentialException;
import com.haobang.apipull.credential.CredentialProvider;
import com.haobang.apipull.credential.CredentialSchemas;
import com.haobang.apipull.credential.CredentialSnapshot;
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
import com.haobang.apipull.security.SafeBaseUrlValidator;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.SocketTimeoutException;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import javax.annotation.PreDestroy;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.ssl.SSLContextBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
/**
* 微步 API 配置文档中的逐条日志查询协议。
*
* <p>固定使用 {@code /api/v1/log/searchBySql},响应 {@code data.data[]} 中每个对象作为一条原始
* 日志进入 WAL/Kafka。该接口没有 offset/page 参数,因此任务强制 {@code maxPagesPerRun=1};通用
* Runner 先读取响应 {@code total},当窗口内总数超过 {@code pageSize} 时只丢弃探测响应并继续缩小
* 时间窗口,直到单次响应能够完整返回。最小一秒窗口仍超限时失败且不推进 checkpoint。
*
* <p>externalEventId 使用厂商日志 {@code id},事件时间使用 {@code time}。事件 ID 只由任务命名空间
* 和厂商稳定 ID 生成,lookback 重叠查询不会制造新事件版本。适配器不把 API JSON 伪装成 Syslog,
* 业务字段标准化由下游针对 API 的泛化规则完成。
*/
@Component
public class WeibuApiAdapter implements ApiPullAdapter {
public static final String TYPE = "WEIBU_V1";
public static final String LOG = "WEIBU_LOG";
static final String PATH = "/api/v1/log/searchBySql";
private static final ZoneId DEVICE_ZONE = ZoneId.of("Asia/Shanghai");
private static final DateTimeFormatter EVENT_ID_DATE =
DateTimeFormatter.ofPattern("yyyyMMdd").withZone(DEVICE_ZONE);
private static final UUID EVENT_NAMESPACE =
UUID.fromString("89a53d59-bb84-5d41-924f-38319448cf46");
private static final long MIN_EVENT_EPOCH_SECOND = 946684800L;
private static final long MAX_EVENT_EPOCH_SECOND = 4102444800L;
private final CredentialProvider credentials;
private final SafeBaseUrlValidator urls;
private final ObjectMapper json;
private final CloseableHttpClient http;
@Autowired
public WeibuApiAdapter(
CredentialProvider credentials,
SafeBaseUrlValidator urls,
ObjectMapper json,
ApiPullProperties properties) {
this(
credentials,
urls,
json,
buildHttpClient(properties.isTlsVerifyEnabled()));
}
WeibuApiAdapter(
CredentialProvider credentials,
SafeBaseUrlValidator urls,
ObjectMapper json,
CloseableHttpClient http) {
this.credentials = credentials;
this.urls = urls;
this.json =
json.copy()
.deactivateDefaultTyping()
.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION);
this.http = http;
}
/**
* 进程级全局开关可为使用自签名证书的内网设备关闭证书链和主机名校验。
* 客户端仍拒绝重定向和非 HTTPS URL;静态主机白名单是否启用由部署配置决定。
*/
static CloseableHttpClient buildHttpClient(boolean tlsVerifyEnabled) {
HttpClientBuilder builder =
HttpClients.custom().disableAutomaticRetries().disableRedirectHandling();
if (tlsVerifyEnabled) return builder.build();
try {
javax.net.ssl.SSLContext sslContext =
SSLContextBuilder.create().loadTrustMaterial((chain, authType) -> true).build();
SSLConnectionSocketFactory socketFactory =
new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
return builder.setSSLSocketFactory(socketFactory).build();
} catch (Exception e) {
throw new IllegalStateException("初始化微步内网 TLS 客户端失败", e);
}
}
@Override
public String type() {
return TYPE;
}
@Override
public int version() {
return 1;
}
@Override
public Set<String> streams() {
return Collections.singleton(LOG);
}
/** 厂商时间字段为 Unix 秒,窗口不能安全拆到亚秒。 */
@Override
public int minimumWindowSeconds() {
return 1;
}
@Override
public boolean requiresDistinctExternalIds() {
return true;
}
@Override
public void validateConfig(DeviceCollectApiConfig c) {
if (c == null || !TYPE.equals(c.getAdapterType())) fail("CONFIG_SCHEMA_ERROR");
if (!LOG.equals(c.getStreamKey())) fail("STREAM_NOT_SUPPORTED");
range(c.getPageSize(), 1, 1000);
if (c.getMaxPagesPerRun() == null || c.getMaxPagesPerRun() != 1)
fail("CONFIG_SCHEMA_ERROR");
range(c.getPollIntervalSeconds(), 1, 86400);
range(c.getWindowSizeSeconds(), 1, 86400);
range(c.getLookbackSeconds(), 0, 86400);
range(c.getSafetyDelaySeconds(), 0, 86400);
range(c.getConnectTimeoutMs(), 100, 30000);
range(c.getRequestTimeoutMs(), 1000, 120000);
if (c.getMaxResponseBytes() == null
|| c.getMaxResponseBytes() < 1024
|| c.getMaxResponseBytes() > 67108864L) fail("CONFIG_SCHEMA_ERROR");
URI base = urls.validate(c.getBaseUrl());
String path = base.getPath();
if (!(path == null || path.isEmpty() || "/".equals(path))) fail("CONFIG_SCHEMA_ERROR");
if (c.getExtraConfigJson() == null || !"{}".equals(c.getExtraConfigJson().trim()))
fail("CONFIG_SCHEMA_ERROR");
}
@Override
public PullPage pull(PullRequest request) {
DeviceCollectApiConfig config = request.getConfig();
validateConfig(config);
validateRequest(request);
Instant vendorStart = request.getQueryStart().truncatedTo(ChronoUnit.SECONDS);
Instant vendorEnd = ceilToSecond(request.getQueryEnd()).minusSeconds(1L);
if (vendorEnd.isBefore(vendorStart)) fail("REQUEST_RANGE_INVALID");
ObjectNode body = json.createObjectNode();
body.putArray("assets_group");
body.put("search_connection", false);
body.putArray("net_data_type").add("attack").add("risk");
body.put("time_from", vendorStart.getEpochSecond());
body.put("time_to", vendorEnd.getEpochSecond());
body.put("sql", "");
body.putArray("columns");
body.put("asc", false);
// 厂商接口只有 size,没有 offset/page。Runner 通过 total 自动缩小窗口,确保该次
// size 足以取回窗口内全部日志,而不是把 size 当作伪分页游标。
body.put("size", request.getLimit());
byte[] responseBytes = null;
HttpPost post = null;
long httpStarted = System.nanoTime();
try (CredentialSnapshot snapshot =
credentials.load(config.getCredentialRef(), TYPE, config.getDeviceId())) {
char[] keyChars = snapshot.require("app_key").copy();
char[] secretChars = snapshot.require("app_secret").copy();
try {
String apiKey = new String(keyChars);
String secret = new String(secretChars);
CredentialSchemas.validate(TYPE, apiKey, secret, null);
long timestamp = Instant.now().getEpochSecond();
String signature = sign(apiKey, secretChars, timestamp);
URI endpoint =
new URIBuilder(endpoint(urls.validate(config.getBaseUrl())))
.addParameter("api_key", apiKey)
.addParameter("auth_timestamp", Long.toString(timestamp))
.addParameter("sign", signature)
.build();
post = new HttpPost(endpoint);
post.setConfig(requestConfig(config));
post.setHeader("Accept", "application/json");
post.setEntity(
new StringEntity(json.writeValueAsString(body), ContentType.APPLICATION_JSON));
try (CloseableHttpResponse response = http.execute(post)) {
int status = response.getStatusLine().getStatusCode();
if (status < 200 || status >= 300) {
String code =
status == 401
? "AUTH_HTTP_401"
: status == 403 ? "AUTH_HTTP_403" : "HTTP_ERROR";
throw new AdapterException(code, "微步 API HTTP 请求失败: " + status, status);
}
responseBytes = readBounded(response.getEntity(), config.getMaxResponseBytes());
long httpMs = elapsed(httpStarted);
long parseStarted = System.nanoTime();
return parsePage(responseBytes, status, httpMs, parseStarted, request);
}
} finally {
Arrays.fill(keyChars, '\0');
Arrays.fill(secretChars, '\0');
}
} catch (AdapterException | CredentialException e) {
throw e;
} catch (SocketTimeoutException | org.apache.http.conn.ConnectTimeoutException e) {
fail("HTTP_TIMEOUT");
return null;
} catch (IOException e) {
fail("DEVICE_API_UNAVAILABLE");
return null;
} catch (Exception e) {
fail("REQUEST_BUILD_ERROR");
return null;
} finally {
if (post != null) post.releaseConnection();
if (responseBytes != null) Arrays.fill(responseBytes, (byte) 0);
}
}
private PullPage parsePage(
byte[] bytes,
int status,
long httpMs,
long parseStarted,
PullRequest request) {
try {
JsonNode root = parseBounded(bytes);
if (!root.isObject()) fail("RESPONSE_SCHEMA_ERROR");
JsonNode responseCode = root.get("response_code");
if (responseCode == null || !responseCode.isIntegralNumber())
fail("RESPONSE_SCHEMA_ERROR");
if (responseCode.intValue() != 0) fail("VENDOR_RESPONSE_ERROR");
JsonNode data = root.get("data");
if (data == null || !data.isObject()) fail("RESPONSE_SCHEMA_ERROR");
JsonNode items = data.get("data");
if (items == null || !items.isArray())
fail("RESPONSE_SCHEMA_ERROR");
long total = nonNegativeLong(data.get("total"));
long expectedItems = Math.min(total, request.getLimit());
if (items.size() != expectedItems) fail("PAGINATION_INCONSISTENT");
ArrayList<PulledEvent> events = new ArrayList<>();
Set<String> pageIds = new HashSet<>();
UUID taskNamespace =
uuidV5(
EVENT_NAMESPACE,
configIdentity(request.getConfig()));
for (JsonNode item : items) {
if (!item.isObject()) fail("RESPONSE_SCHEMA_ERROR");
String external = requiredText(item.get("id"), 512);
if (!pageIds.add(external)) fail("PAGINATION_DUPLICATE_DOCUMENT");
long eventSecond = integral(item.get("time"));
if (eventSecond < MIN_EVENT_EPOCH_SECOND || eventSecond >= MAX_EVENT_EPOCH_SECOND)
fail("EVENT_TIME_INVALID");
Instant eventTime = Instant.ofEpochSecond(eventSecond);
// 请求使用外扩后的整秒闭区间,最终仍严格服从平台的半开窗口。厂商若返回
// 外扩边界记录,只按设备返回数完成完整性校验,不让该记录进入 WAL。
if (eventTime.isBefore(request.getQueryStart())
|| !eventTime.isBefore(request.getQueryEnd())) continue;
String raw = json.writeValueAsString(item);
if (raw.getBytes(StandardCharsets.UTF_8).length > 1048576) fail("EVENT_TOO_LARGE");
String eventId =
EVENT_ID_DATE.format(eventTime) + "-" + uuidV5(taskNamespace, external);
events.add(new PulledEvent(eventId, external, eventTime, raw));
}
return new PullPage(
events,
total,
status,
bytes.length,
httpMs,
elapsed(parseStarted),
Long.toString(total),
items.size());
} catch (AdapterException e) {
throw e;
} catch (Exception e) {
fail("RESPONSE_SCHEMA_ERROR");
return null;
}
}
private void validateRequest(PullRequest r) {
if (r.getConfig().getDeviceId() == null
|| r.getConfig().getCollectTaskId() == null
|| r.getQueryStart() == null
|| r.getQueryEnd() == null
|| !r.getQueryEnd().isAfter(r.getQueryStart())
|| r.getLimit() < 1
|| r.getLimit() > r.getConfig().getPageSize()
|| r.getOffset() != 0L) fail("REQUEST_RANGE_INVALID");
}
private URI endpoint(URI base) throws Exception {
return new URI(base.getScheme(), null, base.getHost(), base.getPort(), PATH, null, null);
}
private RequestConfig requestConfig(DeviceCollectApiConfig c) {
return RequestConfig.custom()
.setConnectTimeout(c.getConnectTimeoutMs())
.setConnectionRequestTimeout(c.getConnectTimeoutMs())
.setSocketTimeout(c.getRequestTimeoutMs())
.setRedirectsEnabled(false)
.build();
}
static String sign(String apiKey, char[] secret, long timestamp) throws Exception {
byte[] secretBytes = new String(secret).getBytes(StandardCharsets.UTF_8);
try {
Mac mac = Mac.getInstance("HmacSHA256");
mac.init(new SecretKeySpec(secretBytes, "HmacSHA256"));
byte[] digest =
mac.doFinal((apiKey + timestamp).getBytes(StandardCharsets.UTF_8));
try {
return Base64.getUrlEncoder().withoutPadding().encodeToString(digest);
} finally {
Arrays.fill(digest, (byte) 0);
}
} finally {
Arrays.fill(secretBytes, (byte) 0);
}
}
static Instant ceilToSecond(Instant value) {
Instant floor = value.truncatedTo(ChronoUnit.SECONDS);
return floor.equals(value) ? floor : floor.plusSeconds(1L);
}
private byte[] readBounded(HttpEntity entity, long maxBytes) throws IOException {
if (entity == null) fail("RESPONSE_SCHEMA_ERROR");
if (entity.getContentLength() > maxBytes) fail("RESPONSE_TOO_LARGE");
try (InputStream in = entity.getContent();
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
byte[] buffer = new byte[8192];
long size = 0;
int n;
while ((n = in.read(buffer)) != -1) {
size += n;
if (size > maxBytes) fail("RESPONSE_TOO_LARGE");
out.write(buffer, 0, n);
}
return out.toByteArray();
}
}
private JsonNode parseBounded(byte[] bytes) {
try {
try (JsonParser parser = json.getFactory().createParser(bytes)) {
int depth = 0;
int tokens = 0;
JsonToken token;
while ((token = parser.nextToken()) != null) {
if (++tokens > 2000000) fail("RESPONSE_SCHEMA_ERROR");
if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) {
if (++depth > 64) fail("RESPONSE_SCHEMA_ERROR");
} else if (token == JsonToken.END_OBJECT || token == JsonToken.END_ARRAY) {
depth--;
} else if ((token == JsonToken.VALUE_STRING && parser.getTextLength() > 1048576)
|| (token == JsonToken.FIELD_NAME && parser.getCurrentName().length() > 512)) {
fail("RESPONSE_SCHEMA_ERROR");
}
}
}
try (JsonParser parser = json.getFactory().createParser(bytes)) {
JsonNode value = json.readTree(parser);
if (value == null || parser.nextToken() != null) fail("RESPONSE_SCHEMA_ERROR");
return value;
}
} catch (AdapterException e) {
throw e;
} catch (Exception e) {
fail("RESPONSE_SCHEMA_ERROR");
return null;
}
}
private String configIdentity(DeviceCollectApiConfig c) {
return c.getDeviceId() + "|" + c.getCollectTaskId() + "|" + TYPE + "|" + LOG;
}
private long nonNegativeLong(JsonNode value) {
long result = integral(value);
if (result < 0) fail("RESPONSE_SCHEMA_ERROR");
return result;
}
private long integral(JsonNode value) {
if (value == null || !value.isIntegralNumber() || !value.canConvertToLong())
fail("RESPONSE_SCHEMA_ERROR");
return value.longValue();
}
private String requiredText(JsonNode value, int max) {
if (value == null
|| !value.isTextual()
|| value.textValue().trim().isEmpty()
|| value.textValue().length() > max) fail("RESPONSE_SCHEMA_ERROR");
return value.textValue();
}
private void range(Integer value, int min, int max) {
if (value == null || value < min || value > max) fail("CONFIG_SCHEMA_ERROR");
}
static UUID uuidV5(UUID namespace, String value) throws Exception {
MessageDigest digest = MessageDigest.getInstance("SHA-1");
digest.update(
ByteBuffer.allocate(16)
.putLong(namespace.getMostSignificantBits())
.putLong(namespace.getLeastSignificantBits())
.array());
byte[] hash = digest.digest(value.getBytes(StandardCharsets.UTF_8));
hash[6] = (byte) ((hash[6] & 15) | 80);
hash[8] = (byte) ((hash[8] & 63) | 128);
ByteBuffer buffer = ByteBuffer.wrap(hash);
return new UUID(buffer.getLong(), buffer.getLong());
}
private long elapsed(long started) {
return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started);
}
private void fail(String code) {
throw new AdapterException(code, "微步 API: " + code);
}
@PreDestroy
public void close() {
try {
http.close();
} catch (IOException ignored) {
// Spring shutdown best effort.
}
}
}
@@ -5,7 +5,7 @@ public final class CredentialSchemas {
private CredentialSchemas() {}
public static void validate(String type, String key, String secret, String token) {
if ("NDRSERVER_V1".equals(type)) {
if ("NDRSERVER_V1".equals(type) || "WEIBU_V1".equals(type)) {
require(key, 512);
require(secret, 2048);
if (token != null) fail();
@@ -14,14 +14,16 @@ import org.springframework.stereotype.Component;
/**
* 设备基础地址的 SSRF 防护边界。
*
* <p>只有显式加入白名单的 HTTPS 主机才能访问;拒绝 user-info、query、fragment 和路径穿越。 主机采用精确匹配,不允许后缀或通配符匹配,避免相似域名绕过。TLS
* 证书链和主机名继续由 JVM 默认校验器验证。
* <p>始终拒绝非 HTTPS、user-info、query、fragment 和路径穿越。现场默认关闭静态主机白名单,
* 避免设备增减必须重启;显式启用后采用精确主机匹配,不允许后缀或通配符匹配。
*/
@Component
public class SafeBaseUrlValidator {
private final boolean hostAllowlistEnabled;
private final Set<String> allowedHosts;
public SafeBaseUrlValidator(ApiPullProperties properties) {
hostAllowlistEnabled = properties.isHostAllowlistEnabled();
String configured = properties.getAllowedHosts();
allowedHosts =
configured == null || configured.trim().isEmpty()
@@ -45,7 +47,7 @@ public class SafeBaseUrlValidator {
throw new AdapterException(
"SSRF_REJECTED", "baseUrl 必须是无 user-info/query/fragment 的 HTTPS URL");
String host = uri.getHost().toLowerCase();
if (allowedHosts.isEmpty() || !allowedHosts.contains(host))
if (hostAllowlistEnabled && (allowedHosts.isEmpty() || !allowedHosts.contains(host)))
throw new AdapterException("SSRF_REJECTED", "设备主机不在 device.api.allowed-hosts 中: " + host);
InetAddress[] addresses = InetAddress.getAllByName(host);
if (addresses.length == 0)
@@ -1,6 +1,7 @@
package com.haobang.dispatch;
import com.common.entity.DeviceDevice;
import com.common.entity.DeviceCollectionMode;
import com.common.entity.DeviceUnknown;
import com.common.service.DeviceDeviceService;
import com.common.service.DeviceUnknownService;
@@ -80,7 +81,7 @@ public class DeviceContextResolver {
for (Map.Entry<String, List<DeviceDevice>> entry : grouped.entrySet()) {
List<DeviceDevice> sameIp = entry.getValue();
Resolution resolution = sameIp.size() == 1
? validateAssignment(sameIp.get(0))
? validateSyslogDevice(sameIp.get(0))
: Resolution.terminal(Type.DUPLICATE_DEVICE, null,
"来源 IP 存在多条设备记录: " + entry.getKey());
resolutions.put(entry.getKey(), resolution);
@@ -147,15 +148,7 @@ public class DeviceContextResolver {
resolution = Resolution.terminal(Type.DUPLICATE_DEVICE, null,
"来源 IP 存在多条设备记录: " + sourceIp);
} else {
DeviceDevice device = devices.get(0);
Integer assignedCollector = device.getDeviceCollectId();
if (assignedCollector == null
|| assignedCollector != properties.getAppServiceDeviceCollectId()) {
resolution = Resolution.terminal(Type.NOT_ASSIGNED, device,
"设备不属于当前采集探针: deviceId=" + device.getId());
} else {
resolution = Resolution.registered(device);
}
resolution = validateSyslogDevice(devices.get(0));
}
cache.put(sourceIp, new CacheEntry(resolution,
now + properties.getSyslogDeviceContextCacheTtlMs()));
@@ -196,9 +189,10 @@ public class DeviceContextResolver {
}
misses.increment();
DeviceDevice device = deviceService.getByIdSafely(deviceId);
Resolution resolution = device == null
Resolution resolution = device == null || device.getDeletedAt() != null
|| (device.getDelFlag() != null && !"0".equals(device.getDelFlag()))
? Resolution.terminal(Type.UNKNOWN_DEVICE, null, "设备不存在: deviceId=" + deviceId)
: validateAssignment(device);
: validateApiDevice(device);
deviceIdCache.put(deviceId, new CacheEntry(resolution,
now + properties.getSyslogDeviceContextCacheTtlMs()));
return resolution;
@@ -214,10 +208,46 @@ public class DeviceContextResolver {
return Resolution.registered(device);
}
private Resolution validateSyslogDevice(DeviceDevice device) {
Resolution assignment = validateAssignment(device);
if (!assignment.isRegistered()) return assignment;
DeviceCollectionMode mode = device.getCollectionMode();
if (!mode.allowsSyslog()) {
return Resolution.terminal(Type.COLLECTION_MODE_DENIED, device,
"设备采集模式不允许 SYSLOG_PUSH: deviceId=" + device.getId()
+ ", collectionMode=" + mode);
}
return assignment;
}
private Resolution validateApiDevice(DeviceDevice device) {
Resolution assignment = validateAssignment(device);
if (!assignment.isRegistered()) return assignment;
DeviceCollectionMode mode = device.getCollectionMode();
if (!mode.allowsApi()) {
return Resolution.terminal(Type.COLLECTION_MODE_DENIED, device,
"设备采集模式不允许 API_PULL: deviceId=" + device.getId()
+ ", collectionMode=" + mode);
}
return assignment;
}
public void invalidate(String sourceIp) {
if (sourceIp != null) cache.remove(sourceIp);
}
/** 清除指定设备的 device_id 和所有关联 source_ip 本地缓存。 */
public void invalidateDevice(Integer deviceId, String currentIp) {
if (deviceId != null) deviceIdCache.remove(deviceId);
cache.forEach((sourceIp, entry) -> {
DeviceDevice cachedDevice = entry.resolution.getDevice();
if ((currentIp != null && currentIp.equals(sourceIp))
|| (deviceId != null && cachedDevice != null && deviceId.equals(cachedDevice.getId()))) {
cache.remove(sourceIp, entry);
}
});
}
public void invalidateAll() { cache.clear(); deviceIdCache.clear(); }
public long getHitCount() { return hits.sum(); }
public long getMissCount() { return misses.sum(); }
@@ -255,7 +285,13 @@ public class DeviceContextResolver {
}
}
public enum Type { REGISTERED, UNKNOWN_DEVICE, DUPLICATE_DEVICE, NOT_ASSIGNED }
public enum Type {
REGISTERED,
UNKNOWN_DEVICE,
DUPLICATE_DEVICE,
NOT_ASSIGNED,
COLLECTION_MODE_DENIED
}
public enum IngressStatus { REGISTERED, INVALID, UNAVAILABLE }
private static final class IngressSnapshot {
@@ -2,6 +2,7 @@ package com.haobang.dispatch;
import com.haobang.config.AppProperties;
import com.haobang.diagnostics.SyslogIpMetricsRegistry;
import com.haobang.wal.IngressType;
import com.haobang.wal.SyslogWalRecord;
import com.haobang.wal.SyslogWalService;
import com.kafka.kafkaProducer;
@@ -30,9 +31,9 @@ import java.util.concurrent.atomic.LongAdder;
/**
* 从 WAL 有界并发异步投递 Kafka,并只按连续 ACK 推进 WAL 游标。
*
* <p>v2/v3 保持原有按 source_ip 识别设备和单时间参数的生产路径;v4 改为按可信 device_id
* 识别设备,并分别传递 receive_time/event_time。兼容分支留在 Dispatcher,Kafka 消息外层结构
* 不因接入方式改变。
* <p>v2/v3 保持原有按 source_ip 识别设备和单时间参数的生产路径;v4 API_PULL 按可信
* device_id 识别设备,并分别传递 receive_time/event_time。Dispatcher 按 ingress_type 选择
* 设备策略,确保模式变化后已经进入 WAL 的记录仍会被最终校验。
*/
@Component
public class SyslogWalDispatcher {
@@ -148,8 +149,10 @@ public class SyslogWalDispatcher {
}
final DeviceContextResolver.Resolution resolution;
try {
// API 的 HTTP 对端可能是代理或网关;v4 必须使用任务写入 WAL 的 device_id。
if (record.getSchemaVersion() >= 4 && record.getDeviceId() != null) {
// API 的 HTTP 对端可能是代理或网关;API_PULL 必须使用任务写入 WAL 的 device_id。
// 不能仅凭 schema/device_id 判断,否则未来携带 device_id 的 SYSLOG_PUSH 会误用 API 权限。
if (record.getSchemaVersion() >= 4
&& IngressType.API_PULL.name().equals(record.getIngressType())) {
resolution = deviceResolver.resolveDeviceId(record.getDeviceId());
} else {
resolution = deviceResolver.resolve(record.getSourceIp(), record.getProtocol());
@@ -9,6 +9,11 @@ import java.util.function.Supplier;
import java.util.LinkedHashMap;
import java.util.List ;
import java.util.stream.Collectors;
import java.util.Arrays;
import java.util.ArrayList;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.ScanOptions;
@Component
public class SafeCacheUtil {
@@ -105,4 +110,39 @@ public class SafeCacheUtil {
return loader.get();
}
}
/**
* 严格删除 SafeCacheUtil 使用的原始 Redis Key。
*
* <p>设备策略更新不能像读缓存一样静默降级,否则接口成功后仍可能读取旧策略。</p>
*/
public void evict(String... keys) {
if (keys == null || keys.length == 0) return;
redisTemplate.delete(Arrays.asList(keys));
}
/** 使用 Redis SCAN 分批删除匹配的原始 Key,避免生产环境执行阻塞式 KEYS。 */
public void evictMatching(String... patterns) {
if (patterns == null || patterns.length == 0) return;
redisTemplate.execute((RedisCallback<Void>) connection -> {
for (String pattern : patterns) {
if (pattern == null || pattern.trim().isEmpty()) continue;
ScanOptions options = ScanOptions.scanOptions().match(pattern).count(500).build();
try (Cursor<byte[]> cursor = connection.scan(options)) {
List<byte[]> batch = new ArrayList<>(500);
while (cursor.hasNext()) {
batch.add(cursor.next());
if (batch.size() == 500) {
connection.del(batch.toArray(new byte[batch.size()][]));
batch.clear();
}
}
if (!batch.isEmpty()) {
connection.del(batch.toArray(new byte[batch.size()][]));
}
}
}
return null;
});
}
}
@@ -7,8 +7,8 @@ server.error.include-message=always
server.error.include-binding-errors=always
# Syslog Server Configuration
# 当前测试只验证设备 API 拉取,关闭 TCP/UDP Netty 监听;可用环境变量重新开启。
syslog.server.enabled=${SYSLOG_SERVER_ENABLED:false}
# 默认同时开启 TCP/UDP Syslog 与设备 API 拉取;设备级 collectionMode 再决定具体接入方式。
syslog.server.enabled=${SYSLOG_SERVER_ENABLED:true}
syslog.tcp.port=514
syslog.udp.port=514
syslog.tcp.backlog=${SYSLOG_TCP_BACKLOG:4096}
@@ -127,7 +127,7 @@ mybatis.mapper-locations=classpath:mapper/*.xml
mybatis.type-aliases-package=com.common.entity
mybatis.configuration.map-underscore-to-camel-case=true
# 当前测试环境开启设备 API 拉取调度。
# 默认开启设备 API 拉取调度;单个任务仍须通过 resume 启用。
device.api.enabled=${DEVICE_API_ENABLED:true}
# 设备 AppKey/AppSecret 由 device_collect_api_credential 表管理,配置文件不再保存厂商凭据。
device.api.scheduler.instance-id=${DEVICE_API_INSTANCE_ID:${HOSTNAME:local}}
@@ -141,9 +141,12 @@ device.api.scheduler.max-pages-per-slice=${DEVICE_API_MAX_PAGES_PER_SLICE:20}
# HTTP + 解析 + 本页 WAL durable 总耗时达到 1 秒即打印一条结构化 WARN。
device.api.page.slow-threshold-ms=${DEVICE_API_SLOW_PAGE_THRESHOLD_MS:1000}
device.api.page.backpressure-timeout-ms=${DEVICE_API_BACKPRESSURE_TIMEOUT_MS:120000}
# 逗号分隔的设备 DNS 名或 IP;必须显式配置,防止管理接口被用作 SSRF 跳板。
# NDR 测试设备证书仅声明 CN=wlza.com;通过宿主机/容器 hosts 将该域名固定映射到 192.168.4.19。
# 现场默认关闭静态主机白名单,新增 API 设备无需修改配置和重启;需要收紧时可显式开启。
device.api.host-allowlist-enabled=${DEVICE_API_HOST_ALLOWLIST_ENABLED:false}
# 白名单开启时使用的逗号分隔 DNS 名或 IP;主机精确匹配,不支持通配符。
device.api.allowed-hosts=${DEVICE_API_ALLOWED_HOSTS:192.168.4.19,wlza.com}
# 所有设备 API 默认关闭证书链和主机名校验;仍只允许 HTTPS,并禁止重定向。
device.api.tls-verify-enabled=${DEVICE_API_TLS_VERIFY_ENABLED:false}
# 开发环境缓存配置
spring.redis.host=localhost
@@ -42,6 +42,9 @@
<result column="today_non_log_count" property="todayNonLogCount" />
<result column="create_dept" property="createDept" />
<result column="device_collect_id" property="deviceCollectId" />
<result column="collection_mode" property="collectionMode"
javaType="com.common.entity.DeviceCollectionMode"
typeHandler="org.apache.ibatis.type.EnumTypeHandler" />
</resultMap>
<!-- 基础查询列 -->
@@ -51,7 +54,8 @@
control_agent_id, license_start_time, license_end_time, is_monitoring,
security_scope_id, owner_id, ssh_config_id, status, created_by_id, decode_type,
miss_policy, tenant_id, create_time, update_time, create_by, update_by, del_flag,
manager_name, today_parse_count, today_non_log_count, create_dept, device_collect_id
manager_name, today_parse_count, today_non_log_count, create_dept, device_collect_id,
collection_mode
</sql>
<!-- 根据ID查询 -->
@@ -235,4 +239,14 @@
WHERE vendor = #{vendor}
</select>
<update id="updateCollectionMode">
UPDATE device_device
SET collection_mode = #{collectionMode,javaType=com.common.entity.DeviceCollectionMode,typeHandler=org.apache.ibatis.type.EnumTypeHandler},
updated_at = SYSDATE,
update_time = SYSDATE
WHERE id = #{id}
AND deleted_at IS NULL
AND (del_flag IS NULL OR del_flag = '0')
</update>
</mapper>
@@ -0,0 +1,131 @@
package com.common.controller;
import com.common.entity.DeviceCollectionMode;
import com.common.entity.DeviceDevice;
import com.common.service.DeviceDeviceService;
import com.haobang.dispatch.DeviceContextResolver;
import org.junit.jupiter.api.Test;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
class DeviceDeviceControllerTest {
@Test
void rejectsInvalidCollectionMode() throws Exception {
DeviceDeviceService service = mock(DeviceDeviceService.class);
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
MockMvc mvc = MockMvcBuilders.standaloneSetup(
new DeviceDeviceController(service, resolver)).build();
mvc.perform(put("/api/devices/40/collection-mode")
.contentType(MediaType.APPLICATION_JSON)
.content("{\"collectionMode\":\"INVALID\"}"))
.andExpect(status().isBadRequest());
verify(service, never()).updateCollectionMode(
org.mockito.ArgumentMatchers.anyInt(), org.mockito.ArgumentMatchers.any());
}
@Test
void rejectsAttemptsToUpdateOtherDeviceFields() throws Exception {
DeviceDeviceService service = mock(DeviceDeviceService.class);
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
MockMvc mvc = MockMvcBuilders.standaloneSetup(
new DeviceDeviceController(service, resolver)).build();
mvc.perform(put("/api/devices/40/collection-mode")
.contentType(MediaType.APPLICATION_JSON)
.content("{\"collectionMode\":\"API_ONLY\",\"name\":\"tampered\"}"))
.andExpect(status().isBadRequest());
verify(service, never()).updateCollectionMode(
org.mockito.ArgumentMatchers.anyInt(), org.mockito.ArgumentMatchers.any());
}
@Test
void updateInvalidatesLocalCachesAndRefreshesIngressSnapshot() throws Exception {
DeviceDeviceService service = mock(DeviceDeviceService.class);
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
DeviceDevice updated = new DeviceDevice();
updated.setId(40);
updated.setIp("10.56.171.164");
updated.setCollectionMode(DeviceCollectionMode.API_ONLY);
when(service.updateCollectionMode(40, DeviceCollectionMode.API_ONLY)).thenReturn(updated);
MockMvc mvc = MockMvcBuilders.standaloneSetup(
new DeviceDeviceController(service, resolver)).build();
mvc.perform(put("/api/devices/40/collection-mode")
.contentType(MediaType.APPLICATION_JSON)
.content("{\"collectionMode\":\"API_ONLY\"}"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.id").value(40))
.andExpect(jsonPath("$.collectionMode").value("API_ONLY"));
verify(resolver).invalidateAll();
verify(resolver).refreshIngressSnapshot();
}
@Test
void returnsNotFoundForMissingOrDeletedDevice() throws Exception {
DeviceDeviceService service = mock(DeviceDeviceService.class);
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
MockMvc mvc = MockMvcBuilders.standaloneSetup(
new DeviceDeviceController(service, resolver)).build();
mvc.perform(put("/api/devices/404/collection-mode")
.contentType(MediaType.APPLICATION_JSON)
.content("{\"collectionMode\":\"API_ONLY\"}"))
.andExpect(status().isNotFound());
verify(resolver, never()).refreshIngressSnapshot();
}
@Test
void clearsAllDeviceCacheLayersAndRefreshesSnapshot() throws Exception {
DeviceDeviceService service = mock(DeviceDeviceService.class);
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
MockMvc mvc = MockMvcBuilders.standaloneSetup(
new DeviceDeviceController(service, resolver)).build();
mvc.perform(delete("/api/devices/cache"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.cleared").value(true))
.andExpect(jsonPath("$.scope").value("ALL"));
verify(service).evictAllDeviceCaches();
verify(resolver).invalidateAll();
verify(resolver).refreshIngressSnapshot();
}
@Test
void clearsSpecifiedDeviceCacheLayersAndRefreshesSnapshot() throws Exception {
DeviceDeviceService service = mock(DeviceDeviceService.class);
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
DeviceDevice device = new DeviceDevice();
device.setId(40);
device.setIp("10.56.171.164");
when(service.evictDeviceCaches(40)).thenReturn(device);
MockMvc mvc = MockMvcBuilders.standaloneSetup(
new DeviceDeviceController(service, resolver)).build();
mvc.perform(delete("/api/devices/40/cache"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.cleared").value(true))
.andExpect(jsonPath("$.scope").value("DEVICE"))
.andExpect(jsonPath("$.deviceId").value(40))
.andExpect(jsonPath("$.ip").value("10.56.171.164"));
verify(service).evictDeviceCaches(40);
verify(resolver).invalidateDevice(40, "10.56.171.164");
verify(resolver).refreshIngressSnapshot();
}
}
@@ -0,0 +1,40 @@
package com.common.mapper;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import org.apache.ibatis.builder.xml.XMLMapperBuilder;
import org.apache.ibatis.io.Resources;
import org.apache.ibatis.mapping.ResultMapping;
import org.apache.ibatis.session.Configuration;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertTrue;
class DeviceDeviceMapperXmlTest {
@Test
void parsesCollectionModeMappingAndDedicatedUpdateStatement() throws Exception {
Configuration configuration = new Configuration();
String resource = "mapper/DeviceDeviceMapper.xml";
try (InputStream input = Resources.getResourceAsStream(resource)) {
new XMLMapperBuilder(input, configuration, resource, configuration.getSqlFragments()).parse();
}
ResultMapping collectionMode = configuration
.getResultMap("com.common.mapper.DeviceDeviceMapper.BaseResultMap")
.getResultMappings().stream()
.filter(mapping -> "collectionMode".equals(mapping.getProperty()))
.findFirst().orElseThrow(() -> new AssertionError("collectionMode mapping missing"));
assertTrue("collection_mode".equalsIgnoreCase(collectionMode.getColumn()));
Map<String, Object> parameters = new HashMap<>();
parameters.put("id", 40);
parameters.put("collectionMode", "API_ONLY");
String sql = configuration
.getMappedStatement("com.common.mapper.DeviceDeviceMapper.updateCollectionMode")
.getBoundSql(parameters).getSql().replaceAll("\\s+", " ").toLowerCase();
assertTrue(sql.contains("set collection_mode = ?"));
assertTrue(sql.contains("where id = ?"));
assertTrue(sql.contains("deleted_at is null"));
}
}
@@ -0,0 +1,79 @@
package com.common.service;
import com.common.entity.DeviceCollectionMode;
import com.common.entity.DeviceDevice;
import com.common.mapper.DeviceDeviceMapper;
import com.common.service.impl.DeviceDeviceServiceImpl;
import com.haobang.util.SafeCacheUtil;
import org.junit.jupiter.api.Test;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class DeviceDeviceServiceImplTest {
@Test
void updateEvictsRawRedisKeysAndEntireSpringDeviceCache() {
DeviceDeviceMapper mapper = mock(DeviceDeviceMapper.class);
SafeCacheUtil safeCache = mock(SafeCacheUtil.class);
CacheManager cacheManager = mock(CacheManager.class);
Cache springCache = mock(Cache.class);
DeviceDevice current = new DeviceDevice();
current.setId(40);
current.setIp("10.56.171.164");
current.setDelFlag("0");
DeviceDevice updated = new DeviceDevice();
updated.setId(40);
updated.setIp(current.getIp());
updated.setCollectionMode(DeviceCollectionMode.API_ONLY);
when(mapper.selectById(40)).thenReturn(current, updated);
when(mapper.updateCollectionMode(40, DeviceCollectionMode.API_ONLY)).thenReturn(1);
when(cacheManager.getCache("device")).thenReturn(springCache);
DeviceDeviceServiceImpl service = new DeviceDeviceServiceImpl(mapper, safeCache, cacheManager);
DeviceDevice result = service.updateCollectionMode(40, DeviceCollectionMode.API_ONLY);
assertEquals(DeviceCollectionMode.API_ONLY, result.getCollectionMode());
verify(safeCache).evict("device:id:40", "device:ip:10.56.171.164");
verify(springCache).clear();
}
@Test
void evictsOnlySpecifiedDeviceIdAndIpEntries() {
DeviceDeviceMapper mapper = mock(DeviceDeviceMapper.class);
SafeCacheUtil safeCache = mock(SafeCacheUtil.class);
CacheManager cacheManager = mock(CacheManager.class);
Cache springCache = mock(Cache.class);
DeviceDevice device = new DeviceDevice();
device.setId(40);
device.setIp("10.56.171.164");
when(mapper.selectById(40)).thenReturn(device);
when(cacheManager.getCache("device")).thenReturn(springCache);
DeviceDeviceServiceImpl service = new DeviceDeviceServiceImpl(mapper, safeCache, cacheManager);
DeviceDevice result = service.evictDeviceCaches(40);
assertEquals(40, result.getId());
verify(safeCache).evict("device:id:40", "device:ip:10.56.171.164");
verify(springCache).evict("device:id:40");
verify(springCache).evict("device:ip:10.56.171.164");
}
@Test
void evictsAllRawRedisAndSpringDeviceCaches() {
DeviceDeviceMapper mapper = mock(DeviceDeviceMapper.class);
SafeCacheUtil safeCache = mock(SafeCacheUtil.class);
CacheManager cacheManager = mock(CacheManager.class);
Cache springCache = mock(Cache.class);
when(cacheManager.getCache("device")).thenReturn(springCache);
DeviceDeviceServiceImpl service = new DeviceDeviceServiceImpl(mapper, safeCache, cacheManager);
service.evictAllDeviceCaches();
verify(safeCache).evictMatching("device:id:*", "device:ip:*");
verify(springCache).clear();
}
}
@@ -0,0 +1,57 @@
package com.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.read.ListAppender;
import org.junit.jupiter.api.Test;
import org.slf4j.LoggerFactory;
import org.springframework.mock.env.MockEnvironment;
class SyslogBuildStartupLoggerTest {
@Test
void logsCodeRevisionAndEffectiveOverrides() {
MockEnvironment environment = new MockEnvironment()
.withProperty("device.api.enabled", "false")
.withProperty("device.api.tls-verify-enabled", "true")
.withProperty("app.service.device_collect_id", "40")
.withProperty("spring.kafka.producer.topic", "prod-syslog");
environment.setActiveProfiles("prod");
SyslogBuildStartupLogger reporter = new SyslogBuildStartupLogger(environment);
Logger logger = (Logger) LoggerFactory.getLogger(SyslogBuildStartupLogger.class);
ListAppender<ILoggingEvent> appender = new ListAppender<>();
appender.start();
logger.addAppender(appender);
try {
reporter.onApplicationReady();
assertEquals(1, appender.list.size());
String message = appender.list.get(0).getFormattedMessage();
assertTrue(message.contains("[SYSLOG_BUILD_READY] codeRevision="
+ SyslogBuildStartupLogger.CODE_REVISION));
assertTrue(message.contains("application=syslog-serve"));
assertTrue(message.contains("profiles=prod"));
assertTrue(message.contains("apiPullEnabled=false"));
assertTrue(message.contains("apiTlsVerifyEnabled=true"));
assertTrue(message.contains("deviceCollectId=40"));
assertTrue(message.contains("kafkaTopic=prod-syslog"));
} finally {
logger.detachAppender(appender);
appender.stop();
}
}
@Test
void usesSafeDefaultsAndIgnoresConfiguredRevision() {
String summary = new SyslogBuildStartupLogger(new MockEnvironment()
.withProperty("codeRevision", "fake")).startupSummary();
assertTrue(summary.contains("codeRevision=" + SyslogBuildStartupLogger.CODE_REVISION));
assertFalse(summary.contains("codeRevision=fake"));
assertTrue(summary.contains("syslogEnabled=true"));
assertTrue(summary.contains("apiPullEnabled=true"));
assertTrue(summary.contains("apiTlsVerifyEnabled=false"));
assertTrue(summary.contains("walEnabled=true"));
}
}
@@ -0,0 +1,15 @@
package com.haobang.apipull;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
class ApiPullPropertiesTest {
@Test
void apiPullIsEnabledByDefault() {
ApiPullProperties properties = new ApiPullProperties();
assertTrue(properties.isEnabled());
assertFalse(properties.isTlsVerifyEnabled());
}
}
@@ -154,6 +154,44 @@ class ApiPullRunnerTest {
verify(fixture.ingress, times(3)).tryAppend(any(IngressEvent.class));
}
@Test
void unpagedAdapterSplitsUntilSingleResponseContainsWholeWindow() {
RunnerFixture fixture = new RunnerFixture();
fixture.config.setPageSize(2);
fixture.config.setMaxPagesPerRun(1);
when(fixture.adapter.pull(any()))
.thenAnswer(
invocation -> {
PullRequest request = invocation.getArgument(0);
long windowSeconds =
Duration.between(WINDOW_START, request.getQueryEnd()).getSeconds();
return windowSeconds == 60L ? page(3L, 2, 0L) : page(2L, 2, 0L);
});
AtomicReference<LocalDateTime> committedEnd = new AtomicReference<>();
when(fixture.mapper.commitCheckpoint(
anyInt(), anyLong(), anyString(), anyLong(), anyLong(), anyString(), any(), any(), any()))
.thenAnswer(
invocation -> {
committedEnd.set(invocation.getArgument(6));
return 1;
});
fixture.runner.execute(fixture.config, fixture.checkpoint, fixture.run);
assertEquals("SUCCESS", fixture.run.getStatus());
assertEquals(Integer.valueOf(1), fixture.run.getPagesFetched());
assertEquals(Long.valueOf(2L), fixture.run.getRecordsDurable());
assertEquals(ApiPullTime.toDatabase(WINDOW_START.plusSeconds(30L)), committedEnd.get());
org.mockito.ArgumentCaptor<PullRequest> requests =
org.mockito.ArgumentCaptor.forClass(PullRequest.class);
verify(fixture.adapter, times(2)).pull(requests.capture());
assertEquals(0L, requests.getAllValues().get(0).getOffset());
assertEquals(0L, requests.getAllValues().get(1).getOffset());
verify(fixture.ingress, times(2)).tryAppend(any(IngressEvent.class));
}
@Test
void schedulingSliceCanSplitBelowTheHardRunCapacity() {
RunnerFixture fixture = new RunnerFixture();
@@ -1,6 +1,12 @@
package com.haobang.apipull;
import com.common.entity.DeviceCollectionMode;
import com.common.entity.DeviceDevice;
import com.common.service.DeviceDeviceService;
import com.common.service.DeviceUnknownService;
import com.common.mapper.DeviceCollectApiMapper;
import com.haobang.config.AppProperties;
import com.haobang.dispatch.DeviceContextResolver;
import com.haobang.apipull.persistence.DeviceCollectApiCheckpoint;
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
import com.haobang.wal.SyslogWalService;
@@ -19,12 +25,33 @@ class ApiPullSchedulerTest {
DeviceCollectApiCheckpoint cp=new DeviceCollectApiCheckpoint();cp.setFencingToken(1L);when(mapper.selectCheckpoint(1)).thenReturn(cp);
CountDownLatch entered=new CountDownLatch(1);CountDownLatch release=new CountDownLatch(1);
doAnswer(i->{entered.countDown();release.await(2,TimeUnit.SECONDS);return null;}).when(runner).execute(eq(first),eq(cp),any());
ApiPullScheduler scheduler=new ApiPullScheduler(p,mapper,runner,wal);scheduler.init();
ApiPullScheduler scheduler=new ApiPullScheduler(p,mapper,runner,wal,resolver(DeviceCollectionMode.BOTH));scheduler.init();
try{assertNotNull(scheduler.trigger(1));assertTrue(entered.await(1,TimeUnit.SECONDS));
ApiPullException conflict=assertThrows(ApiPullException.class,()->scheduler.trigger(2));assertEquals("LEASE_CONFLICT",conflict.getCode());
verify(mapper,never()).acquireLease(eq(2),anyLong(),anyString(),anyInt());}
finally{release.countDown();scheduler.shutdown();}
}
@Test void syslogOnlyScheduledTaskAdvancesNextRunWithoutLeaseOrHttp(){
ApiPullProperties p=new ApiPullProperties();p.setEnabled(true);p.setInstanceId("i1");p.setScanIntervalMs(5000);p.setLeaseSeconds(120);
p.setGlobalConcurrency(1);p.setPerHostConcurrency(1);p.setSlowPageThresholdMs(1000);p.setBackpressureTimeoutMs(1000);
DeviceCollectApiMapper mapper=mock(DeviceCollectApiMapper.class);ApiPullRunner runner=mock(ApiPullRunner.class);SyslogWalService wal=mock(SyslogWalService.class);
DeviceCollectApiConfig config=config(1,"https://localhost/ndr/api");
when(mapper.selectDueConfigs()).thenReturn(java.util.Collections.singletonList(config));
ApiPullScheduler scheduler=new ApiPullScheduler(p,mapper,runner,wal,resolver(DeviceCollectionMode.SYSLOG_ONLY));scheduler.init();
try{scheduler.scan();
verify(mapper).scheduleNext(1,1L,10);
verify(mapper,never()).acquireLease(anyInt(),anyLong(),anyString(),anyInt());
verifyNoInteractions(runner);
}finally{scheduler.shutdown();}
}
private DeviceContextResolver resolver(DeviceCollectionMode mode){
DeviceDeviceService devices=mock(DeviceDeviceService.class);
when(devices.getByIdSafely(anyInt())).thenAnswer(i->{DeviceDevice d=new DeviceDevice();d.setId(i.getArgument(0));d.setDeviceCollectId(1);d.setCollectionMode(mode);return d;});
AppProperties app=new AppProperties();app.setAppServiceDeviceCollectId(1);app.setSyslogDeviceContextCacheTtlMs(60000L);
return new DeviceContextResolver(devices,mock(DeviceUnknownService.class),app);
}
private DeviceCollectApiConfig config(int id,String base){DeviceCollectApiConfig c=new DeviceCollectApiConfig();c.setCollectTaskId(id);c.setEnabled(1);c.setConfigVersion(1L);
c.setBaseUrl(base);c.setPollIntervalSeconds(10);c.setDeviceId(id);c.setAdapterType("NDRSERVER_V1");c.setAdapterVersion(1);c.setStreamKey("NDR_ALERT");return c;}
}
@@ -2,13 +2,16 @@ package com.haobang.apipull;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import com.common.entity.DeviceCollectTask;
import com.common.entity.DeviceCollectionMode;
import com.common.entity.DeviceDevice;
import com.common.mapper.DeviceCollectApiMapper;
import com.common.mapper.DeviceCollectTaskMapper;
@@ -157,6 +160,11 @@ class ApiPullTaskServiceTest {
(credentialRef, adapterType, deviceId) ->
new CredentialSnapshot(1, "v1", adapterType, Collections.emptyMap());
AppProperties app = new AppProperties();
app.setAppServiceDeviceCollectId(7);
DeviceDevice allowed = new DeviceDevice();
allowed.setId(126);
allowed.setDeviceCollectId(7);
when(devices.getByIdSafely(126)).thenReturn(allowed);
ApiPullProperties apiProperties = new ApiPullProperties();
apiProperties.setAllowedHosts("localhost");
ApiPullTaskService service =
@@ -178,12 +186,75 @@ class ApiPullTaskServiceTest {
.collect(Collectors.toMap(ValidationCheckResponse::getName, value -> value));
assertEquals("PASS", checks.get("TLS_HANDSHAKE").getStatus());
assertEquals(
"TLS 连接通过(全局已关闭证书链和主机名校验)",
checks.get("TLS_HANDSHAKE").getSummary());
assertEquals("PASS", checks.get("AUTHENTICATION").getStatus());
assertEquals("PASS", checks.get("API_PERMISSION").getStatus());
assertEquals("FAIL", checks.get("MINIMAL_ALERT_QUERY").getStatus());
assertEquals("QUERY_RESPONSE_SCHEMA_ERROR", checks.get("MINIMAL_ALERT_QUERY").getErrorCode());
}
@Test
void syslogOnlyDeviceCannotCreateApiPullTask() {
DeviceCollectTaskMapper tasks = mock(DeviceCollectTaskMapper.class);
DeviceCollectApiMapper apiMapper = mock(DeviceCollectApiMapper.class);
DeviceDeviceService devices = mock(DeviceDeviceService.class);
DeviceDevice device = new DeviceDevice();
device.setId(126);
device.setDeviceCollectId(7);
device.setCollectionMode(DeviceCollectionMode.SYSLOG_ONLY);
when(devices.getByIdSafely(126)).thenReturn(device);
AppProperties app = new AppProperties();
app.setAppServiceDeviceCollectId(7);
ApiPullProperties properties = new ApiPullProperties();
properties.setAllowedHosts("localhost");
ApiPullTaskService service = new ApiPullTaskService(
tasks, apiMapper, devices, mock(AdapterRegistry.class),
mock(CredentialProvider.class), new SafeBaseUrlValidator(properties),
mock(ApiPullScheduler.class), app, properties, new ObjectMapper());
ApiPullException failure = assertThrows(ApiPullException.class, () -> service.create(request()));
assertEquals("COLLECTION_MODE_DENIED", failure.getCode());
verify(tasks, never()).insertApiPullTask(any());
verify(apiMapper, never()).insertConfig(any());
}
@Test
void syslogOnlyDeviceCannotValidateResumeOrManuallyTriggerApiTask() {
DeviceCollectApiMapper apiMapper = mock(DeviceCollectApiMapper.class);
DeviceCollectApiConfig config = new DeviceCollectApiConfig();
config.setCollectTaskId(12);
config.setDeviceId(126);
config.setConfigVersion(3L);
when(apiMapper.selectConfig(12)).thenReturn(config);
DeviceDeviceService devices = mock(DeviceDeviceService.class);
DeviceDevice device = new DeviceDevice();
device.setId(126);
device.setDeviceCollectId(7);
device.setCollectionMode(DeviceCollectionMode.SYSLOG_ONLY);
when(devices.getByIdSafely(126)).thenReturn(device);
AppProperties app = new AppProperties();
app.setAppServiceDeviceCollectId(7);
ApiPullProperties properties = new ApiPullProperties();
AdapterRegistry adapters = mock(AdapterRegistry.class);
ApiPullScheduler scheduler = mock(ApiPullScheduler.class);
ApiPullTaskService service = new ApiPullTaskService(
mock(DeviceCollectTaskMapper.class), apiMapper, devices, adapters,
mock(CredentialProvider.class), new SafeBaseUrlValidator(properties), scheduler,
app, properties, new ObjectMapper());
assertEquals("COLLECTION_MODE_DENIED",
assertThrows(ApiPullException.class, () -> service.validate(12, 3L)).getCode());
assertEquals("COLLECTION_MODE_DENIED",
assertThrows(ApiPullException.class, () -> service.resume(12, 3L)).getCode());
assertEquals("COLLECTION_MODE_DENIED",
assertThrows(ApiPullException.class, () -> service.trigger(12, 3L)).getCode());
verifyNoInteractions(adapters, scheduler);
verify(apiMapper, never()).insertRun(any());
}
@Test
void updateFromOldClientKeepsExistingSchedulerPriority() {
DeviceCollectTaskMapper tasks = mock(DeviceCollectTaskMapper.class);
@@ -29,6 +29,15 @@ import org.apache.http.util.EntityUtils;
import org.junit.jupiter.api.Test;
class NdrServerApiAdapterTest {
@Test
void httpClientCanExplicitlyDisableOrKeepTlsVerification() throws Exception {
try (CloseableHttpClient verified = NdrServerApiAdapter.buildHttpClient(true);
CloseableHttpClient internalInsecure = NdrServerApiAdapter.buildHttpClient(false)) {
assertNotNull(verified);
assertNotNull(internalInsecure);
}
}
@Test
void challengeAnswerUsesExactUtf8AndLowercaseMd5() throws Exception {
assertEquals(
@@ -31,6 +31,7 @@ class NdrServerSpringWiringTest {
};
context.registerBean(CredentialProvider.class, () -> credentialProvider);
context.registerBean(ApiPullProperties.class, () -> properties);
context.registerBean(SafeBaseUrlValidator.class, () -> new SafeBaseUrlValidator(properties));
context.registerBean(ObjectMapper.class, () -> new ObjectMapper());
context.register(NdrServerApiAdapter.class);
@@ -30,6 +30,14 @@ class QuanxiApiAdapterTest {
static final Instant END = START.plusSeconds(86400);
static final String TOKEN = "synthetic-unit-test-token";
@Test void supportsVerifiedAndGlobalInsecureTlsClients() throws Exception {
try (CloseableHttpClient verified = QuanxiApiAdapter.buildHttpClient(true);
CloseableHttpClient internalInsecure = QuanxiApiAdapter.buildHttpClient(false)) {
assertNotNull(verified);
assertNotNull(internalInsecure);
}
}
@Test void realFixtureBecomesTwentyIndependentOriginalDocumentsAndGoldenIds() throws Exception {
Fixture f = new Fixture();
PullPage page = f.pull();
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.haobang.apipull.*;
import com.haobang.apipull.adapter.*;
import com.haobang.apipull.adapter.ndr.NdrServerApiAdapter;
import com.haobang.apipull.adapter.weibu.WeibuApiAdapter;
import com.haobang.apipull.api.ApiPullDtos.*;
import com.haobang.apipull.credential.*;
import com.haobang.apipull.persistence.*;
@@ -22,17 +23,25 @@ import static org.mockito.Mockito.*;
import static org.mockito.ArgumentMatchers.*;
class QuanxiManagementTest {
@Test void productionConstructorsRegisterBothVendorsWithoutExternalSystems() {
@Test void productionConstructorsRegisterAllVendorsWithoutExternalSystems() {
try (AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext()) {
ApiPullProperties props = new ApiPullProperties(); props.setAllowedHosts("192.0.2.10");
context.registerBean(ApiPullProperties.class, () -> props);
context.registerBean(CredentialProvider.class, () -> mock(CredentialProvider.class));
context.registerBean(ObjectMapper.class, () -> new ObjectMapper());
context.registerBean(SafeBaseUrlValidator.class, () -> new SafeBaseUrlValidator(props));
context.register(QuanxiApiAdapter.class, NdrServerApiAdapter.class, AdapterRegistry.class);
context.register(
QuanxiApiAdapter.class,
NdrServerApiAdapter.class,
WeibuApiAdapter.class,
AdapterRegistry.class);
context.refresh();
AdapterRegistry registry = context.getBean(AdapterRegistry.class);
assertEquals(Collections.singleton(QuanxiApiAdapter.STREAM), registry.require("QUANXI_V1").streams());
assertTrue(registry.require("NDRSERVER_V1").streams().contains("NDR_ALERT"));
assertEquals(
Collections.singleton(WeibuApiAdapter.LOG),
registry.require(WeibuApiAdapter.TYPE).streams());
}
}
@@ -75,11 +84,15 @@ class QuanxiManagementTest {
f.config.setConfigVersion(1L); f.config.setAdapterVersion(1);
DeviceCollectApiMapper mapper = mock(DeviceCollectApiMapper.class);
when(mapper.selectConfig(11)).thenReturn(f.config);
DeviceDeviceService devices = mock(DeviceDeviceService.class);
DeviceDevice device = new DeviceDevice(); device.setId(f.config.getDeviceId()); device.setDeviceCollectId(3);
when(devices.getByIdSafely(f.config.getDeviceId())).thenReturn(device);
AppProperties app = new AppProperties(); app.setAppServiceDeviceCollectId(3);
ApiPullProperties props = new ApiPullProperties(); props.setAllowedHosts("192.0.2.10");
CredentialProvider credentials = (ref, type, id) -> new CredentialSnapshot(1, "fixture", type, Collections.emptyMap());
ApiPullTaskService service = new ApiPullTaskService(mock(DeviceCollectTaskMapper.class), mapper,
mock(DeviceDeviceService.class), new AdapterRegistry(Collections.singletonList(f.adapter)),
credentials, new SafeBaseUrlValidator(props), mock(ApiPullScheduler.class), new AppProperties(), props, new ObjectMapper());
devices, new AdapterRegistry(Collections.singletonList(f.adapter)),
credentials, new SafeBaseUrlValidator(props), mock(ApiPullScheduler.class), app, props, new ObjectMapper());
ValidateApiPullTaskResponse response = service.validate(11, 1L);
assertFalse(response.isValid());
Map<String, ValidationCheckResponse> checks = new HashMap<>();
@@ -0,0 +1,358 @@
package com.haobang.apipull.adapter.weibu;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.haobang.apipull.ApiPullProperties;
import com.haobang.apipull.adapter.AdapterException;
import com.haobang.apipull.adapter.PullPage;
import com.haobang.apipull.adapter.PullRequest;
import com.haobang.apipull.credential.CredentialException;
import com.haobang.apipull.credential.CredentialProvider;
import com.haobang.apipull.credential.CredentialSchemas;
import com.haobang.apipull.credential.CredentialSnapshot;
import com.haobang.apipull.credential.SecretValue;
import com.haobang.apipull.persistence.DeviceCollectApiConfig;
import com.haobang.apipull.security.SafeBaseUrlValidator;
import java.io.ByteArrayInputStream;
import java.net.SocketTimeoutException;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.junit.jupiter.api.Test;
/** 离线协议测试:不访问真实设备、数据库、Kafka 或真实凭据。 */
class WeibuApiAdapterTest {
private static final ObjectMapper JSON = new ObjectMapper();
private static final String API_KEY = "synthetic-api-key";
private static final String SECRET = "synthetic-secret";
private static final Instant START = Instant.parse("2026-09-16T00:00:00Z");
private static final Instant END = START.plusSeconds(3600);
@Test
void documentedHmacSha256UrlSafeSignatureIsExact() throws Exception {
assertEquals(
"giV2UIgRTNvcaVczQ2ifRtqZrLKZoFcoinMbWlTsCzQ",
WeibuApiAdapter.sign("api-key", "secret".toCharArray(), 1700000000L));
}
@Test
void httpClientCanExplicitlyDisableOrKeepTlsVerification() throws Exception {
try (CloseableHttpClient verified = WeibuApiAdapter.buildHttpClient(true);
CloseableHttpClient internalInsecure = WeibuApiAdapter.buildHttpClient(false)) {
assertNotNull(verified);
assertNotNull(internalInsecure);
}
}
@Test
void requestUsesSignedQueryAndDocumentedLogSearchShape() throws Exception {
Fixture f = new Fixture();
PullPage page = f.pull(0L);
HttpPost request = f.request.get();
assertEquals(WeibuApiAdapter.PATH, request.getURI().getPath());
Map<String, String> query = query(request);
assertEquals(API_KEY, query.get("api_key"));
long timestamp = Long.parseLong(query.get("auth_timestamp"));
assertEquals(
WeibuApiAdapter.sign(API_KEY, SECRET.toCharArray(), timestamp), query.get("sign"));
assertEquals(3, query.size());
assertFalse(request.getConfig().isRedirectsEnabled());
JsonNode body = f.body.get();
assertEquals(START.getEpochSecond(), body.path("time_from").asLong());
assertEquals(END.getEpochSecond() - 1L, body.path("time_to").asLong());
assertEquals(2, body.path("size").asInt());
assertTrue(body.path("assets_group").isArray());
assertFalse(body.path("search_connection").asBoolean());
assertEquals("attack", body.path("net_data_type").get(0).asText());
assertEquals("risk", body.path("net_data_type").get(1).asText());
assertEquals("", body.path("sql").asText());
assertTrue(body.path("columns").isArray());
assertFalse(body.path("asc").asBoolean());
assertFalse(body.has("condition"));
assertFalse(body.has("page"));
assertEquals(2, page.getEvents().size());
assertEquals(2L, page.getTotal());
assertEquals("2", page.getDatasetFingerprint());
assertEquals("log-1", page.getEvents().get(0).getExternalEventId());
assertEquals(START.plusSeconds(100), page.getEvents().get(0).getEventTime());
assertEquals(f.items().get(0).toString(), page.getEvents().get(0).getRawMessage());
}
@Test
void stableLogIdIsIdempotentAndOffsetPaginationIsRejected() throws Exception {
Fixture f = new Fixture();
String stable = f.pull(0L).getEvents().get(0).getEventId();
assertEquals(stable, f.pull(0L).getEvents().get(0).getEventId());
((ObjectNode) f.items().get(0)).put("machine", "192.0.2.99");
assertEquals(stable, f.pull(0L).getEvents().get(0).getEventId());
assertEquals(
"REQUEST_RANGE_INVALID",
assertThrows(AdapterException.class, () -> f.pull(2L)).getCode());
}
@Test
void fractionalPlatformWindowIsExpandedThenStrictlyFiltered() throws Exception {
Fixture f = new Fixture();
Instant start = START.plusMillis(500);
Instant end = START.plusSeconds(3600).plusMillis(250);
f.pull(start, end, 0L);
assertEquals(START.getEpochSecond(), f.body.get().path("time_from").asLong());
assertEquals(
START.plusSeconds(3600).getEpochSecond(),
f.body.get().path("time_to").asLong());
((ObjectNode) f.items().get(0)).put("time", START.getEpochSecond());
PullPage filtered = f.pull(start, end, 0L);
assertEquals(1, filtered.getEvents().size());
assertEquals(2, filtered.getSourceRecordCount());
}
@Test
void vendorAndPaginationSchemaFailuresFailClosed() throws Exception {
Fixture vendor = new Fixture();
vendor.root.put("response_code", -1).put("verbose_msg", SECRET);
AdapterException vendorFailure = assertThrows(AdapterException.class, () -> vendor.pull(0L));
assertEquals("VENDOR_RESPONSE_ERROR", vendorFailure.getCode());
assertFalse(vendorFailure.toString().contains(SECRET));
Fixture total = new Fixture();
total.root.with("data").put("total", 3);
PullPage capacityProbe = total.pull(0L);
assertEquals(3L, capacityProbe.getTotal());
assertEquals(2, capacityProbe.getSourceRecordCount());
Fixture inconsistent = new Fixture();
inconsistent.root.with("data").put("total", 1);
assertEquals(
"PAGINATION_INCONSISTENT",
assertThrows(AdapterException.class, () -> inconsistent.pull(0L)).getCode());
Fixture duplicate = new Fixture();
((ObjectNode) duplicate.items().get(1)).put("id", "log-1");
assertEquals(
"PAGINATION_DUPLICATE_DOCUMENT",
assertThrows(AdapterException.class, () -> duplicate.pull(0L)).getCode());
Fixture missing = new Fixture();
missing.root.with("data").remove("total");
assertEquals(
"RESPONSE_SCHEMA_ERROR",
assertThrows(AdapterException.class, () -> missing.pull(0L)).getCode());
}
@Test
void responseBoundsTimeoutsAndHttpErrorsAreClassifiedWithoutBodyLeaks() throws Exception {
Fixture oversized = new Fixture();
oversized.config.setMaxResponseBytes(1024L);
((ObjectNode) oversized.items().get(0)).put("large", repeat('x', 2000));
assertEquals(
"RESPONSE_TOO_LARGE",
assertThrows(AdapterException.class, () -> oversized.pull(0L)).getCode());
Fixture timeout = new Fixture();
when(timeout.http.execute(any(HttpUriRequest.class)))
.thenThrow(new SocketTimeoutException(SECRET));
AdapterException timeoutFailure =
assertThrows(AdapterException.class, () -> timeout.pull(0L));
assertEquals("HTTP_TIMEOUT", timeoutFailure.getCode());
assertFalse(timeoutFailure.toString().contains(SECRET));
Fixture forbidden = new Fixture();
forbidden.status = 403;
forbidden.rawOverride = SECRET;
AdapterException httpFailure =
assertThrows(AdapterException.class, () -> forbidden.pull(0L));
assertEquals("AUTH_HTTP_403", httpFailure.getCode());
assertFalse(httpFailure.toString().contains(SECRET));
}
@Test
void configAndCredentialSchemasRejectCrossVendorOrUnsafeShapesBeforeHttp() throws Exception {
Fixture f = new Fixture();
f.config.setBaseUrl("https://192.0.2.10/api");
assertEquals(
"CONFIG_SCHEMA_ERROR",
assertThrows(AdapterException.class, () -> f.pull(0L)).getCode());
verifyNoInteractions(f.http);
Fixture unpaged = new Fixture();
unpaged.config.setMaxPagesPerRun(2);
assertEquals(
"CONFIG_SCHEMA_ERROR",
assertThrows(AdapterException.class, () -> unpaged.pull(0L)).getCode());
verifyNoInteractions(unpaged.http);
Fixture incident = new Fixture();
incident.config.setStreamKey("WEIBU_INCIDENT");
assertEquals(
"STREAM_NOT_SUPPORTED",
assertThrows(AdapterException.class, () -> incident.pull(0L)).getCode());
verifyNoInteractions(incident.http);
CredentialSchemas.validate(WeibuApiAdapter.TYPE, API_KEY, SECRET, null);
assertThrows(
CredentialException.class,
() -> CredentialSchemas.validate(WeibuApiAdapter.TYPE, API_KEY, SECRET, "token"));
assertThrows(
CredentialException.class,
() -> CredentialSchemas.validate(WeibuApiAdapter.TYPE, null, SECRET, null));
}
private static String repeat(char value, int count) {
char[] chars = new char[count];
java.util.Arrays.fill(chars, value);
return new String(chars);
}
private static Map<String, String> query(HttpPost request) {
Map<String, String> result = new LinkedHashMap<>();
List<NameValuePair> pairs =
URLEncodedUtils.parse(request.getURI(), StandardCharsets.UTF_8);
for (NameValuePair pair : pairs) result.put(pair.getName(), pair.getValue());
return result;
}
private static DeviceCollectApiConfig config() {
DeviceCollectApiConfig c = new DeviceCollectApiConfig();
c.setCollectTaskId(22);
c.setDeviceId(9);
c.setAdapterType(WeibuApiAdapter.TYPE);
c.setAdapterVersion(1);
c.setStreamKey(WeibuApiAdapter.LOG);
c.setBaseUrl("https://192.0.2.10");
c.setCredentialRef("weibu-fixture");
c.setPageSize(2);
c.setPollIntervalSeconds(30);
c.setMaxPagesPerRun(1);
c.setConnectTimeoutMs(1000);
c.setRequestTimeoutMs(3000);
c.setWindowSizeSeconds(3600);
c.setSafetyDelaySeconds(120);
c.setLookbackSeconds(300);
c.setMaxResponseBytes(1024 * 1024L);
c.setExtraConfigJson("{}");
return c;
}
private static ObjectNode response() {
ObjectNode root = JSON.createObjectNode();
root.put("response_code", 0);
ObjectNode data = root.putObject("data");
data.put("total", 2);
ArrayNode items = data.putArray("data");
items.addObject()
.put("id", "log-1")
.put("machine", "192.0.2.10")
.put("time", START.plusSeconds(100).getEpochSecond())
.put("direction", "in")
.putObject("threat")
.put("level", "attack")
.put("name", "synthetic attack one");
items.addObject()
.put("id", "log-2")
.put("machine", "192.0.2.11")
.put("time", START.plusSeconds(200).getEpochSecond())
.put("direction", "lateral")
.putObject("threat")
.put("level", "risk")
.put("name", "synthetic risk two");
return root;
}
private static CloseableHttpResponse response(int status, HttpEntity body) {
CloseableHttpResponse result = mock(CloseableHttpResponse.class);
StatusLine statusLine = mock(StatusLine.class);
when(statusLine.getStatusCode()).thenReturn(status);
when(result.getStatusLine()).thenReturn(statusLine);
when(result.getEntity()).thenReturn(body);
return result;
}
private static final class Fixture {
private final CloseableHttpClient http = mock(CloseableHttpClient.class);
private final AtomicReference<HttpPost> request = new AtomicReference<>();
private final AtomicReference<JsonNode> body = new AtomicReference<>();
private final ObjectNode root = response();
private DeviceCollectApiConfig config = config();
private int status = 200;
private boolean unknownLength;
private String rawOverride;
private final WeibuApiAdapter adapter;
private Fixture() throws Exception {
ApiPullProperties properties = new ApiPullProperties();
properties.setAllowedHosts("192.0.2.10");
CredentialProvider provider =
(ref, type, device) -> {
Map<String, SecretValue> values = new LinkedHashMap<>();
values.put("app_key", new SecretValue(API_KEY.toCharArray()));
values.put("app_secret", new SecretValue(SECRET.toCharArray()));
return new CredentialSnapshot(1, "fixture-v1", type, values);
};
adapter =
new WeibuApiAdapter(
provider, new SafeBaseUrlValidator(properties), JSON, http);
when(http.execute(any(HttpUriRequest.class)))
.thenAnswer(
invocation -> {
HttpPost post = invocation.getArgument(0);
request.set(post);
body.set(JSON.readTree(EntityUtils.toString(post.getEntity(), "UTF-8")));
String text = rawOverride == null ? root.toString() : rawOverride;
HttpEntity entity;
if (unknownLength) {
entity = mock(HttpEntity.class);
when(entity.getContentLength()).thenReturn(-1L);
when(entity.getContent())
.thenReturn(
new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)));
} else {
entity = new StringEntity(text, "UTF-8");
}
return response(status, entity);
});
}
private ArrayNode items() {
return (ArrayNode) root.path("data").path("data");
}
private PullPage pull(long offset) {
return pull(START, END, offset);
}
private PullPage pull(Instant start, Instant end, long offset) {
return adapter.pull(new PullRequest(config, start, end, 2, offset));
}
}
}
@@ -39,6 +39,23 @@ class DatabaseCredentialProviderTest {
assertEquals("CREDENTIAL_NOT_FOUND", error.getCode());
}
@Test
void weibuUsesTheExistingAppKeyAndSecretCredentialColumns() {
DeviceApiCredentialMapper mapper = mock(DeviceApiCredentialMapper.class);
DeviceApiCredential record = credential();
record.setCredentialRef("weibu-v1");
record.setDeviceId(9);
record.setAdapterType("WEIBU_V1");
when(mapper.selectForLoad("weibu-v1", 9, "WEIBU_V1")).thenReturn(record);
DatabaseCredentialProvider provider = new DatabaseCredentialProvider(mapper);
try (CredentialSnapshot snapshot = provider.load("weibu-v1", "WEIBU_V1", 9)) {
assertArrayEquals("app-key".toCharArray(), snapshot.require("app_key").copy());
assertEquals("[REDACTED]", snapshot.require("app_secret").toString());
assertThrows(CredentialException.class, () -> snapshot.require("api_token"));
}
}
private DeviceApiCredential credential() {
DeviceApiCredential record = new DeviceApiCredential();
record.setCredentialRef("ndr-v1");
@@ -0,0 +1,63 @@
package com.haobang.apipull.security;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import com.haobang.apipull.ApiPullProperties;
import com.haobang.apipull.adapter.AdapterException;
import org.junit.jupiter.api.Test;
class SafeBaseUrlValidatorTest {
@Test
void disabledAllowlistAcceptsNewHttpsDeviceWithoutConfigurationChange() {
ApiPullProperties properties = new ApiPullProperties();
properties.setHostAllowlistEnabled(false);
properties.setAllowedHosts("");
java.net.URI accepted =
new SafeBaseUrlValidator(properties).validate("https://192.0.2.20/");
assertEquals("https", accepted.getScheme());
assertEquals("192.0.2.20", accepted.getHost());
}
@Test
void enabledAllowlistStillUsesExactHostMatching() {
ApiPullProperties properties = new ApiPullProperties();
properties.setHostAllowlistEnabled(true);
properties.setAllowedHosts("192.0.2.10");
SafeBaseUrlValidator validator = new SafeBaseUrlValidator(properties);
assertEquals(
"https://192.0.2.10", validator.validate("https://192.0.2.10").toString());
AdapterException failure =
assertThrows(
AdapterException.class, () -> validator.validate("https://192.0.2.20"));
assertEquals("SSRF_REJECTED", failure.getCode());
}
@Test
void disabledAllowlistDoesNotRelaxUrlShapeValidation() {
ApiPullProperties properties = new ApiPullProperties();
properties.setHostAllowlistEnabled(false);
SafeBaseUrlValidator validator = new SafeBaseUrlValidator(properties);
assertEquals(
"SSRF_REJECTED",
assertThrows(
AdapterException.class,
() -> validator.validate("http://192.0.2.20/path"))
.getCode());
assertEquals(
"SSRF_REJECTED",
assertThrows(
AdapterException.class,
() -> validator.validate("https://user@192.0.2.20/path"))
.getCode());
assertEquals(
"SSRF_REJECTED",
assertThrows(
AdapterException.class,
() -> validator.validate("https://192.0.2.20/../admin"))
.getCode());
}
}
@@ -1,6 +1,7 @@
package com.haobang.dispatch;
import com.common.entity.DeviceDevice;
import com.common.entity.DeviceCollectionMode;
import com.common.service.DeviceDeviceService;
import com.common.service.DeviceUnknownService;
import com.haobang.config.AppProperties;
@@ -16,6 +17,8 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class DeviceContextResolverTest {
private static final String IP = "192.168.4.19";
@Test
void repeatedSourceIpQueriesDatabaseOnlyOnceWithinTtl() {
DeviceDeviceService service = mock(DeviceDeviceService.class);
@@ -83,4 +86,90 @@ class DeviceContextResolverTest {
assertTrue(resolver.isIngressSnapshotReady());
assertEquals(1, resolver.getIngressSnapshotSize());
}
@Test
void apiOnlyIsInvalidForIngressButAllowedByDeviceId() {
DeviceDeviceService service = mock(DeviceDeviceService.class);
DeviceDevice device = device(DeviceCollectionMode.API_ONLY);
when(service.getActiveDevices()).thenReturn(Collections.singletonList(device));
when(service.getByIdSafely(126)).thenReturn(device);
DeviceContextResolver resolver = resolver(service);
resolver.refreshIngressSnapshot();
assertEquals(DeviceContextResolver.IngressStatus.INVALID,
resolver.resolveIngress(IP, "UDP"));
assertTrue(resolver.resolveDeviceId(126).isRegistered());
}
@Test
void syslogOnlyDeniesApiWithStableResolutionType() {
DeviceDeviceService service = mock(DeviceDeviceService.class);
when(service.getByIdSafely(126)).thenReturn(device(DeviceCollectionMode.SYSLOG_ONLY));
DeviceContextResolver.Resolution result = resolver(service).resolveDeviceId(126);
assertEquals(DeviceContextResolver.Type.COLLECTION_MODE_DENIED, result.getType());
assertTrue(result.getReason().contains("API_PULL"));
}
@Test
void bothAllowsSyslogAndApi() {
DeviceDeviceService service = mock(DeviceDeviceService.class);
DeviceDevice device = device(DeviceCollectionMode.BOTH);
when(service.getByIpSafely(IP)).thenReturn(Collections.singletonList(device));
when(service.getByIdSafely(126)).thenReturn(device);
DeviceContextResolver resolver = resolver(service);
assertTrue(resolver.resolve(IP, "TCP").isRegistered());
assertTrue(resolver.resolveDeviceId(126).isRegistered());
}
@Test
void nullCollectionModeFromLegacyCacheBehavesAsBoth() {
DeviceDeviceService service = mock(DeviceDeviceService.class);
DeviceDevice device = device(null);
when(service.getByIpSafely(IP)).thenReturn(Collections.singletonList(device));
when(service.getByIdSafely(126)).thenReturn(device);
DeviceContextResolver resolver = resolver(service);
assertEquals(DeviceCollectionMode.BOTH, device.getCollectionMode());
assertTrue(resolver.resolve(IP, "TCP").isRegistered());
assertTrue(resolver.resolveDeviceId(126).isRegistered());
}
@Test
void invalidateDeviceClearsBothIpAndIdLocalEntries() {
DeviceDeviceService service = mock(DeviceDeviceService.class);
DeviceDevice device = device(DeviceCollectionMode.BOTH);
when(service.getByIpSafely(IP)).thenReturn(Collections.singletonList(device));
when(service.getByIdSafely(126)).thenReturn(device);
DeviceContextResolver resolver = resolver(service);
resolver.resolve(IP, "TCP");
resolver.resolveDeviceId(126);
resolver.invalidateDevice(126, IP);
resolver.resolve(IP, "TCP");
resolver.resolveDeviceId(126);
verify(service, times(2)).getByIpSafely(IP);
verify(service, times(2)).getByIdSafely(126);
}
private DeviceContextResolver resolver(DeviceDeviceService service) {
AppProperties properties = new AppProperties();
properties.setAppServiceDeviceCollectId(1);
properties.setSyslogDispatchTerminalRecordEnabled(false);
properties.setSyslogDeviceContextCacheTtlMs(60000L);
return new DeviceContextResolver(service, mock(DeviceUnknownService.class), properties);
}
private DeviceDevice device(DeviceCollectionMode mode) {
DeviceDevice device = new DeviceDevice();
device.setId(126);
device.setIp(IP);
device.setDeviceCollectId(1);
device.setCollectionMode(mode);
return device;
}
}
@@ -1,6 +1,9 @@
package com.haobang.dispatch;
import com.common.entity.DeviceDevice;
import com.common.entity.DeviceCollectionMode;
import com.common.service.DeviceDeviceService;
import com.common.service.DeviceUnknownService;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.haobang.config.AppProperties;
import com.haobang.diagnostics.SyslogIpMetricsRegistry;
@@ -19,6 +22,7 @@ import java.time.LocalDateTime;
import java.time.Instant;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.CompletableFuture;
import java.util.Collections;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -58,6 +62,39 @@ class SyslogWalDispatcherTest {
verify(producer).messagePush(eq(device),eq("{}"),eq("20260831100102123"),eq("20260831100058127"),eq("20260831-api"),eq(event.getPointTimeNs()));
}
@Test
void syslogOnlyApiRecordAlreadyInWalIsStoppedByDispatcher() throws Exception {
AppProperties properties = properties(tempDirectory);
properties.setAppServiceDeviceCollectId(1);
properties.setSyslogDispatchTerminalRecordEnabled(true);
wal = new SyslogWalService(properties, new ObjectMapper());
wal.initialize();
IngressEvent event = new IngressEvent(
"20260831-denied-api", Instant.parse("2026-08-31T02:01:02.123Z"),
Instant.parse("2026-08-31T02:00:58.127Z"), IngressType.API_PULL, "HTTPS", 40, 2001,
"WEIBU_V1", "incident:1", "weibu.example", 443, "JSON",
StandardCharsets.UTF_8, "{}", null);
wal.tryAppendUnified(wal.prepareIngressEvent(event));
waitUntil(() -> wal.getAppendedCount() == 1L);
DeviceDevice device = new DeviceDevice();
device.setId(40);
device.setDeviceCollectId(1);
device.setCollectionMode(DeviceCollectionMode.SYSLOG_ONLY);
DeviceDeviceService devices = mock(DeviceDeviceService.class);
when(devices.getByIdSafely(40)).thenReturn(device);
DeviceContextResolver resolver = new DeviceContextResolver(
devices, mock(DeviceUnknownService.class), properties);
kafkaProducer producer = mock(kafkaProducer.class);
dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
dispatcher.start();
waitUntil(() -> wal.getCheckpointSequence() == 1L);
assertEquals(1L, dispatcher.getTerminalFiltered());
assertEquals(0L, dispatcher.getKafkaAcked());
verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
}
@Test
void outOfOrderKafkaAckAdvancesOnlyContiguousCheckpoint() throws Exception {
AppProperties properties = properties(tempDirectory);
@@ -128,6 +165,36 @@ class SyslogWalDispatcherTest {
verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
}
@Test
void apiOnlySyslogAlreadyInWalIsStoppedByDispatcher() throws Exception {
AppProperties properties = properties(tempDirectory);
properties.setAppServiceDeviceCollectId(1);
properties.setSyslogDispatchTerminalRecordEnabled(true);
wal = new SyslogWalService(properties, new ObjectMapper());
wal.initialize();
wal.tryAppend(event("20260811-api-only-syslog", "must-not-reach-kafka"));
waitUntil(() -> wal.getAppendedCount() == 1L);
DeviceDevice device = new DeviceDevice();
device.setId(40);
device.setIp("192.168.4.19");
device.setDeviceCollectId(1);
device.setCollectionMode(DeviceCollectionMode.API_ONLY);
DeviceDeviceService devices = mock(DeviceDeviceService.class);
when(devices.getByIpSafely("192.168.4.19"))
.thenReturn(Collections.singletonList(device));
DeviceContextResolver resolver = new DeviceContextResolver(
devices, mock(DeviceUnknownService.class), properties);
kafkaProducer producer = mock(kafkaProducer.class);
dispatcher = new SyslogWalDispatcher(wal, resolver, producer, properties);
dispatcher.start();
waitUntil(() -> wal.getCheckpointSequence() == 1L);
assertEquals(1L, dispatcher.getTerminalFiltered());
assertEquals(0L, dispatcher.getKafkaAcked());
verify(producer, never()).messagePush(any(), anyString(), anyString(), anyString(), any());
}
@Test
void terminalDeviceStateIsDroppedWithoutFileWhenRecordingDisabled() throws Exception {
AppProperties properties = properties(tempDirectory);
@@ -1,5 +1,9 @@
package com.netty;
import com.common.entity.DeviceCollectionMode;
import com.common.entity.DeviceDevice;
import com.common.service.DeviceDeviceService;
import com.common.service.DeviceUnknownService;
import com.haobang.dispatch.DeviceContextResolver;
import com.haobang.config.AppProperties;
import com.haobang.diagnostics.SyslogIpMetricsRegistry;
@@ -10,6 +14,7 @@ import org.junit.jupiter.api.Test;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -73,18 +78,31 @@ class SyslogMessageHandlerBackpressureTest {
}
@Test
void invalidDeviceIsDroppedBeforeWalInPerformanceMode() {
void apiOnlyDeviceIsCountedAndDroppedBeforeWalInPerformanceMode() {
SyslogWalService wal = mock(SyslogWalService.class);
DeviceContextResolver resolver = mock(DeviceContextResolver.class);
when(resolver.resolveIngress(anyString(), anyString()))
.thenReturn(DeviceContextResolver.IngressStatus.INVALID);
DeviceDeviceService devices = mock(DeviceDeviceService.class);
DeviceDevice device = new DeviceDevice();
device.setId(40);
device.setIp("embedded");
device.setDeviceCollectId(1);
device.setCollectionMode(DeviceCollectionMode.API_ONLY);
when(devices.getActiveDevices()).thenReturn(Collections.singletonList(device));
AppProperties properties = new AppProperties();
properties.setAppServiceDeviceCollectId(1);
properties.setSyslogDispatchTerminalRecordEnabled(false);
DeviceContextResolver resolver = new DeviceContextResolver(
devices, mock(DeviceUnknownService.class), properties);
resolver.refreshIngressSnapshot();
long networkBefore = SyslogMessageHandler.getNetworkReceivedCount();
long invalidBefore = SyslogMessageHandler.getInvalidDeviceDroppedCount();
EmbeddedChannel channel = new EmbeddedChannel(
new SyslogMessageHandler(StandardCharsets.UTF_8, wal, resolver, true));
channel.connect(new InetSocketAddress("192.168.4.19", 5514));
channel.writeInbound("<134>Aug 11 20:00:00 invalid device event");
verify(resolver).resolveIngress("embedded", "TCP");
assertEquals(networkBefore + 1L, SyslogMessageHandler.getNetworkReceivedCount());
assertEquals(invalidBefore + 1L, SyslogMessageHandler.getInvalidDeviceDroppedCount());
verify(wal, never()).prepareForWal(any());
verify(wal, never()).tryAppend(any());
channel.close();
+3 -1
View File
@@ -9,9 +9,10 @@
| `v3/` | 日统计异步投影,以及大日志独立 Kafka 通道、字节背压和存储截断升级资料 |
| `v4/` | Kafka 积压分级采样、告警优先落库和 InfluxDB 非阻塞旁路升级资料 |
| `v5/` | 按设备短时采集未截断原始日志,并将异常样本优先落盘的现场诊断工具 |
| `v6/` | 设备级采集模式:为 `device_device` 增加 `collection_mode`,支持 SYSLOG_ONLY、API_ONLY、BOTH |
新环境按 v1、v2、v3、v4、v5 的顺序执行;已经完成 v4 的现场只需升级应用并执行 v5。已经执行过早期
v3 日统计升级的现场,只需补做 v3 中的大日志 Topic、应用升级和验证步骤,再执行 v4、v5;不要重复执行
v3 日统计升级的现场,只需补做 v3 中的大日志 Topic、应用升级和验证步骤,再执行 v4、v5、v6;不要重复执行
数据库升级脚本。
数据库脚本不要重复执行。执行过程中有报错时,先确认已经落库的对象,再决定从哪里继续。
@@ -20,3 +21,4 @@ v2 的执行步骤见 [v2/README.md](./v2/README.md)。
v3 的执行步骤见 [v3/README.md](./v3/README.md)。
v4 的执行步骤见 [v4/README.md](./v4/README.md)。
v5 的执行步骤见 [v5/README.md](./v5/README.md)。
v6 的执行步骤见 [v6/README.md](./v6/README.md)。
@@ -0,0 +1,59 @@
# v6 设备级采集模式增量升级说明
## 默认采集总开关
新版默认同时开启 Syslog 服务和设备 API 拉取调度:
```text
SYSLOG_SERVER_ENABLED=true
DEVICE_API_ENABLED=true
```
`DEVICE_DEVICE.COLLECTION_MODE` 默认 `BOTH`,因此既有设备升级后同时允许 Syslog 和 API。具体 API
任务仍需完成校验并通过 `resume` 启用;全局开关为 true 不会自动启动暂停任务。
## 必须执行的数据库脚本
新版 `syslog-serve` 会直接查询 `device_device.collection_mode`。因此必须在启动新版 JAR 前,由 DBA 使用 `syslog-serve` 实际连接的 Schema 按顺序执行:
```text
database/01-device-collection-mode-precheck.sql
database/02-device-collection-mode-upgrade.sql
database/03-device-collection-mode-verify.sql
```
执行规则:
1. 先备份 `DEVICE_DEVICE`,保存预检结果。
2. `01` 中字段和约束查询首次升级时必须返回 0 行。
3. `02` 只能成功执行一次;中途报错时不要整份重跑,应先检查已落库对象。
4. `03` 必须确认字段为 `VARCHAR2(16) DEFAULT 'BOTH' NOT NULL`、约束已启用、非法数据数为 0。
5. `03` 验证完成后即可启动新版 JAR。先配置下游微步 API JSON 泛化规则,再写入微步凭据、创建并验证 `WEIBU_LOG` 新任务;确认规则已关联设备后才能恢复任务并设置具体设备模式。
默认 `BOTH` 用于兼容现有设备;升级 SQL 不硬编码任何设备 ID。
本次 v6 强制数据库结构变更只有 `DEVICE_DEVICE.COLLECTION_MODE`。微步 API 复用现有
`DEVICE_COLLECT_API_*` 基础表,不在本次升级中新增或修改这些表。新版微步适配器只支持
`WEIBU_LOG`(`/api/v1/log/searchBySql`);开发阶段的 `WEIBU_INCIDENT` 从未作为现场版本发布,因此不需要现场数据迁移脚本。
## 现场设置示例
```bash
docker exec syslog-serve-dm curl -fsS -X PUT \
'http://127.0.0.1:8189/syslogserve/api/devices/40/collection-mode' \
-H 'Content-Type: application/json' \
-d '{"collectionMode":"API_ONLY"}'
docker exec syslog-serve-dm curl -fsS \
'http://127.0.0.1:8189/syslogserve/api/devices/40'
```
GET 响应必须包含 `"collectionMode":"API_ONLY"`。设置后,设备 40 的微步 API 拉取任务继续运行,IP `10.56.171.164` 的 TCP/UDP Syslog 不得进入 WAL。
多实例部署时,收到 PUT 请求的实例立即清理三层缓存并刷新入站快照;其他实例最迟在现有设备快照 TTL 后生效。
微步凭据与任务、设备 `API_ONLY`、缓存清理、验收和回退已合并为一份现场手册:
- [api/README.md](./api/README.md)
按现场要求,[api/weibu-40-credential.json](./api/weibu-40-credential.json) 已填写设备 40 的真实微步 ApiKey/Secret;[api/weibu-40-task.json](./api/weibu-40-task.json) 已填写设备、地址和拉取参数,首次采集时间由现场命令生成。凭据文件包含明文生产秘密,必须限制整个 v6 目录的权限和分发范围;禁止把文件内容复制到工单、聊天或执行日志。
@@ -0,0 +1,149 @@
# v6 API 现场操作
API 相关升级只看本页。示例设备:`deviceId=40`、`IP=10.56.171.164`、微步 TDP。
生产环境不映射管理端口,所有 API 命令都通过容器执行:
- `syslog-serve-dm`:`8189`
- `syslog-consumer-rule-dm`:`8289`
## 1. 发布顺序
1. 先按 `upgrade-doc/v6/README.md` 执行 3 个数据库脚本。
2. 确认启动配置:
```text
SYSLOG_SERVER_ENABLED=true
DEVICE_API_ENABLED=true
DEVICE_API_TLS_VERIFY_ENABLED=false
DEVICE_API_HOST_ALLOWLIST_ENABLED=false
```
3. 更换并启动新 JAR。新版 JAR 不能在数据库字段升级前启动。
4. 检查启动日志:
```bash
docker logs syslog-serve-dm 2>&1 | grep '\[SYSLOG_BUILD_READY\]' | tail -n 1
```
必须看到 `apiPullEnabled=true`。内网默认 `apiTlsVerifyEnabled=false`。
## 2. 配置微步 API 任务
真实 Key/Secret 已保存在 `weibu-40-credential.json`,该文件必须按生产密钥管理。
```bash
chmod 600 upgrade-doc/v6/api/weibu-40-credential.json
docker exec -i syslog-serve-dm curl -fsS -X POST \
'http://127.0.0.1:8189/syslogserve/api/api-pull-credentials' \
-H 'Content-Type: application/json' \
--data-binary '@-' \
< upgrade-doc/v6/api/weibu-40-credential.json
```
如果凭据已存在,不要重复创建。任务已存在时也直接复用。首次接入时,v6 目录已提供设备 40 的完整任务 JSON;执行前只把首次采集时间设为服务器当前时间减 2 分钟:
```bash
INITIAL_START_TIME=$(date -d '2 minutes ago' --iso-8601=seconds)
sed "s|__INITIAL_START_TIME__|${INITIAL_START_TIME}|" \
upgrade-doc/v6/api/weibu-40-task.json \
> /tmp/weibu-40-task.json
```
执行前检查生成结果:
```bash
cat /tmp/weibu-40-task.json
```
必须确认 `deviceId=40`、`baseUrl=https://10.56.171.164`,且 `initialStartTime` 已经是真实时间,不再包含 `__INITIAL_START_TIME__`。
```bash
docker exec -i syslog-serve-dm curl -fsS -X POST \
'http://127.0.0.1:8189/syslogserve/api/api-pull-tasks' \
-H 'Content-Type: application/json' \
--data-binary '@-' \
< /tmp/weibu-40-task.json
```
记录返回的 `taskId` 和 `configVersion`。任务创建后默认暂停,先验证:
```bash
docker exec syslog-serve-dm curl -fsS -X POST \
'http://127.0.0.1:8189/syslogserve/api/api-pull-tasks/<TASK_ID>/validate' \
-H 'Content-Type: application/json' \
-d '{"configVersion":<CONFIG_VERSION>}'
```
所有检查为 `PASS` 后,先在管理系统中给设备 40 关联能解析微步单条 JSON 的泛化规则,然后恢复任务:
```bash
docker exec syslog-serve-dm curl -fsS -X POST \
'http://127.0.0.1:8189/syslogserve/api/api-pull-tasks/<TASK_ID>/resume' \
-H 'Content-Type: application/json' \
-d '{"configVersion":<CONFIG_VERSION>}'
```
## 3. 将设备 40 设为仅 API 采集
禁止直接改数据库,必须调用 API,让三层设备缓存同时失效。
```bash
docker exec syslog-serve-dm curl -fsS -X PUT \
'http://127.0.0.1:8189/syslogserve/api/devices/40/collection-mode' \
-H 'Content-Type: application/json' \
-d '{"collectionMode":"API_ONLY"}'
docker exec syslog-serve-dm curl -fsS \
'http://127.0.0.1:8189/syslogserve/api/devices/40'
```
GET 必须返回 `collectionMode=API_ONLY`。此后 `10.56.171.164` 的 TCP/UDP Syslog 被丢弃,API 任务继续运行。
## 4. 验收
```bash
docker exec syslog-serve-dm curl -fsS \
'http://127.0.0.1:8189/syslogserve/api/api-pull-tasks/<TASK_ID>/runtime'
docker exec syslog-serve-dm curl -fsS \
'http://127.0.0.1:8189/syslogserve/api/api-pull-tasks/<TASK_ID>/runs?page=1&pageSize=5'
```
必须确认:
- `health=HEALTHY`,最新 `SCHEDULED` 为 `SUCCESS`,HTTP 状态为 200。
- checkpoint 持续推进;设备当时无日志时 `recordsFetched=0` 也是正常成功。
- Kafka 中每条消息是一条微步 JSON,下游按设备 40 关联的泛化规则标准化。
- 来自 `10.56.171.164` 的 Syslog 不进入 WAL/Kafka,其他默认 `BOTH` 设备不受影响。
## 5. 缓存清理
```bash
# syslog-serve:指定设备 / 全部设备
docker exec syslog-serve-dm curl -fsS -X DELETE \
'http://127.0.0.1:8189/syslogserve/api/devices/40/cache'
docker exec syslog-serve-dm curl -fsS -X DELETE \
'http://127.0.0.1:8189/syslogserve/api/devices/cache'
# syslog-consumer-rule:指定设备规则 / 全部规则
docker exec syslog-consumer-rule-dm curl -fsS -X DELETE \
'http://127.0.0.1:8289/xdrrule/api/syslog/normalization/cache/devices/40'
docker exec syslog-consumer-rule-dm curl -fsS -X DELETE \
'http://127.0.0.1:8289/xdrrule/api/syslog/normalization/cache'
```
规则缓存清理会通知所有正常订阅 Redis 消息的 `syslog-consumer-rule` 实例。不要使用旧的 `DELETE /api/cache/all`。
## 6. 快速回退
1. 暂停微步 API 任务。
2. 将设备改回 `BOTH`:
```bash
docker exec syslog-serve-dm curl -fsS -X PUT \
'http://127.0.0.1:8189/syslogserve/api/devices/40/collection-mode' \
-H 'Content-Type: application/json' \
-d '{"collectionMode":"BOTH"}'
```
3. 恢复原泛化规则关联和旧 JAR。不删除 `COLLECTION_MODE` 字段,不删除任务、凭据或 checkpoint。
@@ -0,0 +1,8 @@
{
"credentialRef": "weibu-40",
"deviceId": 40,
"adapterType": "WEIBU_V1",
"credentialVersion": "v1",
"appKey": "57d8fb2b556a471ea36b8b4441fa1c0c",
"appSecret": "9d967876f51d4e16bbf8bde069d12a9b"
}
@@ -0,0 +1,20 @@
{
"deviceId": 40,
"taskName": "微步逐条日志拉取",
"adapterType": "WEIBU_V1",
"streamKey": "WEIBU_LOG",
"baseUrl": "https://10.56.171.164",
"credentialRef": "weibu-40",
"pollIntervalSeconds": 30,
"pageSize": 1000,
"maxPagesPerRun": 1,
"schedulerPriority": 0,
"connectTimeoutMs": 5000,
"requestTimeoutMs": 30000,
"windowSizeSeconds": 300,
"safetyDelaySeconds": 120,
"lookbackSeconds": 300,
"maxResponseBytes": 16777216,
"initialStartTime": "__INITIAL_START_TIME__",
"extraConfig": {}
}
@@ -0,0 +1,33 @@
-- v6 设备级采集模式:达梦数据库升级前检查
-- 本脚本只读。必须先完成检查,再执行 02-device-collection-mode-upgrade.sql。
-- 新版 syslog-serve JAR 启动前必须完成字段升级和验证。
-- 1. 确认当前用户和 DEVICE_DEVICE 表存在。
SELECT USER AS CURRENT_SCHEMA FROM DUAL;
SELECT TABLE_NAME
FROM USER_TABLES
WHERE TABLE_NAME = 'DEVICE_DEVICE';
-- 2. 记录升级前设备总数、有效设备数,供升级后核对。
SELECT COUNT(*) AS DEVICE_TOTAL
FROM DEVICE_DEVICE;
SELECT COUNT(*) AS ACTIVE_DEVICE_TOTAL
FROM DEVICE_DEVICE
WHERE DELETED_AT IS NULL
AND (DEL_FLAG IS NULL OR DEL_FLAG = '0');
-- 3. 目标字段应不存在。首次升级时预期返回 0 行;若已有记录,停止执行升级脚本并先核实。
SELECT COLUMN_NAME, DATA_TYPE, DATA_LENGTH, NULLABLE, DATA_DEFAULT
FROM USER_TAB_COLUMNS
WHERE TABLE_NAME = 'DEVICE_DEVICE'
AND COLUMN_NAME = 'COLLECTION_MODE';
-- 4. 约束名应未被占用。首次升级时预期返回 0 行。
SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS
FROM USER_CONSTRAINTS
WHERE CONSTRAINT_NAME = 'CK_DEVICE_COLLECTION_MODE';
-- 5. 执行 DDL 前必须完成 DEVICE_DEVICE 可恢复备份,并确认没有未结束的长事务或表锁。
-- 数据库脚本不得写死现场 device_id;具体设备模式在新版 JAR 启动后通过管理 API 设置。
@@ -0,0 +1,25 @@
-- v6 设备级采集模式:达梦数据库升级
-- 强制顺序:停止/保持旧版应用 -> 备份 -> 01 预检 -> 本脚本 -> 03 验证 -> 启动新版 JAR。
-- 新版 syslog-serve 的所有设备查询都会读取 COLLECTION_MODE,本脚本必须在新版 JAR 启动前执行。
-- 本脚本不设置任何现场 device_id;所有既有设备统一回填 BOTH,避免误停原有 Syslog 或 API 任务。
ALTER TABLE DEVICE_DEVICE
ADD COLLECTION_MODE VARCHAR2(16) DEFAULT 'BOTH';
UPDATE DEVICE_DEVICE
SET COLLECTION_MODE = 'BOTH'
WHERE COLLECTION_MODE IS NULL;
COMMIT;
ALTER TABLE DEVICE_DEVICE
MODIFY COLLECTION_MODE VARCHAR2(16) DEFAULT 'BOTH' NOT NULL;
ALTER TABLE DEVICE_DEVICE
ADD CONSTRAINT CK_DEVICE_COLLECTION_MODE
CHECK (COLLECTION_MODE IN ('SYSLOG_ONLY', 'API_ONLY', 'BOTH'));
COMMENT ON COLUMN DEVICE_DEVICE.COLLECTION_MODE IS
'设备级采集模式:SYSLOG_ONLY、API_ONLY、BOTH;默认 BOTH';
COMMIT;
@@ -0,0 +1,28 @@
-- v6 设备级采集模式:达梦数据库升级后验证
-- 本脚本只读。所有检查通过后才允许启动新版 syslog-serve JAR。
-- 1. 预期 1 行:VARCHAR2(16)、NULLABLE=N、DATA_DEFAULT 包含 BOTH。
SELECT COLUMN_NAME, DATA_TYPE, DATA_LENGTH, NULLABLE, DATA_DEFAULT
FROM USER_TAB_COLUMNS
WHERE TABLE_NAME = 'DEVICE_DEVICE'
AND COLUMN_NAME = 'COLLECTION_MODE';
-- 2. 预期 1 行:CONSTRAINT_TYPE=C、STATUS=ENABLED。
SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, STATUS
FROM USER_CONSTRAINTS
WHERE TABLE_NAME = 'DEVICE_DEVICE'
AND CONSTRAINT_NAME = 'CK_DEVICE_COLLECTION_MODE';
-- 3. 预期 INVALID_MODE_TOTAL=0。
SELECT COUNT(*) AS INVALID_MODE_TOTAL
FROM DEVICE_DEVICE
WHERE COLLECTION_MODE IS NULL
OR COLLECTION_MODE NOT IN ('SYSLOG_ONLY', 'API_ONLY', 'BOTH');
-- 4. 首次升级且尚未调用管理 API 时,预期只有 BOTH,数量与预检 DEVICE_TOTAL 一致。
SELECT COLLECTION_MODE, COUNT(*) AS DEVICE_TOTAL
FROM DEVICE_DEVICE
GROUP BY COLLECTION_MODE
ORDER BY COLLECTION_MODE;
-- 验证脚本不创建测试设备,也不绑定任何现场 device_id。