初次提交代码

haobang-security-xdr/syslog-consumer/src/main/java/com/config/AppConfig.java

@@ -0,0 +1,105 @@
+package com.config;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.springframework.context.annotation.Bean;
+
+import java.io.File;
+
+public class AppConfig {
+    private static final Config config;
+
+    static {
+        // 加载配置文件
+        File configFile = new File("application.properties");
+        if (configFile.exists()) {
+            config = ConfigFactory.parseFile(configFile);
+        } else {
+            config = ConfigFactory.load("application.properties");
+        }
+    }
+    // Syslog 配置
+    public static int getSyslogTcpPort() {
+        return config.getInt("syslog.tcp.port");
+    }
+
+    public static int getSyslogUdpPort() {
+        return config.getInt("syslog.udp.port");
+    }
+
+    public static int getSyslogMaxFrameLength() {
+        return config.getInt("syslog.max.frame.length");
+    }
+
+    public static int getSyslogBufferSize() {
+        return config.getInt("syslog.buffer.size");
+    }
+
+    // InfluxDB 配置
+    public static String getInfluxUrl() {
+        return config.getString("influxdb.url");
+    }
+
+    public static String getInfluxToken() {
+        return config.getString("influxdb.token");
+    }
+
+    public static String getInfluxOrg() {
+        return config.getString("influxdb.org");
+    }
+
+    public static String getInfluxBucket() {
+        return config.getString("influxdb.bucket");
+    }
+
+    public static int getInfluxBatchSize() {
+        return config.getInt("influxdb.batch.size");
+    }
+
+    public static int getInfluxFlushInterval() {
+        return config.getInt("influxdb.flush.interval");
+    }
+
+    public static int getInfluxRetryAttempts() {
+        return config.getInt("influxdb.retry.attempts");
+    }
+
+    public static int getInfluxRetryDelay() {
+        return config.getInt("influxdb.retry.delay");
+    }
+
+    // 应用配置
+    public static int getWorkerThreads() {
+        return config.getInt("app.worker.threads");
+    }
+
+    public static int getMaxQueueSize() {
+        return config.getInt("app.max.queue.size");
+    }
+
+    public static boolean isMetricsEnabled() {   return config.getBoolean("app.metrics.enabled");   }
+
+    //kafka consumer应用配置
+
+    public static String getBootstrapServers() {
+        return config.getString("spring.kafka.consumer.bootstrap-servers");
+    }
+
+    public static String getGroupId() {
+        return config.getString("spring.kafka.consumer.group-id");
+    }
+
+    public static String getAutoOffsetReset() {
+        return config.getString("spring.kafka.consumer.auto-offset-reset");
+    }
+
+    public static boolean getEnableAutoCommit() { return config.getBoolean("spring.kafka.consumer.enable-auto-commit");     }
+
+    public static String getAutoCommitIntervalMS() {      return config.getString("spring.kafka.consumer.auto-commit-interval");}
+
+    public static String getTopic() {  return config.getString("spring.kafka.consumer.topic"); }
+
+    public static String geRunEnvironment‌() {  return config.getString("server.run.environment"); }
+
+
+}

haobang-security-xdr/syslog-consumer/src/main/java/com/config/CacheConfig.java

@@ -0,0 +1,120 @@
+package com.config;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.jsontype.BasicPolymorphicTypeValidator;
+import com.fasterxml.jackson.databind.jsontype.PolymorphicTypeValidator;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.redis.cache.RedisCacheConfiguration;
+import org.springframework.data.redis.cache.RedisCacheManager;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
+import org.springframework.data.redis.serializer.RedisSerializationContext;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import java.time.Duration;
+import java.util.Collections;
+import org.springframework.context.annotation.Primary;
+@Configuration
+@EnableCaching  // 启用缓存
+public class CacheConfig {
+
+    /**
+     * 配置支持类型信息的 ObjectMapper
+     */
+    @Bean
+    public ObjectMapper objectMapper() {
+        ObjectMapper mapper = new ObjectMapper();
+
+        // 注册 Java 8 日期时间支持
+        mapper.registerModule(new JavaTimeModule());
+
+        // 禁用将日期序列化为时间戳
+        mapper.disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+
+        // 启用类型信息，解决 LinkedHashMap 转换问题
+        PolymorphicTypeValidator ptv = BasicPolymorphicTypeValidator.builder()
+                .allowIfSubType("com.common.entity.")    // 允许你的实体类包
+                .allowIfSubType("java.util.ArrayList")   // 允许 ArrayList
+                .allowIfSubType("java.util.LinkedList")  // 允许 LinkedList
+                .allowIfBaseType("java.util.List")       // 允许 List 接口
+                .allowIfBaseType("java.lang.Object")     // 允许 Object 类型
+                .build();
+
+        // 忽略未知属性
+        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); // [citation:1][citation:5][citation:7]
+        // 遇到无效子类型时不立即失败，为你自定义反序列化逻辑留出空间
+        mapper.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false); // [citation:2][citation:5]
+
+        // 激活默认类型信息
+        mapper.activateDefaultTyping(
+                ptv,
+                ObjectMapper.DefaultTyping.NON_FINAL,
+                com.fasterxml.jackson.annotation.JsonTypeInfo.As.PROPERTY
+        );
+
+        return mapper;
+    }
+
+
+    /**
+     * Redis 缓存配置
+     */
+    @Bean
+    public RedisCacheConfiguration redisCacheConfiguration() {
+        ObjectMapper mapper = objectMapper();
+        GenericJackson2JsonRedisSerializer serializer = new GenericJackson2JsonRedisSerializer(mapper);
+
+        return RedisCacheConfiguration.defaultCacheConfig()
+                .entryTtl(Duration.ofMinutes(30))
+                .disableCachingNullValues()
+                .serializeKeysWith(RedisSerializationContext.SerializationPair
+                        .fromSerializer(new StringRedisSerializer()))
+                .serializeValuesWith(RedisSerializationContext.SerializationPair
+                        .fromSerializer(serializer));
+    }
+
+    @Bean
+    public CacheManager cacheManager(RedisConnectionFactory factory) {
+        ObjectMapper mapper = objectMapper();
+        GenericJackson2JsonRedisSerializer serializer = new GenericJackson2JsonRedisSerializer(mapper);
+
+        RedisCacheConfiguration deviceConfig = RedisCacheConfiguration.defaultCacheConfig()
+                .entryTtl(Duration.ofHours(1))
+                .serializeKeysWith(RedisSerializationContext.SerializationPair
+                        .fromSerializer(new StringRedisSerializer()))
+                .serializeValuesWith(RedisSerializationContext.SerializationPair
+                        .fromSerializer(serializer));
+
+        RedisCacheConfiguration collectTaskConfig = RedisCacheConfiguration.defaultCacheConfig()
+                .entryTtl(Duration.ofHours(1))
+                .serializeKeysWith(RedisSerializationContext.SerializationPair
+                        .fromSerializer(new StringRedisSerializer()))
+                .serializeValuesWith(RedisSerializationContext.SerializationPair
+                        .fromSerializer(serializer));
+        RedisCacheConfiguration normalizeRuleConfig = RedisCacheConfiguration.defaultCacheConfig()
+                .entryTtl(Duration.ofHours(1))
+                .serializeKeysWith(RedisSerializationContext.SerializationPair
+                        .fromSerializer(new StringRedisSerializer()))
+                .serializeValuesWith(RedisSerializationContext.SerializationPair
+                        .fromSerializer(serializer));
+        return RedisCacheManager.builder(factory)
+                .cacheDefaults(redisCacheConfiguration())
+                .withInitialCacheConfigurations(Collections.singletonMap(
+                        "device", deviceConfig
+                ))
+                .withInitialCacheConfigurations(Collections.singletonMap(
+
+                        "collectTask",collectTaskConfig
+                ))
+                .withInitialCacheConfigurations(Collections.singletonMap(
+
+                        "normalizeRule",normalizeRuleConfig
+                ))
+                .transactionAware()
+                .build();
+    }
+}

haobang-security-xdr/syslog-consumer/src/main/java/com/config/RedisConfig.java

@@ -0,0 +1,59 @@
+package com.config;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+
+@Configuration
+public class RedisConfig {
+
+
+    @Bean
+    public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory factory) {
+        RedisTemplate<String, Object> template = new RedisTemplate<>();
+        template.setConnectionFactory(factory);
+
+        // 使用 Jackson2JsonRedisSerializer 并指定类型
+        Jackson2JsonRedisSerializer<Object> serializer = new Jackson2JsonRedisSerializer<>(Object.class);
+
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.registerModule(new JavaTimeModule());
+        mapper.activateDefaultTyping(
+                mapper.getPolymorphicTypeValidator(),
+                ObjectMapper.DefaultTyping.NON_FINAL
+        );
+        serializer.setObjectMapper(mapper);
+
+        template.setKeySerializer(new StringRedisSerializer());
+        template.setValueSerializer(serializer);
+        template.setHashKeySerializer(new StringRedisSerializer());
+        template.setHashValueSerializer(serializer);
+        template.afterPropertiesSet();
+
+        return template;
+    }
+/**
+   public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory connectionFactory) {
+       RedisTemplate<String, Object> template = new RedisTemplate<>();
+       template.setConnectionFactory(connectionFactory);
+
+       // 使用StringRedisSerializer来序列化和反序列化redis的key值
+       template.setKeySerializer(new StringRedisSerializer());
+       // 使用Jackson2JsonRedisSerializer来序列化和反序列化redis的value值
+       Jackson2JsonRedisSerializer<Object> serializer = new Jackson2JsonRedisSerializer<>(Object.class);
+       template.setValueSerializer(serializer);
+
+       // Hash的key和value也分别设置序列化器
+       template.setHashKeySerializer(new StringRedisSerializer());
+       template.setHashValueSerializer(serializer);
+
+       template.afterPropertiesSet();
+       return template;
+   }
+**/
+}

haobang-security-xdr/syslog-consumer/src/main/java/com/config/RestTemplateConfig.java

@@ -0,0 +1,74 @@
+
+package com.config;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.client.ClientHttpRequestInterceptor;
+import org.springframework.http.client.SimpleClientHttpRequestFactory;
+import org.springframework.web.client.RestTemplate;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Configuration
+public class RestTemplateConfig {
+
+    private static final Logger log = LoggerFactory.getLogger(RestTemplateConfig.class);
+
+    @Bean
+    public RestTemplate restTemplate() {
+        SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
+        factory.setConnectTimeout(30000); // 30秒连接超时
+        factory.setReadTimeout(60000);    // 60秒读取超时
+
+        RestTemplate restTemplate = new RestTemplate(factory);
+
+        // 添加自定义日志拦截器，控制日志输出
+        List<ClientHttpRequestInterceptor> interceptors = new ArrayList<>();
+        interceptors.add(new LoggingInterceptor());
+        restTemplate.setInterceptors(interceptors);
+
+        return restTemplate;
+    }
+
+    /**
+     * 自定义日志拦截器，控制请求/响应日志
+     */
+    public static class LoggingInterceptor implements ClientHttpRequestInterceptor {
+
+        @Override
+        public org.springframework.http.client.ClientHttpResponse intercept(
+                org.springframework.http.HttpRequest request,
+                byte[] body,
+                org.springframework.http.client.ClientHttpRequestExecution execution)
+                throws java.io.IOException {
+
+            // 记录请求信息
+            log.debug("请求URL: {}", request.getURI());
+            log.debug("请求方法: {}", request.getMethod());
+            log.debug("请求头: {}", request.getHeaders());
+
+            // 记录请求体（限制长度）
+            if (log.isDebugEnabled() && body != null) {
+                String requestBody = new String(body, "UTF-8");
+                int printLength = Math.min(200, requestBody.length());
+                log.debug("请求体 (前{}字符): {}", printLength, requestBody.substring(0, printLength));
+            }
+
+            // 执行请求
+            org.springframework.http.client.ClientHttpResponse response = execution.execute(request, body);
+
+            // 记录响应信息
+            log.debug("响应状态码: {}", response.getStatusCode());
+            log.debug("响应头: {}", response.getHeaders());
+
+            // 注意：这里不要直接读取响应体，否则会导致后续无法读取
+            // 使用BufferingClientHttpResponseWrapper包装响应
+
+            return response;
+        }
+    }
+}

haobang-security-xdr/syslog-consumer/src/main/java/com/config/ScheduleConfig.java

@@ -0,0 +1,11 @@
+package com.config;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
+
+@Configuration
+@EnableScheduling
+public class ScheduleConfig {
+    // 启用定时任务支持
+}

haobang-security-xdr/syslog-consumer/src/main/java/com/config/ThreadPoolConfig.java

@@ -0,0 +1,47 @@
+package com.config;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadPoolExecutor;
+import org.springframework.context.annotation.Primary;
+@Configuration
+@EnableAsync
+public class ThreadPoolConfig {
+
+    @Value("${app.processor.thread-pool.core-pool-size:10}")
+    private int corePoolSize;
+
+    @Value("${app.processor.thread-pool.max-pool-size:20}")
+    private int maxPoolSize;
+
+    @Value("${app.processor.thread-pool.queue-capacity:2000}")
+    private int queueCapacity;
+
+    @Value("${app.processor.thread-pool.keep-alive-seconds:60}")
+    private int keepAliveSeconds;
+
+    @Bean("logNormalProcessorExecutor")
+    @Primary  // 添加这个注解
+    public Executor logNormalProcessorExecutor() {
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+        executor.setCorePoolSize(corePoolSize);
+        executor.setMaxPoolSize(maxPoolSize);
+        executor.setQueueCapacity(queueCapacity);
+        executor.setKeepAliveSeconds(keepAliveSeconds);
+        executor.setThreadNamePrefix("log-processor-");
+
+        // 拒绝策略：由调用线程处理该任务（保证不丢失消息）
+        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
+
+        // 等待所有任务结束后再关闭线程池
+        executor.setWaitForTasksToCompleteOnShutdown(true);
+        executor.setAwaitTerminationSeconds(60);
+        executor.initialize();
+        return executor;
+    }
+}

haobang-security-xdr/syslog-consumer/src/main/java/com/config/WebConfig.java

@@ -0,0 +1,56 @@
+package com.config;
+
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.converter.HttpMessageConverter;
+import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+
+import java.util.List;
+
+@Configuration
+public class WebConfig implements WebMvcConfigurer {
+
+    /**
+     * 创建专门用于 HTTP 请求的 ObjectMapper
+     */
+    @Bean
+    public ObjectMapper httpObjectMapper() {
+        ObjectMapper mapper = new ObjectMapper();
+
+        // 注册 Java 8 日期时间支持
+        mapper.registerModule(new JavaTimeModule());
+
+        // 禁用将日期序列化为时间戳
+        mapper.disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+
+        // 忽略未知属性
+        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+        // 确保禁用所有类型信息
+        mapper.deactivateDefaultTyping();
+
+        return mapper;
+    }
+
+    /**
+     * 创建 HTTP 消息转换器
+     */
+    @Bean
+    public MappingJackson2HttpMessageConverter mappingJackson2HttpMessageConverter() {
+        return new MappingJackson2HttpMessageConverter(httpObjectMapper());
+    }
+
+    /**
+     * 注册消息转换器到 Spring MVC
+     */
+    @Override
+    public void configureMessageConverters(List<HttpMessageConverter<?>> converters) {
+        // 将 Jackson 转换器添加到转换器列表的开头
+        converters.add(0, mappingJackson2HttpMessageConverter());
+    }
+}