diff --git a/haobang-security-dm/.idea/workspace.xml b/haobang-security-dm/.idea/workspace.xml index 7035a1c..e789142 100644 --- a/haobang-security-dm/.idea/workspace.xml +++ b/haobang-security-dm/.idea/workspace.xml @@ -1,7 +1,105 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -155,6 +253,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -162,87 +323,87 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - + + - + - - + + + + + + + + + + + + + + + + + + + + - + - - + + - + - - + + - + - - + + + + + + + + + + + + + + + + + + + + @@ -258,18 +419,6 @@ - srcAddress - src_ip - window - close - 192.168.4.26 - firewall.enabled - firewall.api-key - firewall.iptables-path - getAppServieDeviceId - whitelistUsername - probe.platform.api-url - 处理心跳失败 tenant_id syslog-consumer @Scheduled @@ -288,6 +437,18 @@ 补充原始记录日志字段异 补充原始记录日志字段异常 processAlgorithm + spring.datasource.hikari.auto-commit + LISTENER + COMMIT + parseDateTime + properties + containerFactory + topic + org.springframework.kafka + syslog.charset + init + findRequiredFieldsAfterTime + KafkaConsumerProperties com.common. @@ -302,23 +463,8 @@ @@ -392,16 +553,7 @@ - - - - - - - - - - + @@ -416,9 +568,9 @@ diff --git a/haobang-security-dm/syslog-consumer-rule/syslog-consumer-rule.iml b/haobang-security-dm/syslog-consumer-rule/syslog-consumer-rule.iml new file mode 100644 index 0000000..78eb072 --- /dev/null +++ b/haobang-security-dm/syslog-consumer-rule/syslog-consumer-rule.iml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/build_output.txt b/haobang-security-dm/syslog-consumer/build_output.txt new file mode 100644 index 0000000..92a5a13 --- /dev/null +++ b/haobang-security-dm/syslog-consumer/build_output.txt @@ -0,0 +1,258 @@ +[INFO] Scanning for projects... +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/boot/spring-boot-starter-parent/2.7.4/spring-boot-starter-parent-2.7.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter-parent/2.7.4/spring-boot-starter-parent-2.7.4.pom +Progress (1): 4.1/9.2 kB Progress (1): 8.2/9.2 kB Progress (1): 9.2 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter-parent/2.7.4/spring-boot-starter-parent-2.7.4.pom (9.2 kB at 12 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/boot/spring-boot-dependencies/2.7.4/spring-boot-dependencies-2.7.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-dependencies/2.7.4/spring-boot-dependencies-2.7.4.pom +Progress (1): 2.9/108 kB Progress (1): 7.0/108 kB Progress (1): 11/108 kB Progress (1): 15/108 kB Progress (1): 19/108 kB Progress (1): 23/108 kB Progress (1): 27/108 kB Progress (1): 32/108 kB Progress (1): 36/108 kB Progress (1): 40/108 kB Progress (1): 44/108 kB Progress (1): 48/108 kB Progress (1): 52/108 kB Progress (1): 56/108 kB Progress (1): 60/108 kB Progress (1): 64/108 kB Progress (1): 68/108 kB Progress (1): 73/108 kB Progress (1): 77/108 kB Progress (1): 81/108 kB Progress (1): 85/108 kB Progress (1): 89/108 kB Progress (1): 93/108 kB Progress (1): 95/108 kB Progress (1): 99/108 kB Progress (1): 103/108 kB Progress (1): 107/108 kB Progress (1): 108 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-dependencies/2.7.4/spring-boot-dependencies-2.7.4.pom (108 kB at 276 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/datastax/oss/java-driver-bom/4.14.1/java-driver-bom-4.14.1.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/datastax/oss/java-driver-bom/4.14.1/java-driver-bom-4.14.1.pom +Progress (1): 1.9/4.1 kB Progress (1): 4.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/datastax/oss/java-driver-bom/4.14.1/java-driver-bom-4.14.1.pom (4.1 kB at 25 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/io/dropwizard/metrics/metrics-bom/4.2.12/metrics-bom-4.2.12.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/dropwizard/metrics/metrics-bom/4.2.12/metrics-bom-4.2.12.pom +Progress (1): 4.1/6.9 kB Progress (1): 4.8/6.9 kB Progress (1): 6.9 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/dropwizard/metrics/metrics-bom/4.2.12/metrics-bom-4.2.12.pom (6.9 kB at 47 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/io/dropwizard/metrics/metrics-parent/4.2.12/metrics-parent-4.2.12.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/dropwizard/metrics/metrics-parent/4.2.12/metrics-parent-4.2.12.pom +Progress (1): 0.5/20 kB Progress (1): 4.6/20 kB Progress (1): 8.7/20 kB Progress (1): 13/20 kB Progress (1): 17/20 kB Progress (1): 20 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/dropwizard/metrics/metrics-parent/4.2.12/metrics-parent-4.2.12.pom (20 kB at 142 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/codehaus/groovy/groovy-bom/3.0.13/groovy-bom-3.0.13.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/codehaus/groovy/groovy-bom/3.0.13/groovy-bom-3.0.13.pom +Progress (1): 1.9/26 kB Progress (1): 6.0/26 kB Progress (1): 10/26 kB Progress (1): 14/26 kB Progress (1): 18/26 kB Progress (1): 22/26 kB Progress (1): 26 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/codehaus/groovy/groovy-bom/3.0.13/groovy-bom-3.0.13.pom (26 kB at 173 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/infinispan/infinispan-bom/13.0.11.Final/infinispan-bom-13.0.11.Final.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/infinispan/infinispan-bom/13.0.11.Final/infinispan-bom-13.0.11.Final.pom +Progress (1): 4.1/18 kB Progress (1): 7.3/18 kB Progress (1): 11/18 kB Progress (1): 16/18 kB Progress (1): 18 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/infinispan/infinispan-bom/13.0.11.Final/infinispan-bom-13.0.11.Final.pom (18 kB at 48 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/infinispan/infinispan-build-configuration-parent/13.0.11.Final/infinispan-build-configuration-parent-13.0.11.Final.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/infinispan/infinispan-build-configuration-parent/13.0.11.Final/infinispan-build-configuration-parent-13.0.11.Final.pom +Progress (1): 4.1/15 kB Progress (1): 8.2/15 kB Progress (1): 12/15 kB Progress (1): 15 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/infinispan/infinispan-build-configuration-parent/13.0.11.Final/infinispan-build-configuration-parent-13.0.11.Final.pom (15 kB at 33 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/jboss/jboss-parent/36/jboss-parent-36.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/jboss/jboss-parent/36/jboss-parent-36.pom +Progress (1): 1.9/66 kB Progress (1): 6.0/66 kB Progress (1): 10/66 kB Progress (1): 14/66 kB Progress (1): 18/66 kB Progress (1): 22/66 kB Progress (1): 26/66 kB Progress (1): 31/66 kB Progress (1): 35/66 kB Progress (1): 39/66 kB Progress (1): 43/66 kB Progress (1): 47/66 kB Progress (1): 51/66 kB Progress (1): 52/66 kB Progress (1): 54/66 kB Progress (1): 59/66 kB Progress (1): 63/66 kB Progress (1): 63/66 kB Progress (1): 66 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/jboss/jboss-parent/36/jboss-parent-36.pom (66 kB at 506 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/fasterxml/jackson/jackson-bom/2.13.4/jackson-bom-2.13.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/jackson-bom/2.13.4/jackson-bom-2.13.4.pom +Progress (1): 0.5/17 kB Progress (1): 4.5/17 kB Progress (1): 8.6/17 kB Progress (1): 13/17 kB Progress (1): 17/17 kB Progress (1): 17 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/jackson-bom/2.13.4/jackson-bom-2.13.4.pom (17 kB at 119 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/fasterxml/jackson/jackson-parent/2.13/jackson-parent-2.13.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/jackson-parent/2.13/jackson-parent-2.13.pom +Progress (1): 3.4/7.4 kB Progress (1): 7.4 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/jackson-parent/2.13/jackson-parent-2.13.pom (7.4 kB at 53 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/fasterxml/oss-parent/43/oss-parent-43.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/oss-parent/43/oss-parent-43.pom +Progress (1): 4.1/24 kB Progress (1): 7.3/24 kB Progress (1): 11/24 kB Progress (1): 15/24 kB Progress (1): 20/24 kB Progress (1): 24 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/oss-parent/43/oss-parent-43.pom (24 kB at 112 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/glassfish/jersey/jersey-bom/2.35/jersey-bom-2.35.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/glassfish/jersey/jersey-bom/2.35/jersey-bom-2.35.pom +Progress (1): 0.5/19 kB Progress (1): 4.6/19 kB Progress (1): 8.7/19 kB Progress (1): 13/19 kB Progress (1): 17/19 kB Progress (1): 19 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/glassfish/jersey/jersey-bom/2.35/jersey-bom-2.35.pom (19 kB at 125 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/eclipse/ee4j/project/1.0.6/project-1.0.6.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/eclipse/ee4j/project/1.0.6/project-1.0.6.pom +Progress (1): 0.4/13 kB Progress (1): 4.5/13 kB Progress (1): 8.6/13 kB Progress (1): 13/13 kB Progress (1): 13 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/eclipse/ee4j/project/1.0.6/project-1.0.6.pom (13 kB at 92 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/eclipse/jetty/jetty-bom/9.4.49.v20220914/jetty-bom-9.4.49.v20220914.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/eclipse/jetty/jetty-bom/9.4.49.v20220914/jetty-bom-9.4.49.v20220914.pom +Progress (1): 0.5/18 kB Progress (1): 4.6/18 kB Progress (1): 8.7/18 kB Progress (1): 13/18 kB Progress (1): 17/18 kB Progress (1): 18 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/eclipse/jetty/jetty-bom/9.4.49.v20220914/jetty-bom-9.4.49.v20220914.pom (18 kB at 116 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/junit/junit-bom/5.8.2/junit-bom-5.8.2.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/junit/junit-bom/5.8.2/junit-bom-5.8.2.pom +Progress (1): 4.1/5.6 kB Progress (1): 5.6 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/junit/junit-bom/5.8.2/junit-bom-5.8.2.pom (5.6 kB at 41 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/jetbrains/kotlin/kotlin-bom/1.6.21/kotlin-bom-1.6.21.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/jetbrains/kotlin/kotlin-bom/1.6.21/kotlin-bom-1.6.21.pom +Progress (1): 1.9/9.3 kB Progress (1): 6.0/9.3 kB Progress (1): 9.3 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/jetbrains/kotlin/kotlin-bom/1.6.21/kotlin-bom-1.6.21.pom (9.3 kB at 49 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/jetbrains/kotlinx/kotlinx-coroutines-bom/1.6.4/kotlinx-coroutines-bom-1.6.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/jetbrains/kotlinx/kotlinx-coroutines-bom/1.6.4/kotlinx-coroutines-bom-1.6.4.pom +Progress (1): 1.9/4.3 kB Progress (1): 4.3 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/jetbrains/kotlinx/kotlinx-coroutines-bom/1.6.4/kotlinx-coroutines-bom-1.6.4.pom (4.3 kB at 28 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/logging/log4j/log4j-bom/2.17.2/log4j-bom-2.17.2.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/logging/log4j/log4j-bom/2.17.2/log4j-bom-2.17.2.pom +Progress (1): 1.9/8.1 kB Progress (1): 6.0/8.1 kB Progress (1): 8.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/logging/log4j/log4j-bom/2.17.2/log4j-bom-2.17.2.pom (8.1 kB at 72 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/logging/logging-parent/5/logging-parent-5.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/logging/logging-parent/5/logging-parent-5.pom +Progress (1): 1.9/3.3 kB Progress (1): 3.3 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/logging/logging-parent/5/logging-parent-5.pom (3.3 kB at 25 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/apache/24/apache-24.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/apache/24/apache-24.pom +Progress (1): 4.1/20 kB Progress (1): 7.3/20 kB Progress (1): 11/20 kB Progress (1): 16/20 kB Progress (1): 20/20 kB Progress (1): 20 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/apache/24/apache-24.pom (20 kB at 139 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/io/micrometer/micrometer-bom/1.9.4/micrometer-bom-1.9.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/micrometer/micrometer-bom/1.9.4/micrometer-bom-1.9.4.pom +Progress (1): 4.1/7.1 kB Progress (1): 7.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/micrometer/micrometer-bom/1.9.4/micrometer-bom-1.9.4.pom (7.1 kB at 10 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/mockito/mockito-bom/4.5.1/mockito-bom-4.5.1.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/mockito/mockito-bom/4.5.1/mockito-bom-4.5.1.pom +Progress (1): 3.0 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/mockito/mockito-bom/4.5.1/mockito-bom-4.5.1.pom (3.0 kB at 21 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/io/netty/netty-bom/4.1.82.Final/netty-bom-4.1.82.Final.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/netty/netty-bom/4.1.82.Final/netty-bom-4.1.82.Final.pom +Progress (1): 3.4/13 kB Progress (1): 7.5/13 kB Progress (1): 12/13 kB Progress (1): 13 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/netty/netty-bom/4.1.82.Final/netty-bom-4.1.82.Final.pom (13 kB at 34 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/sonatype/oss/oss-parent/7/oss-parent-7.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/sonatype/oss/oss-parent/7/oss-parent-7.pom +Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/sonatype/oss/oss-parent/7/oss-parent-7.pom (0 B at 0 B/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/squareup/okhttp3/okhttp-bom/4.9.3/okhttp-bom-4.9.3.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/squareup/okhttp3/okhttp-bom/4.9.3/okhttp-bom-4.9.3.pom +Progress (1): 1.9/3.0 kB Progress (1): 3.0 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/squareup/okhttp3/okhttp-bom/4.9.3/okhttp-bom-4.9.3.pom (3.0 kB at 7.8 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/oracle/database/jdbc/ojdbc-bom/21.5.0.0/ojdbc-bom-21.5.0.0.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/oracle/database/jdbc/ojdbc-bom/21.5.0.0/ojdbc-bom-21.5.0.0.pom +Progress (1): 1.9/13 kB Progress (1): 6.0/13 kB Progress (1): 10/13 kB Progress (1): 13 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/oracle/database/jdbc/ojdbc-bom/21.5.0.0/ojdbc-bom-21.5.0.0.pom (13 kB at 80 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/io/prometheus/simpleclient_bom/0.15.0/simpleclient_bom-0.15.0.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/prometheus/simpleclient_bom/0.15.0/simpleclient_bom-0.15.0.pom +Progress (1): 1.9/5.8 kB Progress (1): 5.8 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/prometheus/simpleclient_bom/0.15.0/simpleclient_bom-0.15.0.pom (5.8 kB at 38 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/io/prometheus/parent/0.15.0/parent-0.15.0.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/prometheus/parent/0.15.0/parent-0.15.0.pom +Progress (1): 1.9/12 kB Progress (1): 6.0/12 kB Progress (1): 10/12 kB Progress (1): 12 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/prometheus/parent/0.15.0/parent-0.15.0.pom (12 kB at 87 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/querydsl/querydsl-bom/5.0.0/querydsl-bom-5.0.0.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/querydsl/querydsl-bom/5.0.0/querydsl-bom-5.0.0.pom +Progress (1): 3.4/7.2 kB Progress (1): 7.2 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/querydsl/querydsl-bom/5.0.0/querydsl-bom-5.0.0.pom (7.2 kB at 63 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/io/r2dbc/r2dbc-bom/Borca-SR2/r2dbc-bom-Borca-SR2.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/r2dbc/r2dbc-bom/Borca-SR2/r2dbc-bom-Borca-SR2.pom +Progress (1): 3.8 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/r2dbc/r2dbc-bom/Borca-SR2/r2dbc-bom-Borca-SR2.pom (3.8 kB at 9.5 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/io/projectreactor/reactor-bom/2020.0.23/reactor-bom-2020.0.23.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/projectreactor/reactor-bom/2020.0.23/reactor-bom-2020.0.23.pom +Progress (1): 0.5/4.6 kB Progress (1): 4.6/4.6 kB Progress (1): 4.6 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/projectreactor/reactor-bom/2020.0.23/reactor-bom-2020.0.23.pom (4.6 kB at 11 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/io/rest-assured/rest-assured-bom/4.5.1/rest-assured-bom-4.5.1.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/rest-assured/rest-assured-bom/4.5.1/rest-assured-bom-4.5.1.pom +Progress (1): 4.1/5.8 kB Progress (1): 5.8 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/rest-assured/rest-assured-bom/4.5.1/rest-assured-bom-4.5.1.pom (5.8 kB at 33 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/io/rsocket/rsocket-bom/1.1.3/rsocket-bom-1.1.3.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/rsocket/rsocket-bom/1.1.3/rsocket-bom-1.1.3.pom +Progress (1): 2.6 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/io/rsocket/rsocket-bom/1.1.3/rsocket-bom-1.1.3.pom (2.6 kB at 19 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/data/spring-data-bom/2021.2.3/spring-data-bom-2021.2.3.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/data/spring-data-bom/2021.2.3/spring-data-bom-2021.2.3.pom +Progress (1): 0.4/5.7 kB Progress (1): 4.5/5.7 kB Progress (1): 5.7 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/data/spring-data-bom/2021.2.3/spring-data-bom-2021.2.3.pom (5.7 kB at 15 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/spring-framework-bom/5.3.23/spring-framework-bom-5.3.23.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-framework-bom/5.3.23/spring-framework-bom-5.3.23.pom +Progress (1): 0.4/5.7 kB Progress (1): 4.5/5.7 kB Progress (1): 5.7 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-framework-bom/5.3.23/spring-framework-bom-5.3.23.pom (5.7 kB at 39 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/integration/spring-integration-bom/5.5.15/spring-integration-bom-5.5.15.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/integration/spring-integration-bom/5.5.15/spring-integration-bom-5.5.15.pom +Progress (1): 1.9/9.2 kB Progress (1): 6.0/9.2 kB Progress (1): 9.2 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/integration/spring-integration-bom/5.5.15/spring-integration-bom-5.5.15.pom (9.2 kB at 65 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/security/spring-security-bom/5.7.3/spring-security-bom-5.7.3.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/security/spring-security-bom/5.7.3/spring-security-bom-5.7.3.pom +Progress (1): 4.1/5.7 kB Progress (1): 5.7 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/security/spring-security-bom/5.7.3/spring-security-bom-5.7.3.pom (5.7 kB at 38 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/session/spring-session-bom/2021.2.0/spring-session-bom-2021.2.0.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/session/spring-session-bom/2021.2.0/spring-session-bom-2021.2.0.pom +Progress (1): 1.9/3.1 kB Progress (1): 3.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/session/spring-session-bom/2021.2.0/spring-session-bom-2021.2.0.pom (3.1 kB at 21 kB/s) +[INFO] +[INFO] --------------------< com.haobang:syslog-consumer >--------------------- +[INFO] Building syslog-consumer 1.0.0 +[INFO] --------------------------------[ jar ]--------------------------------- +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/boot/spring-boot-maven-plugin/2.7.4/spring-boot-maven-plugin-2.7.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-maven-plugin/2.7.4/spring-boot-maven-plugin-2.7.4.pom +Progress (1): 3.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-maven-plugin/2.7.4/spring-boot-maven-plugin-2.7.4.pom (3.1 kB at 7.4 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/boot/spring-boot-maven-plugin/2.7.4/spring-boot-maven-plugin-2.7.4.jar +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-maven-plugin/2.7.4/spring-boot-maven-plugin-2.7.4.jar +Progress (1): 3.4/107 kB Progress (1): 7.5/107 kB Progress (1): 12/107 kB Progress (1): 16/107 kB Progress (1): 20/107 kB Progress (1): 24/107 kB Progress (1): 28/107 kB Progress (1): 32/107 kB Progress (1): 36/107 kB Progress (1): 40/107 kB Progress (1): 44/107 kB Progress (1): 48/107 kB Progress (1): 53/107 kB Progress (1): 57/107 kB Progress (1): 61/107 kB Progress (1): 65/107 kB Progress (1): 69/107 kB Progress (1): 73/107 kB Progress (1): 77/107 kB Progress (1): 81/107 kB Progress (1): 85/107 kB Progress (1): 89/107 kB Progress (1): 93/107 kB Progress (1): 97/107 kB Progress (1): 100/107 kB Progress (1): 104/107 kB Progress (1): 107 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-maven-plugin/2.7.4/spring-boot-maven-plugin-2.7.4.jar (107 kB at 257 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/maven/plugins/maven-resources-plugin/3.2.0/maven-resources-plugin-3.2.0.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/plugins/maven-resources-plugin/3.2.0/maven-resources-plugin-3.2.0.pom +Progress (1): 3.4/8.1 kB Progress (1): 7.5/8.1 kB Progress (1): 8.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/plugins/maven-resources-plugin/3.2.0/maven-resources-plugin-3.2.0.pom (8.1 kB at 56 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/maven/plugins/maven-plugins/34/maven-plugins-34.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/plugins/maven-plugins/34/maven-plugins-34.pom +Progress (1): 1.9/11 kB Progress (1): 6.0/11 kB Progress (1): 10/11 kB Progress (1): 11 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/plugins/maven-plugins/34/maven-plugins-34.pom (11 kB at 79 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/maven/maven-parent/34/maven-parent-34.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/maven-parent/34/maven-parent-34.pom +Progress (1): 0.4/43 kB Progress (1): 4.5/43 kB Progress (1): 8.6/43 kB Progress (1): 9.2/43 kB Progress (1): 13/43 kB Progress (1): 17/43 kB Progress (1): 18/43 kB Progress (1): 22/43 kB Progress (1): 26/43 kB Progress (1): 30/43 kB Progress (1): 34/43 kB Progress (1): 38/43 kB Progress (1): 43/43 kB Progress (1): 43 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/maven-parent/34/maven-parent-34.pom (43 kB at 300 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/apache/23/apache-23.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/apache/23/apache-23.pom +Progress (1): 0.5/18 kB Progress (1): 4.6/18 kB Progress (1): 8.7/18 kB Progress (1): 13/18 kB Progress (1): 17/18 kB Progress (1): 18 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/apache/23/apache-23.pom (18 kB at 184 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/maven/plugins/maven-resources-plugin/3.2.0/maven-resources-plugin-3.2.0.jar +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/plugins/maven-resources-plugin/3.2.0/maven-resources-plugin-3.2.0.jar +Progress (1): 0.4/33 kB Progress (1): 4.5/33 kB Progress (1): 8.6/33 kB Progress (1): 13/33 kB Progress (1): 17/33 kB Progress (1): 21/33 kB Progress (1): 25/33 kB Progress (1): 29/33 kB Progress (1): 33 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/plugins/maven-resources-plugin/3.2.0/maven-resources-plugin-3.2.0.jar (33 kB at 221 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/maven/plugins/maven-compiler-plugin/3.10.1/maven-compiler-plugin-3.10.1.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/plugins/maven-compiler-plugin/3.10.1/maven-compiler-plugin-3.10.1.pom +Progress (1): 4.1/13 kB Progress (1): 4.8/13 kB Progress (1): 8.9/13 kB Progress (1): 13 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/plugins/maven-compiler-plugin/3.10.1/maven-compiler-plugin-3.10.1.pom (13 kB at 103 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/maven/plugins/maven-compiler-plugin/3.10.1/maven-compiler-plugin-3.10.1.jar +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/plugins/maven-compiler-plugin/3.10.1/maven-compiler-plugin-3.10.1.jar +Progress (1): 1.9/62 kB Progress (1): 6.0/62 kB Progress (1): 10/62 kB Progress (1): 14/62 kB Progress (1): 18/62 kB Progress (1): 22/62 kB Progress (1): 26/62 kB Progress (1): 31/62 kB Progress (1): 35/62 kB Progress (1): 39/62 kB Progress (1): 43/62 kB Progress (1): 47/62 kB Progress (1): 51/62 kB Progress (1): 55/62 kB Progress (1): 59/62 kB Progress (1): 62 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/maven/plugins/maven-compiler-plugin/3.10.1/maven-compiler-plugin-3.10.1.jar (62 kB at 462 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/boot/spring-boot-starter/2.7.4/spring-boot-starter-2.7.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter/2.7.4/spring-boot-starter-2.7.4.pom +Progress (1): 3.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter/2.7.4/spring-boot-starter-2.7.4.pom (3.1 kB at 7.5 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/boot/spring-boot/2.7.4/spring-boot-2.7.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot/2.7.4/spring-boot-2.7.4.pom +Progress (1): 0.5/2.2 kB Progress (1): 2.2 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot/2.7.4/spring-boot-2.7.4.pom (2.2 kB at 5.2 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/spring-core/5.3.23/spring-core-5.3.23.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-core/5.3.23/spring-core-5.3.23.pom +Progress (1): 2.0 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-core/5.3.23/spring-core-5.3.23.pom (2.0 kB at 14 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/spring-jcl/5.3.23/spring-jcl-5.3.23.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-jcl/5.3.23/spring-jcl-5.3.23.pom +Progress (1): 1.8 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-jcl/5.3.23/spring-jcl-5.3.23.pom (1.8 kB at 13 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/spring-context/5.3.23/spring-context-5.3.23.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-context/5.3.23/spring-context-5.3.23.pom +Progress (1): 1.9/2.6 kB Progress (1): 2.6 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-context/5.3.23/spring-context-5.3.23.pom (2.6 kB at 15 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/spring-aop/5.3.23/spring-aop-5.3.23.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-aop/5.3.23/spring-aop-5.3.23.pom +Progress (1): 1.9/2.2 kB Progress (1): 2.2 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-aop/5.3.23/spring-aop-5.3.23.pom (2.2 kB at 15 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/spring-beans/5.3.23/spring-beans-5.3.23.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-beans/5.3.23/spring-beans-5.3.23.pom +Progress (1): 1.9/2.0 kB Progress (1): 2.0 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-beans/5.3.23/spring-beans-5.3.23.pom (2.0 kB at 14 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/spring-expression/5.3.23/spring-expression-5.3.23.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-expression/5.3.23/spring-expression-5.3.23.pom +Progress (1): 0.5/2.1 kB Progress (1): 2.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-expression/5.3.23/spring-expression-5.3.23.pom (2.1 kB at 14 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/boot/spring-boot-autoconfigure/2.7.4/spring-boot-autoconfigure-2.7.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-autoconfigure/2.7.4/spring-boot-autoconfigure-2.7.4.pom +Progress (1): 2.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-autoconfigure/2.7.4/spring-boot-autoconfigure-2.7.4.pom (2.1 kB at 4.8 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/boot/spring-boot-starter-logging/2.7.4/spring-boot-starter-logging-2.7.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter-logging/2.7.4/spring-boot-starter-logging-2.7.4.pom +Progress (1): 2.5 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter-logging/2.7.4/spring-boot-starter-logging-2.7.4.pom (2.5 kB at 6.0 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/ch/qos/logback/logback-classic/1.2.11/logback-classic-1.2.11.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/ch/qos/logback/logback-classic/1.2.11/logback-classic-1.2.11.pom +Progress (1): 4.1/9.7 kB Progress (1): 4.8/9.7 kB Progress (1): 8.9/9.7 kB Progress (1): 9.7 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/ch/qos/logback/logback-classic/1.2.11/logback-classic-1.2.11.pom (9.7 kB at 82 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/ch/qos/logback/logback-parent/1.2.11/logback-parent-1.2.11.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/ch/qos/logback/logback-parent/1.2.11/logback-parent-1.2.11.pom +Progress (1): 4.1/19 kB Progress (1): 7.3/19 kB Progress (1): 11/19 kB Progress (1): 15/19 kB Progress (1): 19 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/ch/qos/logback/logback-parent/1.2.11/logback-parent-1.2.11.pom (19 kB at 131 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/ch/qos/logback/logback-core/1.2.11/logback-core-1.2.11.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/ch/qos/logback/logback-core/1.2.11/logback-core-1.2.11.pom +Progress (1): 1.9/4.2 kB Progress (1): 4.2 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/ch/qos/logback/logback-core/1.2.11/logback-core-1.2.11.pom (4.2 kB at 37 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.pom +Progress (1): 1.9/2.7 kB Progress (1): 2.7 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.pom (2.7 kB at 7.5 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/slf4j/slf4j-parent/1.7.36/slf4j-parent-1.7.36.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/slf4j/slf4j-parent/1.7.36/slf4j-parent-1.7.36.pom +Progress (1): 0.4/14 kB Progress (1): 4.5/14 kB Progress (1): 8.6/14 kB Progress (1): 13/14 kB Progress (1): 14 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/slf4j/slf4j-parent/1.7.36/slf4j-parent-1.7.36.pom (14 kB at 38 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/logging/log4j/log4j-to-slf4j/2.17.2/log4j-to-slf4j-2.17.2.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/logging/log4j/log4j-to-slf4j/2.17.2/log4j-to-slf4j-2.17.2.pom +Progress (1): 4.1/7.3 kB Progress (1): 6.3/7.3 kB Progress (1): 7.3 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/logging/log4j/log4j-to-slf4j/2.17.2/log4j-to-slf4j-2.17.2.pom (7.3 kB at 64 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/logging/log4j/log4j/2.17.2/log4j-2.17.2.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/logging/log4j/log4j/2.17.2/log4j-2.17.2.pom +Progress (1): 1.9/73 kB Progress (1): 6.0/73 kB Progress (1): 10/73 kB Progress (1): 14/73 kB Progress (1): 18/73 kB Progress (1): 22/73 kB Progress (1): 26/73 kB Progress (1): 31/73 kB Progress (1): 35/73 kB Progress (1): 39/73 kB Progress (1): 43/73 kB Progress (1): 47/73 kB Progress (1): 51/73 kB Progress (1): 55/73 kB Progress (1): 59/73 kB Progress (1): 63/73 kB Progress (1): 67/73 kB Progress (1): 72/73 kB Progress (1): 73 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/logging/log4j/log4j/2.17.2/log4j-2.17.2.pom (73 kB at 507 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/logging/log4j/log4j-api/2.17.2/log4j-api-2.17.2.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/logging/log4j/log4j-api/2.17.2/log4j-api-2.17.2.pom +Progress (1): 3.4/14 kB Progress (1): 7.5/14 kB Progress (1): 12/14 kB Progress (1): 14 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/logging/log4j/log4j-api/2.17.2/log4j-api-2.17.2.pom (14 kB at 102 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/slf4j/jul-to-slf4j/1.7.36/jul-to-slf4j-1.7.36.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/slf4j/jul-to-slf4j/1.7.36/jul-to-slf4j-1.7.36.pom +Progress (1): 991 B Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/slf4j/jul-to-slf4j/1.7.36/jul-to-slf4j-1.7.36.pom (991 B at 8.7 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/jakarta/annotation/jakarta.annotation-api/1.3.5/jakarta.annotation-api-1.3.5.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/jakarta/annotation/jakarta.annotation-api/1.3.5/jakarta.annotation-api-1.3.5.pom +Progress (1): 4.1/16 kB Progress (1): 7.3/16 kB Progress (1): 11/16 kB Progress (1): 16/16 kB Progress (1): 16 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/jakarta/annotation/jakarta.annotation-api/1.3.5/jakarta.annotation-api-1.3.5.pom (16 kB at 134 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/jakarta/annotation/ca-parent/1.3.5/ca-parent-1.3.5.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/jakarta/annotation/ca-parent/1.3.5/ca-parent-1.3.5.pom +Progress (1): 1.9/2.8 kB Progress (1): 2.8 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/jakarta/annotation/ca-parent/1.3.5/ca-parent-1.3.5.pom (2.8 kB at 25 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/eclipse/ee4j/project/1.0.5/project-1.0.5.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/eclipse/ee4j/project/1.0.5/project-1.0.5.pom +Progress (1): 0.4/13 kB Progress (1): 4.5/13 kB Progress (1): 8.6/13 kB Progress (1): 13/13 kB Progress (1): 13 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/eclipse/ee4j/project/1.0.5/project-1.0.5.pom (13 kB at 112 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/yaml/snakeyaml/1.30/snakeyaml-1.30.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/yaml/snakeyaml/1.30/snakeyaml-1.30.pom +Progress (1): 0.5/37 kB Progress (1): 4.5/37 kB Progress (1): 6.3/37 kB Progress (1): 10/37 kB Progress (1): 14/37 kB Progress (1): 18/37 kB Progress (1): 22/37 kB Progress (1): 24/37 kB Progress (1): 28/37 kB Progress (1): 32/37 kB Progress (1): 36/37 kB Progress (1): 37 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/yaml/snakeyaml/1.30/snakeyaml-1.30.pom (37 kB at 300 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/boot/spring-boot-starter-web/2.7.4/spring-boot-starter-web-2.7.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter-web/2.7.4/spring-boot-starter-web-2.7.4.pom +Progress (1): 3.0 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter-web/2.7.4/spring-boot-starter-web-2.7.4.pom (3.0 kB at 9.6 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/boot/spring-boot-starter-json/2.7.4/spring-boot-starter-json-2.7.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter-json/2.7.4/spring-boot-starter-json-2.7.4.pom +Progress (1): 0.5/3.1 kB Progress (1): 3.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter-json/2.7.4/spring-boot-starter-json-2.7.4.pom (3.1 kB at 7.5 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/spring-web/5.3.23/spring-web-5.3.23.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-web/5.3.23/spring-web-5.3.23.pom +Progress (1): 2.2 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/spring-web/5.3.23/spring-web-5.3.23.pom (2.2 kB at 14 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/fasterxml/jackson/core/jackson-databind/2.13.4/jackson-databind-2.13.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/core/jackson-databind/2.13.4/jackson-databind-2.13.4.pom +Progress (1): 1.9/16 kB Progress (1): 6.0/16 kB Progress (1): 10/16 kB Progress (1): 14/16 kB Progress (1): 16 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/core/jackson-databind/2.13.4/jackson-databind-2.13.4.pom (16 kB at 118 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/fasterxml/jackson/jackson-base/2.13.4/jackson-base-2.13.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/jackson-base/2.13.4/jackson-base-2.13.4.pom +Progress (1): 4.1/9.9 kB Progress (1): 7.3/9.9 kB Progress (1): 9.9 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/jackson-base/2.13.4/jackson-base-2.13.4.pom (9.9 kB at 65 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/fasterxml/jackson/core/jackson-annotations/2.13.4/jackson-annotations-2.13.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/core/jackson-annotations/2.13.4/jackson-annotations-2.13.4.pom +Progress (1): 4.1/6.1 kB Progress (1): 6.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/core/jackson-annotations/2.13.4/jackson-annotations-2.13.4.pom (6.1 kB at 55 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/fasterxml/jackson/core/jackson-core/2.13.4/jackson-core-2.13.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/core/jackson-core/2.13.4/jackson-core-2.13.4.pom +Progress (1): 4.1/5.5 kB Progress (1): 5.5 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/core/jackson-core/2.13.4/jackson-core-2.13.4.pom (5.5 kB at 52 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/fasterxml/jackson/datatype/jackson-datatype-jdk8/2.13.4/jackson-datatype-jdk8-2.13.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/datatype/jackson-datatype-jdk8/2.13.4/jackson-datatype-jdk8-2.13.4.pom +Progress (1): 1.9/2.6 kB Progress (1): 2.6 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/datatype/jackson-datatype-jdk8/2.13.4/jackson-datatype-jdk8-2.13.4.pom (2.6 kB at 24 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/fasterxml/jackson/module/jackson-modules-java8/2.13.4/jackson-modules-java8-2.13.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/module/jackson-modules-java8/2.13.4/jackson-modules-java8-2.13.4.pom +Progress (1): 1.9/3.2 kB Progress (1): 3.2 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/module/jackson-modules-java8/2.13.4/jackson-modules-java8-2.13.4.pom (3.2 kB at 26 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.13.4/jackson-datatype-jsr310-2.13.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.13.4/jackson-datatype-jsr310-2.13.4.pom +Progress (1): 4.1/4.9 kB Progress (1): 4.9 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.13.4/jackson-datatype-jsr310-2.13.4.pom (4.9 kB at 28 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/com/fasterxml/jackson/module/jackson-module-parameter-names/2.13.4/jackson-module-parameter-names-2.13.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/module/jackson-module-parameter-names/2.13.4/jackson-module-parameter-names-2.13.4.pom +Progress (1): 4.1/4.4 kB Progress (1): 4.4 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/com/fasterxml/jackson/module/jackson-module-parameter-names/2.13.4/jackson-module-parameter-names-2.13.4.pom (4.4 kB at 38 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/springframework/boot/spring-boot-starter-tomcat/2.7.4/spring-boot-starter-tomcat-2.7.4.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter-tomcat/2.7.4/spring-boot-starter-tomcat-2.7.4.pom +Progress (1): 3.1 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/springframework/boot/spring-boot-starter-tomcat/2.7.4/spring-boot-starter-tomcat-2.7.4.pom (3.1 kB at 8.3 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/tomcat/embed/tomcat-embed-core/9.0.65/tomcat-embed-core-9.0.65.pom +Downloading from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/tomcat/embed/tomcat-embed-core/9.0.65/tomcat-embed-core-9.0.65.pom +Progress (1): 1.7 kB Downloaded from nexus-aliyun: http://maven.aliyun.com/nexus/content/groups/public/org/apache/tomcat/embed/tomcat-embed-core/9.0.65/tomcat-embed-core-9.0.65.pom (1.7 kB at 4.0 kB/s) +Downloading from etone-nexus: http://192.168.8.79:8081/repository/maven-public/org/apache/tomcat/embed/tomcat-embed-el/9.0.65/tomcat-embed-el-9.0.65.pom diff --git a/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-nw.yaml b/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-nw.yaml index 5ff4835..f06f212 100644 --- a/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-nw.yaml +++ b/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-nw.yaml @@ -21,83 +21,83 @@ services: - SPRING_PROFILES_ACTIVE=dev - TZ=Asia/Shanghai # 数据库配置 - - spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys - - spring.datasource.username=postgres - - spring.datasource.password=TnLanWaidYSwTSG5 - - spring.datasource.driver-class-name=org.postgresql.Driver + - SPRING_DATASOURCE_URL=jdbc:postgresql://117.72.68.72:54329/ecosys + - SPRING_DATASOURCE_USERNAME=postgres + - SPRING_DATASOURCE_PASSWORD=TnLanWaidYSwTSG5 + - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver # HikariCP 连接池配置 - - spring.datasource.hikari.maximum-pool-size=50 - - spring.datasource.hikari.minimum-idle=5 - - spring.datasource.hikari.connection-timeout=30000 - - spring.datasource.hikari.idle-timeout=600000 - - spring.datasource.hikari.max-lifetime=900000 - - spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer - - spring.datasource.hikari.auto-commit=false - - spring.datasource.hikari.schema=public + - SPRING_DATASOURCE_HIKARI_MAXIMUM_POOL_SIZE=50 + - SPRING_DATASOURCE_HIKARI_MINIMUM_IDLE=5 + - SPRING_DATASOURCE_HIKARI_CONNECTION_TIMEOUT=30000 + - SPRING_DATASOURCE_HIKARI_IDLE_TIMEOUT=600000 + - SPRING_DATASOURCE_HIKARI_MAX_LIFETIME=900000 + - SPRING_DATASOURCE_HIKARI_POOL_NAME=HikariPool-SyslogConsumer + - SPRING_DATASOURCE_HIKARI_AUTO_COMMIT=false + - SPRING_DATASOURCE_HIKARI_SCHEMA=public # Redis配置 - - spring.redis.host=192.168.222.131 - - spring.redis.port=6379 - - spring.redis.password= - - spring.redis.database=0 - - spring.redis.timeout=2000 - - spring.cache.redis.time-to-live=600000 + - SPRING_REDIS_HOST=192.168.222.131 + - SPRING_REDIS_PORT=6379 + - SPRING_REDIS_PASSWORD= + - SPRING_REDIS_DATABASE=0 + - SPRING_REDIS_TIMEOUT=2000 + - SPRING_CACHE_REDIS_TIME_TO_LIVE=600000 # Kafka配置 - - spring.kafka.consumer.bootstrap-servers=192.168.222.130:9092 - - spring.kafka.consumer.group-id=test-group-app - - spring.kafka.consumer.auto-offset-reset=latest - - spring.kafka.consumer.enable-auto-commit=false - - spring.kafka.consumer.topic=test-topic - - spring.kafka.consumer.max-poll-records=1000 - - spring.kafka.listener.ack-mode=manual - - spring.kafka.listener.concurrency=2 - - spring.kafka.listener.type=batch + - SPRING_KAFKA_CONSUMER_BOOTSTRAP_SERVERS=192.168.222.130:9092 + - SPRING_KAFKA_CONSUMER_GROUP_ID=test-group-app + - SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest + - SPRING_KAFKA_CONSUMER_ENABLE_AUTO_COMMIT=false + - SPRING_KAFKA_CONSUMER_TOPIC=test-topic + - SPRING_KAFKA_CONSUMER_MAX_POLL_RECORDS=1000 + - SPRING_KAFKA_LISTENER_ACK_MODE=manual + - SPRING_KAFKA_LISTENER_CONCURRENCY=2 + - SPRING_KAFKA_LISTENER_TYPE=batch # InfluxDB配置 - - influxdb.url=http://192.168.222.131:8086 - - influxdb.token=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA== - - influxdb.org=yelang - - influxdb.bucket=yelangbucket - - influxdb.batch.size=1000 - - influxdb.flush.interval=1000 - - influxdb.connection.timeout=30s - - influxdb.connection.read-timeout=30s - - influxdb.connection.write-timeout=60s + - INFLUXDB_URL=http://192.168.222.131:8086 + - INFLUXDB_TOKEN=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA== + - INFLUXDB_ORG=yelang + - INFLUXDB_BUCKET=yelangbucket + - INFLUXDB_BATCH_SIZE=1000 + - INFLUXDB_FLUSH_INTERVAL=1000 + - INFLUXDB_CONNECTION_TIMEOUT=30s + - INFLUXDB_CONNECTION_READ_TIMEOUT=30s + - INFLUXDB_CONNECTION_WRITE_TIMEOUT=60s # Elasticsearch配置 - - spring.elasticsearch.uris=http://192.168.1.174:9200 - - spring.elasticsearch.username=CONTAINER_NAME - - spring.elasticsearch.password=t2NZCiajmdazxBrF - - spring.elasticsearch.connection-timeout=10s - - spring.elasticsearch.socket-timeout=30s + - SPRING_ELASTICSEARCH_URIS=http://192.168.1.174:9200 + - SPRING_ELASTICSEARCH_USERNAME=CONTAINER_NAME + - SPRING_ELASTICSEARCH_PASSWORD=t2NZCiajmdazxBrF + - SPRING_ELASTICSEARCH_CONNECTION_TIMEOUT=10s + - SPRING_ELASTICSEARCH_SOCKET_TIMEOUT=30s # API配置 - - interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 - - interlocking.api.base-url=http://192.168.222.131:8089/xdrservice/interlocking + - INTERLOCKING_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 + - INTERLOCKING_API_BASE_URL=http://192.168.222.131:8089/xdrservice/interlocking # 探针心跳配置 - - probe.heartbeat.enabled=true - - probe.heartbeat.offline-threshold-minutes=10 - - probe.status.check.cron=0 */10 * * * ? - - probe.heartbeat.tenant-id=000000 - - probe.heartbeat.history.keep-days=10 - - probe.heartbeat.history.cleanup-enabled=true - - probe.history.cleanup.cron=0 0 1 * * ? + - PROBE_HEARTBEAT_ENABLED=true + - PROBE_HEARTBEAT_OFFLINE_THRESHOLD_MINUTES=10 + - PROBE_STATUS_CHECK_CRON=0 */10 * * * ? + - PROBE_HEARTBEAT_TENANT_ID=000000 + - PROBE_HEARTBEAT_HISTORY_KEEP_DAYS=10 + - PROBE_HEARTBEAT_HISTORY_CLEANUP_ENABLED=true + - PROBE_HISTORY_CLEANUP_CRON=0 0 1 * * ? # 告警健康检查配置 - - alarm.health-check.alarm-hours=4 - - alarm.health-check.alarm-visit-hours=2 - - alarm.health-check.enabled=true + - ALARM_HEALTH_CHECK_ALARM_HOURS=4 + - ALARM_HEALTH_CHECK_ALARM_VISIT_HOURS=2 + - ALARM_HEALTH_CHECK_ENABLED=true # 关联分析规则配置 - - analysis.realtime.enabled=true - - analysis.realtime.check-interval-seconds=10 + - ANALYSIS_REALTIME_ENABLED=true + - ANALYSIS_REALTIME_CHECK_INTERVAL_SECONDS=10 # 分区表检查配置 - - partition.check.tomorrow.enabled=true - - partition.check.future.days=7 - - partition.auto.create=true + - PARTITION_CHECK_TOMORROW_ENABLED=true + - PARTITION_CHECK_FUTURE_DAYS=7 + - PARTITION_AUTO_CREATE=true # 定时任务配置 - - spring.task.scheduling.pool.size=10 + - SPRING_TASK_SCHEDULING_POOL_SIZE=10 # 日志配置 - - logging.level.com.common.schedule=INFO - - logging.level.com.common.service=INFO + - LOGGING_LEVEL_COM_COMMON_SCHEDULE=INFO + - LOGGING_LEVEL_COM_COMMON_SERVICE=INFO # ETL配置 - - etl.batch.page-size=1000 - - etl.batch.insert-batch-size=500 - - etl.schedule.cron=0 0 2 * * ? + - ETL_BATCH_PAGE_SIZE=1000 + - ETL_BATCH_INSERT_BATCH_SIZE=500 + - ETL_SCHEDULE_CRON=0 0 2 * * ? # JVM配置 - JAVA_OPTS=-Xms512m -Xmx1024m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 ports: diff --git a/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-prod.yaml b/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-prod.yaml index 6b254ef..18c7b7a 100644 --- a/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-prod.yaml +++ b/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer-prod.yaml @@ -12,120 +12,103 @@ services: # ============================================ syslog-consumer: build: - context: ./syslog-consumer + context: ./consumer-dm dockerfile: Dockerfile - container_name: xdr-syslog-consumer + image: syslog-consumer-dm:v1.2.2 + container_name: syslog-consumer-dm restart: unless-stopped environment: # 环境配置 - SPRING_PROFILES_ACTIVE=prod - TZ=Asia/Shanghai # 数据库配置 - - spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys - - spring.datasource.username=postgres - - spring.datasource.password=TnLanWaidYSwTSG5 - - spring.datasource.driver-class-name=org.postgresql.Driver + - SPRING_DATASOURCE_URL=jdbc:dm://192.168.4.99:5237 + - SPRING_DATASOURCE_USERNAME=SYSDBA + - SPRING_DATASOURCE_PASSWORD=caZ2TcmXNSW8L2Ap + - SPRING_DATASOURCE_DRIVER_CLASS_NAME=dm.jdbc.driver.DmDriver + - SPRING_DATASOURCE_HIKARI_SCHEMA=PUBLIC # HikariCP 连接池配置 - - spring.datasource.hikari.maximum-pool-size=50 - - spring.datasource.hikari.minimum-idle=5 - - spring.datasource.hikari.connection-timeout=30000 - - spring.datasource.hikari.idle-timeout=600000 - - spring.datasource.hikari.max-lifetime=900000 - - spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer - - spring.datasource.hikari.auto-commit=false - - spring.datasource.hikari.schema=public + - SPRING_DATASOURCE_HIKARI_MAXIMUM_POOL_SIZE=50 + - SPRING_DATASOURCE_HIKARI_MINIMUM_IDLE=5 + - SPRING_DATASOURCE_HIKARI_CONNECTION_TIMEOUT=30000 + - SPRING_DATASOURCE_HIKARI_IDLE_TIMEOUT=600000 + - SPRING_DATASOURCE_HIKARI_MAX_LIFETIME=900000 + - SPRING_DATASOURCE_HIKARI_POOL_NAME=HikariPool-SyslogConsumer + - SPRING_DATASOURCE_HIKARI_AUTO-COMMIT=false + # Redis配置 - - spring.redis.host=127.0.0.1 - - spring.redis.port=6379 - - spring.redis.password= - - spring.redis.database=0 - - spring.redis.timeout=2000 - - spring.cache.redis.time-to-live=600000 + - SPRING_REDIS_HOST=192.168.4.99 + - SPRING_REDIS_PORT=6379 + - SPRING_REDIS_PASSWORD=redis_GdGWte + - SPRING_REDIS_DATABASE=0 + - SPRING_REDIS_TIMEOUT=2000 + - SPRING_CACHE_REDIS_TIME_TO_LIVE=600000 # Kafka配置 - - spring.kafka.consumer.bootstrap-servers=192.168.222.130:9092 - - spring.kafka.consumer.group-id=test-group-app - - spring.kafka.consumer.auto-offset-reset=latest - - spring.kafka.consumer.enable-auto-commit=false - - spring.kafka.consumer.topic=test-topic - - spring.kafka.consumer.max-poll-records=1000 - - spring.kafka.listener.ack-mode=manual - - spring.kafka.listener.concurrency=2 - - spring.kafka.listener.type=batch + # Kafka配置 + - SPRING_KAFKA_CONSUMER_BOOTSTRAP_SERVERS=192.168.4.99:9092 + - SPRING_KAFKA_CONSUMER_GROUP_ID=agent-syslog-topic + - SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest + - SPRING_KAFKA_CONSUMER_ENABLE_AUTO_COMMIT=false + - SPRING_KAFKA_CONSUMER_TOPIC=agent-syslog-topic + - SPRING_KAFKA_CONSUMER_MAX_POLL_RECORDS=1000 + - SPRING_KAFKA_LISTENER_ACK_MODE=manual + - SPRING_KAFKA_LISTENER_CONCURRENCY=2 + - SPRING_KAFKA_LISTENER_TYPE=batch # InfluxDB配置 - - influxdb.url=http://192.168.222.131:8086 - - influxdb.token=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA== - - influxdb.org=yelang - - influxdb.bucket=yelangbucket - - influxdb.batch.size=1000 - - influxdb.flush.interval=1000 - - influxdb.connection.timeout=30s - - influxdb.connection.read-timeout=30s - - influxdb.connection.write-timeout=60s + - INFLUXDB_URL=http://192.168.4.99:8087 + - INFLUXDB_TOKEN=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg== + - INFLUXDB_ORG=influxdb + - INFLUXDB_BUCKET=yelangbucket + - INFLUXDB_BATCH_SIZE=1000 + - INFLUXDB_FLUSH_INTERVAL=1000 + - INFLUXDB_CONNECTION_TIMEOUT=30s + - INFLUXDB_CONNECTION_READ_TIMEOUT=30s + - INFLUXDB_CONNECTION_WRITE_TIMEOUT=60s # Elasticsearch配置 - - spring.elasticsearch.uris=http://192.168.1.174:9200 - - spring.elasticsearch.username=CONTAINER_NAME - - spring.elasticsearch.password=t2NZCiajmdazxBrF - - spring.elasticsearch.connection-timeout=10s - - spring.elasticsearch.socket-timeout=30s + - SPRING_ELASTICSEARCH_URIS=http://192.168.1.174:9200 + - SPRING_ELASTICSEARCH_USERNAME=CONTAINER_NAME + - SPRING_ELASTICSEARCH_PASSWORD=t2NZCiajmdazxBrF + - SPRING_ELASTICSEARCH_CONNECTION_TIMEOUT=10s + - SPRING_ELASTICSEARCH_SOCKET_TIMEOUT=30s # API配置 - - interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 - - interlocking.api.base-url=http://localhost:8089/xdrservice/interlocking + - INTERLOCKING_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 + - INTERLOCKING_API_BASE_URL=http://192.168.4.99:8089/xdrservice/interlocking # 探针心跳配置 - - probe.heartbeat.enabled=true - - probe.heartbeat.offline-threshold-minutes=10 - - probe.status.check.cron=0 */10 * * * ? - - probe.heartbeat.tenant-id=000000 - - probe.heartbeat.history.keep-days=10 - - probe.heartbeat.history.cleanup-enabled=true - - probe.history.cleanup.cron=0 0 1 * * ? + - PROBE_HEARTBEAT_ENABLED=true + - PROBE_HEARTBEAT_OFFLINE_THRESHOLD_MINUTES=10 + - PROBE_STATUS_CHECK_CRON=0 */10 * * * ? + - PROBE_HEARTBEAT_TENANT_ID=000000 + - PROBE_HEARTBEAT_HISTORY_KEEP_DAYS=10 + - PROBE_HEARTBEAT_HISTORY_CLEANUP_ENABLED=true + - PROBE_HISTORY_CLEANUP_CRON=0 0 1 * * ? # 告警健康检查配置 - - alarm.health-check.alarm-hours=4 - - alarm.health-check.alarm-visit-hours=2 - - alarm.health-check.enabled=true + - ALARM_HEALTH_CHECK_ALARM-HOURS=4 + - ALARM_HEALTH_CHECK_ALARM-VISIT-HOURS=2 + - ALARM_HEALTH_CHECK_ENABLED=true # 关联分析规则配置 - - analysis.realtime.enabled=true - - analysis.realtime.check-interval-seconds=10 + - ANALYSIS_REALTIME_ENABLED=true + - ANALYSIS_REALTIME_CHECK_INTERVAL_SECONDS=10 # 分区表检查配置 - - partition.check.tomorrow.enabled=true - - partition.check.future.days=7 - - partition.auto.create=true + - PARTITION_CHECK_TOMORROW_ENABLED=true + - PARTITION_CHECK_FUTURE_DAYS=7 + - PARTITION_AUTO_CREATE=true # 定时任务配置 - - spring.task.scheduling.pool.size=10 + - SPRING_TASK_SCHEDULING_POOL_SIZE=10 # 日志配置 - - logging.level.com.common.schedule=INFO - - logging.level.com.common.service=INFO + - LOGGING_LEVEL_COM_COMMON_SCHEDULE=INFO + - LOGGING_LEVEL_COM_COMMON_SERVICE=INFO # ETL配置 - - etl.batch.page-size=1000 - - etl.batch.insert-batch-size=500 - - etl.schedule.cron=0 0 2 * * ? - # JVM配置 - - JAVA_OPTS=-Xms512m -Xmx1024m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 + - ETL_BATCH_PAGE_SIZE=1000 + - ETL_BATCH_INSERT_BATCH_SIZE=500 + - ETL_SCHEDULE_CRON=0 0 2 * * ? + ports: - "8089:8089" volumes: - - syslog-consumer-logs:/app/logs + - /home/syslog/logs:/app/logs networks: - xdr-network - - # ============================================ - # PostgreSQL 数据库 (可选，如已存在可注释掉) - # ============================================ - postgres: - image: postgres:14-alpine - container_name: xdr-postgres - restart: unless-stopped - environment: - POSTGRES_DB: ecosys - POSTGRES_USER: postgres - POSTGRES_PASSWORD: TnLanWaidYSwTSG5 - TZ: Asia/Shanghai - ports: - - "54329:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - networks: - - xdr-network - + privileged: true # ============================================ # 网络配置 # ============================================ @@ -133,11 +116,3 @@ networks: xdr-network: driver: bridge -# ============================================ -# 卷配置 -# ============================================ -volumes: - syslog-consumer-logs: - driver: local - postgres_data: - driver: local diff --git a/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer.yaml b/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer.yaml index 849ec41..4925660 100644 --- a/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer.yaml +++ b/haobang-security-dm/syslog-consumer/docker-compose/docker-compose-consumer.yaml @@ -21,83 +21,79 @@ services: - SPRING_PROFILES_ACTIVE=dev - TZ=Asia/Shanghai # 数据库配置 - - spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys - - spring.datasource.username=postgres - - spring.datasource.password=TnLanWaidYSwTSG5 - - spring.datasource.driver-class-name=org.postgresql.Driver + - SPRING_DATASOURCE_URL=jdbc:postgresql://117.72.68.72:54329/ecosys + - SPRING_DATASOURCE_USERNAME=postgres + - SPRING_DATASOURCE_PASSWORD=TnLanWaidYSwTSG5 + - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver # HikariCP 连接池配置 - - spring.datasource.hikari.maximum-pool-size=50 - - spring.datasource.hikari.minimum-idle=5 - - spring.datasource.hikari.connection-timeout=30000 - - spring.datasource.hikari.idle-timeout=600000 - - spring.datasource.hikari.max-lifetime=900000 - - spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer - - spring.datasource.hikari.auto-commit=false - - spring.datasource.hikari.schema=public + - SPRING_DATASOURCE_HIKARI_MAXIMUM_POOL_SIZE=50 + - SPRING_DATASOURCE_HIKARI_MINIMUM_IDLE=5 + - SPRING_DATASOURCE_HIKARI_CONNECTION_TIMEOUT=30000 + - SPRING_DATASOURCE_HIKARI_IDLE_TIMEOUT=600000 + - SPRING_DATASOURCE_HIKARI_MAX_LIFETIME=900000 + - SPRING_DATASOURCE_HIKARI_POOL_NAME=HikariPool-SyslogConsumer + - SPRING_DATASOURCE_HIKARI_AUTO_COMMIT=false + - SPRING_DATASOURCE_HIKARI_SCHEMA=public # Redis配置 - - spring.redis.host=192.168.222.131 - - spring.redis.port=6379 - - spring.redis.password= - - spring.redis.database=0 - - spring.redis.timeout=2000 - - spring.cache.redis.time-to-live=600000 + - SPRING_REDIS_HOST=192.168.222.131 + - SPRING_REDIS_PORT=6379 + - SPRING_REDIS_PASSWORD= + - SPRING_REDIS_DATABASE=0 + - SPRING_REDIS_TIMEOUT=2000 + - SPRING_CACHE_REDIS_TIME_TO_LIVE=600000 # Kafka配置 - - spring.kafka.consumer.bootstrap-servers=192.168.222.130:9092 - - spring.kafka.consumer.group-id=test-group-app - - spring.kafka.consumer.auto-offset-reset=latest - - spring.kafka.consumer.enable-auto-commit=false - - spring.kafka.consumer.topic=test-topic - - spring.kafka.consumer.max-poll-records=1000 - - spring.kafka.listener.ack-mode=manual - - spring.kafka.listener.concurrency=2 - - spring.kafka.listener.type=batch + - SPRING_KAFKA_CONSUMER_BOOTSTRAP_SERVERS=192.168.222.130:9092 + - SPRING_KAFKA_CONSUMER_GROUP_ID=test-group-app + - SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest + - SPRING_KAFKA_CONSUMER_ENABLE_AUTO_COMMIT=false + - SPRING_KAFKA_CONSUMER_TOPIC=test-topic + - SPRING_KAFKA_CONSUMER_MAX_POLL_RECORDS=1000 + - SPRING_KAFKA_LISTENER_ACK_MODE=manual + - SPRING_KAFKA_LISTENER_CONCURRENCY=2 + - SPRING_KAFKA_LISTENER_TYPE=batch # InfluxDB配置 - - influxdb.url=http://192.168.222.131:8086 - - influxdb.token=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA== - - influxdb.org=yelang - - influxdb.bucket=yelangbucket - - influxdb.batch.size=1000 - - influxdb.flush.interval=1000 - - influxdb.connection.timeout=30s - - influxdb.connection.read-timeout=30s - - influxdb.connection.write-timeout=60s + - INFLUXDB_URL=http://192.168.222.131:8086 + - INFLUXDB_TOKEN=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA== + - INFLUXDB_ORG=yelang + - INFLUXDB_BUCKET=yelangbucket + - INFLUXDB_BATCH_SIZE=1000 + - INFLUXDB_FLUSH_INTERVAL=1000 + - INFLUXDB_CONNECTION_TIMEOUT=30s + - INFLUXDB_CONNECTION_READ_TIMEOUT=30s + - INFLUXDB_CONNECTION_WRITE_TIMEOUT=60s # Elasticsearch配置 - - spring.elasticsearch.uris=http://192.168.1.174:9200 - - spring.elasticsearch.username=CONTAINER_NAME - - spring.elasticsearch.password=t2NZCiajmdazxBrF - - spring.elasticsearch.connection-timeout=10s - - spring.elasticsearch.socket-timeout=30s + - SPRING_ELASTICSEARCH_URIS=http://192.168.1.174:9200 + - SPRING_ELASTICSEARCH_USERNAME=CONTAINER_NAME + - SPRING_ELASTICSEARCH_PASSWORD=t2NZCiajmdazxBrF + - SPRING_ELASTICSEARCH_CONNECTION_TIMEOUT=10s + - SPRING_ELASTICSEARCH_SOCKET_TIMEOUT=30s # API配置 - - interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 - - interlocking.api.base-url=http://192.168.222.131:8089/xdrservice/interlocking + - INTERLOCKING_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 + - INTERLOCKING_API_BASE_URL=http://192.168.222.131:8089/xdrservice/interlocking # 探针心跳配置 - - probe.heartbeat.enabled=true - - probe.heartbeat.offline-threshold-minutes=10 - - probe.status.check.cron=0 */10 * * * ? - - probe.heartbeat.tenant-id=000000 - - probe.heartbeat.history.keep-days=10 - - probe.heartbeat.history.cleanup-enabled=true - - probe.history.cleanup.cron=0 0 1 * * ? + - PROBE_HEARTBEAT_ENABLED=true + - PROBE_HEARTBEAT_OFFLINE_THRESHOLD_MINUTES=10 + - PROBE_STATUS_CHECK_CRON=0 */10 * * * ? + - PROBE_HEARTBEAT_TENANT_ID=000000 + - PROBE_HEARTBEAT_HISTORY_KEEP_DAYS=10 + - PROBE_HEARTBEAT_HISTORY_CLEANUP_ENABLED=true + - PROBE_HISTORY_CLEANUP_CRON=0 0 1 * * ? # 告警健康检查配置 - - alarm.health-check.alarm-hours=4 - - alarm.health-check.alarm-visit-hours=2 - - alarm.health-check.enabled=true + - ALARM_HEALTH_CHECK_ALARM_HOURS=4 + - ALARM_HEALTH_CHECK_ALARM_VISIT_HOURS=2 + - ALARM_HEALTH_CHECK_ENABLED=true # 关联分析规则配置 - - analysis.realtime.enabled=true - - analysis.realtime.check-interval-seconds=10 + - ANALYSIS_REALTIME_ENABLED=true + - ANALYSIS_REALTIME_CHECK_INTERVAL_SECONDS=10 # 分区表检查配置 - - partition.check.tomorrow.enabled=true - - partition.check.future.days=7 - - partition.auto.create=true + - PARTITION_CHECK_TOMORROW_ENABLED=true + - PARTITION_CHECK_FUTURE_DAYS=7 + - PARTITION_AUTO_CREATE=true # 定时任务配置 - - spring.task.scheduling.pool.size=10 + - SPRING_TASK_SCHEDULING_POOL_SIZE=10 # 日志配置 - - logging.level.com.common.schedule=INFO - - logging.level.com.common.service=INFO - # ETL配置 - - etl.batch.page-size=1000 - - etl.batch.insert-batch-size=500 - - etl.schedule.cron=0 0 2 * * ? + - LOGGING_LEVEL_COM_COMMON_SCHEDULE=INFO + - LOGGING_LEVEL_COM_COMMON_SERVICE=INFO # JVM配置 - JAVA_OPTS=-Xms512m -Xmx1024m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 ports: diff --git a/haobang-security-dm/syslog-consumer/pom.xml b/haobang-security-dm/syslog-consumer/pom.xml index 86bdcb4..d4a00ea 100644 --- a/haobang-security-dm/syslog-consumer/pom.xml +++ b/haobang-security-dm/syslog-consumer/pom.xml @@ -46,7 +46,7 @@ org.apache.kafka kafka-clients - 3.4.0 + 3.9.2 org.springframework.kafka @@ -104,8 +104,8 @@ com.dameng - DmJdbcDriver18 - 8.1.2.141 + DmJdbcDriver + 8.1.4.169 diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java index 2d1ab30..b2e163a 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/LogNormalProcessor.java @@ -1,462 +1,361 @@ package com.Modules.NormalData; + import cn.hutool.core.date.DateTime; -import com.common.entity.XdrHoneypot; -import com.common.mapper.XdrHoneypotMapper; -import com.common.service.SyslogNonNormalMessageService; -import com.common.util.*; -import com.config.AppConfig; -import org.apache.ibatis.session.SqlSession; -import org.springframework.beans.factory.annotation.Autowired; -import com.common.entity.SyslogMessage; -import com.influx.SyslogToInfluxApp; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import com.common.service.DmNormalizeRuleService; -import com.common.service.DmColumnService; -import com.common.service.SyslogNormalDataService; -import com.common.entity.DmColumn; -import com.common.service.impl.DmColumnServiceImpl; -import com.common.mapper.DmColumnMapper; -import org.springframework.stereotype.Service; -import org.springframework.web.bind.annotation.RestController; -import com.common.mapper.DmColumnMapper; -import com.common.mapper.DmNormalizeRuleMapper; -import java.sql.Timestamp; -import java.util.*; - -import org.springframework.stereotype.Component; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import org.json.JSONObject; -import com.common.entity.RuleContent.*; -import com.common.entity.SyslogNonNormalMessage; -import org.joda.time.LocalDateTime; -import com.common.service.LogDataFilterService; -import com.common.service.LogDataCompleteService; -import com.common.service.DeviceCollectTaskService; import com.common.entity.DeviceCollectTask; import com.common.entity.DeviceDevice; -import com.common.util.TimeConversionUtils; -import com.common.service.DeviceDeviceService; +import com.common.entity.RuleContent.*; +import com.common.entity.SyslogNonNormalMessage; +import com.common.mapper.DmColumnMapper; +import com.common.mapper.DmNormalizeRuleMapper; +import com.common.service.*; +import com.common.util.*; +import com.config.AppProperties; +import com.config.KafkaConsumerProperties; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.joda.time.LocalDateTime; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import java.sql.Timestamp; +import java.util.*; +import java.util.stream.Collectors; +import com.common.entity.SyslogMessage; + +/** + * 日志标准化处理器 + * 线程安全版本：运行时状态封装为 ProcessContext，每次 process() 调用独立（解决多线程 NPE 问题） + */ +@Service public class LogNormalProcessor { private static final Logger logger = LoggerFactory.getLogger(LogNormalProcessor.class); - private String strLogMsg ; - private String strKafkaMessage; - private String strDeviceInfo; - private String strDataType ="json" ; - private Map messageMap ; - private Map deviceInfoMap ; - private String strSyslogUUID ; - private String strSyslogTopic ; - private boolean isSaveNonNormal =false ; + // ==================== 依赖注入（Spring Bean，单例，线程安全）==================== + private final DmNormalizeRuleService dmNormalizeRuleService; + private final DmColumnService dmColumnService; + private final SyslogNormalDataService syslogNormalDataService; + private final SyslogNonNormalMessageService messageService; + private final LogDataFilterService logDataFilterService; + private final LogDataCompleteService logDataCompleteService; + private final DeviceCollectTaskService deviceCollectTaskService; + private final DeviceDeviceService deviceDeviceService; + private final DmColumnMapper dmColumnMapper; + private final DmNormalizeRuleMapper dmNormalizeRuleMapper; + private final KafkaConsumerProperties kafkaConsumerProperties; + private final AppProperties appProperties; - @Autowired - public DmNormalizeRuleService dmNormalizeRuleService ; - @Autowired - public DmColumnService dmColumnService ; - @Autowired - public SyslogNormalDataService syslogNormalDataService =SpringContextUtil.getBean(SyslogNormalDataService.class); - @Autowired - private SyslogNonNormalMessageService messageService =SpringContextUtil.getBean(SyslogNonNormalMessageService.class); - @Autowired - private LogDataFilterService logDataFilterService= SpringContextUtil.getBean(LogDataFilterService.class); - - @Autowired - private LogDataCompleteService logDataCompleteService= SpringContextUtil.getBean(LogDataCompleteService.class); - @Autowired - private DeviceCollectTaskService deviceCollectTaskService= SpringContextUtil.getBean(DeviceCollectTaskService.class); - @Autowired - private DeviceDeviceService deviceDeviceService= SpringContextUtil.getBean(DeviceDeviceService.class); - - @Autowired - SyslogNonNormalMessage syslogNonNormalMessage=new SyslogNonNormalMessage(); - - @Autowired - DmColumnMapper dmColumnMapper; - @Autowired - DmNormalizeRuleMapper dmNormalizeRuleMapper; - - private List> dmNormalizeRuleList; - private List> dmColumnList; - private LinkedHashMap OrginalColumnMap ; - - public LogNormalProcessor( String LogMsg, String syslogUUID,String syslogTopic) { - - /** 标准化数据处理步骤 */ - //初始化 （获取syslog_normal_data 表全部字段的属性、设备ID对应的规则设置、转化成JSON） - //判断数据解析类型（json、键值、xml、正则表达式、分割符）解析syslogMessage 字段转成 HashMAP - //匹配规则内容，获取字段映射关系配置，抽取命中的字段名称、数值 - //判断字段内容及类型，根据数据类型属性，进行内容转换。 - //生成insert SQL语句、执行入库操作。 - strKafkaMessage=LogMsg; - strSyslogUUID=syslogUUID; - strSyslogTopic=syslogTopic; - - if(!LogMsg.isEmpty()) { - strLogMsg = SyslogParser.substringAfterFirstCloseBracket(LogMsg); - strDeviceInfo=SyslogParser.substringBeforeFirstChar(LogMsg,']'); - } - else{ - LogMsg="[receive_time=20251118165909470 device_id=1 device_name=honeypot vendor=changting data_type=json device_collect_id=1]<14>1 2025-09-24T11:52:26Z 5f46d3be75e1 supermario 128 honeypot_event - {\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}"; - strLogMsg="<14>1 2025-09-24T11:52:26Z 5f46d3be75e1 supermario 128 honeypot_event - {\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}"; - strDeviceInfo=SyslogParser.substringBeforeFirstChar(LogMsg,']'); - } - } /** - * 初始化数据init() + * 请求级处理上下文 —— 每个 process() 调用创建新实例，彻底解决多线程竞态条件 */ - public void init() - { - Map mapdev =SyslogParser.parseKeyValuePairs(strDeviceInfo); - deviceInfoMap=mapdev; - //获取日志对应的设备ID - long deviceID=Long.parseLong(mapdev.get("device_id")); - System.out.println("device_id:"+deviceID ); - strDataType=mapdev.get("data_type"); - //getDeviceID(strLogMsg); - try{ - // 通过工具类获取Service实例 - dmColumnService= SpringContextUtil.getBean(DmColumnService.class); - dmNormalizeRuleService= SpringContextUtil.getBean(DmNormalizeRuleService.class); - if(deviceID>0) { - dmNormalizeRuleList = dmNormalizeRuleService.selectByDeviceIdAuto(deviceID); + private static class ProcessContext { + String strLogMsg; + String strKafkaMessage; + String strDeviceInfo; + String strDataType = "json"; + Map messageMap; + Map deviceInfoMap; + String strSyslogUUID; + String strSyslogTopic; + boolean isSaveNonNormal = false; + SyslogNonNormalMessage syslogNonNormalMessage = new SyslogNonNormalMessage(); + List> dmNormalizeRuleList; + List> dmColumnList; + LinkedHashMap OrginalColumnMap; + } + + // ==================== 构造器注入（Spring 自动完成）==================== + @Autowired + public LogNormalProcessor( + DmNormalizeRuleService dmNormalizeRuleService, + DmColumnService dmColumnService, + SyslogNormalDataService syslogNormalDataService, + SyslogNonNormalMessageService messageService, + LogDataFilterService logDataFilterService, + LogDataCompleteService logDataCompleteService, + DeviceCollectTaskService deviceCollectTaskService, + DeviceDeviceService deviceDeviceService, + DmColumnMapper dmColumnMapper, + DmNormalizeRuleMapper dmNormalizeRuleMapper, + KafkaConsumerProperties kafkaConsumerProperties, + AppProperties appProperties + ) { + this.dmNormalizeRuleService = dmNormalizeRuleService; + this.dmColumnService = dmColumnService; + this.syslogNormalDataService = syslogNormalDataService; + this.messageService = messageService; + this.logDataFilterService = logDataFilterService; + this.logDataCompleteService = logDataCompleteService; + this.deviceCollectTaskService = deviceCollectTaskService; + this.deviceDeviceService = deviceDeviceService; + this.dmColumnMapper = dmColumnMapper; + this.dmNormalizeRuleMapper = dmNormalizeRuleMapper; + this.kafkaConsumerProperties = kafkaConsumerProperties; + this.appProperties = appProperties; + } + + /** + * 处理一条日志消息（线程安全：每次调用创建独立的 ProcessContext） + * + * @param logMsg 完整日志消息（含头部） + * @param syslogUUID 消息UUID + * @param syslogTopic 消息Topic（可传 null，自动使用配置值） + */ + public void process(String logMsg, String syslogUUID, String syslogTopic) { + ProcessContext ctx = new ProcessContext(); + ctx.strSyslogUUID = syslogUUID; + ctx.strSyslogTopic = (syslogTopic != null && !syslogTopic.isEmpty()) + ? syslogTopic + : kafkaConsumerProperties.getTopic(); + ctx.isSaveNonNormal = false; + ctx.syslogNonNormalMessage = new SyslogNonNormalMessage(); + + // 解析消息 + ctx.strKafkaMessage = logMsg; + if (!logMsg.isEmpty()) { + ctx.strLogMsg = SyslogParser.substringAfterFirstCloseBracket(logMsg); + ctx.strDeviceInfo = SyslogParser.substringBeforeFirstChar(logMsg, ']'); + } else { + // 测试用例硬编码数据 + logMsg = "[receive_time=20251118165909470 device_id=1 device_name=honeypot vendor=changting data_type=json device_collect_id=1]<14>1 2025-09-24T11:52:26Z 5f46d3be75e1 supermario 128 honeypot_event - \uFEFF{\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\"}"; + ctx.strLogMsg = "<14>1 2025-09-24T11:52:26Z 5f46d3be75e1 supermario 128 honeypot_event - \uFEFF{\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\"}"; + ctx.strDeviceInfo = SyslogParser.substringBeforeFirstChar(logMsg, ']'); + } + + // 执行处理 + init(ctx); + } + + /** + * 初始化数据处理（所有运行时状态通过 ctx 传递，线程安全） + */ + private void init(ProcessContext ctx) { + Map mapdev = SyslogParser.parseKeyValuePairs(ctx.strDeviceInfo); + ctx.deviceInfoMap = mapdev; + long deviceID = Long.parseLong(mapdev.get("device_id")); + System.out.println("device_id:" + deviceID); + ctx.strDataType = mapdev.get("data_type"); + try { + if (deviceID > 0) { + ctx.dmNormalizeRuleList = dmNormalizeRuleService.selectByDeviceIdAuto(deviceID); } - //dmColumnList=dmColumnService.selectAllNormal(); - //System.out.println("dmColumnList size:"+ dmColumnList.size()); - - //OrginalColumnMap=getMessageToMap(strLogMsg); - //解析SyslogMessage - //SyslogMessage logMsg = SyslogParser.parse(strLogMsg); - //设备对应的规则normal rule Map 进行标准化数据处理 - for (int i = 0; i < dmNormalizeRuleList.size(); i++) { + for (int i = 0; i < ctx.dmNormalizeRuleList.size(); i++) { try { - Map dmNormalizeRule = dmNormalizeRuleList.get(i); + Map dmNormalizeRule = ctx.dmNormalizeRuleList.get(i); String data_type = dmNormalizeRule.get("data_type").toString(); - //数据类型及格式不符，则break; System.out.println("索引: " + i + ", 值: " + dmNormalizeRule); - System.out.println("normalrule ID: " + dmNormalizeRule.get("id") + ", display_name:" + dmNormalizeRule.get("display_name") + " data_type:" + dmNormalizeRule.get("data_type")); - //数据类型不匹配，则跳过规则 - //if (!data_type.equals(strDataType)) break; + System.out.println("normalrule ID: " + dmNormalizeRule.get("id") + + ", display_name:" + dmNormalizeRule.get("display_name") + + " data_type:" + dmNormalizeRule.get("data_type")); - //获取syslog message 文本解析配置项 - OrginalColumnMap = getMessageToMap(dmNormalizeRule, strLogMsg, data_type); - if ((OrginalColumnMap == null) || (OrginalColumnMap.size()==0)) { + ctx.OrginalColumnMap = getMessageToMap(dmNormalizeRule, ctx.strLogMsg, data_type); + if ((ctx.OrginalColumnMap == null) || (ctx.OrginalColumnMap.size() == 0)) { logger.error("OrginalColumnMap 对象获取为空"); - //保存非标日志信息 - syslogNonNormalMessage.setReason("Log解析异常"); - syslogNonNormalMessage.setReasonDetail("log解析异常,返回规则名称："+dmNormalizeRule.get("display_name") +",OrginalColumnMap 对象获取为空"); - if(isSaveNonNormal==false) - SaveNonNormalMessage(deviceID, DateTime.now()); + ctx.syslogNonNormalMessage.setReason("Log解析异常"); + ctx.syslogNonNormalMessage.setReasonDetail("log解析异常,返回规则名称：" + + dmNormalizeRule.get("display_name") + ",OrginalColumnMap 对象获取为空"); + if (ctx.isSaveNonNormal == false) + SaveNonNormalMessage(ctx, deviceID, DateTime.now()); continue; } List> destColumnList = getRuleContentMappers(dmNormalizeRule); - //解析字段匹配已命中的配置规则字段 - List> ruleColumnList = getNormalColumnList(OrginalColumnMap, destColumnList); - //System.out.println("ruleColumnList :"+ ruleColumnList); + List> ruleColumnList = getNormalColumnList(ctx.OrginalColumnMap, destColumnList); Map destMap = getColumnMap(ruleColumnList); - //List> destMap2= getCompleteColumnsList(dmNormalizeRule); - //数据处理-过滤规则 - if (logDataFilterService.evaluateFilterRule(dmNormalizeRule.get("rule_content").toString(), destMap)) + // 数据处理-过滤规则 + if (logDataFilterService.evaluateFilterRule( + dmNormalizeRule.get("rule_content").toString(), destMap)) continue; - //数据处理-补全规则 - logDataCompleteService.processDataCompletion(dmNormalizeRule.get("rule_content").toString(), destMap); - //long ruleid=Long.parseLong(dmNormalizeRule.get("id").toString()); - SaveNormalData(deviceID, DateTime.now(), destMap, Long.parseLong(dmNormalizeRule.get("id").toString()), dmNormalizeRule.get("name").toString()); + // 数据处理-补全规则 + logDataCompleteService.processDataCompletion( + dmNormalizeRule.get("rule_content").toString(), destMap); + SaveNormalData(ctx, deviceID, DateTime.now(), destMap, + Long.parseLong(dmNormalizeRule.get("id").toString()), + dmNormalizeRule.get("name").toString()); } catch (Exception ex) { logger.error("泛化规则处理失败：" + ex.getMessage()); System.out.println(ex.getMessage()); - /** - //保存非标日志信息 - syslogNonNormalMessage.setReason("泛化规则处理失败"); - syslogNonNormalMessage.setReasonDetail("泛化规则处理失败,失败详情："+ ex.getMessage()); - SaveNonNormalMessage(deviceID,DateTime.now()); - **/ - } } - } - catch (Exception ex) - { - logger.error("处理日志消息异常："+ex.getMessage()); + } catch (Exception ex) { + logger.error("处理日志消息异常：" + ex.getMessage()); System.out.println(ex.getMessage()); } - } + // ==================== 以下为原有业务方法 ==================== - //获取数据泛化规则的列表字段，提取 Cropper_params - public Cropper_paramsType getCropperParams( Map dmNormalizeRule ) - { - Cropper_paramsType cropperParams = new Cropper_paramsType(); + public Cropper_paramsType getCropperParams(Map dmNormalizeRule) { + Cropper_paramsType cropperParams = new Cropper_paramsType(); try { - - JSONObject jsonObject = new JSONObject( dmNormalizeRule.get("rule_content").toString()); + JSONObject jsonObject = new JSONObject(dmNormalizeRule.get("rule_content").toString()); if (jsonObject.isEmpty()) return null; - cropperParams.sethead_key(jsonObject.getJSONObject("cropper_params").get("head_key").toString()); - //需要判断是否字符串head_offset是否空 - cropperParams.sethead_offset(Integer.parseInt(jsonObject.getJSONObject("cropper_params").get("head_offset").toString())); + cropperParams.sethead_offset(Integer.parseInt( + jsonObject.getJSONObject("cropper_params").get("head_offset").toString())); cropperParams.settail_key(jsonObject.getJSONObject("cropper_params").get("tail_key").toString()); - //需要判断是否字符串tail_offset是否空 - cropperParams.settail_offset( Integer.parseInt(jsonObject.getJSONObject("cropper_params").get("tail_offset").toString())); + cropperParams.settail_offset(Integer.parseInt( + jsonObject.getJSONObject("cropper_params").get("tail_offset").toString())); return cropperParams; - }catch (Exception ex) { - logger.error("getCropperParams："+ex.getMessage()); + } catch (Exception ex) { + logger.error("getCropperParams：" + ex.getMessage()); return null; } - } - - public List> getRuleContentMappers( Map dmNormalizeRule ) - { - List> filedList =null; + public List> getRuleContentMappers(Map dmNormalizeRule) { + List> filedList = null; ObjectMapper objectMapper = new ObjectMapper(); try { - // 转换为list Map - //Map rule_contentMap = objectMapper.readValue( dmNormalizeRule.get("rule_content").toString(), Map.class); - //if (rule_contentMap.isEmpty()) return null; - JSONObject jsonObject = new JSONObject( dmNormalizeRule.get("rule_content").toString()); + JSONObject jsonObject = new JSONObject(dmNormalizeRule.get("rule_content").toString()); if (jsonObject.isEmpty()) return null; - ObjectMapper objectMapper_mappers = new ObjectMapper(); - filedList =readJsonToList( jsonObject.get("mappers").toString() ); - - }catch (Exception ex) { - logger.error("getRuleContentMappers异常："+ex.getMessage()); + filedList = readJsonToList(jsonObject.get("mappers").toString()); + } catch (Exception ex) { + logger.error("getRuleContentMappers异常：" + ex.getMessage()); } return filedList; } - - - public List> getCompleteColumnsList( Map dmNormalizeRule ) - { - List> filedList =null; - ObjectMapper objectMapper = new ObjectMapper(); + public List> getCompleteColumnsList(Map dmNormalizeRule) { + List> filedList = null; try { - // 转换为list Map - //Map rule_contentMap = objectMapper.readValue( dmNormalizeRule.get("rule_content").toString(), Map.class); - //if (rule_contentMap.isEmpty()) return null; - JSONObject jsonObject = new JSONObject( dmNormalizeRule.get("rule_content").toString()); + JSONObject jsonObject = new JSONObject(dmNormalizeRule.get("rule_content").toString()); if (jsonObject.isEmpty()) return null; - ObjectMapper objectMapper_mappers = new ObjectMapper(); - filedList =readJsonToList( jsonObject.get("complete_columns").toString() ); - - }catch (Exception ex) { - logger.error("getCompleteColumnsList："+ex.getMessage()); + filedList = readJsonToList(jsonObject.get("complete_columns").toString()); + } catch (Exception ex) { + logger.error("getCompleteColumnsList：" + ex.getMessage()); } return filedList; } - /** - * 将JSON 字符串转换为 List> - */ + public static List> readJsonToList(String jsonStr) { ObjectMapper objectMapper = new ObjectMapper(); try { - return objectMapper.readValue( - jsonStr, - new TypeReference>>() {} - ); + return objectMapper.readValue(jsonStr, + new TypeReference>>() {}); } catch (Exception e) { - - logger.error("readJsonToList 解析失败："+jsonStr); - //throw new RuntimeException("JSON 解析失败: " + e.getMessage(), e); + logger.error("readJsonToList 解析失败：" + jsonStr); return null; } } - /**\ - * 判断日志信息的数据类型 - * @param logMsg - * @return - */ - public String getLogDataType( String logMsg) - { - //判断日志信息的数据类型(json、regex、kv、sep、xml） - //默认返回json + + public String getLogDataType(String logMsg) { return "json"; } - public int getDeviceID( String logMsg) - { - //解析日志信息头部，获取device_id - // 默认返回device_id =1 + public int getDeviceID(String logMsg) { return 1; } - //获取log 内容字段及数据值 - public LinkedHashMap getMessageToMap( String MsgContent ) - { - LinkedHashMap linkMap =null; - if(strDataType.equals("json")) { - //json 类型 - //解析SyslogMessage + + public LinkedHashMap getMessageToMap(String MsgContent) { + LinkedHashMap linkMap = null; + if ("json".equals("json")) { // 默认走 json 分支 SyslogMessage msg = SyslogParser.parse(MsgContent); System.out.println("解析结果-log content: " + msg.getMessage().toString()); - String complexJson = "{\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}"; + String complexJson = "{\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\"}"; System.out.println("complexJson content: " + complexJson); - //静态字符串、编码有问题，临时用 静态字符串做测试，流程环境没问题 - if(AppConfig.geRunEnvironment‌().equals("dev")) { - //LinkedHashMap flatMap = JsonParser.parseJsonToFlatMap(complexJson); - LinkedHashMap flatMap =JsonParser.jsonToMap(complexJson); + if ("dev".equals(appProperties.getRunEnvironment())) { + LinkedHashMap flatMap = JsonParser.jsonToMap(complexJson); flatMap.forEach((key, value) -> System.out.println(key + " = " + value)); return flatMap; - } - else - { - //LinkedHashMap flatMap = JsonParser.parseJsonToFlatMap(msg.getMessage()); - LinkedHashMap flatMap =JsonParser.jsonToMap(msg.getMessage()); + } else { + LinkedHashMap flatMap = JsonParser.jsonToMap(msg.getMessage()); flatMap.forEach((key, value) -> System.out.println(key + " = " + value)); return flatMap; } } - else if(strDataType.equals("kv")) //key-value 键值类型 - { - return null; - } - else if(strDataType.equals("sep")) //分隔符 - { - return null; - } - else if(strDataType.equals("xml")) //类型 xml - { - return null; - } - else if(strDataType.equals("regex")) //正则表达式 - { - return null; - } - // List> rulelst=dmNormalizeRuleService.selectByDeviceId((long)1); return linkMap; } - - //获取log message内容字段及数据值 - public LinkedHashMap getMessageToMap( Map dmNormalizeRule, String MsgContent , String dataType) - { + public LinkedHashMap getMessageToMap( + Map dmNormalizeRule, String MsgContent, String dataType) { LinkedHashMap linkMap = new LinkedHashMap<>(); try { - //String decode =dmNormalizeRule.get("decode").toString(); - Cropper_paramsType cropperParams = getCropperParams(dmNormalizeRule); + Cropper_paramsType cropperParams = getCropperParams(dmNormalizeRule); if (dataType.equals("json")) { String strJson = logNormalData.ParserMessageJsonType(MsgContent, cropperParams); LinkedHashMap result1 = NestedJsonParserUtil.safeParseJson(strJson); LinkedHashMap flattened = NestedJsonUtils.flattenNestedJson(result1); return flattened; - //parseJsonToFlatMap 复杂的json转换、解析过程有异常 - //return JsonParser.parseJsonToFlatMap(strJson); - // return JsonParser.jsonToMap(strJson); - } - else if (dataType.equals("kv")) //key-value 键值类型 - { + } else if (dataType.equals("kv")) { String strKeyVal = logNormalData.ParserMessageJsonType(MsgContent, cropperParams); - KvTextParser kvTextParser =new KvTextParser(); - kv_paramsType kvparams=logNormalData.getkv_paramsType( dmNormalizeRule); - linkMap= kvTextParser.parseKvText(strKeyVal,kvparams); + KvTextParser kvTextParser = new KvTextParser(); + kv_paramsType kvparams = logNormalData.getkv_paramsType(dmNormalizeRule); + linkMap = kvTextParser.parseKvText(strKeyVal, kvparams); return linkMap; - } - else if (dataType.equals("sep")) //分隔符 - { - //获取分隔符 - String SepKey=logNormalData.sepType(dmNormalizeRule); + } else if (dataType.equals("sep")) { + String SepKey = logNormalData.sepType(dmNormalizeRule); String strSep = logNormalData.ParserMessageJsonType(MsgContent, cropperParams); - linkMap= TextParserUtil.parseSeparatedText(strSep, SepKey); + linkMap = TextParserUtil.parseSeparatedText(strSep, SepKey); return linkMap; - } - else if (dataType.equals("xml")) //类型 xml - { + } else if (dataType.equals("xml")) { return linkMap; - } - else if (dataType.equals("regex")) //正则表达式 - { + } else if (dataType.equals("regex")) { String strRegex = logNormalData.ParserMessageJsonType(MsgContent, cropperParams); - String regexp= logNormalData.Regexp(dmNormalizeRule); - linkMap= RegexTextParser.parseWithRegex(strRegex,regexp ); + String regexp = logNormalData.Regexp(dmNormalizeRule); + linkMap = RegexTextParser.parseWithRegex(strRegex, regexp); return linkMap; } - }catch (Exception ex) { - logger.error("getMessageToMap："+ex.getMessage()); - return null; + } catch (Exception ex) { + logger.error("getMessageToMap：" + ex.getMessage()); + return null; } return linkMap; } - /** - * 根据设备ID获取配置规则 - * @param device_id - * @return List> - */ - public List> getRuleList( long device_id) - { - List> rulelst=dmNormalizeRuleService.selectByDeviceId((long)1); - if (rulelst!=null) - { + public List> getRuleList(long device_id) { + List> rulelst = dmNormalizeRuleService.selectByDeviceId((long) 1); + if (rulelst != null) { System.out.println("rulelst: " + rulelst); - } - else{ - logger.error(" List> rulelst is null！" ); + } else { + logger.error(" List> rulelst is null！"); } return rulelst; } - /** - * 查找命中配置规则的字段及数值 - * @param destColumnList 泛化目标字段list - * @param destColumnList 日志源字段及数值list - * @return - */ - public List> getNormalColumnList( LinkedHashMap orginColumnMap ,List> destColumnList ) - { - List> columnlist =new ArrayList<>(); - // 原始解析字段遍历泛化规则目标字段 - for (Map.Entry entry : orginColumnMap.entrySet()) { - //System.out.println(entry.getKey() + ": " + entry.getValue()); + public List> getNormalColumnList( + LinkedHashMap orginColumnMap, + List> destColumnList) { + List> columnlist = new ArrayList<>(); + for (Map.Entry entry : orginColumnMap.entrySet()) { for (Map map : destColumnList) { - //System.out.println( "origin_field: " + map.get("origin_field").toString()); - if ( map.get("origin_field").toString().equals(entry.getKey()) ) { + if (map.get("origin_field").toString().equals(entry.getKey())) { System.out.println(map); - Map normalColumMap =new LinkedHashMap<>(); - normalColumMap.put("origin_field",entry.getKey() ); - normalColumMap.put("dest_field",map.get("dest_field").toString()); - normalColumMap.put("action",(HashMap)map.get("action") ); - //normalColumMap.put("mapping ",entry.getKey() ); - normalColumMap.put("origin_field_value",entry.getValue() ); - // System.out.println("action: " + map.get("action").toString()); - if( ((HashMap)map.get("action")).get("type").equals("equal")) - //直接赋值 - normalColumMap.put("dest_field_value",entry.getValue() ); - else if(((HashMap)map.get("action")).get("type").equals("mapping")) - { - //mapping 映射枚举值 - normalColumMap.put("dest_field_value",entry.getValue() ); - normalColumMap.put("action_param",((HashMap)map.get("action")).get("param") ); - HashMap action_param=(HashMap)((HashMap)map.get("action")).get("param") ; - //匹配并获取映射枚举值 - normalColumMap.put("dest_field_value",getMappingValue(action_param ,entry.getValue().toString() )); - } - else if(((HashMap)map.get("action")).get("type").equals("time")) - { - //time 类型 - //normalColumMap.put("dest_field_value",entry.getValue() ); - normalColumMap.put("action_param",((HashMap)map.get("action")).get("param") ); - HashMap action_param=(HashMap)((HashMap)map.get("action")).get("param") ; - //匹配时间格式并转成换整型格式 + Map normalColumMap = new LinkedHashMap<>(); + normalColumMap.put("origin_field", entry.getKey()); + normalColumMap.put("dest_field", map.get("dest_field").toString()); + normalColumMap.put("action", (HashMap) map.get("action")); + normalColumMap.put("origin_field_value", entry.getValue()); + if (((HashMap) map.get("action")).get("type").equals("equal")) { + normalColumMap.put("dest_field_value", entry.getValue()); + } else if (((HashMap) map.get("action")).get("type").equals("mapping")) { + normalColumMap.put("action_param", + ((HashMap) map.get("action")).get("param")); + HashMap action_param = + (HashMap) ((HashMap) map.get("action")).get("param"); + normalColumMap.put("dest_field_value", + getMappingValue(action_param, entry.getValue().toString())); + } else if (((HashMap) map.get("action")).get("type").equals("time")) { + normalColumMap.put("action_param", + ((HashMap) map.get("action")).get("param")); + HashMap action_param = + (HashMap) ((HashMap) map.get("action")).get("param"); try { - long longTime = TimeConversionUtils.convertToMillis(entry.getValue().toString(), action_param.get("timezone").toString()); + long longTime = TimeConversionUtils.convertToMillis( + entry.getValue().toString(), action_param.get("timezone").toString()); normalColumMap.put("dest_field_value", longTime); } catch (Exception e) { - logger.error("时间类型转换错误，源值：" + entry.getValue().toString() + ",java_date_format:" + action_param.get("java_date_format").toString()); + logger.error("时间类型转换错误，源值：" + entry.getValue().toString() + + ",java_date_format:" + action_param.get("java_date_format").toString()); e.printStackTrace(); } - } - columnlist.add((HashMap)normalColumMap); - //System.out.println( "normalColumMap: " +normalColumMap); - //存在源字段配置多个目标字段，使用continue,而不是break + columnlist.add((HashMap) normalColumMap); continue; } } @@ -464,156 +363,125 @@ public class LogNormalProcessor { return columnlist; } - /** - * paramMapping Map匹配查找对应的值 - * @param paramMappingValueMap - * @param - * @return - */ - public Object getMappingValue(HashMap paramMappingValueMap,String value) - { - HashMap Map= (HashMap)paramMappingValueMap.get("mapping"); - for (Map.Entry entry : Map.entrySet()) { - if ( entry.getKey().equals(value) ) { - return entry.getValue(); + public Object getMappingValue(HashMap paramMappingValueMap, String value) { + HashMap Map = (HashMap) paramMappingValueMap.get("mapping"); + for (Map.Entry entry : Map.entrySet()) { + if (entry.getKey().equals(value)) { + return entry.getValue(); } } return null; } - /** - * 获取字段Map，包含字段field及Value值 - * @param normalColumnList - * @return - */ - public Map getColumnMap( List> normalColumnList ) - { - Map columnMap= new HashMap<>(); + public Map getColumnMap(List> normalColumnList) { + Map columnMap = new HashMap<>(); for (Map map : normalColumnList) { - Object destFieldValue = map.get("dest_field_value"); - // 判断 dest_field_value 是否为 String 且包含 "\u0000" - if (destFieldValue instanceof String && ((String) destFieldValue).contains("\u0000")) { - // 替换掉所有 "\u0000" 字符 + if (destFieldValue instanceof String + && ((String) destFieldValue).contains("\u0000")) { destFieldValue = ((String) destFieldValue).replace("\u0000", ""); } columnMap.put(map.get("dest_field").toString(), destFieldValue); - //columnMap.put(map.get("dest_field").toString(),map.get("dest_field_value")); } return columnMap; } - /** - * 保存数据到标准化表 - * @param deviceId - * @param logtime - * @param logColumnMap - */ - public void SaveNormalData(long deviceId , DateTime logtime, Map logColumnMap, long normalizeRuleId ,String normalizeRuleName) - { + public void SaveNormalData(ProcessContext ctx, long deviceId, DateTime logtime, + Map logColumnMap, + long normalizeRuleId, String normalizeRuleName) { try { - if(logColumnMap.isEmpty() ) - { - logger.error("SaveNormalData ->logColumnMap 为空，syslogUUID:" +this.strSyslogUUID); - //保存非标日志信息 - syslogNonNormalMessage.setReason("未命中规则"); - syslogNonNormalMessage.setReasonDetail("失败详情：logColumnMap对象字段为空" ); - if(isSaveNonNormal==false) - SaveNonNormalMessage(deviceId,DateTime.now()); - return ; + if (logColumnMap.isEmpty()) { + logger.error("SaveNormalData ->logColumnMap 为空，syslogUUID:" + ctx.strSyslogUUID); + ctx.syslogNonNormalMessage.setReason("未命中规则"); + ctx.syslogNonNormalMessage.setReasonDetail("失败详情：logColumnMap对象字段为空"); + if (ctx.isSaveNonNormal == false) + SaveNonNormalMessage(ctx, deviceId, DateTime.now()); + return; } Map columnMap = logColumnMap; - //补全设备信息字段 + //补全采集设备信息 CompletionDeviceInfo(columnMap, deviceId); - + //补全采集探针相关信息 + CompletionCollectTaskInfo(columnMap,ctx); + //补全日志基础信息 columnMap.put("device_id", deviceId); columnMap.put("log_time", logtime); columnMap.put("id", UUID.randomUUID().toString()); columnMap.put("normalize_rule_id", normalizeRuleId); columnMap.put("normalize_rule_name", normalizeRuleName); - columnMap.put("syslog_uuid", this.strSyslogUUID); - columnMap.put("syslog_topic", this.strSyslogTopic); - System.out.println("columnMap："+columnMap); + columnMap.put("syslog_uuid", ctx.strSyslogUUID); + columnMap.put("syslog_topic", ctx.strSyslogTopic); + System.out.println("columnMap：" + columnMap); syslogNormalDataService.insertDynamic(columnMap); } catch (Exception e) { - logger.error("SaveNormalData失败 " ); - //保存非标日志信息 - syslogNonNormalMessage.setReason("入库失败"); - syslogNonNormalMessage.setReasonDetail("入库失败,normalizeRuleName :"+normalizeRuleName+",失败详情："+ e.getMessage()); - if(isSaveNonNormal==false) - SaveNonNormalMessage(deviceId,DateTime.now()); + logger.error("SaveNormalData失败 "); + ctx.syslogNonNormalMessage.setReason("入库失败"); + ctx.syslogNonNormalMessage.setReasonDetail("入库失败,normalizeRuleName :" + normalizeRuleName + + ",失败详情：" + e.getMessage()); + if (ctx.isSaveNonNormal == false) + SaveNonNormalMessage(ctx, deviceId, DateTime.now()); throw new RuntimeException("SaveNormalData 失败: " + e.getMessage(), e); } } - - /** - * 补全设备信息字段 - * @param columnMap - * @param deviceID - */ - public void CompletionDeviceInfo(Map columnMap ,Long deviceID) - { + //补全采集设备信息 + public void CompletionDeviceInfo(Map columnMap, Long deviceID) { try { - DeviceDevice devInfo = deviceDeviceService.getByIdSafely( Long.valueOf(deviceID).intValue() ); + DeviceDevice devInfo = deviceDeviceService.getByIdSafely(Long.valueOf(deviceID).intValue()); if (devInfo != null) { columnMap.put("device_ip", devInfo.getIp()); columnMap.put("device_manufacturer", devInfo.getVendor()); columnMap.put("device_name", devInfo.getName()); } } catch (Exception e) { - logger.error("CompletionDeviceInfo 失败！ID："+deviceID ); + logger.error("CompletionDeviceInfo 失败！ID：" + deviceID); } } - /** - * 保存非标日志记录 - * @param deviceId - * @param logtime - */ - public void SaveNonNormalMessage(long deviceId , DateTime logtime) - { + //补全采集探针相关信息（ID、IP、名称) + public void CompletionCollectTaskInfo(Map columnMap,ProcessContext ctx) { + Integer collect_id=-1; try { - SyslogNonNormalMessage normalMessage =new SyslogNonNormalMessage() ; - //获取日志对应的设备ID - Integer collect_id= Integer.parseInt(this.deviceInfoMap.get("device_collect_id")); - DeviceCollectTask deviceCollectTask=deviceCollectTaskService.getById(collect_id); - normalMessage.setId( UUID.randomUUID().toString()); - normalMessage.setDeviceId((int)deviceId); + collect_id = Integer.parseInt(ctx.deviceInfoMap.get("device_collect_id")); + DeviceCollectTask deviceCollectTask = deviceCollectTaskService.getById(collect_id); + if (deviceCollectTask != null) { + columnMap.put("agent_id", collect_id); + columnMap.put("origin_agent_id",collect_id.toString() ); + columnMap.put("probe_ip",deviceCollectTask.getDeviceIp() ); + columnMap.put("agent_name", deviceCollectTask.getTaskName()); + columnMap.put("origin_agent_name", deviceCollectTask.getTaskName()); + } + } catch (Exception e) { + logger.error("CompletionCollectTaskInfo 失败！探针ID：" + collect_id); + } + } + + public void SaveNonNormalMessage(ProcessContext ctx, long deviceId, DateTime logtime) { + try { + SyslogNonNormalMessage normalMessage = new SyslogNonNormalMessage(); + Integer collect_id = Integer.parseInt(ctx.deviceInfoMap.get("device_collect_id")); + DeviceCollectTask deviceCollectTask = deviceCollectTaskService.getById(collect_id); + normalMessage.setId(UUID.randomUUID().toString()); + normalMessage.setDeviceId((int) deviceId); normalMessage.setLogTime(logtime.toLocalDateTime()); normalMessage.setRuleTime(logtime.toLocalDateTime()); - normalMessage.setSyslogMessage(this.strLogMsg); - normalMessage.setSyslogTopic(this.strSyslogTopic); - normalMessage.setSyslogUuid(this.strSyslogUUID); - normalMessage.setHeaderMessage(this.strDeviceInfo); + normalMessage.setSyslogMessage(ctx.strLogMsg); + normalMessage.setSyslogTopic(ctx.strSyslogTopic); + normalMessage.setSyslogUuid(ctx.strSyslogUUID); + normalMessage.setHeaderMessage(ctx.strDeviceInfo); normalMessage.setEtlNode("etlgo"); - normalMessage.setReason( syslogNonNormalMessage.getReason()); - normalMessage.setReasonDetail( syslogNonNormalMessage.getReasonDetail()); - //采集探针名称 + normalMessage.setReason(ctx.syslogNonNormalMessage.getReason()); + normalMessage.setReasonDetail(ctx.syslogNonNormalMessage.getReasonDetail()); normalMessage.setRuleResult("FAIL"); - normalMessage.setDeviceName(deviceInfoMap.get("device_name")); + normalMessage.setDeviceName(ctx.deviceInfoMap.get("device_name")); normalMessage.setCollectTaskId(collect_id); - if(deviceCollectTask!=null) - normalMessage.setCollectTaskName(deviceCollectTask.getTaskName()); - this.messageService.saveMessage(normalMessage ); - isSaveNonNormal=true; - + if (deviceCollectTask != null) + normalMessage.setCollectTaskName(deviceCollectTask.getTaskName()); + this.messageService.saveMessage(normalMessage); + ctx.isSaveNonNormal = true; } catch (Exception e) { - logger.error("SaveNonNormalMessage 失败：" ); + logger.error("SaveNonNormalMessage 失败："); throw new RuntimeException("SaveNonNormalMessage 失败: " + e.getMessage(), e); } } - - public static void main(String[] args) { - - String strlogMessage= "<14>1 2025-09-24T11:52:26Z 5f46d3be75e1 supermario 128 honeypot_event - {\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}"; - String strMsgContent= "{\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}"; - //Map flatMap =(new logNormalData()).getMessageToMap(strMsgContent); - //flatMap.forEach((key, value) -> System.out.println(key + " = " + value)); - //LogNormalProcessor logData =new LogNormalProcessor(strlogMessage,); - //logData.init(); - } } - - - diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java index 6c769b1..8dc2e9e 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/Modules/NormalData/SysLogProcessor.java @@ -2,7 +2,8 @@ package com.Modules.NormalData; import com.influx.InfluxDBClient; import com.common.util.SyslogParser; -import com.config.AppConfig; +import com.config.AppProperties; +import com.config.KafkaConsumerProperties; import com.influxdb.client.domain.WritePrecision; import com.influxdb.client.write.Point; import com.influxdb.client.WriteApi; @@ -29,7 +30,6 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import com.common.util.Sm4Util; -import com.config.AppConfig; @Slf4j @Component public class SysLogProcessor { @@ -44,11 +44,20 @@ public class SysLogProcessor { @Value("${app.processor.process-timeout-ms:30000}") private long processTimeoutMs; - private static String strhexKey=AppConfig.getSM4Key(); + @Autowired + private AppProperties appProperties; + + @Autowired + private KafkaConsumerProperties kafkaConsumerProperties; + + @Autowired + private LogNormalProcessor logNormalProcessor; + + @Autowired + private com.influx.InfluxDBClient influxClient; + private final AtomicInteger totalProcessed = new AtomicInteger(0); private final AtomicInteger currentBatchCount = new AtomicInteger(0); - // 初始化 InfluxDB 客户端 - private final com.influx.InfluxDBClient influxClient = new InfluxDBClient(); @@ -88,7 +97,7 @@ public class SysLogProcessor { CompletableFuture future = CompletableFuture.runAsync(() -> { try { // 异步处理单条消息 - log.info("收到syslogmessage："+ Sm4Util.decryptCbc(record.value(), strhexKey)); + log.info("收到syslogmessage："+ Sm4Util.decryptCbc(record.value(), appProperties.getSm4Key())); processSingleMessageAsync(record); } catch (Exception e) { log.error("处理消息失败, topic: {}, partition: {}, offset: {}", @@ -254,13 +263,8 @@ public class SysLogProcessor { long startTime = System.currentTimeMillis(); try { - // 模拟消息解析 - //String message = parseMessage(record); - // 模拟业务处理 - //processBusinessLogic(message); - //Message进行SM4解密 - String Sm4message=Sm4Util.decryptCbc(record.value(), strhexKey); + String Sm4message=Sm4Util.decryptCbc(record.value(), appProperties.getSm4Key()); System.out.println("Sm4message:"+Sm4message); @@ -273,7 +277,7 @@ public class SysLogProcessor { .addTag("device_id", mapdev.get("device_id")) // 添加设备ID标签 .addTag("device_collect_id", mapdev.get("device_collect_id")) // 添加探针ID标签 .addTag("uuid", sysLogUUID) //syslog uuid - .addTag("topic", AppConfig.getTopic()) //kafka topic + .addTag("topic", kafkaConsumerProperties.getTopic()) //kafka topic .addField("message", Sm4message) // 添加字段 .addField("receive_time", mapdev.get("receive_time")) // 添加字段 .addField("uuid", sysLogUUID) @@ -286,12 +290,10 @@ public class SysLogProcessor { // 日志信息插入pg XdrHoneypot 表 //insertSingleRecord( record.value()); - //String syslogMessage= AppConfig.geRunEnvironment‌().equals("test")? record.value().substring(34) : record.value(); + //String syslogMessage= appProperties.getRunEnvironment().equals("test")? record.value().substring(34) : record.value(); String syslogMessage= Sm4message; - //剔除测试环境本机syslog新增的头部信息 - LogNormalProcessor logNormalProcessor = new LogNormalProcessor(syslogMessage,sysLogUUID,AppConfig.getTopic()); - //LogNormalProcessor logNormalProcessor =new LogNormalProcessor(record.value()); - logNormalProcessor.init(); + //使用注入的 Spring Bean 进行标准化处理 + logNormalProcessor.process(syslogMessage, sysLogUUID, null); System.out.println("insert postgres syslog ,value:"+ record.key()); long costTime = System.currentTimeMillis() - startTime; log.debug("消息处理完成, offset: {}, 耗时: {}ms", record.offset(), costTime); diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/AlarmVisit.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/AlarmVisit.java index 6541f59..96203f3 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/AlarmVisit.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/AlarmVisit.java @@ -75,4 +75,8 @@ public class AlarmVisit { private String[] httpReqBody; private String[] httpRespHeader; private String[] httpRespBody; + private String reason; + private String dtype; + private String originField; + private String originLog; } \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/AlarmVisitProcesslog.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/AlarmVisitProcesslog.java new file mode 100644 index 0000000..cd75356 --- /dev/null +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/AlarmVisitProcesslog.java @@ -0,0 +1,18 @@ +package com.common.entity; + +import lombok.Builder; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@Builder +public class AlarmVisitProcesslog { + private Long id; + private String batchNo; + private Long logCount; + private LocalDateTime startTime; + private LocalDateTime endTime; + private LocalDateTime createdAt; + private String remark; +} diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceCollectTask.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceCollectTask.java index 0517eba..0745a62 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceCollectTask.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/entity/DeviceCollectTask.java @@ -20,6 +20,10 @@ public class DeviceCollectTask { private Integer taskCount; private OffsetDateTime recentDiscoverTime; private Integer epmUpperLimit; + private OffsetDateTime expireTime; + private String remark; + private String deviceIp; + private Long organizationId; // Getter and Setter 方法 public Integer getId() { return id; } @@ -73,6 +77,18 @@ public class DeviceCollectTask { public Integer getEpmUpperLimit() { return epmUpperLimit; } public void setEpmUpperLimit(Integer epmUpperLimit) { this.epmUpperLimit = epmUpperLimit; } + public OffsetDateTime getExpireTime() { return expireTime; } + public void setExpireTime(OffsetDateTime expireTime) { this.expireTime = expireTime; } + + public String getRemark() { return remark; } + public void setRemark(String remark) { this.remark = remark; } + + public String getDeviceIp() { return deviceIp; } + public void setDeviceIp(String deviceIp) { this.deviceIp = deviceIp; } + + public Long getOrganizationId() { return organizationId; } + public void setOrganizationId(Long organizationId) { this.organizationId = organizationId; } + @Override public String toString() { return "DeviceCollectTask{" + @@ -93,6 +109,10 @@ public class DeviceCollectTask { ", taskCount=" + taskCount + ", recentDiscoverTime=" + recentDiscoverTime + ", epmUpperLimit=" + epmUpperLimit + + ", expireTime=" + expireTime + + ", remark='" + remark + '\'' + + ", deviceIp='" + deviceIp + '\'' + + ", organizationId=" + organizationId + '}'; } } \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitMapper.java index 9a20855..2efb696 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitMapper.java @@ -23,7 +23,8 @@ public interface AlarmVisitMapper { "attack_port, victim_port, attack_method, etl_time, log_count, ", "attack_chain_phase, disposition_advice, attack_direction, ", "judged_state, disposed_state, attack_result, fall, payload, dns_info, engine_type, " , - "http_req_header , http_req_body,http_resp_header , http_resp_body ", + "http_req_header , http_req_body,http_resp_header , http_resp_body, ", + "reason, dtype, origin_field, origin_log ", ") VALUES ", "", "(#{item.id}, #{item.createdAt}, #{item.alarmName}, #{item.alarmLevel}, ", @@ -34,7 +35,7 @@ public interface AlarmVisitMapper { "#{item.deviceId, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, ", "#{item.comment}, " , "#{item.originLogIds, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ", - "#{item.logStartAt}, #{item.logEndAt},, #{item.windowTime} #{item.httpStatus}, ", + "#{item.logStartAt}, #{item.logEndAt}, #{item.windowTime}, #{item.httpStatus}, ", "#{item.attackPort, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, ", "#{item.victimPort, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, ", "#{item.attackMethod}, #{item.etlTime}, #{item.logCount}, ", @@ -45,7 +46,8 @@ public interface AlarmVisitMapper { "#{item.httpReqHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ", "#{item.httpReqBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ", "#{item.httpRespHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ", - "#{item.httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}) ", + "#{item.httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ", + "#{item.reason}, #{item.dtype}, #{item.originField}, #{item.originLog}) ", "", ""}) void batchInsert(@Param("list") List alarmList); @@ -60,7 +62,8 @@ public interface AlarmVisitMapper { "attack_port, victim_port, attack_method, etl_time, log_count, " + "attack_chain_phase, disposition_advice, attack_direction, " + "judged_state, disposed_state, attack_result, fall, payload, dns_info,engine_type, " + - "http_req_header , http_req_body,http_resp_header , http_resp_body " + + "http_req_header , http_req_body,http_resp_header , http_resp_body, " + + "reason, dtype, origin_field, origin_log " + ") VALUES (" + "#{id}, #{createdAt}, #{alarmName}, #{alarmLevel}, " + "#{alarmType}, #{alarmMajorType}, #{alarmMinorType}, #{alarmAreaId}, " + @@ -80,7 +83,8 @@ public interface AlarmVisitMapper { "#{httpReqHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " + "#{httpReqBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " + "#{httpRespHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " + - "#{httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler} " + + "#{httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " + + "#{reason}, #{dtype}, #{originField}, #{originLog} " + ")") void insert(AlarmVisit alarm); } \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitProcesslogMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitProcesslogMapper.java new file mode 100644 index 0000000..a19e72c --- /dev/null +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/AlarmVisitProcesslogMapper.java @@ -0,0 +1,27 @@ +package com.common.mapper; + +import com.common.entity.AlarmVisitProcesslog; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Select; + +@Mapper +public interface AlarmVisitProcesslogMapper { + + /** + * 插入处理记录 + */ + @Insert("INSERT INTO alarm_visit_processlog (" + + "batch_no, log_count, start_time, end_time, remark" + + ") VALUES (" + + "#{batchNo}, #{logCount}, #{startTime}, #{endTime}, #{remark}" + + ")") + void insert(AlarmVisitProcesslog processlog); + + /** + * 查询当天最大批次序号 + */ + @Select("SELECT COALESCE(MAX(batch_no), '') FROM alarm_visit_processlog " + + "WHERE batch_no LIKE CONCAT(#{todayPrefix}, '%')") + String getTodayMaxBatchNo(String todayPrefix); +} diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java index 221223e..df2e9ae 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java @@ -66,10 +66,12 @@ public interface DeviceCollectTaskMapper extends BaseMapper{ */ @Insert("INSERT INTO device_collect_task (created_at, updated_at, device_id, method, task_name, " + "first_time, last_success_time, last_failed_time, detail_id, epm, epm_peak, " + - "process_architecture, task_count, recent_discover_time, epm_upper_limit) " + + "process_architecture, task_count, recent_discover_time, epm_upper_limit, " + + "expire_time, remark, device_ip, organization_id) " + "VALUES (NOW(), NOW(), #{deviceId}, #{method}, #{taskName}, #{firstTime}, " + "#{lastSuccessTime}, #{lastFailedTime}, #{detailId}, #{epm}, #{epmPeak}, " + - "#{processArchitecture}, #{taskCount}, #{recentDiscoverTime}, #{epmUpperLimit})") + "#{processArchitecture}, #{taskCount}, #{recentDiscoverTime}, #{epmUpperLimit}, " + + "#{expireTime}, #{remark}, #{deviceIp}, #{organizationId})") @Options(useGeneratedKeys = true, keyProperty = "id") int insert(DeviceCollectTask task); @@ -90,7 +92,11 @@ public interface DeviceCollectTaskMapper extends BaseMapper{ "process_architecture = #{processArchitecture}, " + "task_count = #{taskCount}, " + "recent_discover_time = #{recentDiscoverTime}, " + - "epm_upper_limit = #{epmUpperLimit} " + + "epm_upper_limit = #{epmUpperLimit}, " + + "expire_time = #{expireTime}, " + + "remark = #{remark}, " + + "device_ip = #{deviceIp}, " + + "organization_id = #{organizationId} " + "WHERE id = #{id}") int update(DeviceCollectTask task); @@ -189,4 +195,10 @@ public interface DeviceCollectTaskMapper extends BaseMapper{ */ @Select("SELECT * FROM device_collect_task") List selectAllTasks(); + + /** + * 查询 first_time 为空的设备采集任务 ID 列表（用于首次成功时间填充） + */ + @Select("SELECT id FROM device_collect_task WHERE first_time IS NULL") + List selectTasksWithoutFirstTime(); } \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java index 6124c49..e31e40a 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/DeviceReceiveLogMapper.java @@ -100,13 +100,19 @@ public interface DeviceReceiveLogMapper { List selectDailyFailTimes(); /** - * 查询每个设备的首次成功时间（历史最早） + * 按 ID 列表精确查询首次成功时间（仅查 first_time 为空的任务，避免全表扫描） */ - @Select("SELECT device_collect_id, MIN(created_at) AS first_success_time " + + @Select("") + List selectFirstSuccessTimesByIds(@Param("ids") List ids); /** * 查询指定时间范围内的成功记录 diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalDataMapper.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalDataMapper.java index 7ef7737..7e37f75 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalDataMapper.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/mapper/SyslogNormalDataMapper.java @@ -85,7 +85,7 @@ public interface SyslogNormalDataMapper { * 查询指定时间后的所需字段（使用XML映射） * 注意：参数类型保持为LocalDateTime，但实际映射时使用类型处理器转换 */ - List findRequiredFieldsAfterTime(@Param("startTime") LocalDateTime startTime); + List findRequiredFieldsAfterTime(@Param("startTime") LocalDateTime startTime,@Param("endTime") LocalDateTime endTime ); /** * 分页查询所需字段 diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/ETLOrchestrator.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/ETLOrchestrator.java index cd7a23c..70b9cf0 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/ETLOrchestrator.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/ETLOrchestrator.java @@ -116,9 +116,9 @@ public class ETLOrchestrator { public void cleanupOldLogs() { try { //默认删除7天内接收日志记录 - LocalDateTime cutoffTime = LocalDateTime.now().minusDays(7); + LocalDateTime cutoffTime = LocalDateTime.now().minusDays(2); int deleted = deviceReceiveLogService.deleteOldLogs(cutoffTime); - log.info("定时清理任务完成，删除{}条7天前的日志", deleted); + log.info("定时清理任务完成，删除{}条2天前的日志", deleted); } catch (Exception e) { log.error("定时清理日志失败", e); } diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java index b9ccaec..d80ecac 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/schedule/PartitionTableSchedule.java @@ -6,13 +6,9 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import com.common.service.PartitionTableService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; + import java.util.List; + @Component public class PartitionTableSchedule { @@ -21,11 +17,15 @@ public class PartitionTableSchedule { @Autowired private PartitionTableService partitionTableService; + // ============================================================ + // 原有定时任务 — 每月15日凌晨1点（CREATE TABLE ... PARTITION OF 方式）(取消） + // ============================================================ + /** * 每月15日凌晨1点执行 * cron表达式: 秒 分 时 日 月 周 */ - @Scheduled(cron = "0 0 1 15 * ?") + //@Scheduled(cron = "0 0 1 15 * ?") public void createNextMonthPartitionTables() { logger.info("开始执行创建下个月分区表任务..."); @@ -41,6 +41,61 @@ public class PartitionTableSchedule { } } + // ============================================================ + // 新增定时任务 — 通过达梦存储过程预创建下月分区（凌晨2:00） + // 涉及5个主表: alarm / alarm_visit / syslog_normal_data + // / syslog_normal_alarm / syslog_non_normal_message + // 存储过程: CREATE_TABLE_DAILY_PARTITIONS + // ============================================================ + + /** + * 每月15日凌晨2:00 — 预创建下月分区表 + * cron: 秒 分 时 日 月 周 + */ + @Scheduled(cron = "0 0 2 15 * ?") + public void createNextMonthPartitionsOn15th() { + executeMonthlyPartitionCreation("15日"); + } + + /** + * 每月28日凌晨2:00 — 预创建下月分区表 + */ + @Scheduled(cron = "0 0 2 28 * ?") + public void createNextMonthPartitionsOn28th() { + executeMonthlyPartitionCreation("28日"); + } + + /** + * 每月最后一天凌晨2:00 — 预创建下月分区表 + * Spring CronExpression 支持 L 表示最后一天 + */ + @Scheduled(cron = "0 0 2 L * ?") + public void createNextMonthPartitionsOnLastDay() { + executeMonthlyPartitionCreation("月末最后一天"); + } + + /** + * 统一的月度分区创建执行逻辑 + * 通过达梦存储过程 CREATE_TABLE_DAILY_PARTITIONS 批量创建 + */ + private void executeMonthlyPartitionCreation(String triggerDesc) { + logger.info("==================== 月度分区预创建任务触发 ===================="); + logger.info("触发时机: 每月{} 凌晨2:00", triggerDesc); + + try { + partitionTableService.createNextMonthPartitionsByProcedure(); + logger.info("月度分区预创建任务执行完成 (触发: {})", triggerDesc); + } catch (Exception e) { + logger.error("月度分区预创建任务执行失败 (触发: {}), 错误: {}", triggerDesc, e.getMessage(), e); + } + + logger.info("================================================================"); + } + + // ============================================================ + // 已注释的备选定时任务 + // ============================================================ + /** * 测试任务 - 每分钟执行一次（开发环境使用） * 生产环境注释或删除此方法 diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AccessLogAlertService.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AccessLogAlertService.java index 138a6e4..43271ff 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AccessLogAlertService.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/AccessLogAlertService.java @@ -6,6 +6,7 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.common.entity.AlarmVisit; +import com.common.entity.AlarmVisitProcesslog; import com.common.entity.DeviceDevice; import com.common.entity.SecExceptionAlgorithm; import com.common.entity.SyslogNormalData; @@ -32,6 +33,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.common.mapper.*; import java.net.InetAddress; import com.common.util.AlgorithmResultParser; +import java.time.temporal.ChronoUnit; @Slf4j @Service @EnableScheduling @@ -55,6 +57,9 @@ public class AccessLogAlertService { @Autowired private AlarmVisitMapper alarmVisitMapper; + @Autowired + private AlarmVisitProcesslogMapper alarmVisitProcesslogMapper; + // 记录上次处理的时间戳 private LocalDateTime lastProcessTime; @@ -67,8 +72,8 @@ public class AccessLogAlertService { @PostConstruct public void init() { - // 初始化时设置为当前时间减2分钟 - lastProcessTime = LocalDateTime.now().minusMinutes(1); + // 初始化时设置为当前时间减1分钟 + lastProcessTime = LocalDateTime.now().minusMinutes(1).truncatedTo(ChronoUnit.MINUTES); log.info("初始化AccessLogAlertService，上次处理时间: {}", lastProcessTime); // 加载启用的算法配置到缓存 @@ -111,9 +116,9 @@ public class AccessLogAlertService { } } /** - * 定时任务入口 - 每2分钟执行一次 + * 定时任务入口 - 每1分钟执行一次 */ - @Scheduled(cron = "0 */1 * * * ?") + //@Scheduled(cron = "0 */1 * * * ?") @Async public void processAccessLogAlert() { log.info("开始执行访问日志告警处理任务"); @@ -128,9 +133,9 @@ public class AccessLogAlertService { } // 2. 获取上次处理时间之后的日志数据 - LocalDateTime currentTime = LocalDateTime.now(); + LocalDateTime currentTime = LocalDateTime.now().truncatedTo(ChronoUnit.MINUTES); //List newLogs = syslogNormalDataMapper.findAfterTime(lastProcessTime); - List newLogs = syslogNormalDataMapper.findRequiredFieldsAfterTime(lastProcessTime); + List newLogs = syslogNormalDataMapper.findRequiredFieldsAfterTime(lastProcessTime,currentTime ); //List newLogs =algorithmResultParser.buildNewLogsFromExample(); if (newLogs.isEmpty()) { log.info("没有发现新的日志数据，上次处理时间: {}", lastProcessTime); @@ -140,6 +145,20 @@ public class AccessLogAlertService { log.info("获取到 {} 条新的日志数据，时间范围: {} 到 {}", newLogs.size(), lastProcessTime, currentTime); + // 2.1 记录本次处理批次信息 + try { + String batchNo = generateBatchNo(); + AlarmVisitProcesslog processlog = AlarmVisitProcesslog.builder() + .batchNo(batchNo) + .logCount((long) newLogs.size()) + .startTime(lastProcessTime) + .endTime(currentTime) + .build(); + alarmVisitProcesslogMapper.insert(processlog); + log.info("写入处理批次记录: batchNo={}, logCount={}", batchNo, newLogs.size()); + } catch (Exception e) { + log.error("记录处理批次失败: {}", e.getMessage(), e); + } // 3. 处理每条启用的算法配置 for (SecExceptionAlgorithm algorithm : algorithmCache.values()) { try { @@ -360,6 +379,10 @@ public class AccessLogAlertService { .disposedState(0) .dispositionAdvice("研判后处置") .dnsInfo(alarmResult.getString("host")) + .reason(alarmResult.getString("reason")) + .dtype(alarmResult.getString("dtype")) + .originField(alarmResult.getString("origin_field")) + .originLog(alarmResult.getString("origin_log")) .build(); //补充返回结果的原始日志字段 @@ -419,6 +442,9 @@ public class AccessLogAlertService { log.debug("算法：{},ID:{} ,AlarmNme:{} 没有返回 origin_log节点.",AlgorithmName, alarmVisit.getId(), alarmVisit.getAlarmName()); return false; } + // 存储原始日志JSON字符串到 origin_log 字段 + alarmVisit.setOriginLog(originLogObject.toJSONString()); + // _source.sport/dport 在 JSON 中为字符串类型，需要用 getString() 读取后 parseInt String sportStr = originLogObject.getString("_source.sport"); if (sportStr != null && !sportStr.isEmpty()) { @@ -461,6 +487,28 @@ public class AccessLogAlertService { return deviceList.get(0).getId(); } + /** + * 生成批次号：yyyyMMdd_NNNN（时间戳字符串 + 当天序号） + * 如：20260605_0001 + */ + private String generateBatchNo() { + String todayPrefix = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); + String maxBatchNo = alarmVisitProcesslogMapper.getTodayMaxBatchNo(todayPrefix); + + int seq = 1; + if (maxBatchNo != null && !maxBatchNo.isEmpty()) { + // batch_no 格式: yyyyMMdd_XXXX，解析序号部分 + int underscoreIdx = maxBatchNo.lastIndexOf('_'); + if (underscoreIdx >= 0 && underscoreIdx < maxBatchNo.length() - 1) { + try { + seq = Integer.parseInt(maxBatchNo.substring(underscoreIdx + 1)) + 1; + } catch (NumberFormatException e) { + log.warn("解析批次序号失败: {}, 使用默认序号1", maxBatchNo); + } + } + } + return todayPrefix + "_" + String.format("%04d", seq); + } diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DataTransformer.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DataTransformer.java index 81ffb0b..925c00b 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DataTransformer.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DataTransformer.java @@ -194,6 +194,10 @@ public class DataTransformer { .httpReqBody(groupedData.getHttpReqBodys()) .httpRespHeader(groupedData.getHttpRespHeaders()) .httpRespBody(groupedData.getHttpRespBodys()) + .reason(null) + .dtype(null) + .originField(null) + .originLog(null) .build(); } catch (Exception e) { diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceCollectTaskUpdateService.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceCollectTaskUpdateService.java index df06571..44558fe 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceCollectTaskUpdateService.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceCollectTaskUpdateService.java @@ -7,11 +7,12 @@ import com.common.mapper.DeviceReceiveLogMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionTemplate; import org.springframework.util.CollectionUtils; import java.time.LocalDate; import java.time.LocalDateTime; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -24,11 +25,11 @@ public class DeviceCollectTaskUpdateService { private final DeviceReceiveLogMapper deviceReceiveLogMapper; private final DeviceCollectTaskMapper deviceCollectTaskMapper; + private final TransactionTemplate transactionTemplate; /** * 主处理方法：更新所有设备任务的时间信息 */ - @Transactional(rollbackFor = Exception.class) public void updateAllDeviceTaskTimes() { log.info("开始更新设备采集任务时间，当前时间: {}", LocalDateTime.now()); @@ -97,30 +98,29 @@ public class DeviceCollectTaskUpdateService { /** * 批量更新版本（性能更好） + * 注意：不使用 @Transactional 避免长时间持锁，读操作无需事务， + * 仅批写通过 TransactionTemplate 精确控制事务范围 */ - @Transactional(rollbackFor = Exception.class) public void batchUpdateDeviceTaskTimes() { log.info("开始批量更新设备采集任务时间，当前时间: {}", LocalDateTime.now()); - // 1. 查询所有设备任务 + // 1. 查询所有设备任务（只读，无需事务） List allTasks = deviceCollectTaskMapper.selectAllTasks(); if (CollectionUtils.isEmpty(allTasks)) { return; } - // 2. 查询时间信息 + // 2. 查询时间信息（只读，无需事务） Map successTimes = getDeviceSuccessTimes(); Map failTimes = getDeviceFailTimes(); - // 3. 批量更新 + // 3. 内存中组装更新数据 LocalDateTime updateTime = LocalDateTime.now(); for (DeviceCollectTask task : allTasks) { - String deviceCollectId = task.getId().toString(); - // 获取成功和失败时间 - DeviceCollectTaskTime successTime = successTimes.get(deviceCollectId); - DeviceCollectTaskTime failTime = failTimes.get(deviceCollectId); + DeviceCollectTaskTime successTime = successTimes.get(task.getId().toString()); + DeviceCollectTaskTime failTime = failTimes.get(task.getId().toString()); // 更新成功时间 if (successTime != null) { @@ -143,22 +143,32 @@ public class DeviceCollectTaskUpdateService { task.setUpdatedAt(updateTime.atOffset(ZoneOffset.UTC)); } - // 4. 批量更新到数据库 - int updatedCount = deviceCollectTaskMapper.batchUpdateTimes(allTasks); + // 4. 批量更新到数据库（仅写操作加事务，用完即释放连接） + int updatedCount = transactionTemplate.execute(status -> + deviceCollectTaskMapper.batchUpdateTimes(allTasks)); log.info("批量更新完成，总计: {}，已更新: {}", allTasks.size(), updatedCount); } /** * 获取设备的成功时间信息 + * 优化：first_time 为恒定值，只对 first_time IS NULL 的新任务查询首次成功时间， + * 避免对 device_receive_log进行全表扫描 */ private Map getDeviceSuccessTimes() { - // 查询当天的最新成功时间 + // 1. 查询当天的最新成功时间 List dailySuccess = deviceReceiveLogMapper.selectDailySuccessTimes(); - // 查询首次成功时间（历史） - List firstSuccess = deviceReceiveLogMapper.selectFirstSuccessTimes(); + // 2. 仅查询 first_time 为空的任务 ID，避免全表扫描 + List tasksWithoutFirstTime = deviceCollectTaskMapper.selectTasksWithoutFirstTime(); + List firstSuccess; + if (!CollectionUtils.isEmpty(tasksWithoutFirstTime)) { + log.info("发现 {} 个任务尚未设置首次成功时间，进行精确查询", tasksWithoutFirstTime.size()); + firstSuccess = deviceReceiveLogMapper.selectFirstSuccessTimesByIds(tasksWithoutFirstTime); + } else { + firstSuccess = Collections.emptyList(); + } - // 合并结果 + // 3. 合并结果 Map result = new HashMap<>(); // 处理首次成功时间 @@ -205,7 +215,6 @@ public class DeviceCollectTaskUpdateService { /** * 增量更新：只处理最近一段时间的数据 */ - @Transactional(rollbackFor = Exception.class) public void incrementalUpdate(int hours) { LocalDateTime endTime = LocalDateTime.now(); LocalDateTime startTime = endTime.minusHours(hours); diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java index 75e48a6..c5ade88 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/DeviceStatsUpdateService.java @@ -8,7 +8,7 @@ import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionTemplate; import java.sql.ResultSet; import java.sql.SQLException; @@ -34,6 +34,9 @@ public class DeviceStatsUpdateService { @Autowired private JdbcTemplate jdbcTemplate; + @Autowired + private TransactionTemplate transactionTemplate; + // SQL 模板定义 private static final String COLLECT_DEVICE_STATS_SQL = "WITH device_receive_stats AS ( " + @@ -86,9 +89,10 @@ public class DeviceStatsUpdateService { /** * 每5分钟执行一次设备统计更新（秒：0，分：*，时：*） + * 注意：不使用 @Transactional 避免长时间持锁，统计查询为只读无需事务， + * 写操作通过 TransactionTemplate 精确控制事务范围 */ @Scheduled(cron = "0 */5 * * * ?") - @Transactional public void updateDeviceStats() { long startTime = System.currentTimeMillis(); log.info("开始执行设备统计更新任务..."); @@ -97,22 +101,23 @@ public class DeviceStatsUpdateService { // 获取当前日期，用于动态表名 String today = LocalDateTime.now().format(DATE_FORMATTER); - // 统计每个设备的当天数据 + // 1. 统计每个设备的当天数据（只读，无需事务） List deviceStats = collectDeviceStats(today); - // 批量更新 device_device 表 - batchUpdateDeviceStats(deviceStats); + // 2. 批量更新 device_device 表（通过事务模板控制事务边界，用完即释放连接） + transactionTemplate.executeWithoutResult(status -> { + batchUpdateDeviceStats(deviceStats); + }); long endTime = System.currentTimeMillis(); log.info("设备统计更新完成，处理设备数：{}，耗时：{}ms", deviceStats.size(), (endTime - startTime)); - // 批量更新 device_collect_task 表 + // 3. 批量更新 device_collect_task 表（独立事务，由该 Service 自己管理） updateDeviceCollectTaskTimes(); } catch (Exception e) { log.error("设备统计更新任务执行失败", e); - throw e; } } @@ -120,18 +125,16 @@ public class DeviceStatsUpdateService { * 每天执行一次设备统计数值清零更新（秒：1，分：0，时：0） */ @Scheduled(cron = "1 0 0 * * ?") - @Transactional public void updateDeviceCount() { long startTime = System.currentTimeMillis(); log.info("开始执行设备统计数值清零更新任务..."); try { - int devCount= jdbcTemplate.update(UPDATE_DEVICE_COUNT_SQL); + int devCount = transactionTemplate.execute(status -> + jdbcTemplate.update(UPDATE_DEVICE_COUNT_SQL)); long endTime = System.currentTimeMillis(); - log.info("设备统计数值清零更新完成，处理设备数：{} ，耗时：{}ms",devCount, (endTime - startTime)); - + log.info("设备统计数值清零更新完成，处理设备数：{} ，耗时：{}ms", devCount, (endTime - startTime)); } catch (Exception e) { log.error("设备统计数值清零更新任务执行失败", e); - throw e; } } diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/InfluxSyslogService.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/InfluxSyslogService.java index 40994fb..e4be1fe 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/InfluxSyslogService.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/InfluxSyslogService.java @@ -20,11 +20,8 @@ import java.util.Map; @Service public class InfluxSyslogService { - //@Autowired - //private InfluxDBClient influxDBClient; - // 初始化 InfluxDB 客户端 - //@Autowired - private final com.influx.InfluxDBClient influxDBClient = new InfluxDBClient(); + @Autowired + private com.influx.InfluxDBClient influxDBClient; private static final DateTimeFormatter TABLE_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd"); private static final DateTimeFormatter INPUT_DATE_FORMATTER = DateTimeFormatter.ISO_DATE_TIME; diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/PartitionTableService.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/PartitionTableService.java index fa28f97..c774501 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/PartitionTableService.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/PartitionTableService.java @@ -177,6 +177,58 @@ public class PartitionTableService { } } + // ============================================================ + // 达梦(DM)存储过程方式创建分区表 — 下月预创建 + // 存储过程: CREATE_TABLE_DAILY_PARTITIONS(table_name, startDate, endDate) + // ============================================================ + + /** 需要创建月度分区的主表列表 */ + private static final String[] MONTHLY_PARTITION_TABLES = { + "alarm", + "alarm_visit", + "syslog_normal_data", + "syslog_normal_alarm", + "syslog_non_normal_message" + }; + + /** + * 通过达梦存储过程创建下个月所有分区表（5个主表） + * 存储过程内部已处理"分区已存在"异常，可安全重复调用 + */ + public void createNextMonthPartitionsByProcedure() { + LocalDate nextMonthFirstDay = LocalDate.now() + .plusMonths(1) + .withDayOfMonth(1); + LocalDate nextMonthLastDay = nextMonthFirstDay + .with(TemporalAdjusters.lastDayOfMonth()); + + String startDate = nextMonthFirstDay.format(SQL_DATE_FORMATTER); + String endDate = nextMonthLastDay.format(SQL_DATE_FORMATTER); + + logger.info("开始通过存储过程创建下月分区表: {} ~ {}", startDate, endDate); + + int successCount = 0; + int failCount = 0; + + for (String tableName : MONTHLY_PARTITION_TABLES) { + try { + String sql = String.format( + "CALL CREATE_TABLE_DAILY_PARTITIONS('%s', '%s', '%s')", + tableName, startDate, endDate); + jdbcTemplate.execute(sql); + successCount++; + logger.info("存储过程执行成功: 表={}, 日期范围={} ~ {}", tableName, startDate, endDate); + } catch (Exception e) { + failCount++; + logger.error("存储过程执行失败: 表={}, 日期范围={} ~ {}, 错误={}", + tableName, startDate, endDate, e.getMessage(), e); + } + } + + logger.info("下月分区表创建完成: 成功={}, 失败={}, 总计={}", + successCount, failCount, MONTHLY_PARTITION_TABLES.length); + } + /** * 获取下个月需要创建的分区表数量 */ diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/DeviceReceiveLogServiceImpl.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/DeviceReceiveLogServiceImpl.java index 304c3e6..6888a14 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/DeviceReceiveLogServiceImpl.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/DeviceReceiveLogServiceImpl.java @@ -135,7 +135,7 @@ public class DeviceReceiveLogServiceImpl implements DeviceReceiveLogService { } - LocalDateTime endTime = LocalDateTime.now().minusDays(7); // 默认保留7天 + LocalDateTime endTime = LocalDateTime.now().minusDays(3); // 默认保留3天 if (beforeTime.isAfter(endTime)) { log.warn("删除时间点{}晚于默认保留时间{}，使用默认时间", beforeTime, endTime); beforeTime = endTime; diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java index c1b6407..4bd641c 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/RealtimeAnalysisEngine.java @@ -15,10 +15,17 @@ import java.time.format.DateTimeFormatter; import java.util.*; /** - * 实时分析引擎实现 + * 实时分析引擎实现（平铺SQL模式） + * + * 当 analysis.db.dameng-subquery-mode=true 时，本Bean不会被注册， + * 改为使用 RealtimeAnalysisEngineDm（子查询包装模式）适配达梦安全版。 */ @Slf4j @Service("realtimeAnalysisEngine") +@org.springframework.boot.autoconfigure.condition.ConditionalOnProperty( + name = "analysis.db.dameng-subquery-mode", + havingValue = "false", + matchIfMissing = true) public class RealtimeAnalysisEngine implements AnalysisEngine { @Autowired diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/RealtimeAnalysisEngineDm.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/RealtimeAnalysisEngineDm.java new file mode 100644 index 0000000..e32f532 --- /dev/null +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/RealtimeAnalysisEngineDm.java @@ -0,0 +1,817 @@ +package com.common.service.impl; + +import com.common.entity.*; +import com.common.mapper.*; +import com.common.service.AnalysisEngine; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; + +/** + * 实时分析引擎实现（达梦安全版适配） + * + * 与 RealtimeAnalysisEngine 功能完全相同，唯一差异： + * 注入 SqlGeneratorServiceImplDm（子查询包装模式SQL生成器）， + * 以适配达梦安全版数据库 GROUP BY 不支持自定义函数（TUMBLE/HOP/SESSION）的问题。 + * + * 通过 @ConditionalOnProperty 与 RealtimeAnalysisEngine 二选一注册， + * 由配置 analysis.db.dameng-subquery-mode=true 激活本实现。 + * + * @see RealtimeAnalysisEngine 原始实现（平铺SQL模式） + * @see SqlGeneratorServiceImplDm 子查询包装SQL生成器 + */ +@Slf4j +@Service("realtimeAnalysisEngine") +@org.springframework.boot.autoconfigure.condition.ConditionalOnProperty( + name = "analysis.db.dameng-subquery-mode", + havingValue = "true") +public class RealtimeAnalysisEngineDm implements AnalysisEngine { + + // 直接实例化（非Spring Bean），避免与父类SqlGeneratorServiceImpl产生类型冲突 + private final SqlGeneratorServiceImplDm sqlGeneratorService = new SqlGeneratorServiceImplDm(); + + @Autowired + private AnalysisAnalysisRuleMapper ruleMapper; + + @Autowired + private AnalysisFieldMapper fieldMapper; + + @Autowired + private AnalysisWhereConditionMapper whereConditionMapper; + + @Autowired + private AnalysisGroupByColumnMapper groupByColumnMapper; + + @Autowired + private AnalysisGroupByWindowMapper groupByWindowMapper; + + + @Autowired + private AnalysisFilterMapper filterMapper; + + @Autowired + private AnalysisGroupByHavingMapper groupByHavingMapper; + + @Autowired + private AnalysisTaskHistoryMapper taskHistoryMapper; + + @Autowired + private AlarmMapper alarmMapper; + @Autowired + private AnalysisGroupByMapper groupByMapper; + @Autowired + private JdbcTemplate jdbcTemplate; + + private static final String RUN_MODE = "realtime"; + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + @Override + public Map executeRule(AnalysisAnalysisRule rule) { + String batchNo = generateBatchNo(); + LocalDateTime startTime = LocalDateTime.now().withSecond(0).withNano(0); + LocalDateTime dataEndTime = startTime; + + // 加载分组和窗口配置（在计算时间范围之前） + AnalysisGroupByWindow groupByWindow = null; + List groupByList = groupByMapper.selectByRuleId(rule.getRuleId()); + if (groupByList != null && !groupByList.isEmpty()) { + AnalysisGroupBy groupBy = groupByList.get(0); + if (groupBy.getId() != null) { + groupByWindow = groupByWindowMapper.selectByGroupById(groupBy.getId().intValue()); + } + } + + // 根据窗口类型动态计算数据查询时间范围 + LocalDateTime dataStartTime = calculateDataStartTime(dataEndTime, groupByWindow); + + log.info("开始执行实时规则(DM子查询模式): ruleId={}, ruleName={}, batchNo={}, windowType={}, dataStartTime={}, dataEndTime={}", + rule.getRuleId(), rule.getRuleName(), batchNo, + groupByWindow != null ? groupByWindow.getWindowType() : "NONE", + dataStartTime.format(DATE_FORMATTER), + dataEndTime.format(DATE_FORMATTER)); + + // 创建任务历史记录 + AnalysisTaskHistory history = AnalysisTaskHistory.builder() + .id(System.currentTimeMillis()) + .ruleId(rule.getRuleId()) + .startTime(startTime) + .status("RUNNING") + .progressPercent(0) + .inputCount(0L) + .outputCount(0L) + .delFlag("0") + .createTime(startTime) + .updateTime(startTime) + .tenantId("000000") + .remark("实时分析任务(达梦适配) - " + batchNo) + .build(); + taskHistoryMapper.insert(history); + + Map result = new HashMap<>(); + result.put("ruleId", rule.getRuleId()); + result.put("ruleName", rule.getRuleName()); + result.put("runMode", RUN_MODE); + result.put("batchNo", batchNo); + + try { + // 更新规则状态为运行中 + ruleMapper.updateTaskStatus(rule.getRuleId(), "running", 1L); + + // 加载规则配置 + List fields = fieldMapper.selectByRuleId(rule.getRuleId()); + List whereConditions = whereConditionMapper.selectByRuleId(rule.getRuleId()); + List filters = filterMapper.selectByRuleId(rule.getRuleId()); + List groupByColumns = groupByColumnMapper.selectByRuleId(rule.getRuleId()); + List havingConditions = groupByHavingMapper.selectByRuleId(rule.getRuleId()); + + // 生成SQL（调用达梦适配版SQL生成器，子查询包装模式） + String sql = sqlGeneratorService.generateSql( + rule, + fields, + whereConditions, + filters, + groupByColumns, + havingConditions, + groupByWindow, + dataStartTime.format(DATE_FORMATTER), + dataEndTime.format(DATE_FORMATTER) + ); + + log.info("生成的SQL(达梦子查询模式): {}", sql); + + // 执行SQL + List> queryResult = jdbcTemplate.queryForList(sql); + result.put("queryResult", queryResult); + + // 处理结果，生成告警 + long alarmCount = 0; + if (!queryResult.isEmpty()) { + List alarms = convertToAlarms(rule, queryResult); + if (!alarms.isEmpty()) { + String tableName = "alarm"; + alarmMapper.batchInsert(alarms); + alarmCount = alarms.size(); + } + } + + // 更新任务历史 + LocalDateTime endTime = LocalDateTime.now(); + long durationSeconds = java.time.Duration.between(startTime, endTime).getSeconds(); + history.setEndTime(endTime); + history.setDurationTime(durationSeconds); + history.setProgressPercent(100); + history.setInputCount((long) queryResult.size()); + history.setOutputCount(alarmCount); + history.setStatus("COMPLETED"); + history.setUpdateTime(endTime); + taskHistoryMapper.update(history); + + result.put("processedCount", queryResult.size()); + result.put("alarmCount", alarmCount); + result.put("status", "success"); + + // 更新规则状态为等待 + ruleMapper.updateTaskStatus(rule.getRuleId(), "waiting", 1L); + + log.info("规则执行成功(达梦子查询模式): ruleId={}, processedCount={}, alarmCount={}", + rule.getRuleId(), queryResult.size(), alarmCount); + + } catch (Exception e) { + log.error("规则执行失败: ruleId={}", rule.getRuleId(), e); + + // 更新任务历史 + LocalDateTime endTime = LocalDateTime.now(); + long durationSeconds = java.time.Duration.between(startTime, endTime).getSeconds(); + history.setEndTime(endTime); + history.setDurationTime(durationSeconds); + history.setStatus("FAILED"); + history.setRemark("执行失败: " + (e.getMessage().length() > 480 ? e.getMessage().substring(0, 480) : e.getMessage())); + history.setUpdateTime(endTime); + taskHistoryMapper.update(history); + + // 更新规则状态 + ruleMapper.updateTaskStatus(rule.getRuleId(), "stopped", 1L); + + result.put("status", "failed"); + result.put("errorMsg", e.getMessage()); + } + + return result; + } + + @Override + public List> executeRules(List rules) { + List> results = new ArrayList<>(); + + for (AnalysisAnalysisRule rule : rules) { + try { + Map result = executeRule(rule); + results.add(result); + } catch (Exception e) { + log.error("执行规则失败: ruleId={}", rule.getRuleId(), e); + Map errorResult = new HashMap<>(); + errorResult.put("ruleId", rule.getRuleId()); + errorResult.put("ruleName", rule.getRuleName()); + errorResult.put("status", "failed"); + errorResult.put("errorMsg", e.getMessage()); + results.add(errorResult); + } + } + + return results; + } + + @Override + public void stopRule(String ruleId) { + try { + ruleMapper.updateTaskStatus(ruleId, "stopped", 1L); + log.info("已停止规则: ruleId={}", ruleId); + } catch (Exception e) { + log.error("停止规则失败: ruleId={}", ruleId, e); + } + } + + @Override + public String getRunMode() { + return RUN_MODE; + } + + /** + * 生成批次号 + */ + private String generateBatchNo() { + return LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS")); + } + + /** + * 根据窗口类型动态计算数据查询开始时间 + */ + private LocalDateTime calculateDataStartTime(LocalDateTime dataEndTime, AnalysisGroupByWindow groupByWindow) { + if (groupByWindow == null) { + log.warn("未配置窗口类型，使用默认查询范围：最近30分钟"); + return dataEndTime.minusMinutes(30); + } + + String windowType = groupByWindow.getWindowType(); + + if (windowType == null || windowType.trim().isEmpty()) { + log.warn("窗口类型为空，使用默认查询范围：最近30分钟"); + return dataEndTime.minusMinutes(30); + } + + switch (windowType.toUpperCase()) { + case "TUMBLE": + return calculateTumbleWindowStartTime(dataEndTime, groupByWindow); + case "HOP": + return calculateHopWindowStartTime(dataEndTime, groupByWindow); + case "SESSION": + return calculateSessionWindowStartTime(dataEndTime, groupByWindow); + default: + log.warn("未知窗口类型: {}，使用默认查询范围：最近30分钟", windowType); + return dataEndTime.minusMinutes(30); + } + } + + /** + * 计算滚动窗口的数据查询开始时间 + */ + private LocalDateTime calculateTumbleWindowStartTime(LocalDateTime dataEndTime, AnalysisGroupByWindow groupByWindow) { + Integer windowSize = groupByWindow.getTumbleWindowSize(); + String windowSizeUnit = groupByWindow.getTumbleWindowSizeUnit(); + + if (windowSize == null || windowSize <= 0) { + log.warn("滚动窗口大小配置无效，使用默认值：5分钟"); + windowSize = 5; + windowSizeUnit = "m"; + } + + if (windowSizeUnit == null || windowSizeUnit.trim().isEmpty()) { + windowSizeUnit = "m"; + } + + LocalDateTime dataStartTime; + switch (windowSizeUnit.toLowerCase()) { + case "s": + dataStartTime = dataEndTime.minusSeconds(windowSize); + break; + case "m": + dataStartTime = dataEndTime.minusMinutes(windowSize); + break; + case "h": + dataStartTime = dataEndTime.minusHours(windowSize); + break; + case "d": + dataStartTime = dataEndTime.minusDays(windowSize); + break; + default: + log.warn("滚动窗口单位无效: {}，使用默认单位：分钟", windowSizeUnit); + dataStartTime = dataEndTime.minusMinutes(windowSize); + } + + log.info("滚动窗口查询范围: 窗口大小={}{}，查询时间范围=[{}, {}]", + windowSize, windowSizeUnit, + dataStartTime.format(DATE_FORMATTER), + dataEndTime.format(DATE_FORMATTER)); + + return dataStartTime; + } + + /** + * 计算滑动窗口的数据查询开始时间 + */ + private LocalDateTime calculateHopWindowStartTime(LocalDateTime dataEndTime, AnalysisGroupByWindow groupByWindow) { + Integer windowSize = groupByWindow.getHopWindowSize(); + String windowSizeUnit = groupByWindow.getHopWindowSizeUnit(); + + if (windowSize == null || windowSize <= 0) { + log.warn("滑动窗口大小配置无效，使用默认值：5分钟"); + windowSize = 5; + windowSizeUnit = "m"; + } + + if (windowSizeUnit == null || windowSizeUnit.trim().isEmpty()) { + windowSizeUnit = "m"; + } + + LocalDateTime dataStartTime; + switch (windowSizeUnit.toLowerCase()) { + case "s": + dataStartTime = dataEndTime.minusSeconds(windowSize); + break; + case "m": + dataStartTime = dataEndTime.minusMinutes(windowSize); + break; + case "h": + dataStartTime = dataEndTime.minusHours(windowSize); + break; + case "d": + dataStartTime = dataEndTime.minusDays(windowSize); + break; + default: + log.warn("滑动窗口单位无效: {}，使用默认单位：分钟", windowSizeUnit); + dataStartTime = dataEndTime.minusMinutes(windowSize); + } + + log.info("滑动窗口查询范围: 窗口大小={}{}，查询时间范围=[{}, {}]", + windowSize, windowSizeUnit, + dataStartTime.format(DATE_FORMATTER), + dataEndTime.format(DATE_FORMATTER)); + + return dataStartTime; + } + + /** + * 计算会话窗口的数据查询开始时间 + */ + private LocalDateTime calculateSessionWindowStartTime(LocalDateTime dataEndTime, AnalysisGroupByWindow groupByWindow) { + Integer sessionTimeout = groupByWindow.getSessionWindowSize(); + String sessionTimeoutUnit = groupByWindow.getSessionWindowSizeUnit(); + + if (sessionTimeout == null || sessionTimeout <= 0) { + log.warn("会话窗口超时时间配置无效，使用默认值：30分钟"); + sessionTimeout = 30; + sessionTimeoutUnit = "m"; + } + + if (sessionTimeoutUnit == null || sessionTimeoutUnit.trim().isEmpty()) { + sessionTimeoutUnit = "m"; + } + + LocalDateTime dataStartTime; + switch (sessionTimeoutUnit.toLowerCase()) { + case "s": + dataStartTime = dataEndTime.minusSeconds(sessionTimeout); + break; + case "m": + dataStartTime = dataEndTime.minusMinutes(sessionTimeout); + break; + case "h": + dataStartTime = dataEndTime.minusHours(sessionTimeout); + break; + case "d": + dataStartTime = dataEndTime.minusDays(sessionTimeout); + break; + default: + log.warn("会话窗口超时单位无效: {}，使用默认单位：分钟", sessionTimeoutUnit); + dataStartTime = dataEndTime.minusMinutes(sessionTimeout); + } + + // 会话窗口额外增加1天缓冲时间 + dataStartTime = dataStartTime.minusDays(1); + + log.info("会话窗口查询范围: 超时时间={}{}，额外缓冲1天，查询时间范围=[{}, {}]", + sessionTimeout, sessionTimeoutUnit, + dataStartTime.format(DATE_FORMATTER), + dataEndTime.format(DATE_FORMATTER)); + + return dataStartTime; + } + + /** + * 转换查询结果为告警对象 + */ + private List convertToAlarms(AnalysisAnalysisRule rule, List> queryResult) { + List alarms = new ArrayList<>(); + + for (Map row : queryResult) { + Alarm alarm = Alarm.builder() + .id(UUID.randomUUID().toString()) + .createdAt(LocalDateTime.now()) + .updatedAt(LocalDateTime.now()) + .alarmName(rule.getRuleName()) + .engineType("realtime") + .attackResult(-1) + .focused(false) + .fall(0) + .alarmLevel("未知") + .baseFocused(false) + .isUpdated(false) + .alarmSource(1) + .dispositionAdvice("研判后处置") + .disposedState(0) + .attackDirection("other") + .etlTime(LocalDateTime.now()) + .alarmAreaId(0) + .comment(buildComment(row)) + .attackChainPhase(getIntegerArray(-1)) + .judgedState(0) + .build(); + + // 映射查询结果字段到告警对象 + if (row.containsKey("log_start_at")) { + alarm.setLogStartAt(getTimestampValue(row.get("log_start_at"))); + } + if (row.containsKey("log_end_at")) { + alarm.setLogEndAt(getTimestampValue(row.get("log_end_at"))); + } + if (row.containsKey("alarm_name")) { + alarm.setAlarmName(getStringValue(row.get("alarm_name"))); + } + if (row.containsKey("alarm_type")) { + alarm.setAlarmType(getStringValue(row.get("alarm_type"))); + } + if (row.containsKey("alarm_level")) { + alarm.setAlarmLevel(convertAlarmLevel(getIntegerValue(row.get("alarm_level")))); + } + if (row.containsKey("attack_ip")) { + alarm.setAttackIp(getStringArray(row.get("attack_ip"))); + } + if (row.containsKey("victim_ip")) { + alarm.setVictimIp(getStringArray(row.get("victim_ip"))); + } + if (row.containsKey("victim_web_url")) { + alarm.setVictimWebUrl(getStringArray(row.get("victim_web_url"))); + } + if (row.containsKey("attack_chain_phase")) { + alarm.setAttackChainPhase(getIntegerArray(row.get("attack_chain_phase"))); + } + if (row.containsKey("device_id")) { + alarm.setDeviceId(getIntegerArray(row.get("device_id"))); + } + if (row.containsKey("tag")) { + alarm.setTag(getStringArray(row.get("tag"))); + } + if (row.containsKey("comment")) { + alarm.setComment(getStringValue(row.get("comment"))); + } + if (row.containsKey("origin_log_ids")) { + alarm.setOriginLogIds(getStringArray(row.get("origin_log_ids"))); + } + if (row.containsKey("query_id")) { + alarm.setQueryId(getStringValue(row.get("query_id"))); + } + if (row.containsKey("attack_result")) { + alarm.setAttackResult(getIntegerValue(row.get("attack_result"))); + } + if (row.containsKey("fall")) { + alarm.setFall(getIntegerValue(row.get("fall"))); + } + if (row.containsKey("payload")) { + alarm.setPayload(getBytesValue(row.get("payload"))); + } + if (row.containsKey("operate_event")) { + alarm.setOperateEvent(getIntegerArray(row.get("operate_event"))); + } + if (row.containsKey("attack_port")) { + alarm.setAttackPort(getIntegerArray(row.get("attack_port"))); + } + if (row.containsKey("victim_port")) { + alarm.setVictimPort(getIntegerArray(row.get("victim_port"))); + } + if (row.containsKey("attack_method")) { + alarm.setAttackMethod(getStringValue(row.get("attack_method"))); + } + if (row.containsKey("business_ext")) { + alarm.setBusinessExt(getStringValue(row.get("business_ext"))); + } + if (row.containsKey("http_status")) { + alarm.setHttpStatus(getStringValue(row.get("http_status"))); + } + if (row.containsKey("dns_info")) { + alarm.setDnsInfo(getStringValue(row.get("dns_info"))); + } + if (row.containsKey("account_info")) { + alarm.setAccountInfo(getStringValue(row.get("account_info"))); + } + if (row.containsKey("attacker_info")) { + alarm.setAttackerInfo(getStringValue(row.get("attacker_info"))); + } + if (row.containsKey("victim_info")) { + alarm.setVictimInfo(getStringValue(row.get("victim_info"))); + } + if (row.containsKey("suspicious_action")) { + alarm.setSuspiciousAction(getStringValue(row.get("suspicious_action"))); + } + if (row.containsKey("vuln_info")) { + alarm.setVulnInfo(getStringValue(row.get("vuln_info"))); + } + if (row.containsKey("weak_pwd")) { + alarm.setWeakPwd(getStringValue(row.get("weak_pwd"))); + } + if (row.containsKey("compliance_baseline")) { + alarm.setComplianceBaseline(getStringValue(row.get("compliance_baseline"))); + } + if (row.containsKey("file_info")) { + alarm.setFileInfo(getStringValue(row.get("file_info"))); + } + if (row.containsKey("file_tags")) { + alarm.setFileTags(getStringValue(row.get("file_tags"))); + } + if (row.containsKey("endpoint_info")) { + alarm.setEndpointInfo(getStringValue(row.get("endpoint_info"))); + } + if (row.containsKey("origin_info")) { + alarm.setOriginInfo(getStringValue(row.get("origin_info"))); + } + if (row.containsKey("protocol_info")) { + alarm.setProtocolInfo(getStringValue(row.get("protocol_info"))); + } + if (row.containsKey("email_info")) { + alarm.setEmailInfo(getStringValue(row.get("email_info"))); + } + if (row.containsKey("sensitive_data")) { + alarm.setSensitiveData(getStringValue(row.get("sensitive_data"))); + } + if (row.containsKey("hit_intelligence")) { + alarm.setHitIntelligence(getIntegerValue(row.get("hit_intelligence"))); + } + if (row.containsKey("window_time")) { + alarm.setWindowTime(getStringValue(row.get("window_time"))); + } + if (row.containsKey("attack_ip_pic")) { + alarm.setAttackIpPic(getStringValue(row.get("attack_ip_pic"))); + } + if (row.containsKey("victim_ip_pic")) { + alarm.setVictimIpPic(getStringValue(row.get("victim_ip_pic"))); + } + if (row.containsKey("operation_at")) { + alarm.setOperationAt(getTimestampValue(row.get("operation_at"))); + } + if (row.containsKey("attack_direction")) { + alarm.setAttackDirection(getStringValue(row.get("attack_direction"))); + } + if (row.containsKey("etl_time")) { + alarm.setEtlTime(getTimestampValue(row.get("etl_time"))); + } + if (row.containsKey("log_count")) { + alarm.setLogCount(getIntegerValue(row.get("log_count"))); + } + if (row.containsKey("is_asset_hit")) { + alarm.setIsAssetHit(getIntegerValue(row.get("is_asset_hit"))); + } + if (row.containsKey("http_req_header")) { + alarm.setHttpReqHeader(getStringArray(row.get("http_req_header"))); + } + if (row.containsKey("http_req_body")) { + alarm.setHttpReqBody(getStringArray(row.get("http_req_body"))); + } + if (row.containsKey("http_resp_header")) { + alarm.setHttpRespHeader(getStringArray(row.get("http_resp_header"))); + } + if (row.containsKey("http_resp_body")) { + alarm.setHttpRespBody(getStringArray(row.get("http_resp_body"))); + } + if (row.containsKey("window_time")) { + alarm.setLogEndAt(getTimestampValue(row.get("window_time"))); + } + alarms.add(alarm); + } + + return alarms; + } + + private byte[] getBytesValue(Object value) { + if (value == null) { + return null; + } + if (value instanceof byte[]) { + return (byte[]) value; + } + return value.toString().getBytes(); + } + + private String getStringValue(Object value) { + if (value == null) { + return null; + } + return value.toString(); + } + + private Long getLongValue(Object value) { + if (value == null) { + return 0L; + } + if (value instanceof Number) { + return ((Number) value).longValue(); + } + return Long.parseLong(value.toString()); + } + + private Integer getIntegerValue(Object value) { + if (value == null) { + return -1; + } + if (value instanceof Number) { + return ((Number) value).intValue(); + } + return Integer.parseInt(value.toString()); + } + + private LocalDateTime getTimestampValue(Object value) { + if (value == null) { + return null; + } + if (value instanceof LocalDateTime) { + return (LocalDateTime) value; + } + + String strValue = value.toString().trim(); + + String[] patterns = { + "yyyy-MM-dd HH:mm:ss.SSS", + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ss.SSS", + "yyyy-MM-dd'T'HH:mm:ss", + "yyyy-MM-dd HH:mm:ss.SSSSSS", + "yyyy-MM-dd HH:mm:ss.SSSSS", + "yyyy-MM-dd HH:mm:ss.S", + "yyyy-MM-dd HH:mm:ss.SS", + "yyyy-MM-dd'T'HH:mm:ss.SSSSSSS", + "yyyy-MM-dd", + "yyyy/MM/dd HH:mm:ss", + "yyyy/MM/dd HH:mm:ss.SSS" + }; + + for (String pattern : patterns) { + try { + return LocalDateTime.parse(strValue, java.time.format.DateTimeFormatter.ofPattern(pattern)); + } catch (Exception ignored) { + } + } + + try { + return LocalDateTime.parse(strValue); + } catch (Exception e) { + log.warn("无法解析时间字符串: {}", strValue); + return null; + } + } + + @SuppressWarnings("unchecked") + private String[] getStringArray(Object value) { + if (value == null) { + return new String[0]; + } + if (value instanceof String[]) { + return (String[]) value; + } + if (value instanceof Object[]) { + Object[] arr = (Object[]) value; + String[] result = new String[arr.length]; + for (int i = 0; i < arr.length; i++) { + result[i] = arr[i] != null ? arr[i].toString() : null; + } + return result; + } + String str = value.toString(); + if (str.startsWith("[") && str.endsWith("]")) { + str = str.substring(1, str.length() - 1).trim(); + if (str.isEmpty()) { + return new String[0]; + } + String[] parts = str.split(","); + String[] result = new String[parts.length]; + for (int i = 0; i < parts.length; i++) { + String part = parts[i].trim(); + if (part.startsWith("\"") && part.endsWith("\"")) { + part = part.substring(1, part.length() - 1); + } + result[i] = part; + } + return result; + } + if (str.startsWith("{") && str.endsWith("}")) { + str = str.substring(1, str.length() - 1); + return str.split(","); + } + return new String[]{str}; + } + + @SuppressWarnings("unchecked") + private Integer[] getIntegerArray(Object value) { + if (value == null) { + return new Integer[0]; + } + String[] strArray = getStringArray(value); + Integer[] result = new Integer[strArray.length]; + for (int i = 0; i < strArray.length; i++) { + try { + result[i] = Integer.parseInt(strArray[i]); + } catch (NumberFormatException e) { + result[i] = null; + } + } + return result; + } + + @SuppressWarnings("unchecked") + private byte[][] getByteArrayArray(Object value) { + if (value == null) { + return new byte[0][]; + } + if (value instanceof byte[][]) { + return (byte[][]) value; + } + if (value instanceof Object[]) { + Object[] arr = (Object[]) value; + byte[][] result = new byte[arr.length][]; + for (int i = 0; i < arr.length; i++) { + if (arr[i] instanceof byte[]) { + result[i] = (byte[]) arr[i]; + } else { + result[i] = arr[i] != null ? arr[i].toString().getBytes() : null; + } + } + return result; + } + return new byte[0][]; + } + + private String convertAlarmLevel(Integer eventLevel) { + if (eventLevel == null) return "未知"; + switch (eventLevel) { + case 0: return "安全(无威胁)"; + case 1: return "低危"; + case 2: return "中危"; + case 3: return "高危"; + case 4: return "超危"; + default: return "未知"; + } + } + + private String buildComment(Map row) { + String victimIpsStr; + if (row.containsKey("victim_ip")) { + victimIpsStr = String.join(",", getStringArray(row.get("victim_ip"))); + } else { + victimIpsStr = "未知"; + } + String alarmName = ""; + if (row.containsKey("alarm_name")) { + alarmName = getStringValue(row.get("alarm_name")); + } + String AttackIps = ""; + if (row.containsKey("attack_ip")) { + AttackIps = String.join(",", getStringArray(row.get("attack_ip"))); + } + return String.format( + "24小时内，检测到%s上产生%s告警：\n告警名称：%s\n攻击IP：%s\n攻击结果：%d", + victimIpsStr, + convertAlarmLevel(getIntegerValue(row.get("alarm_level"))), + alarmName, + AttackIps, + determineAttackResult(row) + ); + } + + private String convertAttackIps(String[] attackIps) { + if (attackIps == null || attackIps.length == 0) { + return null; + } + return String.join(",", attackIps); + } + + private Integer determineAttackResult(Map row) { + if (row.containsKey("attack_result")) { + return getIntegerValue(row.get("attack_result")); + } else { + return -1; + } + } +} diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java index 9925ad0..98acd53 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/SqlGeneratorServiceImpl.java @@ -395,8 +395,9 @@ public class SqlGeneratorServiceImpl implements SqlGeneratorService { /** * 构建时间窗口表达式（用于SELECT） + * 改为protected以便子类复用（达梦安全版子查询包装模式） */ - private String buildTimeWindowExpression(AnalysisGroupByWindow groupByWindow) { + protected String buildTimeWindowExpression(AnalysisGroupByWindow groupByWindow) { if (groupByWindow == null) { return ""; } diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/SqlGeneratorServiceImplDm.java b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/SqlGeneratorServiceImplDm.java new file mode 100644 index 0000000..7c0dbef --- /dev/null +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/common/service/impl/SqlGeneratorServiceImplDm.java @@ -0,0 +1,200 @@ +package com.common.service.impl; + +import com.common.entity.*; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.*; + +/** + * 达梦安全版 SQL生成服务实现（子查询包装模式） + * + * 继承自 SqlGeneratorServiceImpl，重写 generateSql() 方法。 + * 当配置了时间窗口（TUMBLE/HOP/SESSION）时，采用子查询包装方式生成SQL， + * 将时间窗口函数 TUMBLE()/HOP()/SESSION() 的计算下沉到内层子查询中， + * 外层 GROUP BY 直接引用别名 window_time，规避达梦安全版 GROUP BY 不支持自定义函数的问题。 + * + * 注意：本类不注册为 Spring Bean（不加 @Service），由 RealtimeAnalysisEngineDm 直接实例化， + * 避免与父类 SqlGeneratorServiceImpl 产生类型冲突影响 OfflineAnalysisEngine。 + * + * 产出示例： + * SELECT ...聚合函数..., window_time + * FROM ( + * SELECT 原始列..., TUMBLE(log_time, INTERVAL '5 MINUTE') AS window_time + * FROM syslog_normal_alarm AS t + * WHERE ... + * ) + * GROUP BY src_ip, dest_ip, window_time + */ +@Slf4j +public class SqlGeneratorServiceImplDm extends SqlGeneratorServiceImpl { + + /** + * 重写：根据规则动态生成SQL + * 有窗口配置 → 子查询包装模式 + * 无窗口配置 → 委托父类原有逻辑 + */ + @Override + public String generateSql(AnalysisAnalysisRule rule, + List fields, + List whereConditions, + List filters, + List groupByColumns, + List havingConditions, + AnalysisGroupByWindow groupByWindow, + String startTime, + String endTime) { + // 如果有时间窗口配置 → 走子查询包装模式（达梦安全版兼容方案） + if (groupByWindow != null && StringUtils.isNotBlank(groupByWindow.getWindowType())) { + return generateSubquerySql(fields, filters, groupByColumns, + havingConditions, groupByWindow, startTime, endTime); + } + // 无窗口配置 → 委托父类原有逻辑（平铺SQL） + return super.generateSql(rule, fields, whereConditions, filters, + groupByColumns, havingConditions, groupByWindow, startTime, endTime); + } + + // ==================== 子查询包装模式核心方法 ==================== + + /** + * 生成子查询包装的完整SQL（达梦安全版兼容方案） + * + * SQL结构： + * SELECT [外层聚合列...], window_time + * FROM ( + * SELECT [内层原始列...], TUMBLE(log_time, ...) AS window_time + * FROM syslog_normal_alarm AS t + * WHERE ... + * ) + * GROUP BY [分组列...], window_time + * HAVING ... + */ + private String generateSubquerySql(List fields, + List filters, + List groupByColumns, + List havingConditions, + AnalysisGroupByWindow groupByWindow, + String startTime, + String endTime) { + // 1. 内层子查询SELECT：原始列 + TUMBLE(...) AS window_time + String innerSelect = buildInnerSelectForWindow(fields, groupByColumns, groupByWindow); + + // 2. FROM + WHERE（在内层子查询中） + String fromClause = generateFromClause(fields, null); + String whereClause = generateWhereClauseFromFilters(filters, startTime, endTime); + + // 3. 外层SELECT：聚合列 + window_time + String outerSelect = buildOuterSelectForWindow(fields); + + // 4. 外层GROUP BY：分组列 + window_time（别名，非函数调用） + String groupByClause = buildGroupByForWindow(groupByColumns); + + // 5. HAVING（不变，复用父类） + String havingClause = generateHavingClauseFromConditions(havingConditions); + + // 6. 拼接最终SQL + return assembleSubquerySql(outerSelect, innerSelect, fromClause, + whereClause, groupByClause, havingClause); + } + + /** + * 构建内层子查询的SELECT子句 + * 产出： + * SELECT src_ip, dest_ip, origin_event_name, ..., TUMBLE(log_time, INTERVAL '5 MINUTE') AS window_time + */ + private String buildInnerSelectForWindow(List fields, + List groupByColumns, + AnalysisGroupByWindow groupByWindow) { + Set columns = new LinkedHashSet<>(); + + // 1. 从 fields 中提取所有原始列名（不含聚合函数） + if (fields != null) { + for (AnalysisField field : fields) { + if (StringUtils.isNotBlank(field.getColumnName())) { + columns.add(field.getColumnName()); + } + } + } + + // 2. 从 groupByColumns 中提取原始列名 + if (groupByColumns != null) { + for (AnalysisGroupByColumn col : groupByColumns) { + if (StringUtils.isNotBlank(col.getColumnName())) { + columns.add(col.getColumnName()); + } + } + } + + // 3. 追加时间窗口表达式（带 AS window_time 别名） + // 复用父类 protected 方法 buildTimeWindowExpression + String windowExpr = buildTimeWindowExpression(groupByWindow); + + List parts = new ArrayList<>(columns); + if (StringUtils.isNotBlank(windowExpr)) { + parts.add(windowExpr); + } + + return "SELECT " + String.join(",\n ", parts); + } + + /** + * 构建外层SELECT子句（聚合层） + * 产出： + * SELECT JSONB_AGG(DISTINCT src_port) AS attack_port, ..., window_time + */ + private String buildOuterSelectForWindow(List fields) { + // 复用父类 generateSelectClause(fields, null) + // 传 null 窗口 → 不追加 TUMBLE(...) AS window_time，仅生成聚合函数列 + String baseSelect = generateSelectClause(fields, null); + + // 手动追加 window_time（此时是普通列引用，不是函数调用） + return baseSelect + ",\n window_time"; + } + + /** + * 构建外层GROUP BY子句 + * 产出： + * GROUP BY origin_event_name, src_ip, dest_ip, window_time + * + * 关键差异：使用别名 window_time，而非 TUMBLE(log_time, ...) 函数调用 + */ + private String buildGroupByForWindow(List groupByColumns) { + // 复用父类 generateGroupByClause(columns, null) + // 传 null 窗口 → 不追加 TUMBLE(...) 函数调用，仅生成业务分组列 + String baseGroupBy = generateGroupByClause(groupByColumns, null); + + if (StringUtils.isBlank(baseGroupBy)) { + return "GROUP BY window_time"; + } + // 在现有 GROUP BY 后追加 window_time + return baseGroupBy + ", window_time"; + } + + /** + * 拼接子查询包装SQL + */ + private String assembleSubquerySql(String outerSelect, String innerSelect, + String fromClause, String whereClause, + String groupByClause, String havingClause) { + StringBuilder sql = new StringBuilder(); + // 外层SELECT + sql.append(outerSelect).append("\n"); + // 子查询开始 + sql.append("FROM (\n"); + sql.append(" ").append(innerSelect).append("\n"); + sql.append(" ").append(fromClause).append("\n"); + if (StringUtils.isNotBlank(whereClause)) { + sql.append(" ").append(whereClause).append("\n"); + } + sql.append(")\n"); + // 外层GROUP BY + if (StringUtils.isNotBlank(groupByClause)) { + sql.append(groupByClause).append("\n"); + } + // HAVING + if (StringUtils.isNotBlank(havingClause)) { + sql.append(havingClause).append("\n"); + } + return sql.toString(); + } +} diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/config/AppProperties.java b/haobang-security-dm/syslog-consumer/src/main/java/com/config/AppProperties.java new file mode 100644 index 0000000..3b788e6 --- /dev/null +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/config/AppProperties.java @@ -0,0 +1,70 @@ +package com.config; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +/** + * 应用配置属性 - 替代静态 AppConfig + * 使用 Spring Boot 标准 @Value 机制，支持 docker-compose 环境变量覆盖 + */ +@Data +@Component +public class AppProperties { + + // ========== Syslog 配置 ========== + @Value("${syslog.tcp.port:514}") + private int syslogTcpPort; + + @Value("${syslog.udp.port:515}") + private int syslogUdpPort; + + @Value("${syslog.max.frame.length:65536}") + private int syslogMaxFrameLength; + + @Value("${syslog.buffer.size:1024}") + private int syslogBufferSize; + + // ========== InfluxDB 配置 ========== + @Value("${influxdb.url:http://localhost:8086}") + private String influxUrl; + + @Value("${influxdb.token:}") + private String influxToken; + + @Value("${influxdb.org:}") + private String influxOrg; + + @Value("${influxdb.bucket:syslog}") + private String influxBucket; + + @Value("${influxdb.batch.size:1000}") + private int influxBatchSize; + + @Value("${influxdb.flush.interval:1000}") + private int influxFlushInterval; + + @Value("${influxdb.retry.attempts:3}") + private int influxRetryAttempts; + + @Value("${influxdb.retry.delay:100}") + private int influxRetryDelay; + + // ========== 应用配置 ========== + @Value("${app.worker.threads:4}") + private int workerThreads; + + @Value("${app.max.queue.size:2000}") + private int maxQueueSize; + + @Value("${app.metrics.enabled:false}") + private boolean metricsEnabled; + + // ========== 运行环境 ========== + @Value("${server.run.environment:prod}") + private String runEnvironment; + + // ========== SM4 加密配置 ========== + @Value("${syslog.sm4.generateKey:}") + private String sm4Key; +} diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/config/KafkaConsumerProperties.java b/haobang-security-dm/syslog-consumer/src/main/java/com/config/KafkaConsumerProperties.java new file mode 100644 index 0000000..5dc9fac --- /dev/null +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/config/KafkaConsumerProperties.java @@ -0,0 +1,27 @@ +package com.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * Kafka Consumer 配置属性类 + * 替代 AppConfig 的静态方法读取，支持 docker-compose 环境变量覆盖 + */ +@Data +@Component +@ConfigurationProperties(prefix = "spring.kafka.consumer") +public class KafkaConsumerProperties { + /** Kafka 集群地址 */ + private String bootstrapServers = "192.168.4.99:9092"; + /** 消费者组ID */ + private String groupId = "agent-syslog-group"; + /** 消费位置重置策略 */ + private String autoOffsetReset = "latest"; + /** 是否自动提交 */ + private boolean enableAutoCommit = false; + /** 自动提交间隔(ms) */ + private String autoCommitInterval = "1000"; + /** 消费Topic */ + private String topic = "agent-syslog-topic"; +} diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/controllers/DmColumnController.java b/haobang-security-dm/syslog-consumer/src/main/java/com/controllers/DmColumnController.java index d4096c5..c18f89f 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/controllers/DmColumnController.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/controllers/DmColumnController.java @@ -28,6 +28,8 @@ public class DmColumnController { private DmColumnService dmColumnService; @Autowired private DmNormalizeRuleService dmNormalizeRuleService; + @Autowired + private LogNormalProcessor logNormalProcessor; private static HashMap OrginalColumnMap ; @@ -53,8 +55,7 @@ public class DmColumnController { //List> columnList=dmColumnService.selectAllNormal(); //List> rulelst= dmNormalizeRuleService.selectByDeviceIdAuto((long)1); //OrginalColumnMap= getMessageToMap(strLogMsg); - LogNormalProcessor logNormalProcessor=new LogNormalProcessor( "", UUID.randomUUID().toString(),"test-topic"); - logNormalProcessor.init(); + logNormalProcessor.process("", UUID.randomUUID().toString(), "test-topic"); return dmColumn.getDisplayName() ; //return dmColumn.getDisplayName() ; diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/controllers/SyslogController.java b/haobang-security-dm/syslog-consumer/src/main/java/com/controllers/SyslogController.java index b5630cb..54cc9fc 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/controllers/SyslogController.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/controllers/SyslogController.java @@ -34,6 +34,8 @@ public class SyslogController { private InfluxSyslogService syslogService; @Autowired private SyslogNonNormalMessageService messageService; + @Autowired + private LogNormalProcessor logNormalProcessor; /** * 接口1：按时间范围查询 * GET /api/syslog/time-range?startTime=2023-11-20T00:00:00Z&endTime=2023-11-20T23:59:59Z @@ -191,10 +193,8 @@ public class SyslogController { log.debug("开始泛化处理，syslog_uuid: {}, topic: {}", syslogUuid, syslogTopic); - // 调用泛化处理类 - LogNormalProcessor logNormalProcessor = new LogNormalProcessor( - combinedMessage, syslogUuid, syslogTopic); - logNormalProcessor.init(); + // 调用泛化处理类（Spring管理的Bean） + logNormalProcessor.process(combinedMessage, syslogUuid, syslogTopic); processSuccess = true; processMessage = "泛化处理成功"; @@ -314,9 +314,8 @@ public class SyslogController { firstMessage.getSyslogMessage() : ""; String combinedMessage = headerMessage + syslogMessage; - LogNormalProcessor processor = new LogNormalProcessor( - combinedMessage, syslogUuid, firstMessage.getSyslogTopic()); - processor.init(); + logNormalProcessor.process(combinedMessage, syslogUuid, + firstMessage.getSyslogTopic()); processSuccess = true; processMessage = "处理成功"; diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/InfluxDBClient.java b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/InfluxDBClient.java index ee2051f..f9728ae 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/InfluxDBClient.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/InfluxDBClient.java @@ -5,9 +5,11 @@ import com.influxdb.client.WriteApi; import com.influxdb.client.WriteApiBlocking; import com.influxdb.client.domain.HealthCheck; import com.influxdb.client.write.Point; -import com.config.AppConfig; +import com.config.AppProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import com.influxdb.query.FluxRecord; import com.influxdb.query.FluxTable; import java.util.ArrayList; @@ -19,6 +21,7 @@ import java.time.Instant; import java.time.format.DateTimeFormatter; +@Component public class InfluxDBClient implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(InfluxDBClient.class); @@ -29,18 +32,22 @@ public class InfluxDBClient implements AutoCloseable { private final String org; private final QueryApi queryApi; - public InfluxDBClient() { + private final InfluxDBConfig influxDBConfig; + + @Autowired + public InfluxDBClient(AppProperties appProperties, InfluxDBConfig influxDBConfig) { + this.influxDBConfig = influxDBConfig; this.influxDB = InfluxDBClientFactory.create( - AppConfig.getInfluxUrl(), - AppConfig.getInfluxToken().toCharArray(), - AppConfig.getInfluxOrg(), - AppConfig.getInfluxBucket() + appProperties.getInfluxUrl(), + appProperties.getInfluxToken().toCharArray(), + appProperties.getInfluxOrg(), + appProperties.getInfluxBucket() ); - this.writeApi = influxDB.makeWriteApi(InfluxDBConfig.getWriteOptions()); + this.writeApi = influxDB.makeWriteApi(influxDBConfig.getWriteOptions()); this.writeApiBlocking = influxDB.getWriteApiBlocking(); - this.bucket = AppConfig.getInfluxBucket(); - this.org = AppConfig.getInfluxOrg(); + this.bucket = appProperties.getInfluxBucket(); + this.org = appProperties.getInfluxOrg(); this.queryApi = influxDB.getQueryApi(); // 检查连接状态 @@ -92,7 +99,7 @@ public class InfluxDBClient implements AutoCloseable { */ public void writeRecord(String lineProtocol) { try { - writeApi.writeRecord(InfluxDBConfig.getWritePrecision(),lineProtocol); + writeApi.writeRecord(influxDBConfig.getWritePrecision(),lineProtocol); } catch (Exception e) { logger.error("Failed to write record to InfluxDB: {}", e.getMessage()); } diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/InfluxDBConfig.java b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/InfluxDBConfig.java index 8184240..5cbc41f 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/InfluxDBConfig.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/InfluxDBConfig.java @@ -1,18 +1,31 @@ package com.influx; + +import com.config.AppProperties; import com.influxdb.client.WriteOptions; import com.influxdb.client.domain.WritePrecision; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +@Component public class InfluxDBConfig { - public static WriteOptions getWriteOptions() { + + private final AppProperties appProperties; + + @Autowired + public InfluxDBConfig(AppProperties appProperties) { + this.appProperties = appProperties; + } + + public WriteOptions getWriteOptions() { return WriteOptions.builder() - .batchSize(com.config.AppConfig.getInfluxBatchSize()) - .flushInterval(com.config.AppConfig.getInfluxFlushInterval()) + .batchSize(appProperties.getInfluxBatchSize()) + .flushInterval(appProperties.getInfluxFlushInterval()) .jitterInterval(1000) - .retryInterval(com.config.AppConfig.getInfluxRetryDelay()) + .retryInterval(appProperties.getInfluxRetryDelay()) .build(); } - public static WritePrecision getWritePrecision() { + public WritePrecision getWritePrecision() { return WritePrecision.MS; // 毫秒精度 } } \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/SyslogToInfluxApp.java b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/SyslogToInfluxApp.java index 851886b..b7cfd35 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/influx/SyslogToInfluxApp.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/influx/SyslogToInfluxApp.java @@ -1,7 +1,7 @@ package com.influx; -import com.config.AppConfig; +import com.common.util.SpringContextUtil; import com.influx.InfluxDBClient; import com.influxdb.client.*; import com.influxdb.client.domain.WritePrecision; @@ -30,7 +30,7 @@ public class SyslogToInfluxApp { logger.info("Starting Syslog to InfluxDB application..."); // 初始化 InfluxDB 客户端 - influxClient = new InfluxDBClient(); + influxClient = SpringContextUtil.getBean(InfluxDBClient.class); influxClient.writeRecord(syslog); Point point = Point.measurement("syslog_normal") .addTag("ip", "localhost") // 添加标签 diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/kafka/kafkalogconsumer.java b/haobang-security-dm/syslog-consumer/src/main/java/com/kafka/kafkalogconsumer.java index 4a02221..3228a3c 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/kafka/kafkalogconsumer.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/kafka/kafkalogconsumer.java @@ -18,7 +18,9 @@ import com.common.util.JsonParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.common.util.JsonParser; -import com.config.AppConfig; +import com.common.util.SpringContextUtil; +import com.config.AppProperties; +import com.config.KafkaConsumerProperties; import com.Modules.NormalData.LogNormalProcessor; import java.time.LocalDate; import java.time.format.DateTimeFormatter; @@ -38,34 +40,28 @@ public class kafkalogconsumer { } public static void Run() { - // 配置消费者属性 - //Properties props = new Properties(); - //props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.222.130:9092"); - // props.put(ConsumerConfig.GROUP_ID_CONFIG, "test-group-app"); + AppProperties appProperties = SpringContextUtil.getBean(AppProperties.class); + KafkaConsumerProperties kafkaProps = SpringContextUtil.getBean(KafkaConsumerProperties.class); + LogNormalProcessor logNormalProcessor = SpringContextUtil.getBean(LogNormalProcessor.class); + Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfig.getBootstrapServers()); - props.put(ConsumerConfig.GROUP_ID_CONFIG, AppConfig.getGroupId()); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProps.getBootstrapServers()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaProps.getGroupId()); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); - // 可选配置 - //props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none"); // 从最早的消息开始消费 - //props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 从最早的消息开始消费 - //props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); // 自动提交偏移量 - //props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000"); // 自动提交间隔 - //props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1000); // 设置单次拉取最大消息数[citation:6] - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, AppConfig.getAutoOffsetReset()); // 从last开始消费 - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, AppConfig.getEnableAutoCommit()); // 自动提交偏移量 - props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,AppConfig.getAutoCommitIntervalMS()); // 自动提交间隔 + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaProps.getAutoOffsetReset()); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, kafkaProps.isEnableAutoCommit()); + props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, kafkaProps.getAutoCommitInterval()); // 创建消费者实例 Consumer consumer = new KafkaConsumer<>(props); try { // 订阅主题 - consumer.subscribe(Collections.singletonList(AppConfig.getTopic())); + consumer.subscribe(Collections.singletonList(kafkaProps.getTopic())); System.out.println("开始消费消息..."); - com.influx.InfluxDBClient influxClient = new InfluxDBClient(); + com.influx.InfluxDBClient influxClient = SpringContextUtil.getBean(com.influx.InfluxDBClient.class); // 持续消费消息 while (true) { // 拉取消息（等待最多100毫秒） @@ -88,7 +84,7 @@ public class kafkalogconsumer { Point point = Point.measurement("syslog_security") .addTag("deviceid", mapdev.get("device_id")) // 添加标签 .addTag("uuid", sysLogUUID) //syslog uuid - .addTag("topic", AppConfig.getTopic()) //kafka topic + .addTag("topic", kafkaProps.getTopic()) //kafka topic .addField("message", record.value()) // 添加字段 .time(System.currentTimeMillis(), WritePrecision.MS) ;// 毫秒级时间戳 influxClient.writePointBlocking(point); @@ -97,12 +93,9 @@ public class kafkalogconsumer { //insertSingleRecord( record.value()); System.out.println("insert postgres syslog ,value:"+ record.key()); - //String syslogMessage= AppConfig.geRunEnvironment‌().equals("test")? record.value().substring(34) : record.value(); String syslogMessage= record.value(); - //剔除测试环境本机syslog新增的头部信息 - LogNormalProcessor logNormalProcessor = new LogNormalProcessor(syslogMessage,sysLogUUID,AppConfig.getTopic()); - //LogNormalProcessor logNormalProcessor =new LogNormalProcessor(record.value()); - logNormalProcessor.init(); + //使用注入的 Spring Bean 进行标准化处理 + logNormalProcessor.process(syslogMessage, sysLogUUID, null); } // 手动提交偏移量（如果禁用自动提交） consumer.commitSync(); diff --git a/haobang-security-dm/syslog-consumer/src/main/java/com/kafka/kafkalogconsumerThead.java b/haobang-security-dm/syslog-consumer/src/main/java/com/kafka/kafkalogconsumerThead.java index f32f89b..5a5283b 100644 --- a/haobang-security-dm/syslog-consumer/src/main/java/com/kafka/kafkalogconsumerThead.java +++ b/haobang-security-dm/syslog-consumer/src/main/java/com/kafka/kafkalogconsumerThead.java @@ -5,8 +5,10 @@ import com.common.entity.XdrHoneypot; import com.common.mapper.XdrHoneypotMapper; import com.common.util.JsonParser; import com.common.util.MyBatisUtil; +import com.common.util.SpringContextUtil; import com.common.util.SyslogParser; -import com.config.AppConfig; +import com.config.AppProperties; +import com.config.KafkaConsumerProperties; import com.influx.InfluxDBClient; import com.influxdb.client.domain.WritePrecision; import com.influxdb.client.write.Point; @@ -36,34 +38,28 @@ public class kafkalogconsumerThead { } public static void Run() { - // 配置消费者属性 - //Properties props = new Properties(); - //props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.222.130:9092"); - // props.put(ConsumerConfig.GROUP_ID_CONFIG, "test-group-app"); + KafkaConsumerProperties kafkaProps = SpringContextUtil.getBean(KafkaConsumerProperties.class); + LogNormalProcessor logNormalProcessor = SpringContextUtil.getBean(LogNormalProcessor.class); + Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfig.getBootstrapServers()); - props.put(ConsumerConfig.GROUP_ID_CONFIG, AppConfig.getGroupId()); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProps.getBootstrapServers()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaProps.getGroupId()); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); // 可选配置 - //props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none"); // 从最早的消息开始消费 - //props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 从最早的消息开始消费 - //props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); // 自动提交偏移量 - //props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000"); // 自动提交间隔 - //props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1000); // 设置单次拉取最大消息数[citation:6] - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, AppConfig.getAutoOffsetReset()); // 从last开始消费 - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, AppConfig.getEnableAutoCommit()); // 自动提交偏移量 - props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,AppConfig.getAutoCommitIntervalMS()); // 自动提交间隔 + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaProps.getAutoOffsetReset()); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, kafkaProps.isEnableAutoCommit()); + props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, kafkaProps.getAutoCommitInterval()); // 创建消费者实例 Consumer consumer = new KafkaConsumer<>(props); try { // 订阅主题 - consumer.subscribe(Collections.singletonList(AppConfig.getTopic())); + consumer.subscribe(Collections.singletonList(kafkaProps.getTopic())); System.out.println("开始消费消息..."); - InfluxDBClient influxClient = new InfluxDBClient(); ; + InfluxDBClient influxClient = SpringContextUtil.getBean(InfluxDBClient.class); // 持续消费消息 while (true) { // 拉取消息（等待最多100毫秒） @@ -86,7 +82,7 @@ public class kafkalogconsumerThead { Point point = Point.measurement("syslog_security") .addTag("deviceid", mapdev.get("device_id")) // 添加标签 .addTag("uuid", sysLogUUID) //syslog uuid - .addTag("topic", AppConfig.getTopic()) //kafka topic + .addTag("topic", kafkaProps.getTopic()) //kafka topic .addField("message", record.value()) // 添加字段 .time(System.currentTimeMillis(), WritePrecision.MS) ;// 毫秒级时间戳 influxClient.writePointBlocking(point); @@ -95,12 +91,9 @@ public class kafkalogconsumerThead { //insertSingleRecord( record.value()); System.out.println("insert postgres syslog ,value:"+ record.key()); - //String syslogMessage= AppConfig.geRunEnvironment‌().equals("test")? record.value().substring(34) : record.value(); String syslogMessage= record.value(); - //剔除测试环境本机syslog新增的头部信息 - LogNormalProcessor logNormalProcessor = new LogNormalProcessor(syslogMessage,sysLogUUID,AppConfig.getTopic()); - //LogNormalProcessor logNormalProcessor =new LogNormalProcessor(record.value()); - logNormalProcessor.init(); + //使用注入的 Spring Bean 进行标准化处理 + logNormalProcessor.process(syslogMessage, sysLogUUID, null); } // 手动提交偏移量（如果禁用自动提交） // consumer.commitSync(); diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application-dev.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application-dev.properties index 8794954..094d0b8 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application-dev.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application-dev.properties @@ -39,7 +39,7 @@ spring.datasource.url=jdbc:dm://192.168.4.99:5237 spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver -spring.datasource.hikari.schema=\"PUBLIC\" +spring.datasource.hikari.schema=ECOSYS # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity @@ -57,7 +57,7 @@ spring.kafka.consumer.group-id=test-group spring.kafka.consumer.auto-offset-reset=latest spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.auto-commit-interval=1000 -spring.kafka.consumer.topic=agent-syslog-topic +spring.kafka.consumer.topic=test-topic spring.kafka.consumer.max-poll-records=1000 spring.kafka.consumer.properties.max.poll.interval.ms=300000 @@ -137,7 +137,7 @@ spring.datasource.hikari.max-lifetime=900000 spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL spring.datasource.hikari.validation-timeout=5000 spring.datasource.hikari.leak-detection-threshold=30000 -spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer +spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-dev spring.datasource.hikari.auto-commit=false # 关联分析规则配置 @@ -145,7 +145,10 @@ analysis.realtime.enabled= true # 检查间隔（秒） - 默认10秒 analysis.realtime.check-interval-seconds: 10 - +# 关联分析规则达梦安全版子查询模式开关（默认false，生产环境达梦安全版设为true） +# true → 使用 RealtimeAnalysisEngineDm，子查询包装SQL，适配 GROUP BY 不支持自定义函数 +# false → 使用 RealtimeAnalysisEngine，平铺SQL（原有行为） +analysis.db.dameng-subquery-mode=true # ============================================ # 探针联动API配置 diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application-pre-nw.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application-pre-nw.properties index 0e5812c..d9e84f3 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application-pre-nw.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application-pre-nw.properties @@ -150,6 +150,9 @@ analysis.realtime.enabled= true # 检查间隔（秒） - 默认10秒 analysis.realtime.check-interval-seconds: 10 +# 达梦安全版子查询模式开关（预发环境若同为达梦安全定制版，按需开启） +analysis.db.dameng-subquery-mode=false + # ============================================ # 探针联动API配置 # ============================================ diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application-prod-zc.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application-prod-zc.properties index fe53bf5..7cb5baa 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application-prod-zc.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application-prod-zc.properties @@ -148,4 +148,7 @@ spring.datasource.hikari.auto-commit=false # 关联分析规则配置 analysis.realtime.enabled= true # 检查间隔（秒） - 默认10秒 -analysis.realtime.check-interval-seconds: 10 \ No newline at end of file +analysis.realtime.check-interval-seconds: 10 + +# 达梦安全版子查询模式开关（生产环境达梦为安全定制版，需开启） +analysis.db.dameng-subquery-mode=true \ No newline at end of file diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application-prod.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application-prod.properties index ffb5c5e..9aa89e1 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application-prod.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application-prod.properties @@ -150,6 +150,12 @@ analysis.realtime.enabled= true # 检查间隔（秒） - 默认10秒 analysis.realtime.check-interval-seconds: 10 + +# 关联分析规则达梦安全版子查询模式开关（默认false，生产环境达梦安全版设为true） +# true → 使用 RealtimeAnalysisEngineDm，子查询包装SQL，适配 GROUP BY 不支持自定义函数 +# false → 使用 RealtimeAnalysisEngine，平铺SQL（原有行为） +analysis.db.dameng-subquery-mode=true + # ============================================ # 探针联动API配置 # ============================================ @@ -162,9 +168,9 @@ interlocking.api.base-url=http://192.168.4.99:8089/xdrservice/interlocking # 告警健康检查配置 # ============================================ # 告警表无数据阈值（小时） -alarm.health-check.alarm-hours=2 +alarm.health-check.alarm-hours=4 # 告警日志表无数据阈值（小时） -alarm.health-check.alarm-visit-hours=4 +alarm.health-check.alarm-visit-hours=2 # 是否启用定时巡检 alarm.health-check.enabled=true diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/application.properties b/haobang-security-dm/syslog-consumer/src/main/resources/application.properties index ffb5c5e..d32fc09 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/application.properties +++ b/haobang-security-dm/syslog-consumer/src/main/resources/application.properties @@ -150,6 +150,12 @@ analysis.realtime.enabled= true # 检查间隔（秒） - 默认10秒 analysis.realtime.check-interval-seconds: 10 +# 关联分析规则达梦安全版子查询模式开关（默认false，生产环境达梦安全版设为true） +# true → 使用 RealtimeAnalysisEngineDm，子查询包装SQL，适配 GROUP BY 不支持自定义函数 +# false → 使用 RealtimeAnalysisEngine，平铺SQL（原有行为） +analysis.db.dameng-subquery-mode=true + + # ============================================ # 探针联动API配置 # ============================================ @@ -162,9 +168,9 @@ interlocking.api.base-url=http://192.168.4.99:8089/xdrservice/interlocking # 告警健康检查配置 # ============================================ # 告警表无数据阈值（小时） -alarm.health-check.alarm-hours=2 +alarm.health-check.alarm-hours=4 # 告警日志表无数据阈值（小时） -alarm.health-check.alarm-visit-hours=4 +alarm.health-check.alarm-visit-hours=2 # 是否启用定时巡检 alarm.health-check.enabled=true diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceCollectTaskMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceCollectTaskMapper.xml index f224846..583b74d 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceCollectTaskMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/DeviceCollectTaskMapper.xml @@ -22,12 +22,17 @@ + + + + id, created_at, updated_at, deleted_at, device_id, method, task_name, first_time, last_success_time, last_failed_time, detail_id, epm, epm_peak, - process_architecture, task_count, recent_discover_time, epm_upper_limit + process_architecture, task_count, recent_discover_time, epm_upper_limit, + expire_time, remark, device_ip, organization_id @@ -60,6 +65,16 @@ AND last_success_time >= #{lastSuccessTime} + + + AND device_ip = #{deviceIp} + + + AND organization_id = #{organizationId} + + + AND remark LIKE CONCAT('%', #{remark}, '%') + ORDER BY updated_at DESC diff --git a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml index bf7ceac..512e3f2 100644 --- a/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml +++ b/haobang-security-dm/syslog-consumer/src/main/resources/mapper/SyslogNormalDataMapper.xml @@ -54,46 +54,47 @@ diff --git a/haobang-security-dm/syslog-consumer/syslog-consumer.iml b/haobang-security-dm/syslog-consumer/syslog-consumer.iml new file mode 100644 index 0000000..78eb072 --- /dev/null +++ b/haobang-security-dm/syslog-consumer/syslog-consumer.iml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-nw.yaml b/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-nw.yaml index 528a7a2..7d5a66c 100644 --- a/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-nw.yaml +++ b/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-nw.yaml @@ -24,38 +24,38 @@ services: # 探针标识 - DEVICE_COLLECT_ID=1 # 数据库配置 - - spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys - - spring.datasource.username=postgres - - spring.datasource.password=TnLanWaidYSwTSG5 - - spring.datasource.driver-class-name=org.postgresql.Driver + - SPRING_DATASOURCE_URL=jdbc:postgresql://117.72.68.72:54329/ecosys + - SPRING_DATASOURCE_USERNAME=postgres + - SPRING_DATASOURCE_PASSWORD=TnLanWaidYSwTSG5 + - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver # Redis配置 - - spring.redis.host=192.168.222.131 - - spring.redis.port=6379 - - spring.redis.password= - - spring.redis.database=0 - - spring.redis.timeout=2000 - - spring.cache.redis.time-to-live=600000 + - SPRING_REDIS_HOST=192.168.222.131 + - SPRING_REDIS_PORT=6379 + - SPRING_REDIS_PASSWORD= + - SPRING_REDIS_DATABASE=0 + - SPRING_REDIS_TIMEOUT=2000 + - SPRING_CACHE_REDIS_TIME_TO_LIVE=600000 # Kafka配置 - - spring.kafka.producer.bootstrap-servers=192.168.222.130:9092 - - spring.kafka.producer.topic=test-topic + - SPRING_KAFKA_PRODUCER_BOOTSTRAP_SERVERS=192.168.222.130:9092 + - SPRING_KAFKA_PRODUCER_TOPIC=test-topic # 防火墙配置 - - firewall.enabled=true - - firewall.api-key=your-secure-api-key + - FIREWALL_ENABLED=true + - FIREWALL_API_KEY=your-secure-api-key # 联动配置 (syslog-consumer 地址) - - interlocking.enabled=true - - interlocking.api.base-url=http://192.168.222.131:8089/xdrservice/interlocking - - interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 + - INTERLOCKING_ENABLED=true + - INTERLOCKING_API_BASE_URL=http://192.168.222.131:8089/xdrservice/interlocking + - INTERLOCKING_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 # 探针心跳配置 - - probe.heartbeat.enabled=true - - probe.heartbeat.interval-seconds=60 - - probe.heartbeat.initial-delay-ms=5000 - - probe.collect-id=probe-001 - - probe.collect-name=日志采集探针 - - probe.app-version=1.0.0 - - probe.platform.api-url=http://192.168.222.131:8089/xdrservice/interlocking/probe/heartbeat - - probe.platform.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 + - PROBE_HEARTBEAT_ENABLED=true + - PROBE_HEARTBEAT_INTERVAL_SECONDS=60 + - PROBE_HEARTBEAT_INITIAL_DELAY_MS=5000 + - PROBE_COLLECT_ID=probe-001 + - PROBE_COLLECT_NAME=日志采集探针 + - PROBE_APP_VERSION=1.0.0 + - PROBE_PLATFORM_API_URL=http://192.168.222.131:8089/xdrservice/interlocking/probe/heartbeat + - PROBE_PLATFORM_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 # 定时任务配置 - - spring.task.scheduling.pool.size=5 + - SPRING_TASK_SCHEDULING_POOL_SIZE=5 # JVM配置 - JAVA_OPTS=-Xms256m -Xmx1024m -XX:+UseG1GC ports: diff --git a/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-prod.yaml b/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-prod.yaml index 5c4b374..d35a225 100644 --- a/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-prod.yaml +++ b/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve-prod.yaml @@ -12,58 +12,57 @@ services: # ============================================ syslog-serve: build: - context: ./syslog-serve + context: ./serve-dm dockerfile: Dockerfile - image: syslog-serve:1.0.1 - container_name: syslog-serve + image: syslog-serve-dm:v1.2.1 + container_name: syslog-serve-dm restart: unless-stopped environment: # 环境配置 - SPRING_PROFILES_ACTIVE=prod - TZ=Asia/Shanghai - # 探针标识 - - DEVICE_COLLECT_ID=probe-001 + # 探针标识ID + - APP_SERVICE_DEVICE_COLLECT_ID=1 # 数据库配置 - - spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys - - spring.datasource.username=postgres - - spring.datasource.password=TnLanWaidYSwTSG5 - - spring.datasource.driver-class-name=org.postgresql.Driver + - SPRING_DATASOURCE_URL=jdbc:dm://192.168.4.99:5237 + - SPRING_DATASOURCE_USERNAME=SYSDBA + - SPRING_DATASOURCE_PASSWORD=caZ2TcmXNSW8L2Ap + - SPRING_DATASOURCE_DRIVER_CLASS_NAME=dm.jdbc.driver.DmDriver + - SPRING_DATASOURCE_HIKARI_SCHEMA=\"PUBLIC\" + # Redis配置 - - spring.redis.host=127.0.0.1 - - spring.redis.port=6379 - - spring.redis.password= - - spring.redis.database=0 - - spring.redis.timeout=2000 - - spring.cache.redis.time-to-live=600000 + - SPRING_REDIS_HOST=192.168.4.99 + - SPRING_REDIS_PORT=6379 + - SPRING_REDIS_PASSWORD=redis_GdGWte + - SPRING_REDIS_DATABASE=0 + - SPRING_REDIS_TIMEOUT=2000 + - SPRING_CACHE_REDIS_TIME_TO_LIVE=600000 # Kafka配置 - - spring.kafka.producer.bootstrap-servers=192.168.222.130:9092 - - spring.kafka.producer.topic=test-topic - # 防火墙配置 - - firewall.enabled=true - - firewall.api-key=your-secure-api-key + - SPRING_KAFKA_PRODUCER_BOOTSTRAP_SERVERS=192.168.4.99:9092 + - SPRING_KAFKA_PRODUCER_TOPIC=agent-01-syslog-topic + # 防火墙配置 + - FIREWALL_ENABLED=true + - FIREWALL_API_KEY=your-secure-api-key # 联动配置 (syslog-consumer 地址) - - interlocking.enabled=true - - interlocking.api.base-url=http://117.72.68.72:8089/xdrservice/interlocking - - interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 + - INTERLOCKING_ENABLED=true + - INTERLOCKING_API_BASE_URL=http://192.168.4.99:8089/xdrservice/interlocking + - INTERLOCKING_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 # 探针心跳配置 - - probe.heartbeat.enabled=true - - probe.heartbeat.interval-seconds=60 - - probe.heartbeat.initial-delay-ms=5000 - - probe.collect-id=probe-001 - - probe.collect-name=日志采集探针 - - probe.app-version=1.0.0 - - probe.platform.api-url=http://117.72.68.72:8089/xdrservice/interlocking/probe/heartbeat - - probe.platform.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 + - PROBE_HEARTBEAT_ENABLED=true + - PROBE_HEARTBEAT_INTERVAL_SECONDS=60 + - PROBE_HEARTBEAT_INITIAL_DELAY_MS=5000 + - PROBE_PLATFORM_API_URL=http://192.168.4.99:8089/xdrservice/interlocking/probe/heartbeat + # 定时任务配置 - - spring.task.scheduling.pool.size=5 + - SPRING_TASK_SCHEDULING_POOL_SIZE=5 # JVM配置 - - JAVA_OPTS=-Xms256m -Xmx512m -XX:+UseG1GC + - JAVA_OPTS=-Xms2048m -Xmx4096m -XX:+UseG1GC ports: - "8189:8189" # API端口 - "514:514/udp" # Syslog UDP - "514:514/tcp" # Syslog TCP volumes: - - syslog-serve-logs:/app/logs + - /home/syslog/logs:/app/logs networks: - xdr-network cap_add: diff --git a/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve.yaml b/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve.yaml index c74b3e9..74afea3 100644 --- a/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve.yaml +++ b/haobang-security-dm/syslog-serve/docker-compose/docker-compose-serve.yaml @@ -24,38 +24,38 @@ services: # 探针标识 - DEVICE_COLLECT_ID=1 # 数据库配置 - - spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys - - spring.datasource.username=postgres - - spring.datasource.password=TnLanWaidYSwTSG5 - - spring.datasource.driver-class-name=org.postgresql.Driver + - SPRING_DATASOURCE_URL=jdbc:postgresql://117.72.68.72:54329/ecosys + - SPRING_DATASOURCE_USERNAME=postgres + - SPRING_DATASOURCE_PASSWORD=TnLanWaidYSwTSG5 + - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver # Redis配置 - - spring.redis.host=192.168.222.131 - - spring.redis.port=6379 - - spring.redis.password= - - spring.redis.database=0 - - spring.redis.timeout=2000 - - spring.cache.redis.time-to-live=600000 + - SPRING_REDIS_HOST=192.168.222.131 + - SPRING_REDIS_PORT=6379 + - SPRING_REDIS_PASSWORD= + - SPRING_REDIS_DATABASE=0 + - SPRING_REDIS_TIMEOUT=2000 + - SPRING_CACHE_REDIS_TIME_TO_LIVE=600000 # Kafka配置 - - spring.kafka.producer.bootstrap-servers=192.168.222.130:9092 - - spring.kafka.producer.topic=test-topic + - SPRING_KAFKA_PRODUCER_BOOTSTRAP_SERVERS=192.168.222.130:9092 + - SPRING_KAFKA_PRODUCER_TOPIC=test-topic # 防火墙配置 - - firewall.enabled=true - - firewall.api-key=your-secure-api-key + - FIREWALL_ENABLED=true + - FIREWALL_API_KEY=your-secure-api-key # 联动配置 (syslog-consumer 地址) - - interlocking.enabled=true - - interlocking.api.base-url=http://192.168.222.131:8089/xdrservice/interlocking - - interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 + - INTERLOCKING_ENABLED=true + - INTERLOCKING_API_BASE_URL=http://192.168.222.131:8089/xdrservice/interlocking + - INTERLOCKING_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 # 探针心跳配置 - - probe.heartbeat.enabled=true - - probe.heartbeat.interval-seconds=60 - - probe.heartbeat.initial-delay-ms=5000 - - probe.collect-id=probe-001 - - probe.collect-name=日志采集探针 - - probe.app-version=1.0.0 - - probe.platform.api-url=http://192.168.222.131:8089/xdrservice/interlocking/probe/heartbeat - - probe.platform.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 + - PROBE_HEARTBEAT_ENABLED=true + - PROBE_HEARTBEAT_INTERVAL_SECONDS=60 + - PROBE_HEARTBEAT_INITIAL_DELAY_MS=5000 + - PROBE_COLLECT_ID=probe-001 + - PROBE_COLLECT_NAME=日志采集探针 + - PROBE_APP_VERSION=1.0.0 + - PROBE_PLATFORM_API_URL=http://192.168.222.131:8089/xdrservice/interlocking/probe/heartbeat + - PROBE_PLATFORM_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 # 定时任务配置 - - spring.task.scheduling.pool.size=5 + - SPRING_TASK_SCHEDULING_POOL_SIZE=5 # JVM配置 - JAVA_OPTS=-Xms256m -Xmx1024m -XX:+UseG1GC ports: diff --git a/haobang-security-dm/syslog-serve/pom.xml b/haobang-security-dm/syslog-serve/pom.xml index 0a0f98d..bef88cb 100644 --- a/haobang-security-dm/syslog-serve/pom.xml +++ b/haobang-security-dm/syslog-serve/pom.xml @@ -56,7 +56,7 @@ org.apache.kafka kafka-clients - 3.4.0 + 3.9.2 @@ -134,8 +134,8 @@ com.dameng - DmJdbcDriver18 - 8.1.2.141 + DmJdbcDriver + 8.1.4.169 diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/Modules/Device/DeviceProcess.java b/haobang-security-dm/syslog-serve/src/main/java/com/Modules/Device/DeviceProcess.java index c07f90f..1a8150f 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/Modules/Device/DeviceProcess.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/Modules/Device/DeviceProcess.java @@ -2,7 +2,7 @@ package com.Modules.Device; import com.common.entity.DeviceDevice; -import com.haobang.config.AppConfig; +import com.haobang.config.AppProperties; import com.kafka.kafkaProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,7 +85,8 @@ public class DeviceProcess { //当前判断是否归属 if(deviceDevice!=null ) { - int deviceCollectId =AppConfig.getDeviceCollectId(); + AppProperties props = SpringContextUtil.getBean(AppProperties.class); + int deviceCollectId = props.getAppServiceDeviceCollectId(); if(deviceDevice.getDeviceCollectId()!=deviceCollectId) { logger.error("设备请求的Host IP不属于当前采集探针任务,请联系管理员核实！"); @@ -110,8 +111,10 @@ public class DeviceProcess { } else { //创建未知设备记录 - DeviceCollectTask deviceCollectTask= deviceCollectTaskService.getById(AppConfig.getDeviceCollectId()); - deviceUnknown.setDeviceCollectId( AppConfig.getDeviceCollectId()); + AppProperties props = SpringContextUtil.getBean(AppProperties.class); + int deviceCollectId = props.getAppServiceDeviceCollectId(); + DeviceCollectTask deviceCollectTask= deviceCollectTaskService.getById(deviceCollectId); + deviceUnknown.setDeviceCollectId(deviceCollectId); deviceUnknown.setDeviceIp( this.sourceIP); deviceUnknown.setFirstTime(currentTime); if(deviceCollectTask!=null) deviceUnknown.setDeviceCollectName(deviceCollectTask.getTaskName()); @@ -141,9 +144,10 @@ public class DeviceProcess { try { // 创建设备接收日志对象 + AppProperties props = SpringContextUtil.getBean(AppProperties.class); devicelog= DeviceReceiveLog.builder() .deviceId(deviceDevice.getId()) - .deviceCollectId(AppConfig.getDeviceCollectId()) + .deviceCollectId(props.getAppServiceDeviceCollectId()) .deviceIp(deviceDevice.getIp()) .receiveTime(now) .receiveTimeStr(timeString) diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/SyslogServeMainApp.java b/haobang-security-dm/syslog-serve/src/main/java/com/SyslogServeMainApp.java index 1acddfe..922f3cf 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/SyslogServeMainApp.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/SyslogServeMainApp.java @@ -1,4 +1,6 @@ package com; + +import com.haobang.config.AppProperties; import com.haobang.config.SyslogConfig; import com.kafka.kafkaProducer; import org.mybatis.spring.annotation.MapperScan; @@ -7,8 +9,9 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.env.Environment; import com.netty.SyslogServer; -import com.haobang.config.AppConfig; import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.EnableScheduling; @@ -23,12 +26,13 @@ public class SyslogServeMainApp { private static final Logger logger = LoggerFactory.getLogger(SyslogServeMainApp.class); public static void main(String[] args) { - SpringApplication.run(SyslogServeMainApp.class, args); + ConfigurableApplicationContext context = SpringApplication.run(SyslogServeMainApp.class, args); + Environment env = context.getEnvironment(); try { - //System.out.println("syslogConfig TcpPort: "+ AppConfig.getSyslogTcpPort()); - int tcpPort = args.length > 0 ? Integer.parseInt(args[0]) : AppConfig.getSyslogTcpPort() ; - int udpPort = args.length > 1 ? Integer.parseInt(args[1]) : AppConfig.getSyslogUdpPort(); - //SyslogServer server = new SyslogServer(tcpPort, udpPort); + int tcpPort = args.length > 0 ? Integer.parseInt(args[0]) + : Integer.parseInt(env.getProperty("syslog.tcp.port", "514")); + int udpPort = args.length > 1 ? Integer.parseInt(args[1]) + : Integer.parseInt(env.getProperty("syslog.udp.port", "515")); SyslogServerBoth server = new SyslogServerBoth(tcpPort, udpPort); logger.info("Application SyslogServer start !"); server.start(); diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/common/entity/DeviceCollectTask.java b/haobang-security-dm/syslog-serve/src/main/java/com/common/entity/DeviceCollectTask.java index 9d28b64..70b1456 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/common/entity/DeviceCollectTask.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/common/entity/DeviceCollectTask.java @@ -21,6 +21,10 @@ public class DeviceCollectTask { private Integer taskCount; private OffsetDateTime recentDiscoverTime; private Integer epmUpperLimit; + private OffsetDateTime expireTime; + private String remark; + private String deviceIp; + private Long organizationId; // Getter and Setter 方法 public Integer getId() { return id; } @@ -74,6 +78,18 @@ public class DeviceCollectTask { public Integer getEpmUpperLimit() { return epmUpperLimit; } public void setEpmUpperLimit(Integer epmUpperLimit) { this.epmUpperLimit = epmUpperLimit; } + public OffsetDateTime getExpireTime() { return expireTime; } + public void setExpireTime(OffsetDateTime expireTime) { this.expireTime = expireTime; } + + public String getRemark() { return remark; } + public void setRemark(String remark) { this.remark = remark; } + + public String getDeviceIp() { return deviceIp; } + public void setDeviceIp(String deviceIp) { this.deviceIp = deviceIp; } + + public Long getOrganizationId() { return organizationId; } + public void setOrganizationId(Long organizationId) { this.organizationId = organizationId; } + @Override public String toString() { return "DeviceCollectTask{" + @@ -94,6 +110,10 @@ public class DeviceCollectTask { ", taskCount=" + taskCount + ", recentDiscoverTime=" + recentDiscoverTime + ", epmUpperLimit=" + epmUpperLimit + + ", expireTime=" + expireTime + + ", remark='" + remark + '\'' + + ", deviceIp='" + deviceIp + '\'' + + ", organizationId=" + organizationId + '}'; } } \ No newline at end of file diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java b/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java index e7bf3c6..332dfc8 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/common/mapper/DeviceCollectTaskMapper.java @@ -60,10 +60,12 @@ public interface DeviceCollectTaskMapper { */ @Insert("INSERT INTO device_collect_task (created_at, updated_at, device_id, method, task_name, " + "first_time, last_success_time, last_failed_time, detail_id, epm, epm_peak, " + - "process_architecture, task_count, recent_discover_time, epm_upper_limit) " + + "process_architecture, task_count, recent_discover_time, epm_upper_limit, " + + "expire_time, remark, device_ip, organization_id) " + "VALUES (NOW(), NOW(), #{deviceId}, #{method}, #{taskName}, #{firstTime}, " + "#{lastSuccessTime}, #{lastFailedTime}, #{detailId}, #{epm}, #{epmPeak}, " + - "#{processArchitecture}, #{taskCount}, #{recentDiscoverTime}, #{epmUpperLimit})") + "#{processArchitecture}, #{taskCount}, #{recentDiscoverTime}, #{epmUpperLimit}, " + + "#{expireTime}, #{remark}, #{deviceIp}, #{organizationId})") @Options(useGeneratedKeys = true, keyProperty = "id") int insert(DeviceCollectTask task); @@ -84,7 +86,11 @@ public interface DeviceCollectTaskMapper { "process_architecture = #{processArchitecture}, " + "task_count = #{taskCount}, " + "recent_discover_time = #{recentDiscoverTime}, " + - "epm_upper_limit = #{epmUpperLimit} " + + "epm_upper_limit = #{epmUpperLimit}, " + + "expire_time = #{expireTime}, " + + "remark = #{remark}, " + + "device_ip = #{deviceIp}, " + + "organization_id = #{organizationId} " + "WHERE id = #{id}") int update(DeviceCollectTask task); diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java new file mode 100644 index 0000000..2e77fbe --- /dev/null +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/config/AppProperties.java @@ -0,0 +1,60 @@ +package com.haobang.config; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +/** + * 应用配置属性 - 替代静态 AppConfig（syslog-serve 模块） + * 使用 Spring Boot 标准 @Value 机制，支持 docker-compose 环境变量覆盖 + */ +@Data +@Component +public class AppProperties { + + // ========== Syslog 配置 ========== + @Value("${syslog.tcp.port:514}") + private int syslogTcpPort; + + @Value("${syslog.udp.port:515}") + private int syslogUdpPort; + + @Value("${syslog.max.frame.length:65536}") + private int syslogMaxFrameLength; + + @Value("${syslog.buffer.size:1024}") + private int syslogBufferSize; + + @Value("${syslog.charset:GBK}") + private String syslogCharset; + + // ========== App Service 配置（探针设备信息）========== + @Value("${app.service.device_id:}") + private String appServiceDeviceId; + + @Value("${app.service.device_name:}") + private String appServiceDeviceName; + + @Value("${app.service.vendor:}") + private String appServiceVendor; + + @Value("${app.service.product_name:}") + private String appServiceProductName; + + @Value("${app.service.data_type:json}") + private String appServiceDataType; + + @Value("${app.service.device_collect_id:1}") + private int appServiceDeviceCollectId; + + // ========== Kafka Producer 配置 ========== + @Value("${spring.kafka.producer.bootstrap-servers:localhost:9092}") + private String kafkaProducerBootstrap; + + @Value("${spring.kafka.producer.topic:agent-syslog-topic}") + private String kafkaProducerTopic; + + // ========== SM4 加密配置 ========== + @Value("${syslog.sm4.generateKey:}") + private String sm4Key; +} diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/util/DeviceInfoUtil.java b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/util/DeviceInfoUtil.java index 40f5a2d..bdd76a0 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/haobang/util/DeviceInfoUtil.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/haobang/util/DeviceInfoUtil.java @@ -3,7 +3,8 @@ package com.haobang.util; import com.common.entity.DeviceDevice; import com.fasterxml.jackson.databind.ObjectMapper; -import com.haobang.config.AppConfig; +import com.haobang.config.AppProperties; +import com.haobang.util.SpringContextUtil; import java.util.LinkedHashMap; import java.util.Map; @@ -11,6 +12,10 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; public class DeviceInfoUtil { + private static AppProperties getProps() { + return SpringContextUtil.getBean(AppProperties.class); + } + /** * 补充设备配置信息 key value值 * @param strSyslog @@ -18,6 +23,7 @@ public class DeviceInfoUtil { */ public static String getFullLogString(String strSyslog) { + AppProperties appProps = getProps(); ObjectMapper objectMapper = new ObjectMapper(); // 创建设备配置信息MAP // 获取当前时间（精度到毫秒） @@ -29,11 +35,11 @@ public class DeviceInfoUtil { Map map = new LinkedHashMap<>(); map.put("receive_time",timeString); - map.put("device_id", AppConfig.getAppServieDeviceId()); - map.put("device_name", AppConfig.getAppServieDeviceName()); - map.put("vendor", AppConfig.getAppServieVendor()); - map.put("data_type", AppConfig.getAppServieDataType()); - map.put("device_collect_id", Long.toString(AppConfig.getDeviceCollectId())); + map.put("device_id", appProps.getAppServiceDeviceId()); + map.put("device_name", appProps.getAppServiceDeviceName()); + map.put("vendor", appProps.getAppServiceVendor()); + map.put("data_type", appProps.getAppServiceDataType()); + map.put("device_collect_id", Long.toString(appProps.getAppServiceDeviceCollectId())); String formattedString = formatDeviceInfo(map); //return formattedString + "" + strSyslog.substring(34); @@ -48,6 +54,7 @@ public class DeviceInfoUtil { */ public static String getFullLogString(DeviceDevice deviceDevice,String strSyslog) { + AppProperties appProps = getProps(); ObjectMapper objectMapper = new ObjectMapper(); // 创建设备配置信息MAP // 获取当前时间（精度到毫秒） @@ -61,8 +68,8 @@ public class DeviceInfoUtil { map.put("device_id",deviceDevice.getId().toString()); map.put("device_name", deviceDevice.getName()); map.put("vendor", deviceDevice.getVendor()); - map.put("data_type", AppConfig.getAppServieDataType()); - map.put("device_collect_id", Long.toString(AppConfig.getDeviceCollectId())); + map.put("data_type", appProps.getAppServiceDataType()); + map.put("device_collect_id", Long.toString(appProps.getAppServiceDeviceCollectId())); String formattedString = formatDeviceInfo(map); //return formattedString + "" + strSyslog.substring(34); //测试环境截取34位之后字符串 @@ -71,6 +78,7 @@ public class DeviceInfoUtil { public static String getFullLogString(DeviceDevice deviceDevice,String strSyslog,String receive_time) { + AppProperties appProps = getProps(); ObjectMapper objectMapper = new ObjectMapper(); // 创建设备配置信息MAP // 获取当前时间（精度到毫秒） @@ -84,8 +92,8 @@ public class DeviceInfoUtil { map.put("device_id",deviceDevice.getId().toString()); map.put("device_name", deviceDevice.getName()); map.put("vendor", deviceDevice.getVendor()); - map.put("data_type", AppConfig.getAppServieDataType()); - map.put("device_collect_id", Long.toString(AppConfig.getDeviceCollectId())); + map.put("data_type", appProps.getAppServiceDataType()); + map.put("device_collect_id", Long.toString(appProps.getAppServiceDeviceCollectId())); String formattedString = formatDeviceInfo(map); //return formattedString + "" + strSyslog.substring(34); //测试环境截取34位之后字符串 diff --git a/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java b/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java index 069f1cd..ed8843a 100644 --- a/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java +++ b/haobang-security-dm/syslog-serve/src/main/java/com/kafka/kafkaProducer.java @@ -1,5 +1,6 @@ package com.kafka; import com.common.entity.DeviceDevice; +import com.haobang.config.AppProperties; import com.haobang.util.Sm4Util; import org.apache.kafka.clients.producer.*; import org.apache.kafka.common.serialization.StringSerializer; @@ -13,21 +14,20 @@ import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.haobang.util.DeviceInfoUtil; - +import com.haobang.util.SpringContextUtil; public class kafkaProducer { private static final Logger logger = LoggerFactory.getLogger(kafkaProducer.class); - private static String strhexKey=AppConfig.getSM4Key(); + + private static AppProperties getProps() { + return SpringContextUtil.getBean(AppProperties.class); + } public static void main(String[] args) { - // System.out.println(getFullLogString("syslogmessage")); - - - + AppProperties appProps = getProps(); // 配置生产者属性 Properties props = new Properties(); - //props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.222.130:9092"); - props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfig.getKafkaProducerBootstrap()); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProps.getKafkaProducerBootstrap()); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); @@ -47,7 +47,7 @@ public class kafkaProducer { // 创建生产者记录 ProducerRecord record = - new ProducerRecord<>(AppConfig.getKafkaProducerTopic(), key, value); + new ProducerRecord<>(appProps.getKafkaProducerTopic(), key, value); // 发送消息（异步方式） producer.send(record, new Callback() { @@ -82,9 +82,10 @@ public class kafkaProducer { public static void messagePush(DeviceDevice deviceDevice, String strSyslog,String strReceiveTime) { + AppProperties appProps = getProps(); // 配置生产者属性 Properties props = new Properties(); - props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfig.getKafkaProducerBootstrap()); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, appProps.getKafkaProducerBootstrap()); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); @@ -103,12 +104,12 @@ public class kafkaProducer { String value = DeviceInfoUtil.getFullLogString(deviceDevice,strSyslog,strReceiveTime); //Message进行SM4加密写入kafka - String Sm4message= Sm4Util.encryptCbc(value, strhexKey); - System.out.println("Sm4message:"+Sm4message); + String Sm4message= Sm4Util.encryptCbc(value, appProps.getSm4Key()); + //System.out.println("Sm4message:"+Sm4message); // 创建生产者记录 ProducerRecord record = - new ProducerRecord<>(AppConfig.getKafkaProducerTopic(), key, Sm4message); + new ProducerRecord<>(appProps.getKafkaProducerTopic(), key, Sm4message); // 发送消息（异步方式） producer.send(record, new Callback() { @@ -148,10 +149,11 @@ public class kafkaProducer { ObjectMapper objectMapper = new ObjectMapper(); // 创建设备配置信息MAP Map map = new LinkedHashMap<>(); - map.put("device_id",AppConfig.getAppServieDeviceId() ); - map.put("device_name", AppConfig.getAppServieDeviceName()); - map.put("vendor", AppConfig.getAppServieVendor()); - map.put("data_type", AppConfig.getAppServieDataType()); + AppProperties appProps = getProps(); + map.put("device_id", appProps.getAppServiceDeviceId()); + map.put("device_name", appProps.getAppServiceDeviceName()); + map.put("vendor", appProps.getAppServiceVendor()); + map.put("data_type", appProps.getAppServiceDataType()); // 添加syslogMessage map.put("syslogMessage", strSyslog); try { diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties index c316fea..00f0459 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-dev.properties @@ -21,7 +21,7 @@ app.service.device_name=honeypot app.service.vendor=changting app.service.product_name=diting #采集探针ID -app.service.device_collect_id=${DEVICE_COLLECT_ID:1} +app.service.device_collect_id=${DEVICE_COLLECT_ID:7} app.service.version=${APP_SERVICE_VER:V1.0.0-20260527} app.service.device_collect_name=${DEVICE_COLLECT_NAME:DevCollect-01} # syslog message data_type @@ -29,7 +29,7 @@ app.service.data_type=json # kafka Configuration spring.kafka.producer.bootstrap-servers=192.168.4.99:9092 -spring.kafka.producer.topic =${KAFKA_PRODUCER_TOPIC:agent-syslog-topic} +spring.kafka.producer.topic =${KAFKA_PRODUCER_TOPIC:test-topic} # kafka Configuration 新增优化配置 spring.kafka.producer.properties.retries=10 spring.kafka.producer.properties.retry.backoff.ms=500 @@ -43,7 +43,7 @@ spring.datasource.url=jdbc:dm://192.168.4.99:5237 spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver -spring.datasource.hikari.schema=\"PUBLIC\" +spring.datasource.hikari.schema=ECOSYS # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties index 08fb189..40e4cce 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-prod-zc.properties @@ -30,7 +30,7 @@ spring.kafka.producer.bootstrap-servers=10.11.2.142:9092 spring.kafka.producer.topic =agent-syslog-topic #database Configuration -spring.datasource.url=jdbc:dm://192.163.4.99:5237/DM_ecosys +spring.datasource.url=jdbc:dm://192.163.4.99:5237 spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties b/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties index 6da6d00..3f5b10d 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application-prod.properties @@ -12,7 +12,7 @@ syslog.udp.port=514 syslog.max.frame.length=262144 syslog.buffer.size=1000 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 -syslog.charset=GBK +syslog.charset=UTF-8 syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a # APP Service Configuration app.service.device_id=1 @@ -37,7 +37,7 @@ spring.kafka.producer.properties.socket.keepalive.enable=true spring.kafka.producer.properties.request.timeout.ms=30000 spring.kafka.producer.properties.delivery.timeout.ms=120000 #database Configuration -spring.datasource.url=jdbc:dm://192.163.4.99:5237 +spring.datasource.url=jdbc:dm://192.168.4.99:5237 spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver @@ -52,7 +52,7 @@ mybatis.configuration.map-underscore-to-camel-case=true spring.redis.host=192.168.4.99 spring.redis.port=6379 # 密码（如果没有设置密码，可以省略） -spring.redis.password=123456 +spring.redis.password=redis_GdGWte spring.redis.database=0 spring.redis.timeout=5000 #spring.redis.password=${REDIS_PASSWORD:default_prod_password} diff --git a/haobang-security-dm/syslog-serve/src/main/resources/application.properties b/haobang-security-dm/syslog-serve/src/main/resources/application.properties index 6da6d00..00f0459 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/application.properties +++ b/haobang-security-dm/syslog-serve/src/main/resources/application.properties @@ -1,4 +1,4 @@ -#Server Configuration [192.168.4.26] +#Server Configuration server.port=8189 server.servlet.context-path=/syslogserve #server.address=0.0.0.0 @@ -14,21 +14,22 @@ syslog.buffer.size=1000 # syslog 消息字符编码，国内安全设备普遍使用 GBK，如需 UTF-8 改为 syslog.charset=UTF-8 syslog.charset=GBK syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a + # APP Service Configuration app.service.device_id=1 app.service.device_name=honeypot app.service.vendor=changting app.service.product_name=diting -# syslog message data_type -app.service.data_type=json #采集探针ID -app.service.device_collect_id=${DEVICE_COLLECT_ID:1} +app.service.device_collect_id=${DEVICE_COLLECT_ID:7} app.service.version=${APP_SERVICE_VER:V1.0.0-20260527} app.service.device_collect_name=${DEVICE_COLLECT_NAME:DevCollect-01} +# syslog message data_type +app.service.data_type=json # kafka Configuration spring.kafka.producer.bootstrap-servers=192.168.4.99:9092 -spring.kafka.producer.topic =agent-01-syslog-topic +spring.kafka.producer.topic =${KAFKA_PRODUCER_TOPIC:test-topic} # kafka Configuration 新增优化配置 spring.kafka.producer.properties.retries=10 spring.kafka.producer.properties.retry.backoff.ms=500 @@ -36,50 +37,48 @@ spring.kafka.producer.properties.connections.max.idle.ms=600000 spring.kafka.producer.properties.socket.keepalive.enable=true spring.kafka.producer.properties.request.timeout.ms=30000 spring.kafka.producer.properties.delivery.timeout.ms=120000 + #database Configuration -spring.datasource.url=jdbc:dm://192.163.4.99:5237 +spring.datasource.url=jdbc:dm://192.168.4.99:5237 spring.datasource.username=SYSDBA spring.datasource.password=caZ2TcmXNSW8L2Ap spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver -spring.datasource.hikari.schema=\"PUBLIC\" +spring.datasource.hikari.schema=ECOSYS # mybatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml mybatis.type-aliases-package=com.common.entity mybatis.configuration.map-underscore-to-camel-case=true -# 生产环境缓存配置 -spring.redis.host=192.168.4.99 +# 开发环境缓存配置 +spring.redis.host=localhost spring.redis.port=6379 # 密码（如果没有设置密码，可以省略） -spring.redis.password=123456 +spring.redis.password= spring.redis.database=0 -spring.redis.timeout=5000 -#spring.redis.password=${REDIS_PASSWORD:default_prod_password} +spring.redis.timeout=2000 -spring.redis.lettuce.pool.max-active=20 -spring.redis.lettuce.pool.max-wait=5000 -spring.redis.lettuce.pool.max-idle=10 -spring.redis.lettuce.pool.min-idle=5 - -# 生产环境缓存时间较长 -spring.cache.redis.time-to-live=3600000 +spring.redis.lettuce.pool.max-active=8 +spring.redis.lettuce.pool.max-wait=-1 +spring.redis.lettuce.pool.max-idle=8 +spring.redis.lettuce.pool.min-idle=0 +# 开发环境缓存时间较短，方便调试 +spring.cache.redis.time-to-live=600000 # 黑名单API配置 blacklist.api.url= https://103.43.84.11/api/v3/Objects/Blacklist blacklist.api.username=apt-admin103 blacklist.api.password=C9W2xYgfc%SN1 + # 白名单API配置 -whitelist.api.url=https://103.43.84.11/api/v3/Policies/GlobalWhitelist - - +whitelist.api.url=https://103.43.84.11/api/v3/Policies/GlobalWhitelist # ============================================ # 探针联动配置 # ============================================ # 是否启用联动功能 interlocking.enabled=true # syslog-consumer API基础URL（安全平台外网映射地址） -interlocking.api.base-url=http://192.168.4.99:8089/xdrservice/interlocking +interlocking.api.base-url=http://localhost:8089/xdrservice/interlocking # API-KEY认证（32位，需与syslog-consumer配置一致） interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 # 定时任务执行间隔（毫秒），默认30秒 @@ -94,8 +93,11 @@ probe.heartbeat.enabled=true probe.heartbeat.interval-seconds=60 # 心跳发送初始延迟（毫秒） probe.heartbeat.initial-delay-ms=5000 + # 平台端心跳接收接口URL -probe.platform.api-url=http://192.168.4.99:8089/xdrservice/interlocking/probe/heartbeat +probe.platform.api-url=http://localhost:8089/xdrservice/interlocking/probe/heartbeat +# 平台API Key +probe.platform.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 # ============================================ # 定时任务配置 diff --git a/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceCollectTaskMapper.xml b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceCollectTaskMapper.xml index f224846..583b74d 100644 --- a/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceCollectTaskMapper.xml +++ b/haobang-security-dm/syslog-serve/src/main/resources/mapper/DeviceCollectTaskMapper.xml @@ -22,12 +22,17 @@ + + + + id, created_at, updated_at, deleted_at, device_id, method, task_name, first_time, last_success_time, last_failed_time, detail_id, epm, epm_peak, - process_architecture, task_count, recent_discover_time, epm_upper_limit + process_architecture, task_count, recent_discover_time, epm_upper_limit, + expire_time, remark, device_ip, organization_id @@ -60,6 +65,16 @@ AND last_success_time >= #{lastSuccessTime} + + + AND device_ip = #{deviceIp} + + + AND organization_id = #{organizationId} + + + AND remark LIKE CONCAT('%', #{remark}, '%') + ORDER BY updated_at DESC diff --git a/haobang-security-dm/syslog-serve/src/test/java/com/haobang/syslog/DeviceDeviceTest.java b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/syslog/DeviceDeviceTest.java new file mode 100644 index 0000000..5a38f5c --- /dev/null +++ b/haobang-security-dm/syslog-serve/src/test/java/com/haobang/syslog/DeviceDeviceTest.java @@ -0,0 +1,174 @@ +package com.haobang.syslog; + +import com.common.entity.DeviceDevice; +import com.common.mapper.DeviceDeviceMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; + +import java.util.List; + +/** + * 达梦数据库连接测试 — 查询 device_device 表 + */ +@SpringBootTest +@ActiveProfiles("dev") +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class DeviceDeviceTest { + + @Autowired + private DeviceDeviceMapper deviceDeviceMapper; + + // ==================== 基础查询测试 ==================== + + @Test + @Order(1) + @DisplayName("查询设备总数") + void testCount() { + System.out.println("\n========== 查询设备总数 =========="); + Long count = deviceDeviceMapper.count(); + System.out.println("device_device 表数据总量: " + count); + assert count != null && count >= 0; + } + + @Test + @Order(2) + @DisplayName("查询所有设备") + void testSelectAll() { + System.out.println("\n========== 查询所有设备 =========="); + List list = deviceDeviceMapper.selectAll(); + System.out.println("查询到 " + list.size() + " 条记录："); + for (int i = 0; i < Math.min(list.size(), 10); i++) { + DeviceDevice d = list.get(i); + System.out.printf(" [%d] id=%-6d name=%-20s ip=%-18s status=%-4s vendor=%-15s delFlag=%s%n", + i + 1, d.getId(), d.getName(), d.getIp(), d.getStatus(), d.getVendor(), d.getDelFlag()); + } + if (list.size() > 10) { + System.out.println(" ... 仅展示前10条，共 " + list.size() + " 条"); + } + assert list != null; + } + + @Test + @Order(3) + @DisplayName("查询未删除的有效设备 (del_flag='0')") + void testSelectActiveDevices() { + System.out.println("\n========== 查询有效设备 (del_flag='0') =========="); + List list = deviceDeviceMapper.selectActiveDevices(); + System.out.println("查询到 " + list.size() + " 条有效记录："); + for (int i = 0; i < Math.min(list.size(), 10); i++) { + DeviceDevice d = list.get(i); + System.out.printf(" [%d] id=%-6d name=%-20s ip=%-18s status=%-4s vendor=%-15s%n", + i + 1, d.getId(), d.getName(), d.getIp(), d.getStatus(), d.getVendor()); + } + if (list.size() > 10) { + System.out.println(" ... 仅展示前10条，共 " + list.size() + " 条"); + } + assert list != null; + } + + @Test + @Order(4) + @DisplayName("分页查询 — 前5条") + void testSelectByPage() { + System.out.println("\n========== 分页查询 (第1页, 每页5条) =========="); + List list = deviceDeviceMapper.selectByPage(0, 5); + System.out.println("查询到 " + list.size() + " 条记录："); + for (DeviceDevice d : list) { + System.out.println(" " + d); + } + assert list != null && list.size() <= 5; + } + + // ==================== 精确查询测试 ==================== + + @Test + @Order(5) + @DisplayName("根据ID查询设备") + void testSelectById() { + System.out.println("\n========== 根据ID查询 =========="); + // 先查所有取第一条的ID + List all = deviceDeviceMapper.selectAll(); + if (all != null && !all.isEmpty()) { + Integer firstId = all.get(0).getId(); + DeviceDevice d = deviceDeviceMapper.selectById(firstId); + System.out.println("ID=" + firstId + " 查询结果："); + System.out.println(" " + d); + assert d != null && d.getId().equals(firstId); + } else { + System.out.println("device_device 表无数据，跳过按ID查询"); + } + } + + // ==================== 分页查询测试 ==================== + + @Test + @Order(6) + @DisplayName("分页查询 — 第2页") + void testSelectByPage2() { + System.out.println("\n========== 分页查询 (第2页, 每页5条) =========="); + List list = deviceDeviceMapper.selectByPage(5, 5); + System.out.println("查询到 " + list.size() + " 条记录："); + for (DeviceDevice d : list) { + System.out.printf(" id=%-6d name=%-20s ip=%-18s%n", d.getId(), d.getName(), d.getIp()); + } + assert list != null; + } + + // ==================== 完整字段展示 ==================== + + @Test + @Order(7) + @DisplayName("完整字段展示 — 查询1条设备的所有字段") + void testShowFullFields() { + System.out.println("\n========== 完整字段展示 =========="); + List all = deviceDeviceMapper.selectAll(); + if (all != null && !all.isEmpty()) { + DeviceDevice d = all.get(0); + System.out.println("┌─────────────────────────────────────────────┐"); + System.out.printf("│ %-12s: %-30s │%n", "id", d.getId()); + System.out.printf("│ %-12s: %-30s │%n", "name", d.getName()); + System.out.printf("│ %-12s: %-30s │%n", "ip", d.getIp()); + System.out.printf("│ %-12s: %-30s │%n", "vendor", d.getVendor()); + System.out.printf("│ %-12s: %-30s │%n", "productName", d.getProductName()); + System.out.printf("│ %-12s: %-30s │%n", "deviceGroup", d.getDeviceGroup()); + System.out.printf("│ %-12s: %-30s │%n", "deviceType", d.getDeviceType()); + System.out.printf("│ %-12s: %-30s │%n", "status", d.getStatus()); + System.out.printf("│ %-12s: %-30s │%n", "isMonitoring", d.getIsMonitoring()); + System.out.printf("│ %-12s: %-30s │%n", "organizationId", d.getOrganizationId()); + System.out.printf("│ %-12s: %-30s │%n", "agentId", d.getAgentId()); + System.out.printf("│ %-12s: %-30s │%n", "tenantId", d.getTenantId()); + System.out.printf("│ %-12s: %-30s │%n", "delFlag", d.getDelFlag()); + System.out.printf("│ %-12s: %-30s │%n", "managerName", d.getManagerName()); + System.out.printf("│ %-12s: %-30s │%n", "createdAt", d.getCreatedAt()); + System.out.printf("│ %-12s: %-30s │%n", "updatedAt", d.getUpdatedAt()); + System.out.printf("│ %-12s: %-30s │%n", "lastReceiveTime", d.getLastReceiveTime()); + System.out.printf("│ %-12s: %-30s │%n", "createTime", d.getCreateTime()); + System.out.printf("│ %-12s: %-30s │%n", "updateTime", d.getUpdateTime()); + System.out.printf("│ %-12s: %-30s │%n", "licenseStartTime", d.getLicenseStartTime()); + System.out.printf("│ %-12s: %-30s │%n", "licenseEndTime", d.getLicenseEndTime()); + System.out.printf("│ %-12s: %-30s │%n", "decodeType", d.getDecodeType()); + System.out.printf("│ %-12s: %-30s │%n", "missPolicy", d.getMissPolicy()); + System.out.printf("│ %-12s: %-30s │%n", "securityScopeId", d.getSecurityScopeId()); + System.out.printf("│ %-12s: %-30s │%n", "ownerId", d.getOwnerId()); + System.out.printf("│ %-12s: %-30s │%n", "sshConfigId", d.getSshConfigId()); + System.out.printf("│ %-12s: %-30s │%n", "createdById", d.getCreatedById()); + System.out.printf("│ %-12s: %-30s │%n", "createBy", d.getCreateBy()); + System.out.printf("│ %-12s: %-30s │%n", "updateBy", d.getUpdateBy()); + System.out.printf("│ %-12s: %-30s │%n", "createDept", d.getCreateDept()); + System.out.printf("│ %-12s: %-30s │%n", "deviceCollectId", d.getDeviceCollectId()); + System.out.printf("│ %-12s: %-30s │%n", "todayParseCount", d.getTodayParseCount()); + System.out.printf("│ %-12s: %-30s │%n", "todayNonLogCount", d.getTodayNonLogCount()); + System.out.printf("│ %-12s: %-30s │%n", "detailId", d.getDetailId()); + System.out.printf("│ %-12s: %-30s │%n", "controlAgentId", d.getControlAgentId()); + System.out.println("└─────────────────────────────────────────────┘"); + } else { + System.out.println("device_device 表无数据"); + } + } +}