1、适配部署环境变量自动注入
2、适配南网达梦数据库安全版SQL语句Group by 不支持自定义函数 3、达梦数据库驱动改为达梦提供的指定版本 4、修改syslog-consumer模块yaml 文件kafka 变量名称 5、修改访问日志告警表新增4个字段，完善异常行为详情页面显示字段
This commit is contained in:
+64
-64
@@ -21,83 +21,83 @@ services:
|
||||
- SPRING_PROFILES_ACTIVE=dev
|
||||
- TZ=Asia/Shanghai
|
||||
# 数据库配置
|
||||
- spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys
|
||||
- spring.datasource.username=postgres
|
||||
- spring.datasource.password=TnLanWaidYSwTSG5
|
||||
- spring.datasource.driver-class-name=org.postgresql.Driver
|
||||
- SPRING_DATASOURCE_URL=jdbc:postgresql://117.72.68.72:54329/ecosys
|
||||
- SPRING_DATASOURCE_USERNAME=postgres
|
||||
- SPRING_DATASOURCE_PASSWORD=TnLanWaidYSwTSG5
|
||||
- SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
|
||||
# HikariCP 连接池配置
|
||||
- spring.datasource.hikari.maximum-pool-size=50
|
||||
- spring.datasource.hikari.minimum-idle=5
|
||||
- spring.datasource.hikari.connection-timeout=30000
|
||||
- spring.datasource.hikari.idle-timeout=600000
|
||||
- spring.datasource.hikari.max-lifetime=900000
|
||||
- spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer
|
||||
- spring.datasource.hikari.auto-commit=false
|
||||
- spring.datasource.hikari.schema=public
|
||||
- SPRING_DATASOURCE_HIKARI_MAXIMUM_POOL_SIZE=50
|
||||
- SPRING_DATASOURCE_HIKARI_MINIMUM_IDLE=5
|
||||
- SPRING_DATASOURCE_HIKARI_CONNECTION_TIMEOUT=30000
|
||||
- SPRING_DATASOURCE_HIKARI_IDLE_TIMEOUT=600000
|
||||
- SPRING_DATASOURCE_HIKARI_MAX_LIFETIME=900000
|
||||
- SPRING_DATASOURCE_HIKARI_POOL_NAME=HikariPool-SyslogConsumer
|
||||
- SPRING_DATASOURCE_HIKARI_AUTO_COMMIT=false
|
||||
- SPRING_DATASOURCE_HIKARI_SCHEMA=public
|
||||
# Redis配置
|
||||
- spring.redis.host=192.168.222.131
|
||||
- spring.redis.port=6379
|
||||
- spring.redis.password=
|
||||
- spring.redis.database=0
|
||||
- spring.redis.timeout=2000
|
||||
- spring.cache.redis.time-to-live=600000
|
||||
- SPRING_REDIS_HOST=192.168.222.131
|
||||
- SPRING_REDIS_PORT=6379
|
||||
- SPRING_REDIS_PASSWORD=
|
||||
- SPRING_REDIS_DATABASE=0
|
||||
- SPRING_REDIS_TIMEOUT=2000
|
||||
- SPRING_CACHE_REDIS_TIME_TO_LIVE=600000
|
||||
# Kafka配置
|
||||
- spring.kafka.consumer.bootstrap-servers=192.168.222.130:9092
|
||||
- spring.kafka.consumer.group-id=test-group-app
|
||||
- spring.kafka.consumer.auto-offset-reset=latest
|
||||
- spring.kafka.consumer.enable-auto-commit=false
|
||||
- spring.kafka.consumer.topic=test-topic
|
||||
- spring.kafka.consumer.max-poll-records=1000
|
||||
- spring.kafka.listener.ack-mode=manual
|
||||
- spring.kafka.listener.concurrency=2
|
||||
- spring.kafka.listener.type=batch
|
||||
- SPRING_KAFKA_CONSUMER_BOOTSTRAP_SERVERS=192.168.222.130:9092
|
||||
- SPRING_KAFKA_CONSUMER_GROUP_ID=test-group-app
|
||||
- SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest
|
||||
- SPRING_KAFKA_CONSUMER_ENABLE_AUTO_COMMIT=false
|
||||
- SPRING_KAFKA_CONSUMER_TOPIC=test-topic
|
||||
- SPRING_KAFKA_CONSUMER_MAX_POLL_RECORDS=1000
|
||||
- SPRING_KAFKA_LISTENER_ACK_MODE=manual
|
||||
- SPRING_KAFKA_LISTENER_CONCURRENCY=2
|
||||
- SPRING_KAFKA_LISTENER_TYPE=batch
|
||||
# InfluxDB配置
|
||||
- influxdb.url=http://192.168.222.131:8086
|
||||
- influxdb.token=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA==
|
||||
- influxdb.org=yelang
|
||||
- influxdb.bucket=yelangbucket
|
||||
- influxdb.batch.size=1000
|
||||
- influxdb.flush.interval=1000
|
||||
- influxdb.connection.timeout=30s
|
||||
- influxdb.connection.read-timeout=30s
|
||||
- influxdb.connection.write-timeout=60s
|
||||
- INFLUXDB_URL=http://192.168.222.131:8086
|
||||
- INFLUXDB_TOKEN=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA==
|
||||
- INFLUXDB_ORG=yelang
|
||||
- INFLUXDB_BUCKET=yelangbucket
|
||||
- INFLUXDB_BATCH_SIZE=1000
|
||||
- INFLUXDB_FLUSH_INTERVAL=1000
|
||||
- INFLUXDB_CONNECTION_TIMEOUT=30s
|
||||
- INFLUXDB_CONNECTION_READ_TIMEOUT=30s
|
||||
- INFLUXDB_CONNECTION_WRITE_TIMEOUT=60s
|
||||
# Elasticsearch配置
|
||||
- spring.elasticsearch.uris=http://192.168.1.174:9200
|
||||
- spring.elasticsearch.username=CONTAINER_NAME
|
||||
- spring.elasticsearch.password=t2NZCiajmdazxBrF
|
||||
- spring.elasticsearch.connection-timeout=10s
|
||||
- spring.elasticsearch.socket-timeout=30s
|
||||
- SPRING_ELASTICSEARCH_URIS=http://192.168.1.174:9200
|
||||
- SPRING_ELASTICSEARCH_USERNAME=CONTAINER_NAME
|
||||
- SPRING_ELASTICSEARCH_PASSWORD=t2NZCiajmdazxBrF
|
||||
- SPRING_ELASTICSEARCH_CONNECTION_TIMEOUT=10s
|
||||
- SPRING_ELASTICSEARCH_SOCKET_TIMEOUT=30s
|
||||
# API配置
|
||||
- interlocking.api-key=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
|
||||
- interlocking.api.base-url=http://192.168.222.131:8089/xdrservice/interlocking
|
||||
- INTERLOCKING_API_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
|
||||
- INTERLOCKING_API_BASE_URL=http://192.168.222.131:8089/xdrservice/interlocking
|
||||
# 探针心跳配置
|
||||
- probe.heartbeat.enabled=true
|
||||
- probe.heartbeat.offline-threshold-minutes=10
|
||||
- probe.status.check.cron=0 */10 * * * ?
|
||||
- probe.heartbeat.tenant-id=000000
|
||||
- probe.heartbeat.history.keep-days=10
|
||||
- probe.heartbeat.history.cleanup-enabled=true
|
||||
- probe.history.cleanup.cron=0 0 1 * * ?
|
||||
- PROBE_HEARTBEAT_ENABLED=true
|
||||
- PROBE_HEARTBEAT_OFFLINE_THRESHOLD_MINUTES=10
|
||||
- PROBE_STATUS_CHECK_CRON=0 */10 * * * ?
|
||||
- PROBE_HEARTBEAT_TENANT_ID=000000
|
||||
- PROBE_HEARTBEAT_HISTORY_KEEP_DAYS=10
|
||||
- PROBE_HEARTBEAT_HISTORY_CLEANUP_ENABLED=true
|
||||
- PROBE_HISTORY_CLEANUP_CRON=0 0 1 * * ?
|
||||
# 告警健康检查配置
|
||||
- alarm.health-check.alarm-hours=4
|
||||
- alarm.health-check.alarm-visit-hours=2
|
||||
- alarm.health-check.enabled=true
|
||||
- ALARM_HEALTH_CHECK_ALARM_HOURS=4
|
||||
- ALARM_HEALTH_CHECK_ALARM_VISIT_HOURS=2
|
||||
- ALARM_HEALTH_CHECK_ENABLED=true
|
||||
# 关联分析规则配置
|
||||
- analysis.realtime.enabled=true
|
||||
- analysis.realtime.check-interval-seconds=10
|
||||
- ANALYSIS_REALTIME_ENABLED=true
|
||||
- ANALYSIS_REALTIME_CHECK_INTERVAL_SECONDS=10
|
||||
# 分区表检查配置
|
||||
- partition.check.tomorrow.enabled=true
|
||||
- partition.check.future.days=7
|
||||
- partition.auto.create=true
|
||||
- PARTITION_CHECK_TOMORROW_ENABLED=true
|
||||
- PARTITION_CHECK_FUTURE_DAYS=7
|
||||
- PARTITION_AUTO_CREATE=true
|
||||
# 定时任务配置
|
||||
- spring.task.scheduling.pool.size=10
|
||||
- SPRING_TASK_SCHEDULING_POOL_SIZE=10
|
||||
# 日志配置
|
||||
- logging.level.com.common.schedule=INFO
|
||||
- logging.level.com.common.service=INFO
|
||||
- LOGGING_LEVEL_COM_COMMON_SCHEDULE=INFO
|
||||
- LOGGING_LEVEL_COM_COMMON_SERVICE=INFO
|
||||
# ETL配置
|
||||
- etl.batch.page-size=1000
|
||||
- etl.batch.insert-batch-size=500
|
||||
- etl.schedule.cron=0 0 2 * * ?
|
||||
- ETL_BATCH_PAGE_SIZE=1000
|
||||
- ETL_BATCH_INSERT_BATCH_SIZE=500
|
||||
- ETL_SCHEDULE_CRON=0 0 2 * * ?
|
||||
# JVM配置
|
||||
- JAVA_OPTS=-Xms512m -Xmx1024m -XX:+UseG1GC -XX:MaxGCPauseMillis=200
|
||||
ports:
|
||||
|
||||
+39
-39
@@ -21,51 +21,51 @@ services:
|
||||
- SPRING_PROFILES_ACTIVE=prod
|
||||
- TZ=Asia/Shanghai
|
||||
# 数据库配置
|
||||
- spring.datasource.url=jdbc:postgresql://192.168.4.26:5432/ecosys
|
||||
- spring.datasource.username=postgres
|
||||
- spring.datasource.password=caZ2TcmXNSW8L2Ap
|
||||
- spring.datasource.driver-class-name=org.postgresql.Driver
|
||||
- SPRING_DATASOURCE_URL=jdbc:postgresql://192.168.4.26:5432/ecosys
|
||||
- SPRING_DATASOURCE_USERNAME=postgres
|
||||
- SPRING_DATASOURCE_PASSWORD=caZ2TcmXNSW8L2Ap
|
||||
- SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
|
||||
# HikariCP 连接池配置
|
||||
- spring.datasource.hikari.maximum-pool-size=50
|
||||
- spring.datasource.hikari.minimum-idle=5
|
||||
- spring.datasource.hikari.connection-timeout=30000
|
||||
- spring.datasource.hikari.idle-timeout=600000
|
||||
- spring.datasource.hikari.max-lifetime=900000
|
||||
- spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer
|
||||
- spring.datasource.hikari.auto-commit=false
|
||||
- spring.datasource.hikari.schema=public
|
||||
- SPRING_DATASOURCE_HIKARI_MAXIMUM_POOL_SIZE=50
|
||||
- SPRING_DATASOURCE_HIKARI_MINIMUM_IDLE=5
|
||||
- SPRING_DATASOURCE_HIKARI_CONNECTION_TIMEOUT=30000
|
||||
- SPRING_DATASOURCE_HIKARI_IDLE_TIMEOUT=600000
|
||||
- SPRING_DATASOURCE_HIKARI_MAX_LIFETIME=900000
|
||||
- SPRING_DATASOURCE_HIKARI_POOL_NAME=HikariPool-SyslogConsumer
|
||||
- SPRING_DATASOURCE_HIKARI_AUTO_COMMIT=false
|
||||
- SPRING_DATASOURCE_HIKARI_SCHEMA=public
|
||||
# Redis配置
|
||||
- spring.redis.host=192.168.4.26
|
||||
- spring.redis.port=6379
|
||||
- spring.redis.password=123456
|
||||
- spring.redis.database=0
|
||||
- spring.redis.timeout=2000
|
||||
- spring.cache.redis.time-to-live=600000
|
||||
- SPRING_REDIS_HOST=192.168.4.26
|
||||
- SPRING_REDIS_PORT=6379
|
||||
- SPRING_REDIS_PASSWORD=123456
|
||||
- SPRING_REDIS_DATABASE=0
|
||||
- SPRING_REDIS_TIMEOUT=2000
|
||||
- SPRING_CACHE_REDIS_TIME_TO_LIVE=600000
|
||||
# Kafka配置
|
||||
- spring.kafka.consumer.bootstrap-servers=192.168.4.26:9092
|
||||
- spring.kafka.consumer.group-id=agent-01-syslog-group
|
||||
- spring.kafka.consumer.auto-offset-reset=latest
|
||||
- spring.kafka.consumer.enable-auto-commit=false
|
||||
- spring.kafka.consumer.topic=agenet-01-syslog-topic
|
||||
- spring.kafka.consumer.max-poll-records=1000
|
||||
- spring.kafka.listener.ack-mode=manual
|
||||
- spring.kafka.listener.concurrency=2
|
||||
- spring.kafka.listener.type=batch
|
||||
- SPRING_KAFKA_CONSUMER_BOOTSTRAP_SERVERS=192.168.4.26:9092
|
||||
- SPRING_KAFKA_CONSUMER_GROUP_ID=agent-syslog-group
|
||||
- SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest
|
||||
- SPRING_KAFKA_CONSUMER_ENABLE_AUTO_COMMIT=false
|
||||
- SPRING_KAFKA_CONSUMER_TOPIC=agenet-syslog-topic
|
||||
- SPRING_KAFKA_CONSUMER_MAX_POLL_RECORDS=1000
|
||||
- SPRING_KAFKA_LISTENER_ACK_MODE=manual
|
||||
- SPRING_KAFKA_LISTENER_CONCURRENCY=2
|
||||
- SPRING_KAFKA_LISTENER_TYPE=batch
|
||||
# InfluxDB配置
|
||||
- influxdb.url=http://192.168.4.26:8087
|
||||
- influxdb.token=LFjXZyRxTf1V84oN-wwjhSjS4qIK-ZMoHzQJB67ir3qHNSBVJbMcTkPuNmM0cNxvzFEDWLYNzrz1VJKMitY5hw==
|
||||
- influxdb.org=influxdb
|
||||
- influxdb.bucket=yelangbucket
|
||||
- influxdb.batch.size=1000
|
||||
- influxdb.flush.interval=1000
|
||||
- influxdb.connection.timeout=30s
|
||||
- influxdb.connection.read-timeout=30s
|
||||
- influxdb.connection.write-timeout=60s
|
||||
- INFLUXDB_URL=http://192.168.4.26:8087
|
||||
- INFLUXDB_TOKEN=LFjXZyRxTf1V84oN-wwjhSjS4qIK-ZMoHzQJB67ir3qHNSBVJbMcTkPuNmM0cNxvzFEDWLYNzrz1VJKMitY5hw==
|
||||
- INFLUXDB_ORG=influxdb
|
||||
- INFLUXDB_BUCKET=yelangbucket
|
||||
- INFLUXDB_BATCH_SIZE=1000
|
||||
- INFLUXDB_FLUSH_INTERVAL=1000
|
||||
- INFLUXDB_CONNECTION_TIMEOUT=30s
|
||||
- INFLUXDB_CONNECTION_READ_TIMEOUT=30s
|
||||
- INFLUXDB_CONNECTION_WRITE_TIMEOUT=60s
|
||||
# 定时任务配置
|
||||
- spring.task.scheduling.pool.size=10
|
||||
- SPRING_TASK_SCHEDULING_POOL_SIZE=10
|
||||
# 日志配置
|
||||
- logging.level.com.common.schedule=INFO
|
||||
- logging.level.com.common.service=INFO
|
||||
- LOGGING_LEVEL_COM_COMMON_SCHEDULE=INFO
|
||||
- LOGGING_LEVEL_COM_COMMON_SERVICE=INFO
|
||||
|
||||
# JVM配置
|
||||
- JAVA_OPTS=-Xms1024m -Xmx4096m -XX:+UseG1GC -XX:MaxGCPauseMillis=200
|
||||
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
# ============================================
|
||||
# Syslog Consumer 部署配置
|
||||
# ============================================
|
||||
# 使用方法: docker compose -f docker-compose-consumer-rule-dm.yaml up -d
|
||||
# ============================================
|
||||
|
||||
|
||||
services:
|
||||
# ============================================
|
||||
# Syslog Consumer-rule - 数据消费规则服务(平台端)
|
||||
# ============================================
|
||||
syslog-consumer-rule:
|
||||
build:
|
||||
context: ./consumer-rule-dm
|
||||
dockerfile: Dockerfile
|
||||
image: syslog-consumer-rule-dm:v1.2.2
|
||||
container_name: syslog-consumer-rule-dm
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# 环境配置
|
||||
- SPRING_PROFILES_ACTIVE=prod
|
||||
- TZ=Asia/Shanghai
|
||||
# 数据库配置
|
||||
- SPRING_DATASOURCE_URL=jdbc:dm://192.168.4.99:5237
|
||||
- SPRING_DATASOURCE_USERNAME=SYSDBA
|
||||
- SPRING_DATASOURCE_PASSWORD=caZ2TcmXNSW8L2Ap
|
||||
- SPRING_DATASOURCE_DRIVER-CLASS-NAME=dm.jdbc.driver.DmDriver
|
||||
- SPRING_DATASOURCE_HIKARI_SCHEMA=\"PUBLIC\"
|
||||
# HikariCP 连接池配置
|
||||
- SPRING_DATASOURCE_HIKARI_MAXIMUM-POOL-SIZE=50
|
||||
- SPRING_DATASOURCE_HIKARI_MINIMUM-IDLE=5
|
||||
- SPRING_DATASOURCE_HIKARI_CONNECTION-TIMEOUT=30000
|
||||
- SPRING_DATASOURCE_HIKARI_IDLE-TIMEOUT=600000
|
||||
- SPRING_DATASOURCE_HIKARI_MAX-LIFETIME=900000
|
||||
- SPRING_DATASOURCE_HIKARI_POOL-NAME=HikariPool-SyslogConsumer-rule
|
||||
- SPRING_DATASOURCE_HIKARI_AUTO-COMMIT=false
|
||||
|
||||
# Redis配置
|
||||
- SPRING_REDIS_HOST=192.168.4.99
|
||||
- SPRING_REDIS_PORT=6379
|
||||
- SPRING_REDIS_PASSWORD=redis_GdGWte
|
||||
- SPRING_REDIS_DATABASE=0
|
||||
- SPRING_REDIS_TIMEOUT=2000
|
||||
- SPRING_CACHE_REDIS_TIME-TO-LIVE=600000
|
||||
# Kafka配置
|
||||
- SPRING_KAFKA_CONSUMER_BOOTSTRAP-SERVERS=192.168.4.99:9092
|
||||
- SPRING_KAFKA_CONSUMER_GROUP-ID=agent-01-syslog-group-dm
|
||||
- SPRING_KAFKA_CONSUMER_AUTO-OFFSET-RESET=latest
|
||||
- SPRING_KAFKA_CONSUMER_ENABLE-AUTO-COMMIT=false
|
||||
- SPRING_KAFKA_CONSUMER_TOPIC=agent-01-syslog-topic
|
||||
- SPRING_KAFKA_CONSUMER_MAX-POLL-RECORDS=1000
|
||||
- SPRING_KAFKA_LISTENER_ACK-MODE=manual
|
||||
- SPRING_KAFKA_LISTENER_CONCURRENCY=2
|
||||
- SPRING_KAFKA_LISTENER_TYPE=batch
|
||||
# InfluxDB配置
|
||||
- INFLUXDB_URL=http://192.168.4.99:8087
|
||||
- INFLUXDB_TOKEN=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg==
|
||||
- INFLUXDB_ORG=influxdb
|
||||
- INFLUXDB_BUCKET=yelangbucket
|
||||
- INFLUXDB_BATCH_SIZE=1000
|
||||
- INFLUXDB_FLUSH_INTERVAL=1000
|
||||
- INFLUXDB_CONNECTION_TIMEOUT=30s
|
||||
- INFLUXDB_CONNECTION_READ-TIMEOUT=30s
|
||||
- INFLUXDB_CONNECTION_WRITE-TIMEOUT=60s
|
||||
# 定时任务配置
|
||||
- SPRING_TASK_SCHEDULING_POOL_SIZE=10
|
||||
# 日志配置
|
||||
- LOGGING_LEVEL_COM_COMMON_SCHEDULE=INFO
|
||||
- LOGGING_LEVEL_COM_COMMON_SERVICE=INFO
|
||||
|
||||
volumes:
|
||||
- /home/syslog/logs:/app/logs
|
||||
networks:
|
||||
- xdr-network
|
||||
privileged: true
|
||||
# ============================================
|
||||
# 网络配置
|
||||
# ============================================
|
||||
networks:
|
||||
xdr-network:
|
||||
driver: bridge
|
||||
|
||||
+39
-39
@@ -21,52 +21,52 @@ services:
|
||||
- SPRING_PROFILES_ACTIVE=dev
|
||||
- TZ=Asia/Shanghai
|
||||
# 数据库配置
|
||||
- spring.datasource.url=jdbc:postgresql://117.72.68.72:54329/ecosys
|
||||
- spring.datasource.username=postgres
|
||||
- spring.datasource.password=TnLanWaidYSwTSG5
|
||||
- spring.datasource.driver-class-name=org.postgresql.Driver
|
||||
- SPRING_DATASOURCE_URL=jdbc:postgresql://117.72.68.72:54329/ecosys
|
||||
- SPRING_DATASOURCE_USERNAME=postgres
|
||||
- SPRING_DATASOURCE_PASSWORD=TnLanWaidYSwTSG5
|
||||
- SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
|
||||
# HikariCP 连接池配置
|
||||
- spring.datasource.hikari.maximum-pool-size=50
|
||||
- spring.datasource.hikari.minimum-idle=5
|
||||
- spring.datasource.hikari.connection-timeout=30000
|
||||
- spring.datasource.hikari.idle-timeout=600000
|
||||
- spring.datasource.hikari.max-lifetime=900000
|
||||
- spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer
|
||||
- spring.datasource.hikari.auto-commit=false
|
||||
- spring.datasource.hikari.schema=public
|
||||
- SPRING_DATASOURCE_HIKARI_MAXIMUM_POOL_SIZE=50
|
||||
- SPRING_DATASOURCE_HIKARI_MINIMUM_IDLE=5
|
||||
- SPRING_DATASOURCE_HIKARI_CONNECTION_TIMEOUT=30000
|
||||
- SPRING_DATASOURCE_HIKARI_IDLE_TIMEOUT=600000
|
||||
- SPRING_DATASOURCE_HIKARI_MAX_LIFETIME=900000
|
||||
- SPRING_DATASOURCE_HIKARI_POOL_NAME=HikariPool-SyslogConsumer
|
||||
- SPRING_DATASOURCE_HIKARI_AUTO_COMMIT=false
|
||||
- SPRING_DATASOURCE_HIKARI_SCHEMA=public
|
||||
# Redis配置
|
||||
- spring.redis.host=192.168.222.131
|
||||
- spring.redis.port=6379
|
||||
- spring.redis.password=
|
||||
- spring.redis.database=0
|
||||
- spring.redis.timeout=2000
|
||||
- spring.cache.redis.time-to-live=600000
|
||||
- SPRING_REDIS_HOST=192.168.222.131
|
||||
- SPRING_REDIS_PORT=6379
|
||||
- SPRING_REDIS_PASSWORD=
|
||||
- SPRING_REDIS_DATABASE=0
|
||||
- SPRING_REDIS_TIMEOUT=2000
|
||||
- SPRING_CACHE_REDIS_TIME_TO_LIVE=600000
|
||||
# Kafka配置
|
||||
- spring.kafka.consumer.bootstrap-servers=192.168.222.130:9092
|
||||
- spring.kafka.consumer.group-id=test-group-app
|
||||
- spring.kafka.consumer.auto-offset-reset=latest
|
||||
- spring.kafka.consumer.enable-auto-commit=false
|
||||
- spring.kafka.consumer.topic=test-topic
|
||||
- spring.kafka.consumer.max-poll-records=1000
|
||||
- spring.kafka.listener.ack-mode=manual
|
||||
- spring.kafka.listener.concurrency=2
|
||||
- spring.kafka.listener.type=batch
|
||||
- SPRING_KAFKA_CONSUMER_BOOTSTRAP_SERVERS=192.168.222.130:9092
|
||||
- SPRING_KAFKA_CONSUMER_GROUP_ID=test-group-app
|
||||
- SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest
|
||||
- SPRING_KAFKA_CONSUMER_ENABLE_AUTO_COMMIT=false
|
||||
- SPRING_KAFKA_CONSUMER_TOPIC=test-topic
|
||||
- SPRING_KAFKA_CONSUMER_MAX_POLL_RECORDS=1000
|
||||
- SPRING_KAFKA_LISTENER_ACK_MODE=manual
|
||||
- SPRING_KAFKA_LISTENER_CONCURRENCY=2
|
||||
- SPRING_KAFKA_LISTENER_TYPE=batch
|
||||
# InfluxDB配置
|
||||
- influxdb.url=http://192.168.222.131:8086
|
||||
- influxdb.token=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA==
|
||||
- influxdb.org=yelang
|
||||
- influxdb.bucket=yelangbucket
|
||||
- influxdb.batch.size=1000
|
||||
- influxdb.flush.interval=1000
|
||||
- influxdb.connection.timeout=30s
|
||||
- influxdb.connection.read-timeout=30s
|
||||
- influxdb.connection.write-timeout=60s
|
||||
- INFLUXDB_URL=http://192.168.222.131:8086
|
||||
- INFLUXDB_TOKEN=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA==
|
||||
- INFLUXDB_ORG=yelang
|
||||
- INFLUXDB_BUCKET=yelangbucket
|
||||
- INFLUXDB_BATCH_SIZE=1000
|
||||
- INFLUXDB_FLUSH_INTERVAL=1000
|
||||
- INFLUXDB_CONNECTION_TIMEOUT=30s
|
||||
- INFLUXDB_CONNECTION_READ_TIMEOUT=30s
|
||||
- INFLUXDB_CONNECTION_WRITE_TIMEOUT=60s
|
||||
|
||||
# 定时任务配置
|
||||
- spring.task.scheduling.pool.size=10
|
||||
- SPRING_TASK_SCHEDULING_POOL_SIZE=10
|
||||
# 日志配置
|
||||
- logging.level.com.common.schedule=INFO
|
||||
- logging.level.com.common.service=INFO
|
||||
- LOGGING_LEVEL_COM_COMMON_SCHEDULE=INFO
|
||||
- LOGGING_LEVEL_COM_COMMON_SERVICE=INFO
|
||||
# JVM配置
|
||||
- JAVA_OPTS=-Xms1024m -Xmx4096m -XX:+UseG1GC -XX:MaxGCPauseMillis=200
|
||||
ports:
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
<version>3.4.0</version>
|
||||
<version>3.9.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.kafka</groupId>
|
||||
@@ -104,8 +104,8 @@
|
||||
<!-- 达梦数据库驱动 JDK1.8 -->
|
||||
<dependency>
|
||||
<groupId>com.dameng</groupId>
|
||||
<artifactId>DmJdbcDriver18</artifactId>
|
||||
<version>8.1.2.141</version>
|
||||
<artifactId>DmJdbcDriver</artifactId>
|
||||
<version>8.1.4.169</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
+313
-447
@@ -1,461 +1,354 @@
|
||||
package com.Modules.NormalData;
|
||||
|
||||
import cn.hutool.core.date.DateTime;
|
||||
import com.common.entity.XdrHoneypot;
|
||||
import com.common.mapper.XdrHoneypotMapper;
|
||||
import com.common.service.SyslogNonNormalMessageService;
|
||||
import com.common.util.*;
|
||||
import com.config.AppConfig;
|
||||
import org.apache.ibatis.session.SqlSession;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import com.common.entity.SyslogMessage;
|
||||
import com.influx.SyslogToInfluxApp;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import com.common.service.DmNormalizeRuleService;
|
||||
import com.common.service.DmColumnService;
|
||||
import com.common.service.SyslogNormalDataService;
|
||||
import com.common.entity.DmColumn;
|
||||
import com.common.service.impl.DmColumnServiceImpl;
|
||||
import com.common.mapper.DmColumnMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import com.common.mapper.DmColumnMapper;
|
||||
import com.common.mapper.DmNormalizeRuleMapper;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.*;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import com.common.entity.RuleContent.*;
|
||||
import com.common.entity.SyslogNonNormalMessage;
|
||||
import org.joda.time.LocalDateTime;
|
||||
import com.common.service.LogDataFilterService;
|
||||
import com.common.service.LogDataCompleteService;
|
||||
import com.common.service.DeviceCollectTaskService;
|
||||
import com.common.entity.DeviceCollectTask;
|
||||
import com.common.entity.DeviceDevice;
|
||||
import com.common.util.TimeConversionUtils;
|
||||
import com.common.service.DeviceDeviceService;
|
||||
import com.common.entity.RuleContent.*;
|
||||
import com.common.entity.SyslogNonNormalMessage;
|
||||
import com.common.mapper.DmColumnMapper;
|
||||
import com.common.mapper.DmNormalizeRuleMapper;
|
||||
import com.common.service.*;
|
||||
import com.common.util.*;
|
||||
import com.config.AppProperties;
|
||||
import com.config.KafkaConsumerProperties;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import com.common.entity.SyslogMessage;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* 日志标准化处理器（规则引擎模块 - Spring 管理的单例 Bean）
|
||||
* 线程安全版本：运行时状态封装为 ProcessContext，每次 process() 调用独立（解决多线程 NPE 问题）
|
||||
*/
|
||||
@Service
|
||||
public class LogNormalProcessor {
|
||||
private static final Logger logger = LoggerFactory.getLogger(LogNormalProcessor.class);
|
||||
|
||||
private String strLogMsg ;
|
||||
private String strKafkaMessage;
|
||||
private String strDeviceInfo;
|
||||
private String strDataType ="json" ;
|
||||
private Map<String, Object> messageMap ;
|
||||
private Map<String, String> deviceInfoMap ;
|
||||
private String strSyslogUUID ;
|
||||
private String strSyslogTopic ;
|
||||
private boolean isSaveNonNormal =false ;
|
||||
// ==================== 依赖注入（Spring Bean，单例，线程安全）====================
|
||||
private final DmNormalizeRuleService dmNormalizeRuleService;
|
||||
private final DmColumnService dmColumnService;
|
||||
private final SyslogNormalDataService syslogNormalDataService;
|
||||
private final SyslogNonNormalMessageService messageService;
|
||||
private final LogDataFilterService logDataFilterService;
|
||||
private final LogDataCompleteService logDataCompleteService;
|
||||
private final DeviceCollectTaskService deviceCollectTaskService;
|
||||
private final DeviceDeviceService deviceDeviceService;
|
||||
private final DmColumnMapper dmColumnMapper;
|
||||
private final DmNormalizeRuleMapper dmNormalizeRuleMapper;
|
||||
private final KafkaConsumerProperties kafkaConsumerProperties;
|
||||
private final AppProperties appProperties;
|
||||
|
||||
@Autowired
|
||||
public DmNormalizeRuleService dmNormalizeRuleService ;
|
||||
@Autowired
|
||||
public DmColumnService dmColumnService ;
|
||||
@Autowired
|
||||
public SyslogNormalDataService syslogNormalDataService =SpringContextUtil.getBean(SyslogNormalDataService.class);
|
||||
@Autowired
|
||||
private SyslogNonNormalMessageService messageService =SpringContextUtil.getBean(SyslogNonNormalMessageService.class);
|
||||
@Autowired
|
||||
private LogDataFilterService logDataFilterService= SpringContextUtil.getBean(LogDataFilterService.class);
|
||||
|
||||
@Autowired
|
||||
private LogDataCompleteService logDataCompleteService= SpringContextUtil.getBean(LogDataCompleteService.class);
|
||||
@Autowired
|
||||
private DeviceCollectTaskService deviceCollectTaskService= SpringContextUtil.getBean(DeviceCollectTaskService.class);
|
||||
@Autowired
|
||||
private DeviceDeviceService deviceDeviceService= SpringContextUtil.getBean(DeviceDeviceService.class);
|
||||
|
||||
@Autowired
|
||||
SyslogNonNormalMessage syslogNonNormalMessage=new SyslogNonNormalMessage();
|
||||
|
||||
@Autowired
|
||||
DmColumnMapper dmColumnMapper;
|
||||
@Autowired
|
||||
DmNormalizeRuleMapper dmNormalizeRuleMapper;
|
||||
|
||||
private List<Map<String, Object>> dmNormalizeRuleList;
|
||||
private List<Map<String, Object>> dmColumnList;
|
||||
private LinkedHashMap<String, Object> OrginalColumnMap ;
|
||||
|
||||
public LogNormalProcessor( String LogMsg, String syslogUUID,String syslogTopic) {
|
||||
|
||||
/** 标准化数据处理步骤 */
|
||||
//初始化 （获取syslog_normal_data 表全部字段的属性、设备ID对应的规则设置、转化成JSON）
|
||||
//判断数据解析类型（json、键值、xml、正则表达式、分割符）解析syslogMessage 字段转成 HashMAP
|
||||
//匹配规则内容，获取字段映射关系配置，抽取命中的字段名称、数值
|
||||
//判断字段内容及类型，根据数据类型属性，进行内容转换。
|
||||
//生成insert SQL语句、执行入库操作。
|
||||
strKafkaMessage=LogMsg;
|
||||
strSyslogUUID=syslogUUID;
|
||||
strSyslogTopic=syslogTopic;
|
||||
|
||||
if(!LogMsg.isEmpty()) {
|
||||
strLogMsg = SyslogParser.substringAfterFirstCloseBracket(LogMsg);
|
||||
strDeviceInfo=SyslogParser.substringBeforeFirstChar(LogMsg,']');
|
||||
}
|
||||
else{
|
||||
LogMsg="[receive_time=20251118165909470 device_id=1 device_name=honeypot vendor=changting data_type=json device_collect_id=1]<14>1 2025-09-24T11:52:26Z 5f46d3be75e1 supermario 128 honeypot_event - {\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}";
|
||||
strLogMsg="<14>1 2025-09-24T11:52:26Z 5f46d3be75e1 supermario 128 honeypot_event - {\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}";
|
||||
strDeviceInfo=SyslogParser.substringBeforeFirstChar(LogMsg,']');
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 初始化数据init()
|
||||
* 请求级处理上下文 —— 每个 process() 调用创建新实例，彻底解决多线程竞态条件
|
||||
*/
|
||||
public void init()
|
||||
{
|
||||
Map<String,String> mapdev =SyslogParser.parseKeyValuePairs(strDeviceInfo);
|
||||
deviceInfoMap=mapdev;
|
||||
//获取日志对应的设备ID
|
||||
long deviceID=Long.parseLong(mapdev.get("device_id"));
|
||||
System.out.println("device_id:"+deviceID );
|
||||
strDataType=mapdev.get("data_type");
|
||||
//getDeviceID(strLogMsg);
|
||||
try{
|
||||
// 通过工具类获取Service实例
|
||||
dmColumnService= SpringContextUtil.getBean(DmColumnService.class);
|
||||
dmNormalizeRuleService= SpringContextUtil.getBean(DmNormalizeRuleService.class);
|
||||
if(deviceID>0) {
|
||||
dmNormalizeRuleList = dmNormalizeRuleService.selectByDeviceIdAuto(deviceID);
|
||||
private static class ProcessContext {
|
||||
String strLogMsg;
|
||||
String strKafkaMessage;
|
||||
String strDeviceInfo;
|
||||
String strDataType = "json";
|
||||
Map<String, Object> messageMap;
|
||||
Map<String, String> deviceInfoMap;
|
||||
String strSyslogUUID;
|
||||
String strSyslogTopic;
|
||||
boolean isSaveNonNormal = false;
|
||||
SyslogNonNormalMessage syslogNonNormalMessage = new SyslogNonNormalMessage();
|
||||
List<Map<String, Object>> dmNormalizeRuleList;
|
||||
List<Map<String, Object>> dmColumnList;
|
||||
LinkedHashMap<String, Object> OrginalColumnMap;
|
||||
}
|
||||
|
||||
// ==================== 构造器注入（Spring 自动完成）====================
|
||||
@Autowired
|
||||
public LogNormalProcessor(
|
||||
DmNormalizeRuleService dmNormalizeRuleService,
|
||||
DmColumnService dmColumnService,
|
||||
SyslogNormalDataService syslogNormalDataService,
|
||||
SyslogNonNormalMessageService messageService,
|
||||
LogDataFilterService logDataFilterService,
|
||||
LogDataCompleteService logDataCompleteService,
|
||||
DeviceCollectTaskService deviceCollectTaskService,
|
||||
DeviceDeviceService deviceDeviceService,
|
||||
DmColumnMapper dmColumnMapper,
|
||||
DmNormalizeRuleMapper dmNormalizeRuleMapper,
|
||||
KafkaConsumerProperties kafkaConsumerProperties,
|
||||
AppProperties appProperties
|
||||
) {
|
||||
this.dmNormalizeRuleService = dmNormalizeRuleService;
|
||||
this.dmColumnService = dmColumnService;
|
||||
this.syslogNormalDataService = syslogNormalDataService;
|
||||
this.messageService = messageService;
|
||||
this.logDataFilterService = logDataFilterService;
|
||||
this.logDataCompleteService = logDataCompleteService;
|
||||
this.deviceCollectTaskService = deviceCollectTaskService;
|
||||
this.deviceDeviceService = deviceDeviceService;
|
||||
this.dmColumnMapper = dmColumnMapper;
|
||||
this.dmNormalizeRuleMapper = dmNormalizeRuleMapper;
|
||||
this.kafkaConsumerProperties = kafkaConsumerProperties;
|
||||
this.appProperties = appProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理一条日志消息（线程安全：每次调用创建独立的 ProcessContext）
|
||||
*
|
||||
* @param logMsg 完整日志消息（含头部）
|
||||
* @param syslogUUID 消息UUID
|
||||
* @param syslogTopic 消息Topic（可传 null，自动使用配置值）
|
||||
*/
|
||||
public void process(String logMsg, String syslogUUID, String syslogTopic) {
|
||||
ProcessContext ctx = new ProcessContext();
|
||||
ctx.strSyslogUUID = syslogUUID;
|
||||
ctx.strSyslogTopic = (syslogTopic != null && !syslogTopic.isEmpty())
|
||||
? syslogTopic
|
||||
: kafkaConsumerProperties.getTopic();
|
||||
ctx.isSaveNonNormal = false;
|
||||
ctx.syslogNonNormalMessage = new SyslogNonNormalMessage();
|
||||
|
||||
// 解析消息
|
||||
ctx.strKafkaMessage = logMsg;
|
||||
if (!logMsg.isEmpty()) {
|
||||
ctx.strLogMsg = SyslogParser.substringAfterFirstCloseBracket(logMsg);
|
||||
ctx.strDeviceInfo = SyslogParser.substringBeforeFirstChar(logMsg, ']');
|
||||
} else {
|
||||
logMsg = "[receive_time=20251118165909470 device_id=1 device_name=honeypot vendor=changting data_type=json device_collect_id=1]<14>1 2025-09-24T11:52:26Z 5f46d3be75e1 supermario 128 honeypot_event - \uFEFF{\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\"}";
|
||||
ctx.strLogMsg = "<14>1 2025-09-24T11:52:26Z 5f46d3be75e1 supermario 128 honeypot_event - \uFEFF{\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\"}";
|
||||
ctx.strDeviceInfo = SyslogParser.substringBeforeFirstChar(logMsg, ']');
|
||||
}
|
||||
|
||||
// 执行处理
|
||||
init(ctx);
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化数据处理（所有运行时状态通过 ctx 传递，线程安全）
|
||||
*/
|
||||
private void init(ProcessContext ctx) {
|
||||
Map<String, String> mapdev = SyslogParser.parseKeyValuePairs(ctx.strDeviceInfo);
|
||||
ctx.deviceInfoMap = mapdev;
|
||||
long deviceID = Long.parseLong(mapdev.get("device_id"));
|
||||
System.out.println("device_id:" + deviceID);
|
||||
ctx.strDataType = mapdev.get("data_type");
|
||||
try {
|
||||
if (deviceID > 0) {
|
||||
ctx.dmNormalizeRuleList = dmNormalizeRuleService.selectByDeviceIdAuto(deviceID);
|
||||
}
|
||||
//dmColumnList=dmColumnService.selectAllNormal();
|
||||
//System.out.println("dmColumnList size:"+ dmColumnList.size());
|
||||
|
||||
//OrginalColumnMap=getMessageToMap(strLogMsg);
|
||||
//解析SyslogMessage
|
||||
//SyslogMessage logMsg = SyslogParser.parse(strLogMsg);
|
||||
//设备对应的规则normal rule Map 进行标准化数据处理
|
||||
for (int i = 0; i < dmNormalizeRuleList.size(); i++) {
|
||||
for (int i = 0; i < ctx.dmNormalizeRuleList.size(); i++) {
|
||||
try {
|
||||
Map<String, Object> dmNormalizeRule = dmNormalizeRuleList.get(i);
|
||||
Map<String, Object> dmNormalizeRule = ctx.dmNormalizeRuleList.get(i);
|
||||
String data_type = dmNormalizeRule.get("data_type").toString();
|
||||
//数据类型及格式不符，则break;
|
||||
System.out.println("索引: " + i + ", 值: " + dmNormalizeRule);
|
||||
System.out.println("normalrule ID: " + dmNormalizeRule.get("id") + ", display_name:" + dmNormalizeRule.get("display_name") + " data_type:" + dmNormalizeRule.get("data_type"));
|
||||
//数据类型不匹配，则跳过规则
|
||||
//if (!data_type.equals(strDataType)) break;
|
||||
System.out.println("normalrule ID: " + dmNormalizeRule.get("id")
|
||||
+ ", display_name:" + dmNormalizeRule.get("display_name")
|
||||
+ " data_type:" + dmNormalizeRule.get("data_type"));
|
||||
|
||||
//获取syslog message 文本解析配置项
|
||||
OrginalColumnMap = getMessageToMap(dmNormalizeRule, strLogMsg, data_type);
|
||||
if ((OrginalColumnMap == null) || (OrginalColumnMap.size()==0)) {
|
||||
ctx.OrginalColumnMap = getMessageToMap(dmNormalizeRule, ctx.strLogMsg, data_type);
|
||||
if ((ctx.OrginalColumnMap == null) || (ctx.OrginalColumnMap.size() == 0)) {
|
||||
logger.error("OrginalColumnMap 对象获取为空");
|
||||
//保存非标日志信息
|
||||
syslogNonNormalMessage.setReason("Log解析异常");
|
||||
syslogNonNormalMessage.setReasonDetail("log解析异常,返回规则名称："+dmNormalizeRule.get("display_name") +",OrginalColumnMap 对象获取为空");
|
||||
if(isSaveNonNormal==false)
|
||||
SaveNonNormalMessage(deviceID, DateTime.now());
|
||||
ctx.syslogNonNormalMessage.setReason("Log解析异常");
|
||||
ctx.syslogNonNormalMessage.setReasonDetail("log解析异常,返回规则名称："
|
||||
+ dmNormalizeRule.get("display_name") + ",OrginalColumnMap 对象获取为空");
|
||||
if (ctx.isSaveNonNormal == false)
|
||||
SaveNonNormalMessage(ctx, deviceID, DateTime.now());
|
||||
continue;
|
||||
}
|
||||
List<HashMap<String, Object>> destColumnList = getRuleContentMappers(dmNormalizeRule);
|
||||
//解析字段匹配已命中的配置规则字段
|
||||
List<HashMap<String, Object>> ruleColumnList = getNormalColumnList(OrginalColumnMap, destColumnList);
|
||||
//System.out.println("ruleColumnList :"+ ruleColumnList);
|
||||
List<HashMap<String, Object>> ruleColumnList = getNormalColumnList(ctx.OrginalColumnMap, destColumnList);
|
||||
Map<String, Object> destMap = getColumnMap(ruleColumnList);
|
||||
//List<HashMap<String ,Object>> destMap2= getCompleteColumnsList(dmNormalizeRule);
|
||||
|
||||
//数据处理-过滤规则
|
||||
if (logDataFilterService.evaluateFilterRule(dmNormalizeRule.get("rule_content").toString(), destMap))
|
||||
// 数据处理-过滤规则
|
||||
if (logDataFilterService.evaluateFilterRule(
|
||||
dmNormalizeRule.get("rule_content").toString(), destMap))
|
||||
continue;
|
||||
|
||||
//数据处理-补全规则
|
||||
logDataCompleteService.processDataCompletion(dmNormalizeRule.get("rule_content").toString(), destMap);
|
||||
//long ruleid=Long.parseLong(dmNormalizeRule.get("id").toString());
|
||||
SaveNormalData(deviceID, DateTime.now(), destMap, Long.parseLong(dmNormalizeRule.get("id").toString()), dmNormalizeRule.get("name").toString());
|
||||
// 数据处理-补全规则
|
||||
logDataCompleteService.processDataCompletion(
|
||||
dmNormalizeRule.get("rule_content").toString(), destMap);
|
||||
SaveNormalData(ctx, deviceID, DateTime.now(), destMap,
|
||||
Long.parseLong(dmNormalizeRule.get("id").toString()),
|
||||
dmNormalizeRule.get("name").toString());
|
||||
|
||||
} catch (Exception ex) {
|
||||
logger.error("泛化规则处理失败：" + ex.getMessage());
|
||||
System.out.println(ex.getMessage());
|
||||
/**
|
||||
//保存非标日志信息
|
||||
syslogNonNormalMessage.setReason("泛化规则处理失败");
|
||||
syslogNonNormalMessage.setReasonDetail("泛化规则处理失败,失败详情："+ ex.getMessage());
|
||||
SaveNonNormalMessage(deviceID,DateTime.now());
|
||||
**/
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.error("处理初始化异常："+ex.getMessage());
|
||||
} catch (Exception ex) {
|
||||
logger.error("处理日志消息异常：" + ex.getMessage());
|
||||
System.out.println(ex.getMessage());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// ==================== 以下为原有业务方法 ====================
|
||||
|
||||
//获取数据泛化规则的列表字段，提取 Cropper_params
|
||||
public Cropper_paramsType getCropperParams( Map<String, Object> dmNormalizeRule )
|
||||
{
|
||||
Cropper_paramsType cropperParams = new Cropper_paramsType();
|
||||
public Cropper_paramsType getCropperParams(Map<String, Object> dmNormalizeRule) {
|
||||
Cropper_paramsType cropperParams = new Cropper_paramsType();
|
||||
try {
|
||||
JSONObject jsonObject = new JSONObject( dmNormalizeRule.get("rule_content").toString());
|
||||
JSONObject jsonObject = new JSONObject(dmNormalizeRule.get("rule_content").toString());
|
||||
if (jsonObject.isEmpty()) return null;
|
||||
|
||||
cropperParams.sethead_key(jsonObject.getJSONObject("cropper_params").get("head_key").toString());
|
||||
//需要判断是否字符串head_offset是否空
|
||||
cropperParams.sethead_offset(Integer.parseInt(jsonObject.getJSONObject("cropper_params").get("head_offset").toString()));
|
||||
cropperParams.sethead_offset(Integer.parseInt(
|
||||
jsonObject.getJSONObject("cropper_params").get("head_offset").toString()));
|
||||
cropperParams.settail_key(jsonObject.getJSONObject("cropper_params").get("tail_key").toString());
|
||||
//需要判断是否字符串tail_offset是否空
|
||||
cropperParams.settail_offset( Integer.parseInt(jsonObject.getJSONObject("cropper_params").get("tail_offset").toString()));
|
||||
cropperParams.settail_offset(Integer.parseInt(
|
||||
jsonObject.getJSONObject("cropper_params").get("tail_offset").toString()));
|
||||
return cropperParams;
|
||||
}catch (Exception ex) {
|
||||
logger.error("getCropperParams："+ex.getMessage());
|
||||
} catch (Exception ex) {
|
||||
logger.error("getCropperParams：" + ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public List<HashMap<String ,Object>> getRuleContentMappers( Map<String, Object> dmNormalizeRule )
|
||||
{
|
||||
List<HashMap<String, Object>> filedList =null;
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
public List<HashMap<String, Object>> getRuleContentMappers(Map<String, Object> dmNormalizeRule) {
|
||||
List<HashMap<String, Object>> filedList = null;
|
||||
try {
|
||||
// 转换为list Map
|
||||
//Map<String, Object> rule_contentMap = objectMapper.readValue( dmNormalizeRule.get("rule_content").toString(), Map.class);
|
||||
//if (rule_contentMap.isEmpty()) return null;
|
||||
JSONObject jsonObject = new JSONObject( dmNormalizeRule.get("rule_content").toString());
|
||||
JSONObject jsonObject = new JSONObject(dmNormalizeRule.get("rule_content").toString());
|
||||
if (jsonObject.isEmpty()) return null;
|
||||
ObjectMapper objectMapper_mappers = new ObjectMapper();
|
||||
filedList =readJsonToList( jsonObject.get("mappers").toString() );
|
||||
|
||||
}catch (Exception ex) {
|
||||
logger.error("getRuleContentMappers异常："+ex.getMessage());
|
||||
filedList = readJsonToList(jsonObject.get("mappers").toString());
|
||||
} catch (Exception ex) {
|
||||
logger.error("getRuleContentMappers异常：" + ex.getMessage());
|
||||
}
|
||||
return filedList;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public List<HashMap<String, Object>> getCompleteColumnsList( Map<String, Object> dmNormalizeRule )
|
||||
{
|
||||
List<HashMap<String, Object>> filedList =null;
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
public List<HashMap<String, Object>> getCompleteColumnsList(Map<String, Object> dmNormalizeRule) {
|
||||
List<HashMap<String, Object>> filedList = null;
|
||||
try {
|
||||
// 转换为list Map
|
||||
//Map<String, Object> rule_contentMap = objectMapper.readValue( dmNormalizeRule.get("rule_content").toString(), Map.class);
|
||||
//if (rule_contentMap.isEmpty()) return null;
|
||||
JSONObject jsonObject = new JSONObject( dmNormalizeRule.get("rule_content").toString());
|
||||
JSONObject jsonObject = new JSONObject(dmNormalizeRule.get("rule_content").toString());
|
||||
if (jsonObject.isEmpty()) return null;
|
||||
ObjectMapper objectMapper_mappers = new ObjectMapper();
|
||||
filedList =readJsonToList( jsonObject.get("complete_columns").toString() );
|
||||
|
||||
}catch (Exception ex) {
|
||||
logger.error("getCompleteColumnsList："+ex.getMessage());
|
||||
filedList = readJsonToList(jsonObject.get("complete_columns").toString());
|
||||
} catch (Exception ex) {
|
||||
logger.error("getCompleteColumnsList：" + ex.getMessage());
|
||||
}
|
||||
return filedList;
|
||||
}
|
||||
/**
|
||||
* 将JSON 字符串转换为 List<HashMap<String, Object>>
|
||||
*/
|
||||
|
||||
public static List<HashMap<String, Object>> readJsonToList(String jsonStr) {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
try {
|
||||
return objectMapper.readValue(
|
||||
jsonStr,
|
||||
new TypeReference<List<HashMap<String, Object>>>() {}
|
||||
);
|
||||
return objectMapper.readValue(jsonStr,
|
||||
new TypeReference<List<HashMap<String, Object>>>() {});
|
||||
} catch (Exception e) {
|
||||
|
||||
logger.error("readJsonToList 解析失败："+jsonStr);
|
||||
//throw new RuntimeException("JSON 解析失败: " + e.getMessage(), e);
|
||||
LoggerFactory.getLogger(LogNormalProcessor.class).error("readJsonToList 解析失败：" + jsonStr);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
/**\
|
||||
* 判断日志信息的数据类型
|
||||
* @param logMsg
|
||||
* @return
|
||||
*/
|
||||
public String getLogDataType( String logMsg)
|
||||
{
|
||||
//判断日志信息的数据类型(json、regex、kv、sep、xml）
|
||||
//默认返回json
|
||||
|
||||
public String getLogDataType(String logMsg) {
|
||||
return "json";
|
||||
}
|
||||
|
||||
public int getDeviceID( String logMsg)
|
||||
{
|
||||
//解析日志信息头部，获取device_id
|
||||
// 默认返回device_id =1
|
||||
public int getDeviceID(String logMsg) {
|
||||
return 1;
|
||||
}
|
||||
//获取log 内容字段及数据值
|
||||
public LinkedHashMap<String, Object> getMessageToMap( String MsgContent )
|
||||
{
|
||||
LinkedHashMap<String, Object> linkMap =null;
|
||||
if(strDataType.equals("json")) {
|
||||
//json 类型
|
||||
//解析SyslogMessage
|
||||
|
||||
public LinkedHashMap<String, Object> getMessageToMap(String MsgContent) {
|
||||
LinkedHashMap<String, Object> linkMap = null;
|
||||
if ("json".equals("json")) { // 默认走 json 分支，已无实例字段 strDataType
|
||||
SyslogMessage msg = SyslogParser.parse(MsgContent);
|
||||
System.out.println("解析结果-log content: " + msg.getMessage().toString());
|
||||
String complexJson = "{\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}";
|
||||
String complexJson = "{\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\"}";
|
||||
System.out.println("complexJson content: " + complexJson);
|
||||
//静态字符串、编码有问题，临时用 静态字符串做测试，流程环境没问题
|
||||
if(AppConfig.geRunEnvironment().equals("dev")) {
|
||||
//LinkedHashMap<String, Object> flatMap = JsonParser.parseJsonToFlatMap(complexJson);
|
||||
LinkedHashMap<String, Object> flatMap =JsonParser.jsonToMap(complexJson);
|
||||
if ("dev".equals(appProperties.getRunEnvironment())) {
|
||||
LinkedHashMap<String, Object> flatMap = JsonParser.jsonToMap(complexJson);
|
||||
flatMap.forEach((key, value) -> System.out.println(key + " = " + value));
|
||||
return flatMap;
|
||||
}
|
||||
else
|
||||
{
|
||||
//LinkedHashMap<String, Object> flatMap = JsonParser.parseJsonToFlatMap(msg.getMessage());
|
||||
LinkedHashMap<String, Object> flatMap =JsonParser.jsonToMap(msg.getMessage());
|
||||
} else {
|
||||
LinkedHashMap<String, Object> flatMap = JsonParser.jsonToMap(msg.getMessage());
|
||||
flatMap.forEach((key, value) -> System.out.println(key + " = " + value));
|
||||
return flatMap;
|
||||
}
|
||||
}
|
||||
else if(strDataType.equals("kv")) //key-value 键值类型
|
||||
{
|
||||
return null;
|
||||
}
|
||||
else if(strDataType.equals("sep")) //分隔符
|
||||
{
|
||||
return null;
|
||||
}
|
||||
else if(strDataType.equals("xml")) //类型 xml
|
||||
{
|
||||
return null;
|
||||
}
|
||||
else if(strDataType.equals("regex")) //正则表达式
|
||||
{
|
||||
return null;
|
||||
}
|
||||
// List<Map<String, Object>> rulelst=dmNormalizeRuleService.selectByDeviceId((long)1);
|
||||
return linkMap;
|
||||
}
|
||||
|
||||
|
||||
//获取log message内容字段及数据值
|
||||
public LinkedHashMap<String, Object> getMessageToMap( Map<String, Object> dmNormalizeRule, String MsgContent , String dataType)
|
||||
{
|
||||
public LinkedHashMap<String, Object> getMessageToMap(
|
||||
Map<String, Object> dmNormalizeRule, String MsgContent, String dataType) {
|
||||
LinkedHashMap<String, Object> linkMap = new LinkedHashMap<>();
|
||||
try {
|
||||
//String decode =dmNormalizeRule.get("decode").toString();
|
||||
Cropper_paramsType cropperParams = getCropperParams(dmNormalizeRule);
|
||||
Cropper_paramsType cropperParams = getCropperParams(dmNormalizeRule);
|
||||
if (dataType.equals("json")) {
|
||||
String strJson = logNormalData.ParserMessageJsonType(MsgContent, cropperParams);
|
||||
LinkedHashMap<String, Object> result1 = NestedJsonParserUtil.safeParseJson(strJson);
|
||||
LinkedHashMap<String, Object> flattened = NestedJsonUtils.flattenNestedJson(result1);
|
||||
return flattened;
|
||||
//parseJsonToFlatMap 复杂的json转换、解析过程有异常
|
||||
//return JsonParser.parseJsonToFlatMap(strJson);
|
||||
// return JsonParser.jsonToMap(strJson);
|
||||
}
|
||||
else if (dataType.equals("kv")) //key-value 键值类型
|
||||
{
|
||||
} else if (dataType.equals("kv")) {
|
||||
String strKeyVal = logNormalData.ParserMessageJsonType(MsgContent, cropperParams);
|
||||
KvTextParser kvTextParser =new KvTextParser();
|
||||
kv_paramsType kvparams=logNormalData.getkv_paramsType( dmNormalizeRule);
|
||||
linkMap= kvTextParser.parseKvText(strKeyVal,kvparams);
|
||||
KvTextParser kvTextParser = new KvTextParser();
|
||||
kv_paramsType kvparams = logNormalData.getkv_paramsType(dmNormalizeRule);
|
||||
linkMap = kvTextParser.parseKvText(strKeyVal, kvparams);
|
||||
return linkMap;
|
||||
}
|
||||
else if (dataType.equals("sep")) //分隔符
|
||||
{
|
||||
//获取分隔符
|
||||
String SepKey=logNormalData.sepType(dmNormalizeRule);
|
||||
} else if (dataType.equals("sep")) {
|
||||
String SepKey = logNormalData.sepType(dmNormalizeRule);
|
||||
String strSep = logNormalData.ParserMessageJsonType(MsgContent, cropperParams);
|
||||
linkMap= TextParserUtil.parseSeparatedText(strSep, SepKey);
|
||||
linkMap = TextParserUtil.parseSeparatedText(strSep, SepKey);
|
||||
return linkMap;
|
||||
}
|
||||
else if (dataType.equals("xml")) //类型 xml
|
||||
{
|
||||
} else if (dataType.equals("xml")) {
|
||||
return linkMap;
|
||||
}
|
||||
else if (dataType.equals("regex")) //正则表达式
|
||||
{
|
||||
} else if (dataType.equals("regex")) {
|
||||
String strRegex = logNormalData.ParserMessageJsonType(MsgContent, cropperParams);
|
||||
String regexp= logNormalData.Regexp(dmNormalizeRule);
|
||||
linkMap= RegexTextParser.parseWithRegex(strRegex,regexp );
|
||||
String regexp = logNormalData.Regexp(dmNormalizeRule);
|
||||
linkMap = RegexTextParser.parseWithRegex(strRegex, regexp);
|
||||
return linkMap;
|
||||
}
|
||||
}catch (Exception ex) {
|
||||
logger.error("getMessageToMap："+ex.getMessage());
|
||||
return null;
|
||||
} catch (Exception ex) {
|
||||
logger.error("getMessageToMap：" + ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
return linkMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据设备ID获取配置规则
|
||||
* @param device_id
|
||||
* @return List<Map<String, Object>>
|
||||
*/
|
||||
public List<Map<String, Object>> getRuleList( long device_id)
|
||||
{
|
||||
List<Map<String, Object>> rulelst=dmNormalizeRuleService.selectByDeviceId((long)1);
|
||||
if (rulelst!=null)
|
||||
{
|
||||
public List<Map<String, Object>> getRuleList(long device_id) {
|
||||
List<Map<String, Object>> rulelst = dmNormalizeRuleService.selectByDeviceId((long) 1);
|
||||
if (rulelst != null) {
|
||||
System.out.println("rulelst: " + rulelst);
|
||||
}
|
||||
else{
|
||||
logger.error(" List<Map<String, Object>> rulelst is null！" );
|
||||
} else {
|
||||
logger.error(" List<Map<String, Object>> rulelst is null！");
|
||||
}
|
||||
return rulelst;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查找命中配置规则的字段及数值
|
||||
* @param destColumnList 泛化目标字段list
|
||||
* @param destColumnList 日志源字段及数值list
|
||||
* @return
|
||||
*/
|
||||
public List<HashMap<String,Object>> getNormalColumnList( LinkedHashMap<String, Object> orginColumnMap ,List<HashMap<String, Object>> destColumnList )
|
||||
{
|
||||
List<HashMap<String, Object>> columnlist =new ArrayList<>();
|
||||
// 原始解析字段遍历泛化规则目标字段
|
||||
for (Map.Entry<String, Object > entry : orginColumnMap.entrySet()) {
|
||||
//System.out.println(entry.getKey() + ": " + entry.getValue());
|
||||
public List<HashMap<String, Object>> getNormalColumnList(
|
||||
LinkedHashMap<String, Object> orginColumnMap,
|
||||
List<HashMap<String, Object>> destColumnList) {
|
||||
List<HashMap<String, Object>> columnlist = new ArrayList<>();
|
||||
for (Map.Entry<String, Object> entry : orginColumnMap.entrySet()) {
|
||||
for (Map<String, Object> map : destColumnList) {
|
||||
//System.out.println( "origin_field: " + map.get("origin_field").toString());
|
||||
if ( map.get("origin_field").toString().equals(entry.getKey()) ) {
|
||||
if (map.get("origin_field").toString().equals(entry.getKey())) {
|
||||
System.out.println(map);
|
||||
Map<String, Object> normalColumMap =new LinkedHashMap<>();
|
||||
normalColumMap.put("origin_field",entry.getKey() );
|
||||
normalColumMap.put("dest_field",map.get("dest_field").toString());
|
||||
normalColumMap.put("action",(HashMap<String, Object>)map.get("action") );
|
||||
//normalColumMap.put("mapping ",entry.getKey() );
|
||||
normalColumMap.put("origin_field_value",entry.getValue() );
|
||||
// System.out.println("action: " + map.get("action").toString());
|
||||
if( ((HashMap<String, Object>)map.get("action")).get("type").equals("equal"))
|
||||
//直接赋值
|
||||
normalColumMap.put("dest_field_value",entry.getValue() );
|
||||
else if(((HashMap<String, Object>)map.get("action")).get("type").equals("mapping"))
|
||||
{
|
||||
//mapping 映射枚举值
|
||||
normalColumMap.put("dest_field_value",entry.getValue() );
|
||||
normalColumMap.put("action_param",((HashMap<String, Object>)map.get("action")).get("param") );
|
||||
HashMap<String, Object> action_param=(HashMap<String, Object>)((HashMap<String, Object>)map.get("action")).get("param") ;
|
||||
//匹配并获取映射枚举值
|
||||
normalColumMap.put("dest_field_value",getMappingValue(action_param ,entry.getValue().toString() ));
|
||||
}
|
||||
else if(((HashMap<String, Object>)map.get("action")).get("type").equals("time"))
|
||||
{
|
||||
//time 类型
|
||||
//normalColumMap.put("dest_field_value",entry.getValue() );
|
||||
normalColumMap.put("action_param",((HashMap<String, Object>)map.get("action")).get("param") );
|
||||
HashMap<String, Object> action_param=(HashMap<String, Object>)((HashMap<String, Object>)map.get("action")).get("param") ;
|
||||
//匹配时间格式并转成换整型格式
|
||||
Map<String, Object> normalColumMap = new LinkedHashMap<>();
|
||||
normalColumMap.put("origin_field", entry.getKey());
|
||||
normalColumMap.put("dest_field", map.get("dest_field").toString());
|
||||
normalColumMap.put("action", (HashMap<String, Object>) map.get("action"));
|
||||
normalColumMap.put("origin_field_value", entry.getValue());
|
||||
if (((HashMap<String, Object>) map.get("action")).get("type").equals("equal")) {
|
||||
normalColumMap.put("dest_field_value", entry.getValue());
|
||||
} else if (((HashMap<String, Object>) map.get("action")).get("type").equals("mapping")) {
|
||||
normalColumMap.put("action_param",
|
||||
((HashMap<String, Object>) map.get("action")).get("param"));
|
||||
HashMap<String, Object> action_param =
|
||||
(HashMap<String, Object>) ((HashMap<String, Object>) map.get("action")).get("param");
|
||||
normalColumMap.put("dest_field_value",
|
||||
getMappingValue(action_param, entry.getValue().toString()));
|
||||
} else if (((HashMap<String, Object>) map.get("action")).get("type").equals("time")) {
|
||||
normalColumMap.put("action_param",
|
||||
((HashMap<String, Object>) map.get("action")).get("param"));
|
||||
HashMap<String, Object> action_param =
|
||||
(HashMap<String, Object>) ((HashMap<String, Object>) map.get("action")).get("param");
|
||||
try {
|
||||
long longTime = TimeConversionUtils.convertToMillis(entry.getValue().toString(), action_param.get("timezone").toString());
|
||||
long longTime = TimeConversionUtils.convertToMillis(
|
||||
entry.getValue().toString(), action_param.get("timezone").toString());
|
||||
normalColumMap.put("dest_field_value", longTime);
|
||||
} catch (Exception e) {
|
||||
logger.error("时间类型转换错误，源值：" + entry.getValue().toString() + ",java_date_format:" + action_param.get("java_date_format").toString());
|
||||
logger.error("时间类型转换错误，源值：" + entry.getValue().toString()
|
||||
+ ",java_date_format:" + action_param.get("java_date_format").toString());
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
columnlist.add((HashMap<String, Object>)normalColumMap);
|
||||
//System.out.println( "normalColumMap: " +normalColumMap);
|
||||
//存在源字段配置多个目标字段，使用continue,而不是break
|
||||
columnlist.add((HashMap<String, Object>) normalColumMap);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -463,156 +356,129 @@ public class LogNormalProcessor {
|
||||
return columnlist;
|
||||
}
|
||||
|
||||
/**
|
||||
* paramMapping Map匹配查找对应的值
|
||||
* @param paramMappingValueMap
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
public Object getMappingValue(HashMap<String, Object> paramMappingValueMap,String value)
|
||||
{
|
||||
HashMap<String, Object> Map= (HashMap<String, Object>)paramMappingValueMap.get("mapping");
|
||||
for (Map.Entry<String, Object > entry : Map.entrySet()) {
|
||||
if ( entry.getKey().equals(value) ) {
|
||||
return entry.getValue();
|
||||
public Object getMappingValue(HashMap<String, Object> paramMappingValueMap, String value) {
|
||||
HashMap<String, Object> Map = (HashMap<String, Object>) paramMappingValueMap.get("mapping");
|
||||
for (Map.Entry<String, Object> entry : Map.entrySet()) {
|
||||
if (entry.getKey().equals(value)) {
|
||||
return entry.getValue();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取字段Map，包含字段field及Value值
|
||||
* @param normalColumnList
|
||||
* @return
|
||||
*/
|
||||
public Map<String, Object > getColumnMap( List<HashMap<String,Object>> normalColumnList )
|
||||
{
|
||||
Map<String, Object > columnMap= new HashMap<>();
|
||||
public Map<String, Object> getColumnMap(List<HashMap<String, Object>> normalColumnList) {
|
||||
Map<String, Object> columnMap = new HashMap<>();
|
||||
for (Map<String, Object> map : normalColumnList) {
|
||||
|
||||
Object destFieldValue = map.get("dest_field_value");
|
||||
// 判断 dest_field_value 是否为 String 且包含 "\u0000"
|
||||
if (destFieldValue instanceof String && ((String) destFieldValue).contains("\u0000")) {
|
||||
// 替换掉所有 "\u0000" 字符
|
||||
if (destFieldValue instanceof String
|
||||
&& ((String) destFieldValue).contains("\u0000")) {
|
||||
destFieldValue = ((String) destFieldValue).replace("\u0000", "");
|
||||
}
|
||||
columnMap.put(map.get("dest_field").toString(), destFieldValue);
|
||||
//columnMap.put(map.get("dest_field").toString(),map.get("dest_field_value"));
|
||||
}
|
||||
return columnMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存数据到标准化表
|
||||
* @param deviceId
|
||||
* @param logtime
|
||||
* @param logColumnMap
|
||||
*/
|
||||
public void SaveNormalData(long deviceId , DateTime logtime, Map<String, Object > logColumnMap, long normalizeRuleId ,String normalizeRuleName)
|
||||
{
|
||||
public void SaveNormalData(ProcessContext ctx, long deviceId, DateTime logtime,
|
||||
Map<String, Object> logColumnMap,
|
||||
long normalizeRuleId, String normalizeRuleName) {
|
||||
try {
|
||||
if(logColumnMap.isEmpty() )
|
||||
{
|
||||
logger.error("SaveNormalData ->logColumnMap 为空，syslogUUID:" +this.strSyslogUUID);
|
||||
//保存非标日志信息
|
||||
syslogNonNormalMessage.setReason("未命中规则");
|
||||
syslogNonNormalMessage.setReasonDetail("失败详情：logColumnMap对象字段为空" );
|
||||
if(isSaveNonNormal==false)
|
||||
SaveNonNormalMessage(deviceId,DateTime.now());
|
||||
return ;
|
||||
if (logColumnMap.isEmpty()) {
|
||||
logger.error("SaveNormalData ->logColumnMap 为空，syslogUUID:" + ctx.strSyslogUUID);
|
||||
ctx.syslogNonNormalMessage.setReason("未命中规则");
|
||||
ctx.syslogNonNormalMessage.setReasonDetail("失败详情：logColumnMap对象字段为空");
|
||||
if (ctx.isSaveNonNormal == false)
|
||||
SaveNonNormalMessage(ctx, deviceId, DateTime.now());
|
||||
return;
|
||||
}
|
||||
Map<String, Object> columnMap = logColumnMap;
|
||||
//补全设备信息字段
|
||||
//补全采集设备信息
|
||||
CompletionDeviceInfo(columnMap, deviceId);
|
||||
|
||||
//补全采集探针相关信息
|
||||
CompletionCollectTaskInfo(columnMap,ctx);
|
||||
//补全日志基础信息
|
||||
columnMap.put("device_id", deviceId);
|
||||
columnMap.put("log_time", logtime);
|
||||
columnMap.put("id", UUID.randomUUID().toString());
|
||||
columnMap.put("normalize_rule_id", normalizeRuleId);
|
||||
columnMap.put("normalize_rule_name", normalizeRuleName);
|
||||
columnMap.put("syslog_uuid", this.strSyslogUUID);
|
||||
columnMap.put("syslog_topic", this.strSyslogTopic);
|
||||
System.out.println("columnMap："+columnMap);
|
||||
columnMap.put("syslog_uuid", ctx.strSyslogUUID);
|
||||
columnMap.put("syslog_topic", ctx.strSyslogTopic);
|
||||
System.out.println("columnMap：" + columnMap);
|
||||
syslogNormalDataService.insertDynamic(columnMap);
|
||||
} catch (Exception e) {
|
||||
logger.error("SaveNormalData失败 " );
|
||||
//保存非标日志信息
|
||||
syslogNonNormalMessage.setReason("入库失败");
|
||||
syslogNonNormalMessage.setReasonDetail("入库失败,normalizeRuleName :"+normalizeRuleName+",失败详情："+ e.getMessage());
|
||||
if(isSaveNonNormal==false)
|
||||
SaveNonNormalMessage(deviceId,DateTime.now());
|
||||
logger.error("SaveNormalData失败 ");
|
||||
ctx.syslogNonNormalMessage.setReason("入库失败");
|
||||
ctx.syslogNonNormalMessage.setReasonDetail("入库失败,normalizeRuleName :" + normalizeRuleName
|
||||
+ ",失败详情：" + e.getMessage());
|
||||
if (ctx.isSaveNonNormal == false)
|
||||
SaveNonNormalMessage(ctx, deviceId, DateTime.now());
|
||||
throw new RuntimeException("SaveNormalData 失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 补全设备信息字段
|
||||
* @param columnMap
|
||||
* @param deviceID
|
||||
*/
|
||||
public void CompletionDeviceInfo(Map<String, Object> columnMap ,Long deviceID)
|
||||
{
|
||||
//补全采集设备信息
|
||||
public void CompletionDeviceInfo(Map<String, Object> columnMap, Long deviceID) {
|
||||
try {
|
||||
DeviceDevice devInfo = deviceDeviceService.getByIdSafely( Long.valueOf(deviceID).intValue() );
|
||||
DeviceDevice devInfo = deviceDeviceService.getByIdSafely(Long.valueOf(deviceID).intValue());
|
||||
if (devInfo != null) {
|
||||
columnMap.put("device_ip", devInfo.getIp());
|
||||
columnMap.put("device_manufacturer", devInfo.getVendor());
|
||||
columnMap.put("device_name", devInfo.getName());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("CompletionDeviceInfo 失败！ID："+deviceID );
|
||||
logger.error("CompletionDeviceInfo 失败！ID：" + deviceID);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存非标日志记录
|
||||
* @param deviceId
|
||||
* @param logtime
|
||||
*/
|
||||
public void SaveNonNormalMessage(long deviceId , DateTime logtime)
|
||||
{
|
||||
//补全采集探针相关信息（ID、IP、名称)
|
||||
public void CompletionCollectTaskInfo(Map<String, Object> columnMap,ProcessContext ctx) {
|
||||
Integer collect_id=-1;
|
||||
try {
|
||||
SyslogNonNormalMessage normalMessage =new SyslogNonNormalMessage() ;
|
||||
//获取日志对应的设备ID
|
||||
Integer collect_id= Integer.parseInt(this.deviceInfoMap.get("device_collect_id"));
|
||||
DeviceCollectTask deviceCollectTask=deviceCollectTaskService.getById(collect_id);
|
||||
normalMessage.setId( UUID.randomUUID().toString());
|
||||
normalMessage.setDeviceId((int)deviceId);
|
||||
collect_id = Integer.parseInt(ctx.deviceInfoMap.get("device_collect_id"));
|
||||
DeviceCollectTask deviceCollectTask = deviceCollectTaskService.getById(collect_id);
|
||||
if (deviceCollectTask != null) {
|
||||
columnMap.put("agent_id", collect_id);
|
||||
columnMap.put("origin_agent_id",collect_id.toString() );
|
||||
columnMap.put("probe_ip",deviceCollectTask.getDeviceIp() );
|
||||
columnMap.put("agent_name", deviceCollectTask.getTaskName());
|
||||
columnMap.put("origin_agent_name", deviceCollectTask.getTaskName());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("CompletionCollectTaskInfo 失败！探针ID：" + collect_id);
|
||||
}
|
||||
}
|
||||
|
||||
public void SaveNonNormalMessage(ProcessContext ctx, long deviceId, DateTime logtime) {
|
||||
try {
|
||||
SyslogNonNormalMessage normalMessage = new SyslogNonNormalMessage();
|
||||
Integer collect_id = Integer.parseInt(ctx.deviceInfoMap.get("device_collect_id"));
|
||||
DeviceCollectTask deviceCollectTask = deviceCollectTaskService.getById(collect_id);
|
||||
normalMessage.setId(UUID.randomUUID().toString());
|
||||
normalMessage.setDeviceId((int) deviceId);
|
||||
normalMessage.setLogTime(logtime.toLocalDateTime());
|
||||
normalMessage.setRuleTime(logtime.toLocalDateTime());
|
||||
normalMessage.setSyslogMessage(this.strLogMsg);
|
||||
normalMessage.setSyslogTopic(this.strSyslogTopic);
|
||||
normalMessage.setSyslogUuid(this.strSyslogUUID);
|
||||
normalMessage.setHeaderMessage(this.strDeviceInfo);
|
||||
normalMessage.setSyslogMessage(ctx.strLogMsg);
|
||||
normalMessage.setSyslogTopic(ctx.strSyslogTopic);
|
||||
normalMessage.setSyslogUuid(ctx.strSyslogUUID);
|
||||
normalMessage.setHeaderMessage(ctx.strDeviceInfo);
|
||||
normalMessage.setEtlNode("etlgo");
|
||||
normalMessage.setReason( syslogNonNormalMessage.getReason());
|
||||
normalMessage.setReasonDetail( syslogNonNormalMessage.getReasonDetail());
|
||||
//采集探针名称
|
||||
normalMessage.setReason(ctx.syslogNonNormalMessage.getReason());
|
||||
normalMessage.setReasonDetail(ctx.syslogNonNormalMessage.getReasonDetail());
|
||||
normalMessage.setRuleResult("FAIL");
|
||||
normalMessage.setDeviceName(deviceInfoMap.get("device_name"));
|
||||
normalMessage.setDeviceName(ctx.deviceInfoMap.get("device_name"));
|
||||
normalMessage.setCollectTaskId(collect_id);
|
||||
if(deviceCollectTask!=null)
|
||||
normalMessage.setCollectTaskName(deviceCollectTask.getTaskName());
|
||||
this.messageService.saveMessage(normalMessage );
|
||||
isSaveNonNormal=true;
|
||||
|
||||
if (deviceCollectTask != null)
|
||||
normalMessage.setCollectTaskName(deviceCollectTask.getTaskName());
|
||||
this.messageService.saveMessage(normalMessage);
|
||||
ctx.isSaveNonNormal = true;
|
||||
} catch (Exception e) {
|
||||
logger.error("SaveNonNormalMessage 失败：" );
|
||||
logger.error("SaveNonNormalMessage 失败：");
|
||||
throw new RuntimeException("SaveNonNormalMessage 失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
String strlogMessage= "<14>1 2025-09-24T11:52:26Z 5f46d3be75e1 supermario 128 honeypot_event - {\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}";
|
||||
String strMsgContent= "{\"source\":\"honeypot1\",\"id\":\"f6a13c35-bf9d-4da6-a181-50ce23e7ef6a\",\"start_time\":\"2023-09-03T11:07:02.50167643Z\",\"time\":\"2023-09-03T11:16:18.883885281Z\",\"risk_level\":4,\"connection\":\"b18f3fbe-3fbf-4495-815f-ff26f6fb0bdf\",\"file_info\":null,\"extra\":{\"payload\":{\"format\":\"line\",\"name\":{\"cn\":\"攻击载荷\",\"en\":\"payload\"},\"value\":\"\"},\"uid\":{\"format\":\"line\",\"name\":{\"cn\":\"\",\"en\":\"\"},\"uid\":\"b4cbc73c-25d0-4429-ae1b-a856cdf1a651\",\"value\":\"\"}},\"type\":\"WEB_ATTACK_SCANNER\",\"agent_sn\":\"caa7da42-0cca-4cb1-b501-1f1eb2b588d5\",\"agent_name\":\" 教育局蜜罐探针\",\"honeypot_id\":\"11a9ac6bdf38ae2aaa49ec4f1b4a921bff71952cb9f175bdd8ee1f0497057bc6\",\"honeypot_name\":\"茂名市中小学管理平台管理后台\",\"src_ip\":\"117.50.189.7\",\"src_port\":58512,\"src_mac\":\"\",\"dest_ip\":\"192.168.222.2\",\"dest_port\":9200,\"proxy_ip\":null,\"node\":\"WRx3\"}";
|
||||
//Map<String, Object> flatMap =(new logNormalData()).getMessageToMap(strMsgContent);
|
||||
//flatMap.forEach((key, value) -> System.out.println(key + " = " + value));
|
||||
//LogNormalProcessor logData =new LogNormalProcessor(strlogMessage,);
|
||||
//logData.init();
|
||||
// 旧入口保留，仅用于本地测试
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+20
-23
@@ -2,7 +2,8 @@ package com.Modules.NormalData;
|
||||
|
||||
import com.influx.InfluxDBClient;
|
||||
import com.common.util.SyslogParser;
|
||||
import com.config.AppConfig;
|
||||
import com.config.AppProperties;
|
||||
import com.config.KafkaConsumerProperties;
|
||||
import com.influxdb.client.domain.WritePrecision;
|
||||
import com.influxdb.client.write.Point;
|
||||
import com.influxdb.client.WriteApi;
|
||||
@@ -29,7 +30,6 @@ import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
import com.common.util.Sm4Util;
|
||||
import com.config.AppConfig;
|
||||
@Slf4j
|
||||
@Component
|
||||
public class SysLogProcessor {
|
||||
@@ -44,11 +44,20 @@ public class SysLogProcessor {
|
||||
@Value("${app.processor.process-timeout-ms:30000}")
|
||||
private long processTimeoutMs;
|
||||
|
||||
private static String strhexKey=AppConfig.getSM4Key();
|
||||
@Autowired
|
||||
private AppProperties appProperties;
|
||||
|
||||
@Autowired
|
||||
private KafkaConsumerProperties kafkaConsumerProperties;
|
||||
|
||||
@Autowired
|
||||
private LogNormalProcessor logNormalProcessor;
|
||||
|
||||
@Autowired
|
||||
private com.influx.InfluxDBClient influxClient;
|
||||
|
||||
private final AtomicInteger totalProcessed = new AtomicInteger(0);
|
||||
private final AtomicInteger currentBatchCount = new AtomicInteger(0);
|
||||
// 初始化 InfluxDB 客户端
|
||||
private final com.influx.InfluxDBClient influxClient = new InfluxDBClient();
|
||||
|
||||
|
||||
|
||||
@@ -70,7 +79,7 @@ public class SysLogProcessor {
|
||||
}
|
||||
@KafkaListener(
|
||||
topics = "${spring.kafka.consumer.topic:agent-syslog-topic}",
|
||||
concurrency = "2" // 多线程消费
|
||||
concurrency = "${spring.kafka.listener.concurrency}" // 多线程消费
|
||||
)
|
||||
public void consumeAndParallelProcess(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
|
||||
if (records.isEmpty()) {
|
||||
@@ -88,7 +97,7 @@ public class SysLogProcessor {
|
||||
CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
|
||||
try {
|
||||
// 异步处理单条消息
|
||||
log.info("收到syslogmessage："+ Sm4Util.decryptCbc(record.value(), strhexKey));
|
||||
log.info("收到syslogmessage："+ Sm4Util.decryptCbc(record.value(), appProperties.getSm4Key()));
|
||||
processSingleMessageAsync(record);
|
||||
} catch (Exception e) {
|
||||
log.error("处理消息失败, topic: {}, partition: {}, offset: {}",
|
||||
@@ -254,15 +263,8 @@ public class SysLogProcessor {
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
try {
|
||||
// 模拟消息解析
|
||||
//String message = parseMessage(record);
|
||||
// 模拟业务处理
|
||||
//processBusinessLogic(message);
|
||||
|
||||
//Message进行SM4解密
|
||||
String Sm4message=Sm4Util.decryptCbc(record.value(), strhexKey);
|
||||
System.out.println("Sm4message:"+Sm4message);
|
||||
|
||||
String Sm4message=Sm4Util.decryptCbc(record.value(), appProperties.getSm4Key());
|
||||
|
||||
String sysLogUUID =getSysLogUUID();
|
||||
String strDeviceInfo= SyslogParser.substringBeforeFirstChar(Sm4message,']');
|
||||
@@ -273,25 +275,20 @@ public class SysLogProcessor {
|
||||
.addTag("device_id", mapdev.get("device_id")) // 添加设备ID标签
|
||||
.addTag("device_collect_id", mapdev.get("device_collect_id")) // 添加探针ID标签
|
||||
.addTag("uuid", sysLogUUID) //syslog uuid
|
||||
.addTag("topic", AppConfig.getTopic()) //kafka topic
|
||||
.addTag("topic", kafkaConsumerProperties.getTopic()) //kafka topic
|
||||
.addField("message", Sm4message) // 添加字段
|
||||
.addField("receive_time", mapdev.get("receive_time")) // 添加字段
|
||||
.addField("uuid", sysLogUUID)
|
||||
.time(System.currentTimeMillis(), WritePrecision.MS) ;// 毫秒级时间戳
|
||||
//influxClient.writePointBlocking(point);
|
||||
///写入单点（异步，立即返回）
|
||||
influxClient.writePoint(point);
|
||||
|
||||
System.out.println("influxdb wirte syslog ,value:"+ record.key());
|
||||
// 日志信息插入pg XdrHoneypot 表
|
||||
//insertSingleRecord( record.value());
|
||||
|
||||
//String syslogMessage= AppConfig.geRunEnvironment().equals("test")? record.value().substring(34) : record.value();
|
||||
String syslogMessage= Sm4message;
|
||||
//剔除测试环境本机syslog新增的头部信息
|
||||
LogNormalProcessor logNormalProcessor = new LogNormalProcessor(syslogMessage,sysLogUUID,AppConfig.getTopic());
|
||||
//LogNormalProcessor logNormalProcessor =new LogNormalProcessor(record.value());
|
||||
logNormalProcessor.init();
|
||||
//使用注入的 Spring Bean 进行标准化处理
|
||||
logNormalProcessor.process(syslogMessage, sysLogUUID, null);
|
||||
System.out.println("insert postgres syslog ,value:"+ record.key());
|
||||
long costTime = System.currentTimeMillis() - startTime;
|
||||
log.debug("消息处理完成, offset: {}, 耗时: {}ms", record.offset(), costTime);
|
||||
|
||||
+4
@@ -75,4 +75,8 @@ public class AlarmVisit {
|
||||
private String[] httpReqBody;
|
||||
private String[] httpRespHeader;
|
||||
private String[] httpRespBody;
|
||||
private String reason;
|
||||
private String dtype;
|
||||
private String originField;
|
||||
private String originLog;
|
||||
}
|
||||
+20
@@ -20,6 +20,10 @@ public class DeviceCollectTask {
|
||||
private Integer taskCount;
|
||||
private OffsetDateTime recentDiscoverTime;
|
||||
private Integer epmUpperLimit;
|
||||
private OffsetDateTime expireTime;
|
||||
private String remark;
|
||||
private String deviceIp;
|
||||
private Long organizationId;
|
||||
|
||||
// Getter and Setter 方法
|
||||
public Integer getId() { return id; }
|
||||
@@ -73,6 +77,18 @@ public class DeviceCollectTask {
|
||||
public Integer getEpmUpperLimit() { return epmUpperLimit; }
|
||||
public void setEpmUpperLimit(Integer epmUpperLimit) { this.epmUpperLimit = epmUpperLimit; }
|
||||
|
||||
public OffsetDateTime getExpireTime() { return expireTime; }
|
||||
public void setExpireTime(OffsetDateTime expireTime) { this.expireTime = expireTime; }
|
||||
|
||||
public String getRemark() { return remark; }
|
||||
public void setRemark(String remark) { this.remark = remark; }
|
||||
|
||||
public String getDeviceIp() { return deviceIp; }
|
||||
public void setDeviceIp(String deviceIp) { this.deviceIp = deviceIp; }
|
||||
|
||||
public Long getOrganizationId() { return organizationId; }
|
||||
public void setOrganizationId(Long organizationId) { this.organizationId = organizationId; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "DeviceCollectTask{" +
|
||||
@@ -93,6 +109,10 @@ public class DeviceCollectTask {
|
||||
", taskCount=" + taskCount +
|
||||
", recentDiscoverTime=" + recentDiscoverTime +
|
||||
", epmUpperLimit=" + epmUpperLimit +
|
||||
", expireTime=" + expireTime +
|
||||
", remark='" + remark + '\'' +
|
||||
", deviceIp='" + deviceIp + '\'' +
|
||||
", organizationId=" + organizationId +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
+9
-5
@@ -23,7 +23,8 @@ public interface AlarmVisitMapper {
|
||||
"attack_port, victim_port, attack_method, etl_time, log_count, ",
|
||||
"attack_chain_phase, disposition_advice, attack_direction, ",
|
||||
"judged_state, disposed_state, attack_result, fall, payload, dns_info, engine_type, " ,
|
||||
"http_req_header , http_req_body,http_resp_header , http_resp_body ",
|
||||
"http_req_header , http_req_body,http_resp_header , http_resp_body, ",
|
||||
"reason, dtype, origin_field, origin_log ",
|
||||
") VALUES ",
|
||||
"<foreach collection='list' item='item' separator=','>",
|
||||
"(#{item.id}, #{item.createdAt}, #{item.alarmName}, #{item.alarmLevel}, ",
|
||||
@@ -34,7 +35,7 @@ public interface AlarmVisitMapper {
|
||||
"#{item.deviceId, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, ",
|
||||
"#{item.comment}, " ,
|
||||
"#{item.originLogIds, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
|
||||
"#{item.logStartAt}, #{item.logEndAt},, #{item.windowTime} #{item.httpStatus}, ",
|
||||
"#{item.logStartAt}, #{item.logEndAt}, #{item.windowTime}, #{item.httpStatus}, ",
|
||||
"#{item.attackPort, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, ",
|
||||
"#{item.victimPort, typeHandler=com.Modules.etl.handler.ArrayIntegerTypeHandler}, ",
|
||||
"#{item.attackMethod}, #{item.etlTime}, #{item.logCount}, ",
|
||||
@@ -45,7 +46,8 @@ public interface AlarmVisitMapper {
|
||||
"#{item.httpReqHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
|
||||
"#{item.httpReqBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
|
||||
"#{item.httpRespHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
|
||||
"#{item.httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}) ",
|
||||
"#{item.httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, ",
|
||||
"#{item.reason}, #{item.dtype}, #{item.originField}, #{item.originLog}) ",
|
||||
"</foreach>",
|
||||
"</script>"})
|
||||
void batchInsert(@Param("list") List<AlarmVisit> alarmList);
|
||||
@@ -60,7 +62,8 @@ public interface AlarmVisitMapper {
|
||||
"attack_port, victim_port, attack_method, etl_time, log_count, " +
|
||||
"attack_chain_phase, disposition_advice, attack_direction, " +
|
||||
"judged_state, disposed_state, attack_result, fall, payload, dns_info,engine_type, " +
|
||||
"http_req_header , http_req_body,http_resp_header , http_resp_body " +
|
||||
"http_req_header , http_req_body,http_resp_header , http_resp_body, " +
|
||||
"reason, dtype, origin_field, origin_log " +
|
||||
") VALUES (" +
|
||||
"#{id}, #{createdAt}, #{alarmName}, #{alarmLevel}, " +
|
||||
"#{alarmType}, #{alarmMajorType}, #{alarmMinorType}, #{alarmAreaId}, " +
|
||||
@@ -80,7 +83,8 @@ public interface AlarmVisitMapper {
|
||||
"#{httpReqHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
|
||||
"#{httpReqBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
|
||||
"#{httpRespHeader, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
|
||||
"#{httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler} " +
|
||||
"#{httpRespBody, typeHandler=com.Modules.etl.handler.ArrayStringTypeHandler}, " +
|
||||
"#{reason}, #{dtype}, #{originField}, #{originLog} " +
|
||||
")")
|
||||
void insert(AlarmVisit alarm);
|
||||
}
|
||||
+9
-3
@@ -66,10 +66,12 @@ public interface DeviceCollectTaskMapper extends BaseMapper<DeviceCollectTask>{
|
||||
*/
|
||||
@Insert("INSERT INTO device_collect_task (created_at, updated_at, device_id, method, task_name, " +
|
||||
"first_time, last_success_time, last_failed_time, detail_id, epm, epm_peak, " +
|
||||
"process_architecture, task_count, recent_discover_time, epm_upper_limit) " +
|
||||
"process_architecture, task_count, recent_discover_time, epm_upper_limit, " +
|
||||
"expire_time, remark, device_ip, organization_id) " +
|
||||
"VALUES (NOW(), NOW(), #{deviceId}, #{method}, #{taskName}, #{firstTime}, " +
|
||||
"#{lastSuccessTime}, #{lastFailedTime}, #{detailId}, #{epm}, #{epmPeak}, " +
|
||||
"#{processArchitecture}, #{taskCount}, #{recentDiscoverTime}, #{epmUpperLimit})")
|
||||
"#{processArchitecture}, #{taskCount}, #{recentDiscoverTime}, #{epmUpperLimit}, " +
|
||||
"#{expireTime}, #{remark}, #{deviceIp}, #{organizationId})")
|
||||
@Options(useGeneratedKeys = true, keyProperty = "id")
|
||||
int insert(DeviceCollectTask task);
|
||||
|
||||
@@ -90,7 +92,11 @@ public interface DeviceCollectTaskMapper extends BaseMapper<DeviceCollectTask>{
|
||||
"process_architecture = #{processArchitecture}, " +
|
||||
"task_count = #{taskCount}, " +
|
||||
"recent_discover_time = #{recentDiscoverTime}, " +
|
||||
"epm_upper_limit = #{epmUpperLimit} " +
|
||||
"epm_upper_limit = #{epmUpperLimit}, " +
|
||||
"expire_time = #{expireTime}, " +
|
||||
"remark = #{remark}, " +
|
||||
"device_ip = #{deviceIp}, " +
|
||||
"organization_id = #{organizationId} " +
|
||||
"WHERE id = #{id}")
|
||||
int update(DeviceCollectTask task);
|
||||
|
||||
|
||||
+4
@@ -194,6 +194,10 @@ public class DataTransformer {
|
||||
.httpReqBody(groupedData.getHttpReqBodys())
|
||||
.httpRespHeader(groupedData.getHttpRespHeaders())
|
||||
.httpRespBody(groupedData.getHttpRespBodys())
|
||||
.reason(null)
|
||||
.dtype(null)
|
||||
.originField(null)
|
||||
.originLog(null)
|
||||
.build();
|
||||
|
||||
} catch (Exception e) {
|
||||
|
||||
+2
-5
@@ -20,11 +20,8 @@ import java.util.Map;
|
||||
@Service
|
||||
public class InfluxSyslogService {
|
||||
|
||||
//@Autowired
|
||||
//private InfluxDBClient influxDBClient;
|
||||
// 初始化 InfluxDB 客户端
|
||||
//@Autowired
|
||||
private final com.influx.InfluxDBClient influxDBClient = new InfluxDBClient();
|
||||
@Autowired
|
||||
private com.influx.InfluxDBClient influxDBClient;
|
||||
|
||||
private static final DateTimeFormatter TABLE_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
|
||||
private static final DateTimeFormatter INPUT_DATE_FORMATTER = DateTimeFormatter.ISO_DATE_TIME;
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 应用配置属性 - 替代静态 AppConfig
|
||||
* 使用 Spring Boot 标准 @Value 机制，支持 docker-compose 环境变量覆盖
|
||||
*/
|
||||
@Data
|
||||
@Component
|
||||
public class AppProperties {
|
||||
|
||||
// ========== Syslog 配置 ==========
|
||||
@Value("${syslog.tcp.port:514}")
|
||||
private int syslogTcpPort;
|
||||
|
||||
@Value("${syslog.udp.port:515}")
|
||||
private int syslogUdpPort;
|
||||
|
||||
@Value("${syslog.max.frame.length:65536}")
|
||||
private int syslogMaxFrameLength;
|
||||
|
||||
@Value("${syslog.buffer.size:1024}")
|
||||
private int syslogBufferSize;
|
||||
|
||||
// ========== InfluxDB 配置 ==========
|
||||
@Value("${influxdb.url:http://localhost:8086}")
|
||||
private String influxUrl;
|
||||
|
||||
@Value("${influxdb.token:}")
|
||||
private String influxToken;
|
||||
|
||||
@Value("${influxdb.org:}")
|
||||
private String influxOrg;
|
||||
|
||||
@Value("${influxdb.bucket:syslog}")
|
||||
private String influxBucket;
|
||||
|
||||
@Value("${influxdb.batch.size:1000}")
|
||||
private int influxBatchSize;
|
||||
|
||||
@Value("${influxdb.flush.interval:1000}")
|
||||
private int influxFlushInterval;
|
||||
|
||||
@Value("${influxdb.retry.attempts:3}")
|
||||
private int influxRetryAttempts;
|
||||
|
||||
@Value("${influxdb.retry.delay:100}")
|
||||
private int influxRetryDelay;
|
||||
|
||||
// ========== 应用配置 ==========
|
||||
@Value("${app.worker.threads:4}")
|
||||
private int workerThreads;
|
||||
|
||||
@Value("${app.max.queue.size:2000}")
|
||||
private int maxQueueSize;
|
||||
|
||||
@Value("${app.metrics.enabled:false}")
|
||||
private boolean metricsEnabled;
|
||||
|
||||
// ========== 运行环境 ==========
|
||||
@Value("${server.run.environment:prod}")
|
||||
private String runEnvironment;
|
||||
|
||||
// ========== SM4 加密配置 ==========
|
||||
@Value("${syslog.sm4.generateKey:}")
|
||||
private String sm4Key;
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Kafka Consumer 配置属性类
|
||||
* 替代 AppConfig 的静态方法读取，支持 docker-compose 环境变量覆盖
|
||||
*/
|
||||
@Data
|
||||
@Component
|
||||
@ConfigurationProperties(prefix = "spring.kafka.consumer")
|
||||
public class KafkaConsumerProperties {
|
||||
/** Kafka 集群地址 */
|
||||
private String bootstrapServers = "192.168.4.99:9092";
|
||||
/** 消费者组ID */
|
||||
private String groupId = "agent-syslog-group";
|
||||
/** 消费位置重置策略 */
|
||||
private String autoOffsetReset = "latest";
|
||||
/** 是否自动提交 */
|
||||
private boolean enableAutoCommit = false;
|
||||
/** 自动提交间隔(ms) */
|
||||
private String autoCommitInterval = "1000";
|
||||
/** 消费Topic */
|
||||
private String topic = "agent-syslog-topic";
|
||||
}
|
||||
+3
-2
@@ -28,6 +28,8 @@ public class DmColumnController {
|
||||
private DmColumnService dmColumnService;
|
||||
@Autowired
|
||||
private DmNormalizeRuleService dmNormalizeRuleService;
|
||||
@Autowired
|
||||
private LogNormalProcessor logNormalProcessor;
|
||||
|
||||
private static HashMap OrginalColumnMap ;
|
||||
|
||||
@@ -53,8 +55,7 @@ public class DmColumnController {
|
||||
//List<Map<String, Object>> columnList=dmColumnService.selectAllNormal();
|
||||
//List<Map<String, Object>> rulelst= dmNormalizeRuleService.selectByDeviceIdAuto((long)1);
|
||||
//OrginalColumnMap= getMessageToMap(strLogMsg);
|
||||
LogNormalProcessor logNormalProcessor=new LogNormalProcessor( "", UUID.randomUUID().toString(),"test-topic");
|
||||
logNormalProcessor.init();
|
||||
logNormalProcessor.process("", UUID.randomUUID().toString(), "test-topic");
|
||||
|
||||
return dmColumn.getDisplayName() ;
|
||||
//return dmColumn.getDisplayName() ;
|
||||
|
||||
+16
-10
@@ -5,9 +5,11 @@ import com.influxdb.client.WriteApi;
|
||||
import com.influxdb.client.WriteApiBlocking;
|
||||
import com.influxdb.client.domain.HealthCheck;
|
||||
import com.influxdb.client.write.Point;
|
||||
import com.config.AppConfig;
|
||||
import com.config.AppProperties;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
import com.influxdb.query.FluxRecord;
|
||||
import com.influxdb.query.FluxTable;
|
||||
import java.util.ArrayList;
|
||||
@@ -19,6 +21,7 @@ import java.time.Instant;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
|
||||
@Component
|
||||
public class InfluxDBClient implements AutoCloseable {
|
||||
private static final Logger logger = LoggerFactory.getLogger(InfluxDBClient.class);
|
||||
|
||||
@@ -28,19 +31,22 @@ public class InfluxDBClient implements AutoCloseable {
|
||||
private final String bucket;
|
||||
private final String org;
|
||||
private final QueryApi queryApi;
|
||||
private final InfluxDBConfig influxDBConfig;
|
||||
|
||||
public InfluxDBClient() {
|
||||
@Autowired
|
||||
public InfluxDBClient(AppProperties appProperties, InfluxDBConfig influxDBConfig) {
|
||||
this.influxDBConfig = influxDBConfig;
|
||||
|
||||
this.influxDB = InfluxDBClientFactory.create(
|
||||
AppConfig.getInfluxUrl(),
|
||||
AppConfig.getInfluxToken().toCharArray(),
|
||||
AppConfig.getInfluxOrg(),
|
||||
AppConfig.getInfluxBucket()
|
||||
appProperties.getInfluxUrl(),
|
||||
appProperties.getInfluxToken().toCharArray(),
|
||||
appProperties.getInfluxOrg(),
|
||||
appProperties.getInfluxBucket()
|
||||
);
|
||||
this.writeApi = influxDB.makeWriteApi(InfluxDBConfig.getWriteOptions());
|
||||
this.writeApi = influxDB.makeWriteApi(influxDBConfig.getWriteOptions());
|
||||
this.writeApiBlocking = influxDB.getWriteApiBlocking();
|
||||
this.bucket = AppConfig.getInfluxBucket();
|
||||
this.org = AppConfig.getInfluxOrg();
|
||||
this.bucket = appProperties.getInfluxBucket();
|
||||
this.org = appProperties.getInfluxOrg();
|
||||
this.queryApi = influxDB.getQueryApi();
|
||||
|
||||
// 检查连接状态
|
||||
@@ -92,7 +98,7 @@ public class InfluxDBClient implements AutoCloseable {
|
||||
*/
|
||||
public void writeRecord(String lineProtocol) {
|
||||
try {
|
||||
writeApi.writeRecord(InfluxDBConfig.getWritePrecision(),lineProtocol);
|
||||
writeApi.writeRecord(influxDBConfig.getWritePrecision(),lineProtocol);
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to write record to InfluxDB: {}", e.getMessage());
|
||||
}
|
||||
|
||||
+19
-6
@@ -1,18 +1,31 @@
|
||||
package com.influx;
|
||||
|
||||
import com.config.AppProperties;
|
||||
import com.influxdb.client.WriteOptions;
|
||||
import com.influxdb.client.domain.WritePrecision;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class InfluxDBConfig {
|
||||
public static WriteOptions getWriteOptions() {
|
||||
|
||||
private final AppProperties appProperties;
|
||||
|
||||
@Autowired
|
||||
public InfluxDBConfig(AppProperties appProperties) {
|
||||
this.appProperties = appProperties;
|
||||
}
|
||||
|
||||
public WriteOptions getWriteOptions() {
|
||||
return WriteOptions.builder()
|
||||
.batchSize(com.config.AppConfig.getInfluxBatchSize())
|
||||
.flushInterval(com.config.AppConfig.getInfluxFlushInterval())
|
||||
.batchSize(appProperties.getInfluxBatchSize())
|
||||
.flushInterval(appProperties.getInfluxFlushInterval())
|
||||
.jitterInterval(1000)
|
||||
.retryInterval(com.config.AppConfig.getInfluxRetryDelay())
|
||||
.retryInterval(appProperties.getInfluxRetryDelay())
|
||||
.build();
|
||||
}
|
||||
|
||||
public static WritePrecision getWritePrecision() {
|
||||
public WritePrecision getWritePrecision() {
|
||||
return WritePrecision.MS; // 毫秒精度
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
package com.influx;
|
||||
|
||||
|
||||
import com.config.AppConfig;
|
||||
import com.common.util.SpringContextUtil;
|
||||
import com.influx.InfluxDBClient;
|
||||
import com.influxdb.client.*;
|
||||
import com.influxdb.client.domain.WritePrecision;
|
||||
@@ -30,7 +30,7 @@ public class SyslogToInfluxApp {
|
||||
logger.info("Starting Syslog to InfluxDB application...");
|
||||
|
||||
// 初始化 InfluxDB 客户端
|
||||
influxClient = new InfluxDBClient();
|
||||
influxClient = SpringContextUtil.getBean(InfluxDBClient.class);
|
||||
influxClient.writeRecord(syslog);
|
||||
Point point = Point.measurement("syslog_normal")
|
||||
.addTag("ip", "localhost") // 添加标签
|
||||
|
||||
+16
-24
@@ -18,7 +18,9 @@ import com.common.util.JsonParser;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import com.common.util.JsonParser;
|
||||
import com.config.AppConfig;
|
||||
import com.common.util.SpringContextUtil;
|
||||
import com.config.AppProperties;
|
||||
import com.config.KafkaConsumerProperties;
|
||||
import com.Modules.NormalData.LogNormalProcessor;
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
@@ -38,34 +40,27 @@ public class kafkalogconsumer {
|
||||
}
|
||||
public static void Run()
|
||||
{
|
||||
// 配置消费者属性
|
||||
//Properties props = new Properties();
|
||||
//props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.222.130:9092");
|
||||
// props.put(ConsumerConfig.GROUP_ID_CONFIG, "test-group-app");
|
||||
KafkaConsumerProperties kafkaProps = SpringContextUtil.getBean(KafkaConsumerProperties.class);
|
||||
LogNormalProcessor logNormalProcessor = SpringContextUtil.getBean(LogNormalProcessor.class);
|
||||
|
||||
Properties props = new Properties();
|
||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfig.getBootstrapServers());
|
||||
props.put(ConsumerConfig.GROUP_ID_CONFIG, AppConfig.getGroupId());
|
||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProps.getBootstrapServers());
|
||||
props.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaProps.getGroupId());
|
||||
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
|
||||
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
|
||||
|
||||
// 可选配置
|
||||
//props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none"); // 从最早的消息开始消费
|
||||
//props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 从最早的消息开始消费
|
||||
//props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); // 自动提交偏移量
|
||||
//props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000"); // 自动提交间隔
|
||||
//props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1000); // 设置单次拉取最大消息数[citation:6]
|
||||
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, AppConfig.getAutoOffsetReset()); // 从last开始消费
|
||||
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, AppConfig.getEnableAutoCommit()); // 自动提交偏移量
|
||||
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,AppConfig.getAutoCommitIntervalMS()); // 自动提交间隔
|
||||
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaProps.getAutoOffsetReset());
|
||||
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, kafkaProps.isEnableAutoCommit());
|
||||
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, kafkaProps.getAutoCommitInterval());
|
||||
|
||||
// 创建消费者实例
|
||||
Consumer<String, String> consumer = new KafkaConsumer<>(props);
|
||||
try {
|
||||
// 订阅主题
|
||||
consumer.subscribe(Collections.singletonList(AppConfig.getTopic()));
|
||||
consumer.subscribe(Collections.singletonList(kafkaProps.getTopic()));
|
||||
|
||||
System.out.println("开始消费消息...");
|
||||
com.influx.InfluxDBClient influxClient = new InfluxDBClient();
|
||||
com.influx.InfluxDBClient influxClient = SpringContextUtil.getBean(com.influx.InfluxDBClient.class);
|
||||
// 持续消费消息
|
||||
while (true) {
|
||||
// 拉取消息（等待最多100毫秒）
|
||||
@@ -88,7 +83,7 @@ public class kafkalogconsumer {
|
||||
Point point = Point.measurement("syslog_security")
|
||||
.addTag("deviceid", mapdev.get("device_id")) // 添加标签
|
||||
.addTag("uuid", sysLogUUID) //syslog uuid
|
||||
.addTag("topic", AppConfig.getTopic()) //kafka topic
|
||||
.addTag("topic", kafkaProps.getTopic()) //kafka topic
|
||||
.addField("message", record.value()) // 添加字段
|
||||
.time(System.currentTimeMillis(), WritePrecision.MS) ;// 毫秒级时间戳
|
||||
influxClient.writePointBlocking(point);
|
||||
@@ -97,12 +92,9 @@ public class kafkalogconsumer {
|
||||
//insertSingleRecord( record.value());
|
||||
System.out.println("insert postgres syslog ,value:"+ record.key());
|
||||
|
||||
//String syslogMessage= AppConfig.geRunEnvironment().equals("test")? record.value().substring(34) : record.value();
|
||||
String syslogMessage= record.value();
|
||||
//剔除测试环境本机syslog新增的头部信息
|
||||
LogNormalProcessor logNormalProcessor = new LogNormalProcessor(syslogMessage,sysLogUUID,AppConfig.getTopic());
|
||||
//LogNormalProcessor logNormalProcessor =new LogNormalProcessor(record.value());
|
||||
logNormalProcessor.init();
|
||||
//使用注入的 Spring Bean 进行标准化处理
|
||||
logNormalProcessor.process(syslogMessage, sysLogUUID, null);
|
||||
}
|
||||
// 手动提交偏移量（如果禁用自动提交）
|
||||
consumer.commitSync();
|
||||
|
||||
+16
-24
@@ -5,8 +5,10 @@ import com.common.entity.XdrHoneypot;
|
||||
import com.common.mapper.XdrHoneypotMapper;
|
||||
import com.common.util.JsonParser;
|
||||
import com.common.util.MyBatisUtil;
|
||||
import com.common.util.SpringContextUtil;
|
||||
import com.common.util.SyslogParser;
|
||||
import com.config.AppConfig;
|
||||
import com.config.AppProperties;
|
||||
import com.config.KafkaConsumerProperties;
|
||||
import com.influx.InfluxDBClient;
|
||||
import com.influxdb.client.domain.WritePrecision;
|
||||
import com.influxdb.client.write.Point;
|
||||
@@ -36,34 +38,27 @@ public class kafkalogconsumerThead {
|
||||
}
|
||||
public static void Run()
|
||||
{
|
||||
// 配置消费者属性
|
||||
//Properties props = new Properties();
|
||||
//props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.222.130:9092");
|
||||
// props.put(ConsumerConfig.GROUP_ID_CONFIG, "test-group-app");
|
||||
KafkaConsumerProperties kafkaProps = SpringContextUtil.getBean(KafkaConsumerProperties.class);
|
||||
LogNormalProcessor logNormalProcessor = SpringContextUtil.getBean(LogNormalProcessor.class);
|
||||
|
||||
Properties props = new Properties();
|
||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfig.getBootstrapServers());
|
||||
props.put(ConsumerConfig.GROUP_ID_CONFIG, AppConfig.getGroupId());
|
||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProps.getBootstrapServers());
|
||||
props.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaProps.getGroupId());
|
||||
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
|
||||
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
|
||||
|
||||
// 可选配置
|
||||
//props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none"); // 从最早的消息开始消费
|
||||
//props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // 从最早的消息开始消费
|
||||
//props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); // 自动提交偏移量
|
||||
//props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000"); // 自动提交间隔
|
||||
//props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1000); // 设置单次拉取最大消息数[citation:6]
|
||||
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, AppConfig.getAutoOffsetReset()); // 从last开始消费
|
||||
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, AppConfig.getEnableAutoCommit()); // 自动提交偏移量
|
||||
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,AppConfig.getAutoCommitIntervalMS()); // 自动提交间隔
|
||||
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaProps.getAutoOffsetReset());
|
||||
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, kafkaProps.isEnableAutoCommit());
|
||||
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, kafkaProps.getAutoCommitInterval());
|
||||
|
||||
// 创建消费者实例
|
||||
Consumer<String, String> consumer = new KafkaConsumer<>(props);
|
||||
try {
|
||||
// 订阅主题
|
||||
consumer.subscribe(Collections.singletonList(AppConfig.getTopic()));
|
||||
consumer.subscribe(Collections.singletonList(kafkaProps.getTopic()));
|
||||
|
||||
System.out.println("开始消费消息...");
|
||||
InfluxDBClient influxClient = new InfluxDBClient(); ;
|
||||
InfluxDBClient influxClient = SpringContextUtil.getBean(InfluxDBClient.class);
|
||||
// 持续消费消息
|
||||
while (true) {
|
||||
// 拉取消息（等待最多100毫秒）
|
||||
@@ -86,7 +81,7 @@ public class kafkalogconsumerThead {
|
||||
Point point = Point.measurement("syslog_security")
|
||||
.addTag("deviceid", mapdev.get("device_id")) // 添加标签
|
||||
.addTag("uuid", sysLogUUID) //syslog uuid
|
||||
.addTag("topic", AppConfig.getTopic()) //kafka topic
|
||||
.addTag("topic", kafkaProps.getTopic()) //kafka topic
|
||||
.addField("message", record.value()) // 添加字段
|
||||
.time(System.currentTimeMillis(), WritePrecision.MS) ;// 毫秒级时间戳
|
||||
influxClient.writePointBlocking(point);
|
||||
@@ -95,12 +90,9 @@ public class kafkalogconsumerThead {
|
||||
//insertSingleRecord( record.value());
|
||||
System.out.println("insert postgres syslog ,value:"+ record.key());
|
||||
|
||||
//String syslogMessage= AppConfig.geRunEnvironment().equals("test")? record.value().substring(34) : record.value();
|
||||
String syslogMessage= record.value();
|
||||
//剔除测试环境本机syslog新增的头部信息
|
||||
LogNormalProcessor logNormalProcessor = new LogNormalProcessor(syslogMessage,sysLogUUID,AppConfig.getTopic());
|
||||
//LogNormalProcessor logNormalProcessor =new LogNormalProcessor(record.value());
|
||||
logNormalProcessor.init();
|
||||
//使用注入的 Spring Bean 进行标准化处理
|
||||
logNormalProcessor.process(syslogMessage, sysLogUUID, null);
|
||||
}
|
||||
// 手动提交偏移量（如果禁用自动提交）
|
||||
// consumer.commitSync();
|
||||
|
||||
+23
-23
@@ -15,10 +15,11 @@ syslog.udp.port=514
|
||||
syslog.max.frame.length=65536
|
||||
syslog.buffer.size=1000
|
||||
syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a
|
||||
|
||||
# InfluxDB 2.7 Configuration
|
||||
influxdb.url=http://192.168.222.131:8086
|
||||
influxdb.token=3Tvu-IZWtaY03UDkbUDlufD0kxn85keo9LhYQcv2Cxk0LJmXqqHkNVrO664DbaJAYwoGI7UIg904KqZC7Q_ZFA==
|
||||
influxdb.org=yelang
|
||||
influxdb.url=http://192.168.4.99:8087
|
||||
influxdb.token=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg==
|
||||
influxdb.org=influxdb
|
||||
influxdb.bucket=yelangbucket
|
||||
influxdb.batch.size=1000
|
||||
influxdb.flush.interval=1000
|
||||
@@ -33,17 +34,15 @@ app.worker.threads=8
|
||||
app.max.queue.size=10000
|
||||
app.metrics.enabled=true
|
||||
|
||||
|
||||
#database Configuration
|
||||
spring.datasource.url=jdbc:dm://192.163.4.99:5237/DM_ecosys
|
||||
spring.datasource.url=jdbc:dm://192.168.4.99:5237
|
||||
spring.datasource.username=SYSDBA
|
||||
spring.datasource.password=caZ2TcmXNSW8L2Ap
|
||||
spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver
|
||||
spring.datasource.hikari.schema=\"PUBLIC\"
|
||||
spring.datasource.hikari.schema=ECOSYS
|
||||
# mybatis Configuration
|
||||
mybatis.mapper-locations=classpath:mapper/*.xml
|
||||
mybatis.type-aliases-package=com.common.entity
|
||||
|
||||
#mybatis handler 类
|
||||
mybatis.configuration.default-statement-timeout=30
|
||||
mybatis.configuration.default-fetch-size=1000
|
||||
@@ -51,23 +50,23 @@ mybatis.configuration.map-underscore-to-camel-case=true
|
||||
mybatis.type-handlers-package=com.Modules.etl.handler
|
||||
mybatis-plus.configuration.map-underscore-to-camel-case=true
|
||||
mybatis-plus.type-handlers-package=com.Modules.etl.handler
|
||||
|
||||
# kafka Configuration
|
||||
spring.kafka.consumer.bootstrap-servers=192.168.222.130:9092
|
||||
spring.kafka.consumer.group-id=agent-syslog-group
|
||||
spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092
|
||||
spring.kafka.consumer.group-id=test-group-1
|
||||
spring.kafka.consumer.auto-offset-reset=latest
|
||||
spring.kafka.consumer.enable-auto-commit=false
|
||||
spring.kafka.consumer.auto-commit-interval=1000
|
||||
spring.kafka.consumer.topic=agent-syslog-topic
|
||||
spring.kafka.consumer.topic=test-topic
|
||||
|
||||
|
||||
spring.kafka.consumer.max-poll-records=1000
|
||||
spring.kafka.consumer.properties.max.poll.interval.ms=300000
|
||||
spring.kafka.consumer.properties.session.timeout.ms=45000
|
||||
|
||||
#spring.kafka.consumer.key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
|
||||
#spring.kafka.consumer.value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
|
||||
spring.kafka.consumer.fetch-min-size= 1048576
|
||||
spring.kafka.listener.ack-mode= manual
|
||||
spring.kafka.listener.concurrency= 2
|
||||
spring.kafka.listener.concurrency= 2
|
||||
spring.kafka.listener.type=batch
|
||||
|
||||
|
||||
@@ -83,21 +82,23 @@ partition.check.tomorrow.enabled=true
|
||||
partition.check.future.days=7
|
||||
partition.auto.create=true
|
||||
|
||||
# 开发环境缓存配置
|
||||
|
||||
# 生产环境缓存配置
|
||||
spring.redis.host=localhost
|
||||
spring.redis.port=6379
|
||||
# 密码（如果没有设置密码，可以省略）
|
||||
spring.redis.password=
|
||||
spring.redis.database=0
|
||||
spring.redis.timeout=2000
|
||||
spring.redis.timeout=5000
|
||||
#spring.redis.password=${REDIS_PASSWORD:default_prod_password}
|
||||
|
||||
spring.redis.lettuce.pool.max-active=8
|
||||
spring.redis.lettuce.pool.max-wait=-1
|
||||
spring.redis.lettuce.pool.max-idle=8
|
||||
spring.redis.lettuce.pool.min-idle=0
|
||||
# 开发环境缓存时间较短，方便调试
|
||||
spring.cache.redis.time-to-live=600000
|
||||
spring.redis.lettuce.pool.max-active=20
|
||||
spring.redis.lettuce.pool.max-wait=5000
|
||||
spring.redis.lettuce.pool.max-idle=10
|
||||
spring.redis.lettuce.pool.min-idle=5
|
||||
|
||||
# 生产环境缓存时间较长
|
||||
spring.cache.redis.time-to-live=3600000
|
||||
|
||||
# 应用处理器配置
|
||||
app.processor.thread-pool.core-pool-size=10
|
||||
@@ -124,5 +125,4 @@ spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL
|
||||
spring.datasource.hikari.validation-timeout=5000
|
||||
spring.datasource.hikari.leak-detection-threshold=30000
|
||||
spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule
|
||||
spring.datasource.hikari.auto-commit=false
|
||||
|
||||
spring.datasource.hikari.auto-commit=false
|
||||
+3
-3
@@ -52,8 +52,8 @@ mybatis-plus.configuration.map-underscore-to-camel-case=true
|
||||
mybatis-plus.type-handlers-package=com.Modules.etl.handler
|
||||
|
||||
# kafka Configuration
|
||||
spring.kafka.consumer.bootstrap-servers=192.168.4.26:9092
|
||||
spring.kafka.consumer.group-id=agent-01-syslog-group-dm
|
||||
spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092
|
||||
spring.kafka.consumer.group-id=agent-01-syslog-group
|
||||
spring.kafka.consumer.auto-offset-reset=latest
|
||||
spring.kafka.consumer.enable-auto-commit=false
|
||||
spring.kafka.consumer.auto-commit-interval=1000
|
||||
@@ -63,7 +63,7 @@ spring.kafka.consumer.topic=agent-01-syslog-topic
|
||||
spring.kafka.consumer.max-poll-records=1000
|
||||
spring.kafka.consumer.properties.max.poll.interval.ms=300000
|
||||
spring.kafka.consumer.properties.session.timeout.ms=45000
|
||||
|
||||
spring.kafka.consumer.properties.partition.assignment.strategy=org.apache.kafka.clients.consumer.RoundRobinAssignor
|
||||
spring.kafka.consumer.fetch-min-size= 1048576
|
||||
spring.kafka.listener.ack-mode= manual
|
||||
spring.kafka.listener.concurrency= 2
|
||||
|
||||
@@ -7,7 +7,7 @@ server.error.include-message=always
|
||||
server.error.include-binding-errors=always
|
||||
|
||||
#run.environment: dev|test|pro
|
||||
server.run.environment=pro
|
||||
server.run.environment=dev
|
||||
|
||||
# Syslog Server Configuration
|
||||
syslog.tcp.port=514
|
||||
@@ -39,7 +39,7 @@ spring.datasource.url=jdbc:dm://192.168.4.99:5237
|
||||
spring.datasource.username=SYSDBA
|
||||
spring.datasource.password=caZ2TcmXNSW8L2Ap
|
||||
spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver
|
||||
spring.datasource.hikari.schema=\"PUBLIC\"
|
||||
spring.datasource.hikari.schema=ECOSYS
|
||||
# mybatis Configuration
|
||||
mybatis.mapper-locations=classpath:mapper/*.xml
|
||||
mybatis.type-aliases-package=com.common.entity
|
||||
@@ -52,12 +52,12 @@ mybatis-plus.configuration.map-underscore-to-camel-case=true
|
||||
mybatis-plus.type-handlers-package=com.Modules.etl.handler
|
||||
|
||||
# kafka Configuration
|
||||
spring.kafka.consumer.bootstrap-servers=192.168.4.26:9092
|
||||
spring.kafka.consumer.group-id=agent-01-syslog-group-dm
|
||||
spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092
|
||||
spring.kafka.consumer.group-id=test-group-1
|
||||
spring.kafka.consumer.auto-offset-reset=latest
|
||||
spring.kafka.consumer.enable-auto-commit=false
|
||||
spring.kafka.consumer.auto-commit-interval=1000
|
||||
spring.kafka.consumer.topic=agent-01-syslog-topic
|
||||
spring.kafka.consumer.topic=test-topic
|
||||
|
||||
|
||||
spring.kafka.consumer.max-poll-records=1000
|
||||
@@ -66,7 +66,7 @@ spring.kafka.consumer.properties.session.timeout.ms=45000
|
||||
|
||||
spring.kafka.consumer.fetch-min-size= 1048576
|
||||
spring.kafka.listener.ack-mode= manual
|
||||
spring.kafka.listener.concurrency= 2
|
||||
spring.kafka.listener.concurrency= 2
|
||||
spring.kafka.listener.type=batch
|
||||
|
||||
|
||||
@@ -84,10 +84,10 @@ partition.auto.create=true
|
||||
|
||||
|
||||
# 生产环境缓存配置
|
||||
spring.redis.host=192.168.4.99
|
||||
spring.redis.host=localhost
|
||||
spring.redis.port=6379
|
||||
# 密码（如果没有设置密码，可以省略）
|
||||
spring.redis.password=redis_GdGWte
|
||||
spring.redis.password=
|
||||
spring.redis.database=0
|
||||
spring.redis.timeout=5000
|
||||
#spring.redis.password=${REDIS_PASSWORD:default_prod_password}
|
||||
|
||||
+16
-1
@@ -22,12 +22,17 @@
|
||||
<result column="task_count" property="taskCount" />
|
||||
<result column="recent_discover_time" property="recentDiscoverTime" />
|
||||
<result column="epm_upper_limit" property="epmUpperLimit" />
|
||||
<result column="expire_time" property="expireTime" />
|
||||
<result column="remark" property="remark" />
|
||||
<result column="device_ip" property="deviceIp" />
|
||||
<result column="organization_id" property="organizationId" />
|
||||
</resultMap>
|
||||
|
||||
<sql id="Base_Column_List">
|
||||
id, created_at, updated_at, deleted_at, device_id, method, task_name,
|
||||
first_time, last_success_time, last_failed_time, detail_id, epm, epm_peak,
|
||||
process_architecture, task_count, recent_discover_time, epm_upper_limit
|
||||
process_architecture, task_count, recent_discover_time, epm_upper_limit,
|
||||
expire_time, remark, device_ip, organization_id
|
||||
</sql>
|
||||
|
||||
<!-- 多条件组合查询 -->
|
||||
@@ -60,6 +65,16 @@
|
||||
<if test="lastSuccessTime != null">
|
||||
AND last_success_time >= #{lastSuccessTime}
|
||||
</if>
|
||||
<!-- 新增字段查询 -->
|
||||
<if test="deviceIp != null and deviceIp != ''">
|
||||
AND device_ip = #{deviceIp}
|
||||
</if>
|
||||
<if test="organizationId != null">
|
||||
AND organization_id = #{organizationId}
|
||||
</if>
|
||||
<if test="remark != null and remark != ''">
|
||||
AND remark LIKE CONCAT('%', #{remark}, '%')
|
||||
</if>
|
||||
</where>
|
||||
ORDER BY updated_at DESC
|
||||
</select>
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
|
||||
<component name="FacetManager">
|
||||
<facet type="Spring" name="Spring">
|
||||
<configuration />
|
||||
</facet>
|
||||
<facet type="web" name="Web">
|
||||
<configuration>
|
||||
<webroots />
|
||||
<sourceRoots>
|
||||
<root url="file://$MODULE_DIR$/src/main/java" />
|
||||
<root url="file://$MODULE_DIR$/src/main/resources" />
|
||||
</sourceRoots>
|
||||
</configuration>
|
||||
</facet>
|
||||
</component>
|
||||
</module>
|
||||
Reference in New Issue
Block a user