王绍宏
29445c6825
- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
103 lines
4.2 KiB
YAML
103 lines
4.2 KiB
YAML
# ============================================
|
|
# Syslog Consumer 部署配置
|
|
# ============================================
|
|
# 使用方法: docker compose -f docker-compose-consumer-rule-dm.yaml up -d
|
|
# ============================================
|
|
|
|
|
|
services:
|
|
# ============================================
|
|
# Syslog Consumer-rule - 数据消费规则服务(平台端)
|
|
# ============================================
|
|
syslog-consumer-rule:
|
|
build:
|
|
context: ./consumer-rule-dm
|
|
dockerfile: Dockerfile
|
|
image: syslog-consumer-rule-dm:v1.2.2
|
|
container_name: syslog-consumer-rule-dm
|
|
restart: unless-stopped
|
|
environment:
|
|
# 环境配置
|
|
- SPRING_PROFILES_ACTIVE=prod
|
|
- TZ=Asia/Shanghai
|
|
# 全停升级且已完成增量统计 DDL/基线后，所有 rule 节点统一开启捕获。
|
|
- STATS_INCREMENTAL_CAPTURE_ENABLED=true
|
|
- STATS_INGRESS_METADATA_ENABLED=true
|
|
- STATS_INCREMENTAL_READ_ENABLED=true
|
|
- STATS_DEVICE_RECEIVE_CHANGE_CAPTURE_ENABLED=false
|
|
# 数据库配置
|
|
- SPRING_DATASOURCE_URL=jdbc:dm://192.168.4.99:5237
|
|
- SPRING_DATASOURCE_USERNAME=SYSDBA
|
|
- SPRING_DATASOURCE_PASSWORD=caZ2TcmXNSW8L2Ap
|
|
- SPRING_DATASOURCE_DRIVER-CLASS-NAME=dm.jdbc.driver.DmDriver
|
|
- SPRING_DATASOURCE_HIKARI_SCHEMA=\"PUBLIC\"
|
|
# HikariCP 连接池配置
|
|
- SPRING_DATASOURCE_HIKARI_MAXIMUM-POOL-SIZE=50
|
|
- SPRING_DATASOURCE_HIKARI_MINIMUM-IDLE=5
|
|
- SPRING_DATASOURCE_HIKARI_CONNECTION-TIMEOUT=30000
|
|
- SPRING_DATASOURCE_HIKARI_IDLE-TIMEOUT=600000
|
|
- SPRING_DATASOURCE_HIKARI_MAX-LIFETIME=900000
|
|
- SPRING_DATASOURCE_HIKARI_POOL-NAME=HikariPool-SyslogConsumer-rule
|
|
- SPRING_DATASOURCE_HIKARI_AUTO-COMMIT=false
|
|
|
|
# Redis配置
|
|
- SPRING_REDIS_HOST=192.168.4.99
|
|
- SPRING_REDIS_PORT=6379
|
|
- SPRING_REDIS_PASSWORD=redis_GdGWte
|
|
- SPRING_REDIS_DATABASE=0
|
|
- SPRING_REDIS_TIMEOUT=2000
|
|
- SPRING_CACHE_REDIS_TIME-TO-LIVE=600000
|
|
# Kafka配置
|
|
- SPRING_KAFKA_CONSUMER_BOOTSTRAP_SERVERS=192.168.4.99:9092
|
|
# 保留客户现有生产消费组名称，多节点必须完全一致，不可临时换组。
|
|
- SPRING_KAFKA_CONSUMER_GROUP_ID=agent-01-syslog-group-dm
|
|
- SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest
|
|
- SPRING_KAFKA_CONSUMER_ENABLE_AUTO_COMMIT=false
|
|
- SPRING_KAFKA_CONSUMER_TOPIC=agent-01-syslog-topic
|
|
# consumer-rule 是活跃原始日志消费节点，多实例通过同一消费组分担分区。
|
|
- SYSLOG_RAW_CONSUMER_ENABLED=true
|
|
# Retry/Replay 与 Raw 隔离，避免未到期重试占用实时消费线程。
|
|
- SYSLOG_RELIABILITY_CONSUMER_GROUP_ID=agent-01-syslog-group-dm-retry
|
|
- SYSLOG_RELIABILITY_CONSUMER_ENABLED=true
|
|
- SYSLOG_RELIABILITY_LISTENER_CONCURRENCY=1
|
|
- SPRING_KAFKA_CONSUMER_MAX_POLL_RECORDS=500
|
|
- SPRING_KAFKA_CONSUMER_PROPERTIES_MAX_POLL_INTERVAL_MS=900000
|
|
- SPRING_KAFKA_LISTENER_ACK_MODE=manual
|
|
- SPRING_KAFKA_LISTENER_CONCURRENCY=2
|
|
- SPRING_KAFKA_LISTENER_TYPE=batch
|
|
# InfluxDB配置
|
|
- INFLUXDB_URL=http://192.168.4.99:8087
|
|
- INFLUXDB_TOKEN=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg==
|
|
- INFLUXDB_ORG=influxdb
|
|
- INFLUXDB_BUCKET=yelangbucket
|
|
- INFLUXDB_RAW_WRITE_SCHEMA=V2
|
|
- INFLUXDB_RAW_READ_MODE=V1_V2
|
|
- INFLUXDB_RAW_V1_BUCKET=yelangbucket
|
|
- INFLUXDB_RAW_V2_BUCKET=yelangbucket_v2
|
|
- INFLUXDB_RAW_UUID_BUCKET_COUNT=4096
|
|
- INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED=true
|
|
- INFLUXDB_RAW_QUERY_MAX_ROWS=10000
|
|
- INFLUXDB_RAW_QUERY_MAX_RESPONSE_BYTES=10485760
|
|
- INFLUXDB_BATCH_SIZE=1000
|
|
- INFLUXDB_FLUSH_INTERVAL=1000
|
|
- INFLUXDB_CONNECTION_TIMEOUT=30s
|
|
- INFLUXDB_CONNECTION_READ-TIMEOUT=30s
|
|
- INFLUXDB_CONNECTION_WRITE-TIMEOUT=60s
|
|
# 定时任务配置
|
|
- SPRING_TASK_SCHEDULING_POOL_SIZE=10
|
|
# 日志配置
|
|
- LOGGING_LEVEL_COM_COMMON_SCHEDULE=INFO
|
|
- LOGGING_LEVEL_COM_COMMON_SERVICE=INFO
|
|
|
|
volumes:
|
|
- /home/syslog/logs:/app/logs
|
|
networks:
|
|
- xdr-network
|
|
privileged: true
|
|
# ============================================
|
|
# 网络配置
|
|
# ============================================
|
|
networks:
|
|
xdr-network:
|
|
driver: bridge
|