王绍宏
29445c6825
- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
225 lines
9.3 KiB
Properties
225 lines
9.3 KiB
Properties
#Server Configuration
|
||
server.port=8289
|
||
server.servlet.context-path=/xdrrule
|
||
#server.address=0.0.0.0
|
||
server.tomcat.uri-encoding=UTF-8
|
||
server.error.include-message=always
|
||
server.error.include-binding-errors=always
|
||
|
||
#run.environment: dev|test|pro
|
||
server.run.environment=dev
|
||
|
||
# Syslog Server Configuration
|
||
syslog.tcp.port=514
|
||
syslog.udp.port=514
|
||
syslog.max.frame.length=65536
|
||
syslog.buffer.size=1000
|
||
syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a
|
||
|
||
# InfluxDB 2.7 Configuration
|
||
influxdb.url=http://192.168.4.99:8087
|
||
influxdb.token=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg==
|
||
influxdb.org=influxdb
|
||
influxdb.bucket=yelangbucket
|
||
influxdb.batch.size=1000
|
||
influxdb.flush.interval=1000
|
||
influxdb.retry.attempts=3
|
||
influxdb.retry.delay=1000
|
||
# 原始 Syslog V2 分阶段开关。兼容部署先保持 V1/V1_ONLY，切换时统一写 V2。
|
||
influxdb.raw.write-schema=${INFLUXDB_RAW_WRITE_SCHEMA:V2}
|
||
influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V1_V2}
|
||
influxdb.raw.v1-bucket=${INFLUXDB_RAW_V1_BUCKET:${influxdb.bucket}}
|
||
influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket_v2}
|
||
influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
|
||
influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
|
||
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
|
||
influxdb.raw.query-max-response-bytes=${INFLUXDB_RAW_QUERY_MAX_RESPONSE_BYTES:10485760}
|
||
# InfluxDB 2.7 连接超时配置
|
||
influxdb.connection.timeout=30s
|
||
influxdb.connection.read-timeout=30s
|
||
influxdb.connection.write-timeout=60s
|
||
# Application Configuration
|
||
app.worker.threads=8
|
||
app.max.queue.size=10000
|
||
app.metrics.enabled=true
|
||
|
||
#database Configuration
|
||
spring.datasource.url=jdbc:dm://192.168.4.99:5237
|
||
spring.datasource.username=SYSDBA
|
||
spring.datasource.password=caZ2TcmXNSW8L2Ap
|
||
spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver
|
||
spring.datasource.hikari.schema=ECOSYS
|
||
# mybatis Configuration
|
||
mybatis.mapper-locations=classpath:mapper/*.xml
|
||
mybatis.type-aliases-package=com.common.entity
|
||
#mybatis handler 类
|
||
mybatis.configuration.default-statement-timeout=30
|
||
mybatis.configuration.default-fetch-size=1000
|
||
mybatis.configuration.map-underscore-to-camel-case=true
|
||
mybatis.type-handlers-package=com.Modules.etl.handler
|
||
mybatis-plus.configuration.map-underscore-to-camel-case=true
|
||
mybatis-plus.type-handlers-package=com.Modules.etl.handler
|
||
|
||
# kafka Configuration
|
||
spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092
|
||
spring.kafka.consumer.group-id=test-group-1
|
||
spring.kafka.consumer.auto-offset-reset=latest
|
||
spring.kafka.consumer.enable-auto-commit=false
|
||
spring.kafka.consumer.auto-commit-interval=1000
|
||
spring.kafka.consumer.topic=test-topic
|
||
# consumer-rule 是当前活跃的 Kafka 原始日志标准化节点，默认启动监听。
|
||
syslog.raw-consumer.enabled=true
|
||
# Raw Topic 当前有 3 个分区，使用独立工厂显式创建 3 个 consumer；可通过环境变量灰度调整。
|
||
syslog.raw-listener.concurrency=${SYSLOG_RAW_LISTENER_CONCURRENCY:3}
|
||
# P0：Retry/DLT 发布与原始消费使用同一 Kafka 集群；Topic 名称全部可由环境变量覆盖。
|
||
spring.kafka.bootstrap-servers=${spring.kafka.consumer.bootstrap-servers}
|
||
spring.kafka.producer.acks=all
|
||
spring.kafka.producer.properties.enable.idempotence=true
|
||
spring.kafka.producer.properties.delivery.timeout.ms=120000
|
||
spring.kafka.producer.properties.request.timeout.ms=30000
|
||
syslog.reliability.retry-1m-topic=${spring.kafka.consumer.topic}.RETRY.1M
|
||
syslog.reliability.retry-10m-topic=${spring.kafka.consumer.topic}.RETRY.10M
|
||
syslog.reliability.retry-1h-topic=${spring.kafka.consumer.topic}.RETRY.1H
|
||
syslog.reliability.dlt-topic=${spring.kafka.consumer.topic}.DLT
|
||
syslog.reliability.replay-topic=${spring.kafka.consumer.topic}.REPLAY
|
||
syslog.reliability.publish-timeout-ms=30000
|
||
syslog.reliability.consumer-version=${APP_VERSION:1.0.0}
|
||
# Retry/Replay 使用独立消费组和线程，未到期重试不会占用 Raw listener。
|
||
syslog.reliability.consumer-group-id=${spring.kafka.consumer.group-id}-retry
|
||
syslog.reliability.consumer.enabled=true
|
||
syslog.reliability.listener.concurrency=1
|
||
|
||
|
||
spring.kafka.consumer.max-poll-records=500
|
||
spring.kafka.consumer.properties.max.poll.interval.ms=900000
|
||
spring.kafka.consumer.properties.session.timeout.ms=45000
|
||
|
||
spring.kafka.consumer.fetch-min-size= 1048576
|
||
spring.kafka.listener.ack-mode= manual
|
||
spring.kafka.listener.concurrency= 2
|
||
spring.kafka.listener.type=batch
|
||
|
||
|
||
# 定时任务配置
|
||
spring.task.scheduling.pool.size=10
|
||
|
||
# 日志配置
|
||
logging.level.com.common.schedule=INFO
|
||
logging.level.com.common.service=INFO
|
||
|
||
# 分区表检查配置
|
||
partition.check.tomorrow.enabled=true
|
||
partition.check.future.days=7
|
||
partition.auto.create=true
|
||
|
||
|
||
# 生产环境缓存配置
|
||
spring.redis.host=localhost
|
||
spring.redis.port=6379
|
||
# 密码（如果没有设置密码，可以省略）
|
||
spring.redis.password=
|
||
spring.redis.database=0
|
||
spring.redis.timeout=5000
|
||
#spring.redis.password=${REDIS_PASSWORD:default_prod_password}
|
||
|
||
spring.redis.lettuce.pool.max-active=20
|
||
spring.redis.lettuce.pool.max-wait=5000
|
||
spring.redis.lettuce.pool.max-idle=10
|
||
spring.redis.lettuce.pool.min-idle=5
|
||
|
||
# 生产环境缓存时间较长
|
||
spring.cache.redis.time-to-live=3600000
|
||
|
||
# 应用处理器配置
|
||
app.processor.thread-pool.core-pool-size=10
|
||
app.processor.thread-pool.max-pool-size=20
|
||
app.processor.thread-pool.queue-capacity=2000
|
||
app.processor.thread-pool.keep-alive-seconds=60
|
||
app.processor.batch-size=100
|
||
app.processor.process-timeout-ms=120000
|
||
|
||
# 达梦标准化表默认启用批量写入；现场需要紧急回退时可显式切换为 SINGLE。
|
||
# BATCH 模式按目标表、log_time 日期和动态列集合分组，失败时只对数据类错误有限二分；
|
||
# 连接、超时或提交结果未知会立即停止并保留 Kafka offset，禁止放大为逐条请求风暴。
|
||
dm.standardized-write.mode=BATCH
|
||
dm.standardized-write.batch-size=100
|
||
dm.standardized-write.writer-concurrency=2
|
||
dm.standardized-write.writer-queue-capacity=100
|
||
dm.standardized-write.transaction-timeout-seconds=45
|
||
dm.standardized-write.kafka-batch-deadline-ms=90000
|
||
dm.standardized-write.max-split-depth=10
|
||
dm.standardized-write.max-fallback-operations=100
|
||
dm.standardized-write.circuit-breaker-failure-threshold=3
|
||
dm.standardized-write.circuit-breaker-backoff-ms=30000
|
||
|
||
# ETL配置
|
||
etl.batch.page-size=1000
|
||
etl.batch.insert-batch-size=500
|
||
etl.schedule.cron=0 0 2 * * ?
|
||
|
||
# ============================================
|
||
# HikariCP Connection Pool Configuration
|
||
# ============================================
|
||
spring.datasource.hikari.maximum-pool-size=50
|
||
spring.datasource.hikari.minimum-idle=5
|
||
spring.datasource.hikari.connection-timeout=30000
|
||
spring.datasource.hikari.idle-timeout=600000
|
||
spring.datasource.hikari.max-lifetime=900000
|
||
spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL
|
||
spring.datasource.hikari.validation-timeout=5000
|
||
spring.datasource.hikari.leak-detection-threshold=30000
|
||
spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule
|
||
spring.datasource.hikari.auto-commit=false
|
||
# 统计写入采用事实同事务直写，不再为每条结果写 change_log + pending。
|
||
# 首次部署 DIRECT 前必须先执行 08/09/10/11 脚本并按设计文档排空旧 pending。
|
||
stats.write-mode=DIRECT
|
||
stats.incremental.capture-enabled=false
|
||
stats.ingress-metadata.enabled=true
|
||
stats.incremental.read-enabled=true
|
||
stats.incremental.transaction-timeout-seconds=45
|
||
stats.device-receive.change-capture-enabled=false
|
||
stats.direct.device-table=device_daily_stats
|
||
stats.direct.rule-table=normalize_rule_daily_stats
|
||
stats.direct.shadow-device-table=device_daily_stats_direct_shadow
|
||
stats.direct.shadow-rule-table=normalize_rule_daily_stats_direct_shadow
|
||
stats.direct.control-table=stats_direct_control
|
||
stats.direct.today-stats-date-enabled=true
|
||
stats.direct.max-concurrency-retries=1
|
||
stats.direct.concurrency-retry-backoff-ms=20
|
||
stats.direct.transaction-log-enabled=false
|
||
stats.direct.day-rollover-check-cron=5 * * * * ?
|
||
stats.business-zone=Asia/Shanghai
|
||
stats.fact-only.confirm-rebuild-required=false
|
||
|
||
# 标准化规则编译与批次缓存灰度开关
|
||
normalization.jetcache.enabled=true
|
||
normalization.rule-source-cache.enabled=true
|
||
normalization.compiled-rule.enabled=true
|
||
normalization.compiled-rule.shadow-compare-enabled=false
|
||
normalization.compiled-rule.shadow-sample-rate=0
|
||
normalization.compiled-rule.local-limit=10000
|
||
normalization.compiled-rule.expire-after-access-minutes=30
|
||
normalization.compiled-rule.failure-cache-seconds=10
|
||
normalization.compiled-rule.shadow-timeout-ms=1000
|
||
normalization.compiled-rule.rule-detail-log-enabled=true
|
||
normalization.batch-enrichment-cache.enabled=true
|
||
normalization.message-parse-cache.enabled=true
|
||
normalization.rule-source-cache.local-ttl-seconds=30
|
||
normalization.rule-source-cache.remote-ttl-seconds=300
|
||
normalization.rule-source-cache.empty-ttl-seconds=10
|
||
normalization.enrichment-cache.local-ttl-seconds=60
|
||
normalization.enrichment-cache.remote-ttl-seconds=3600
|
||
|
||
# JetCache Redis URI 由 JetCacheRedisEnvironmentPostProcessor 从 spring.redis.* 派生。
|
||
jetcache.statIntervalMinutes=15
|
||
jetcache.areaInCacheName=false
|
||
jetcache.penetrationProtect=true
|
||
jetcache.enableMethodCache=true
|
||
jetcache.local.default.type=caffeine
|
||
jetcache.local.default.keyConvertor=fastjson2
|
||
jetcache.remote.default.type=redis.lettuce
|
||
jetcache.remote.default.keyConvertor=fastjson2
|
||
jetcache.remote.default.valueEncoder=KRYO5
|
||
jetcache.remote.default.valueDecoder=KRYO5
|
||
jetcache.remote.default.broadcastChannel=xdr:norm:v1:cache-invalidate
|