王绍宏
8173c2ff3f
- 入口统计改用独立 Kafka 消费组，移除主消费同步统计写库 - DIRECT 模式入库计数改为事务提交后异步投递，取消事实事务内统计 Outbox 写入 - 采样审计、告警分钟桶通知改为后台处理，异常不阻断主消费或回滚告警 - RAW 基础业务批量默认 500，按保留率动态扩大，上限 3000 - 限制业务块字节数及规则任务数量，保留告警全量处理和整次 poll 完成后提交 - 细化 SLOW_SYSLOG_BATCH，补充批次关联、线程排队、数据库执行及提交耗时 - 更新 v4 升级说明，明确异步统计损失风险及现场验证方法 验证：JDK 8 构建成功，232 项测试通过。 升级：复用现有表和 Kafka Topic，无新增建表或建 Topic 脚本要求。
362 lines
22 KiB
Properties
362 lines
22 KiB
Properties
#Server Configuration
|
||
server.port=8289
|
||
server.servlet.context-path=/xdrrule
|
||
#server.address=0.0.0.0
|
||
server.tomcat.uri-encoding=UTF-8
|
||
server.error.include-message=always
|
||
server.error.include-binding-errors=always
|
||
|
||
#run.environment: dev|test|pro
|
||
server.run.environment=dev
|
||
|
||
# Syslog Server Configuration
|
||
syslog.tcp.port=514
|
||
syslog.udp.port=514
|
||
syslog.max.frame.length=65536
|
||
syslog.buffer.size=1000
|
||
syslog.sm4.generateKey=f79548ab6fa8a304fc0115e17230358a
|
||
|
||
# InfluxDB 2.7 Configuration
|
||
influxdb.url=http://192.168.4.99:8087
|
||
influxdb.token=JsUyvU8vhQEFlMM_el4Drm87fyh707IhwJNsPBucPghSdbVmdQ-UvmPcyP5NTzWxsRfEz0T51Rw4ebZUuUrmZg==
|
||
influxdb.org=influxdb
|
||
influxdb.bucket=yelangbucket
|
||
influxdb.batch.size=1000
|
||
influxdb.flush.interval=1000
|
||
influxdb.retry.attempts=3
|
||
influxdb.retry.delay=1000
|
||
# 原始 Syslog V2 分阶段开关。兼容部署先保持 V1/V1_ONLY，切换时统一写 V2。
|
||
influxdb.raw.write-schema=${INFLUXDB_RAW_WRITE_SCHEMA:V2}
|
||
influxdb.raw.read-mode=${INFLUXDB_RAW_READ_MODE:V1_V2}
|
||
influxdb.raw.v1-bucket=${INFLUXDB_RAW_V1_BUCKET:${influxdb.bucket}}
|
||
influxdb.raw.v2-bucket=${INFLUXDB_RAW_V2_BUCKET:yelangbucket_v2}
|
||
influxdb.raw.uuid-bucket-count=${INFLUXDB_RAW_UUID_BUCKET_COUNT:4096}
|
||
influxdb.raw.legacy-point-time-fallback-enabled=${INFLUXDB_RAW_LEGACY_POINT_TIME_FALLBACK_ENABLED:true}
|
||
influxdb.raw.query-max-rows=${INFLUXDB_RAW_QUERY_MAX_ROWS:10000}
|
||
influxdb.raw.query-max-response-bytes=${INFLUXDB_RAW_QUERY_MAX_RESPONSE_BYTES:10485760}
|
||
# 0 表示不限制；启用后最终 message（含截断标记）不超过该 UTF-8 字节数。
|
||
influxdb.raw.message-max-utf8-bytes=${INFLUXDB_RAW_MESSAGE_MAX_UTF8_BYTES:262144}
|
||
# Influx 同步写按 Line Protocol 估算字节切批，避免少量大日志形成超大 HTTP 请求。
|
||
influxdb.raw.write-batch-max-bytes=${INFLUXDB_RAW_WRITE_BATCH_MAX_BYTES:8388608}
|
||
# InfluxDB 2.7 连接超时配置
|
||
influxdb.connection.timeout=30s
|
||
influxdb.connection.read-timeout=30s
|
||
influxdb.connection.write-timeout=60s
|
||
# 原文写是旁路：单阶段最多等待 5 秒，失败后 30 秒内快速跳过，不能阻塞 Kafka 主消费。
|
||
influxdb.raw.write-request-timeout=${INFLUXDB_RAW_WRITE_REQUEST_TIMEOUT:5s}
|
||
influxdb.raw.failure-backoff-ms=${INFLUXDB_RAW_FAILURE_BACKOFF_MS:30000}
|
||
# Application Configuration
|
||
app.worker.threads=8
|
||
app.max.queue.size=10000
|
||
app.metrics.enabled=true
|
||
|
||
#database Configuration
|
||
spring.datasource.url=jdbc:dm://192.168.4.99:5237
|
||
spring.datasource.username=SYSDBA
|
||
spring.datasource.password=caZ2TcmXNSW8L2Ap
|
||
spring.datasource.driver-class-name=dm.jdbc.driver.DmDriver
|
||
spring.datasource.hikari.schema=ECOSYS
|
||
# mybatis Configuration
|
||
mybatis.mapper-locations=classpath:mapper/*.xml
|
||
mybatis.type-aliases-package=com.common.entity
|
||
#mybatis handler 类
|
||
mybatis.configuration.default-statement-timeout=30
|
||
mybatis.configuration.default-fetch-size=1000
|
||
mybatis.configuration.map-underscore-to-camel-case=true
|
||
# 批次列并集写入会为部分行绑定空字段；达梦对未指定类型的 NULL 参数兼容性较差，统一使用 JDBC NULL。
|
||
mybatis.configuration.jdbc-type-for-null=NULL
|
||
mybatis.type-handlers-package=com.Modules.etl.handler
|
||
mybatis-plus.configuration.map-underscore-to-camel-case=true
|
||
mybatis-plus.configuration.jdbc-type-for-null=NULL
|
||
mybatis-plus.type-handlers-package=com.Modules.etl.handler
|
||
|
||
# kafka Configuration
|
||
spring.kafka.consumer.bootstrap-servers=192.168.4.99:9092
|
||
spring.kafka.consumer.group-id=test-group-1
|
||
spring.kafka.consumer.auto-offset-reset=latest
|
||
spring.kafka.consumer.enable-auto-commit=false
|
||
spring.kafka.consumer.auto-commit-interval=1000
|
||
spring.kafka.consumer.topic=test-topic
|
||
spring.kafka.consumer.large-topic=${KAFKA_CONSUMER_LARGE_TOPIC:${spring.kafka.consumer.topic}.large}
|
||
# consumer-rule 是当前活跃的 Kafka 原始日志标准化节点，默认启动监听。
|
||
syslog.raw-consumer.enabled=true
|
||
# Raw Topic 当前有 3 个分区，使用独立工厂显式创建 3 个 consumer；可通过环境变量灰度调整。
|
||
syslog.raw-listener.concurrency=${SYSLOG_RAW_LISTENER_CONCURRENCY:3}
|
||
# 端到端批次耗时达到该阈值时，打印业务各阶段及 Kafka offset 同步提交耗时；0 表示关闭。
|
||
syslog.slow-batch.threshold-ms=${SYSLOG_SLOW_BATCH_THRESHOLD_MS:500}
|
||
# 普通业务基础批量500（保留旧配置名）；不再表示底层poll上限。已有同名环境变量仍覆盖基础值。
|
||
# 目标业务批量=ceil(基础条数*10000/max(普通保留万分比,非标保留万分比))，并受下面3000上限限制。
|
||
syslog.raw-listener.max-poll-records=${SYSLOG_RAW_LISTENER_MAX_POLL_RECORDS:500}
|
||
# 底层RAW max.poll.records固定为此值；无积压时也可能poll到3000，再按500分块，全部处理后才ACK。
|
||
# 基础值必须大于0且不超过上限；Retry/大日志/独立统计不采用此自适应策略。
|
||
syslog.raw-listener.adaptive-max-records=${SYSLOG_RAW_LISTENER_ADAPTIVE_MAX_RECORDS:3000}
|
||
# 大批次按 Kafka value 序列化字节拆成16MiB业务块，全部完成后提交本次poll；单条不会切断。
|
||
syslog.raw-listener.processing-batch-max-bytes=${SYSLOG_RAW_PROCESSING_BATCH_MAX_BYTES:16777216}
|
||
# 每个业务块只向共享规则线程池提交8个worker，避免2000条对应2000个排队任务。
|
||
syslog.normalization.workers-per-batch=${SYSLOG_NORMALIZATION_WORKERS_PER_BATCH:8}
|
||
# 连续性优先：特殊/系统性异常优先写 DLT；即使 DLT 失败也记录告警并推进原 offset，避免阻塞分区。
|
||
syslog.failure-continuity.enabled=${SYSLOG_FAILURE_CONTINUITY_ENABLED:true}
|
||
# DLT 故障时最多快速等待 1 秒；首次失败后整批直接记录并放行。
|
||
syslog.failure-continuity.dlt-publish-timeout-ms=${SYSLOG_FAILURE_CONTINUITY_DLT_PUBLISH_TIMEOUT_MS:1000}
|
||
# Kafka 积压降级采样默认开启；升级应用前必须先创建 syslog_backlog_retention_hourly。
|
||
syslog.backlog-sampling.enabled=${SYSLOG_BACKLOG_SAMPLING_ENABLED:true}
|
||
# 使用 syslog-serve 生成的 receive_time 判断端到端延迟；15 分钟内全量保留。
|
||
# 调试修改：如需改为3分钟，将默认值 900000 改为 180000；若容器设置了
|
||
# SYSLOG_BACKLOG_SAMPLING_START_LAG_MS，则环境变量优先，必须同步修改或删除该环境变量。
|
||
syslog.backlog-sampling.start-lag-ms=${SYSLOG_BACKLOG_SAMPLING_START_LAG_MS:900000}
|
||
# 从免采样窗口结束后开始分级，延迟达到 1 小时时降到最低保留率。
|
||
syslog.backlog-sampling.lag-threshold-ms=${SYSLOG_BACKLOG_SAMPLING_LAG_THRESHOLD_MS:3600000}
|
||
# 万分比：这是达到延迟阈值后的最低保留率；识别出的告警永远全量保留。
|
||
syslog.backlog-sampling.normal-retain-bps=${SYSLOG_BACKLOG_SAMPLING_NORMAL_RETAIN_BPS:1000}
|
||
syslog.backlog-sampling.non-normal-retain-bps=${SYSLOG_BACKLOG_SAMPLING_NON_NORMAL_RETAIN_BPS:2000}
|
||
# 默认十级下降：15 分钟到 1 小时的 45 分钟区间，每 4.5 分钟调整一级。
|
||
syslog.backlog-sampling.degrade-steps=${SYSLOG_BACKLOG_SAMPLING_DEGRADE_STEPS:10}
|
||
# 调整保留率时同步修改策略版本，便于小时审计区分不同策略。
|
||
syslog.backlog-sampling.policy-version=${SYSLOG_BACKLOG_SAMPLING_POLICY_VERSION:v2}
|
||
syslog.backlog-sampling.audit-timeout-seconds=${SYSLOG_BACKLOG_SAMPLING_AUDIT_TIMEOUT_SECONDS:5}
|
||
syslog.backlog-sampling.audit-retry-backoff-ms=${SYSLOG_BACKLOG_SAMPLING_AUDIT_RETRY_BACKOFF_MS:60000}
|
||
# 大日志使用独立 Topic、消费组和 ConsumerFactory；建议生产环境独立 JVM 部署。
|
||
syslog.large-listener.enabled=${SYSLOG_LARGE_LISTENER_ENABLED:true}
|
||
syslog.large-listener.group-id=${SYSLOG_LARGE_LISTENER_GROUP_ID:${spring.kafka.consumer.group-id}-large}
|
||
syslog.large-listener.concurrency=${SYSLOG_LARGE_LISTENER_CONCURRENCY:1}
|
||
syslog.large-listener.max-poll-records=${SYSLOG_LARGE_LISTENER_MAX_POLL_RECORDS:5}
|
||
syslog.large-listener.max-partition-fetch-bytes=${SYSLOG_LARGE_LISTENER_MAX_PARTITION_FETCH_BYTES:33554432}
|
||
syslog.large-listener.fetch-max-bytes=${SYSLOG_LARGE_LISTENER_FETCH_MAX_BYTES:67108864}
|
||
syslog.large-listener.processing-batch-max-bytes=${SYSLOG_LARGE_PROCESSING_BATCH_MAX_BYTES:8388608}
|
||
# 大日志规则计算不复用普通日志线程池，过载时由 Kafka offset 背压。
|
||
syslog.large-processor.thread-pool.core-pool-size=${SYSLOG_LARGE_PROCESSOR_CORE_POOL_SIZE:1}
|
||
syslog.large-processor.thread-pool.max-pool-size=${SYSLOG_LARGE_PROCESSOR_MAX_POOL_SIZE:2}
|
||
syslog.large-processor.thread-pool.queue-capacity=${SYSLOG_LARGE_PROCESSOR_QUEUE_CAPACITY:10}
|
||
# P0：Retry/DLT 发布与原始消费使用同一 Kafka 集群；Topic 名称全部可由环境变量覆盖。
|
||
spring.kafka.bootstrap-servers=${spring.kafka.consumer.bootstrap-servers}
|
||
spring.kafka.producer.acks=all
|
||
spring.kafka.producer.properties.enable.idempotence=true
|
||
spring.kafka.producer.properties.delivery.timeout.ms=120000
|
||
spring.kafka.producer.properties.request.timeout.ms=30000
|
||
# Retry/DLT 信封也可能携带大日志密文，不能保留 Kafka 默认 1MiB。
|
||
spring.kafka.producer.properties.max.request.size=${KAFKA_RELIABILITY_MAX_REQUEST_SIZE:33554432}
|
||
syslog.reliability.retry-1m-topic=${spring.kafka.consumer.topic}.RETRY.1M
|
||
syslog.reliability.retry-10m-topic=${spring.kafka.consumer.topic}.RETRY.10M
|
||
syslog.reliability.retry-1h-topic=${spring.kafka.consumer.topic}.RETRY.1H
|
||
syslog.reliability.dlt-topic=${spring.kafka.consumer.topic}.DLT
|
||
syslog.reliability.replay-topic=${spring.kafka.consumer.topic}.REPLAY
|
||
syslog.reliability.publish-timeout-ms=30000
|
||
syslog.reliability.consumer-version=${APP_VERSION:1.0.0}
|
||
# Retry/Replay 使用独立消费组和线程，未到期重试不会占用 Raw listener。
|
||
syslog.reliability.consumer-group-id=${spring.kafka.consumer.group-id}-retry
|
||
syslog.reliability.consumer.enabled=true
|
||
syslog.reliability.listener.concurrency=1
|
||
# 大日志 Retry/DLT 不回流普通重试通道，避免再次形成队头阻塞。
|
||
syslog.large-reliability.retry-1m-topic=${spring.kafka.consumer.large-topic}.RETRY.1M
|
||
syslog.large-reliability.retry-10m-topic=${spring.kafka.consumer.large-topic}.RETRY.10M
|
||
syslog.large-reliability.retry-1h-topic=${spring.kafka.consumer.large-topic}.RETRY.1H
|
||
syslog.large-reliability.dlt-topic=${spring.kafka.consumer.large-topic}.DLT
|
||
syslog.large-reliability.replay-topic=${spring.kafka.consumer.large-topic}.REPLAY
|
||
syslog.large-reliability.consumer-group-id=${spring.kafka.consumer.group-id}-large-retry
|
||
syslog.large-reliability.consumer.enabled=true
|
||
|
||
|
||
# 通用消费者（如Retry）的上限；RAW底层上限用adaptive-max-records，业务基础值用syslog.raw-listener.max-poll-records。
|
||
# 使用 prod profile 时 application-prod.properties 中存在同名配置，必须同步修改，否则 prod 值会覆盖本项。
|
||
spring.kafka.consumer.max-poll-records=500
|
||
spring.kafka.consumer.properties.max.poll.interval.ms=900000
|
||
spring.kafka.consumer.properties.session.timeout.ms=45000
|
||
|
||
spring.kafka.consumer.fetch-min-size= 1048576
|
||
spring.kafka.listener.ack-mode= manual
|
||
spring.kafka.listener.concurrency=2
|
||
spring.kafka.listener.type=batch
|
||
|
||
|
||
# 定时任务配置
|
||
spring.task.scheduling.pool.size=10
|
||
|
||
# 标准化告警事实与迟到重放通知使用同一个达梦事务。
|
||
analysis.late-replay.outbox-enabled=${ANALYSIS_LATE_REPLAY_OUTBOX_ENABLED:true}
|
||
# 告警提交后后台写分钟桶的事务超时；不在主消费线程等待。JDBC/连接池自身超时仍独立生效。
|
||
analysis.late-replay.notify-timeout-seconds=${ANALYSIS_LATE_REPLAY_NOTIFY_TIMEOUT_SECONDS:5}
|
||
|
||
# 日志配置
|
||
logging.level.com.common.schedule=INFO
|
||
logging.level.com.common.service=INFO
|
||
|
||
# 分区表检查配置
|
||
partition.check.tomorrow.enabled=true
|
||
partition.check.future.days=7
|
||
partition.auto.create=true
|
||
|
||
|
||
# 生产环境缓存配置
|
||
spring.redis.host=localhost
|
||
spring.redis.port=6379
|
||
# 密码（如果没有设置密码，可以省略）
|
||
spring.redis.password=
|
||
spring.redis.database=0
|
||
spring.redis.timeout=5000
|
||
#spring.redis.password=${REDIS_PASSWORD:default_prod_password}
|
||
|
||
spring.redis.lettuce.pool.max-active=20
|
||
spring.redis.lettuce.pool.max-wait=5000
|
||
spring.redis.lettuce.pool.max-idle=10
|
||
spring.redis.lettuce.pool.min-idle=5
|
||
|
||
# 生产环境缓存时间较长
|
||
spring.cache.redis.time-to-live=3600000
|
||
|
||
# 应用处理器配置
|
||
app.processor.thread-pool.core-pool-size=10
|
||
app.processor.thread-pool.max-pool-size=20
|
||
app.processor.thread-pool.queue-capacity=2000
|
||
app.processor.thread-pool.keep-alive-seconds=60
|
||
app.processor.batch-size=100
|
||
app.processor.process-timeout-ms=120000
|
||
|
||
# 达梦标准化表默认启用批量写入；现场需要紧急回退时可显式切换为 SINGLE。
|
||
# BATCH 模式按目标表和 log_time 日期分组，按小批次实际非空字段并集执行 JDBC Batch；
|
||
# 失败时只对数据类错误有限二分；
|
||
# 连接、超时或提交结果未知会立即停止并保留 Kafka offset，禁止放大为逐条请求风暴。
|
||
dm.standardized-write.mode=BATCH
|
||
dm.standardized-write.batch-size=${DM_STANDARDIZED_WRITE_BATCH_SIZE:500}
|
||
dm.standardized-write.writer-concurrency=2
|
||
dm.standardized-write.writer-queue-capacity=100
|
||
# 标准化大字段和非标原文均在达梦持久化边界截断；0 表示不限制。
|
||
dm.normal.large-field-max-utf8-bytes=${DM_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES:262144}
|
||
dm.normal.large-fields=${DM_NORMAL_LARGE_FIELDS:http_req_body,http_resp_body,http_req_header,http_resp_header,detail,description,cmdline,shell_cmdline,parent_cmdline,mail_message}
|
||
dm.non-normal.large-field-max-utf8-bytes=${DM_NON_NORMAL_LARGE_FIELD_MAX_UTF8_BYTES:262144}
|
||
dm.standardized-write.transaction-timeout-seconds=45
|
||
dm.standardized-write.kafka-batch-deadline-ms=90000
|
||
dm.standardized-write.max-split-depth=10
|
||
dm.standardized-write.max-fallback-operations=100
|
||
dm.standardized-write.circuit-breaker-failure-threshold=3
|
||
dm.standardized-write.circuit-breaker-backoff-ms=30000
|
||
dm.non-normal-write.batch-size=${DM_NON_NORMAL_WRITE_BATCH_SIZE:500}
|
||
dm.non-normal-write.queue-capacity=${DM_NON_NORMAL_WRITE_QUEUE_CAPACITY:500}
|
||
dm.non-normal-write.max-concurrency-retries=${DM_NON_NORMAL_WRITE_MAX_CONCURRENCY_RETRIES:3}
|
||
dm.non-normal-write.concurrency-retry-backoff-ms=${DM_NON_NORMAL_WRITE_CONCURRENCY_RETRY_BACKOFF_MS:50}
|
||
|
||
# ETL配置
|
||
etl.batch.page-size=1000
|
||
etl.batch.insert-batch-size=500
|
||
etl.schedule.cron=0 0 2 * * ?
|
||
|
||
# ============================================
|
||
# HikariCP Connection Pool Configuration
|
||
# ============================================
|
||
spring.datasource.hikari.maximum-pool-size=50
|
||
spring.datasource.hikari.minimum-idle=5
|
||
spring.datasource.hikari.connection-timeout=30000
|
||
spring.datasource.hikari.idle-timeout=600000
|
||
spring.datasource.hikari.max-lifetime=900000
|
||
spring.datasource.hikari.connection-test-query=SELECT 1 FROM DUAL
|
||
spring.datasource.hikari.validation-timeout=5000
|
||
spring.datasource.hikari.leak-detection-threshold=30000
|
||
spring.datasource.hikari.pool-name=HikariPool-SyslogConsumer-rule
|
||
spring.datasource.hikari.auto-commit=false
|
||
# DIRECT 连续消费优先：事实事务不再写统计Outbox/日统计表，提交后通过有界内存队列异步发布统计。
|
||
# 首次部署 DIRECT 前必须先执行 08/09/10/11 脚本并按设计文档排空旧 pending。
|
||
stats.write-mode=DIRECT
|
||
stats.incremental.capture-enabled=false
|
||
stats.ingress-metadata.enabled=true
|
||
# 入口统计独立消费原始Topic，直接聚合写统计表；主消费不再写入口Outbox。
|
||
# 各节点必须使用同一组名，且不得与业务主消费组相同。
|
||
stats.ingress-metadata.group-id=${STATS_INGRESS_GROUP_ID:${spring.kafka.consumer.group-id}-ingress-v1}
|
||
stats.ingress-metadata.large-group-id=${STATS_INGRESS_LARGE_GROUP_ID:${stats.ingress-metadata.group-id}-large}
|
||
stats.ingress-metadata.max-poll-records=${STATS_INGRESS_MAX_POLL_RECORDS:2000}
|
||
# 新组默认从Kafka仍保留的最早消息开始，确保不跳过历史接收时间；已有offset继续原位置。
|
||
stats.ingress-metadata.auto-offset-reset=${STATS_INGRESS_AUTO_OFFSET_RESET:earliest}
|
||
stats.incremental.read-enabled=true
|
||
stats.incremental.transaction-timeout-seconds=45
|
||
stats.device-receive.change-capture-enabled=false
|
||
stats.direct.device-table=device_daily_stats
|
||
stats.direct.rule-table=normalize_rule_daily_stats
|
||
stats.direct.shadow-device-table=device_daily_stats_direct_shadow
|
||
stats.direct.shadow-rule-table=normalize_rule_daily_stats_direct_shadow
|
||
stats.direct.control-table=stats_direct_control
|
||
stats.direct.today-stats-date-enabled=true
|
||
stats.direct.max-concurrency-retries=${STATS_DIRECT_MAX_CONCURRENCY_RETRIES:3}
|
||
stats.direct.concurrency-retry-backoff-ms=${STATS_DIRECT_CONCURRENCY_RETRY_BACKOFF_MS:50}
|
||
stats.direct.transaction-log-enabled=${STATS_DIRECT_TRANSACTION_LOG_ENABLED:false}
|
||
stats.direct.day-rollover-check-cron=5 * * * * ?
|
||
stats.business-zone=Asia/Shanghai
|
||
stats.fact-only.confirm-rebuild-required=false
|
||
# 日统计投影完成后按维度脏键异步刷新业务展示字段。
|
||
# 三个实例必须配置相同 topic/group-id，Topic 分区数建议与原始 Topic 相同。
|
||
stats.materialization.enabled=${STATS_MATERIALIZATION_ENABLED:true}
|
||
stats.materialization.topic=${STATS_MATERIALIZATION_TOPIC:${spring.kafka.consumer.topic}.STATS.MATERIALIZE}
|
||
stats.materialization.group-id=${STATS_MATERIALIZATION_GROUP_ID:${spring.kafka.consumer.group-id}-stats-materialize}
|
||
# 每个实例一个物化 consumer；现场三个实例合计三个，足以处理低基数脏键并减少达梦并发。
|
||
stats.materialization.listener-concurrency=${STATS_MATERIALIZATION_LISTENER_CONCURRENCY:1}
|
||
stats.materialization.publish-interval-ms=${STATS_MATERIALIZATION_PUBLISH_INTERVAL_MS:200}
|
||
stats.materialization.publish-timeout-ms=${STATS_MATERIALIZATION_PUBLISH_TIMEOUT_MS:10000}
|
||
stats.materialization.publish-max-batch-size=${STATS_MATERIALIZATION_PUBLISH_MAX_BATCH_SIZE:1000}
|
||
stats.materialization.max-concurrency-retries=${STATS_MATERIALIZATION_MAX_CONCURRENCY_RETRIES:3}
|
||
stats.materialization.concurrency-retry-backoff-ms=${STATS_MATERIALIZATION_CONCURRENCY_RETRY_BACKOFF_MS:50}
|
||
stats.materialization.reconcile-initial-delay-ms=${STATS_MATERIALIZATION_RECONCILE_INITIAL_DELAY_MS:30000}
|
||
stats.materialization.reconcile-interval-ms=${STATS_MATERIALIZATION_RECONCILE_INTERVAL_MS:60000}
|
||
|
||
# 事实提交后只做内存offer，独立线程发送到统计Kafka；满队列/进程退出可能少统计，不阻碍事实消费。
|
||
# enabled=false 会跳过统计并记录损失，绝不回退为同步统计写库。正常保持true。
|
||
# 旧Outbox继续后台排空，applied表仍用于统计事件去重，不要删表。
|
||
stats.projection-async.enabled=${STATS_PROJECTION_ASYNC_ENABLED:true}
|
||
# 队列最多1000个批次聚合事件；含正在重试事件的聚合分组总数最多50000，限制内存占用。
|
||
stats.projection-async.memory-queue-capacity=${STATS_PROJECTION_MEMORY_QUEUE_CAPACITY:1000}
|
||
stats.projection-async.memory-max-pending-groups=${STATS_PROJECTION_MEMORY_MAX_PENDING_GROUPS:50000}
|
||
stats.projection-async.outbox-table=${STATS_PROJECTION_OUTBOX_TABLE:stats_projection_outbox}
|
||
stats.projection-async.applied-table=${STATS_PROJECTION_APPLIED_TABLE:stats_projection_applied_event}
|
||
stats.projection-async.topic=${STATS_PROJECTION_TOPIC:${spring.kafka.consumer.topic}.STATS.PROJECTION}
|
||
stats.projection-async.group-id=${STATS_PROJECTION_GROUP_ID:${spring.kafka.consumer.group-id}-stats-projection}
|
||
stats.projection-async.relay-interval-ms=${STATS_PROJECTION_RELAY_INTERVAL_MS:200}
|
||
stats.projection-async.relay-batch-size=${STATS_PROJECTION_RELAY_BATCH_SIZE:100}
|
||
stats.projection-async.publish-timeout-ms=${STATS_PROJECTION_PUBLISH_TIMEOUT_MS:10000}
|
||
stats.projection-async.retry-backoff-ms=${STATS_PROJECTION_RETRY_BACKOFF_MS:1000}
|
||
stats.projection-async.redelivery-interval-ms=${STATS_PROJECTION_REDELIVERY_INTERVAL_MS:60000}
|
||
stats.projection-async.max-concurrency-retries=${STATS_PROJECTION_MAX_CONCURRENCY_RETRIES:3}
|
||
stats.projection-async.concurrency-retry-backoff-ms=${STATS_PROJECTION_CONCURRENCY_RETRY_BACKOFF_MS:50}
|
||
stats.projection-async.retention-days=${STATS_PROJECTION_RETENTION_DAYS:14}
|
||
stats.projection-async.cleanup-initial-delay-ms=${STATS_PROJECTION_CLEANUP_INITIAL_DELAY_MS:3600000}
|
||
stats.projection-async.cleanup-interval-ms=${STATS_PROJECTION_CLEANUP_INTERVAL_MS:3600000}
|
||
|
||
# 标准化规则编译与批次缓存灰度开关
|
||
normalization.jetcache.enabled=true
|
||
normalization.rule-source-cache.enabled=true
|
||
normalization.compiled-rule.enabled=true
|
||
normalization.compiled-rule.shadow-compare-enabled=false
|
||
normalization.compiled-rule.shadow-sample-rate=0
|
||
normalization.compiled-rule.local-limit=10000
|
||
normalization.compiled-rule.expire-after-access-minutes=30
|
||
normalization.compiled-rule.failure-cache-seconds=10
|
||
normalization.compiled-rule.shadow-timeout-ms=1000
|
||
normalization.compiled-rule.rule-detail-log-enabled=true
|
||
normalization.batch-enrichment-cache.enabled=true
|
||
normalization.message-parse-cache.enabled=true
|
||
# 同设备多规则唯一归属：默认开启，最终只允许一条胜出规则进入映射和入库。
|
||
normalization.rule-selection.enabled=true
|
||
# 仅在 enabled=false 时用于后台观察新选择结果，不改变旧路径的实际输出。
|
||
normalization.rule-selection.shadow-enabled=true
|
||
# 并列策略：LOWEST_RULE_ID=稳定选择最小规则ID；STRICT=业务结果冲突时转为一条非标记录。
|
||
normalization.rule-selection.ambiguous-fallback=LOWEST_RULE_ID
|
||
# 只记录 event hash、候选规则ID和分数等脱敏详情，不记录原始日志。
|
||
normalization.rule-selection.detail-log-enabled=false
|
||
# 优先使用 rule_content.match_params 中的显式格式条件。
|
||
normalization.rule-selection.explicit-enabled=true
|
||
# 没有显式命中时，允许未配置 match_params 的历史规则按自动字段指纹选优。
|
||
normalization.rule-selection.automatic-fallback-enabled=true
|
||
normalization.rule-source-cache.local-ttl-seconds=30
|
||
normalization.rule-source-cache.remote-ttl-seconds=300
|
||
normalization.rule-source-cache.empty-ttl-seconds=10
|
||
normalization.enrichment-cache.local-ttl-seconds=60
|
||
normalization.enrichment-cache.remote-ttl-seconds=3600
|
||
|
||
# JetCache Redis URI 由 JetCacheRedisEnvironmentPostProcessor 从 spring.redis.* 派生。
|
||
jetcache.statIntervalMinutes=15
|
||
jetcache.areaInCacheName=false
|
||
jetcache.penetrationProtect=true
|
||
jetcache.enableMethodCache=true
|
||
jetcache.local.default.type=caffeine
|
||
jetcache.local.default.keyConvertor=fastjson2
|
||
jetcache.remote.default.type=redis.lettuce
|
||
jetcache.remote.default.keyConvertor=fastjson2
|
||
jetcache.remote.default.valueEncoder=KRYO5
|
||
jetcache.remote.default.valueDecoder=KRYO5
|
||
jetcache.remote.default.broadcastChannel=xdr:norm:v1:cache-invalidate
|