feat(syslog): 完成高吞吐可靠链路与统计存储优化
- 新增接入 WAL、检查点、背压恢复及可靠分发机制，完善 Kafka 幂等发布、分区、重试、DLT 和补偿流程 - 引入规则预编译、批内快照及 JetCache/Caffeine 两级缓存，支持缓存失效广播、影子对账和执行指标 - 升级 InfluxDB V2 原始日志模型，统一接收时间与 Point 时间，支持按日 measurement、UUID 分桶及 V1/V2 双读 - 重构达梦标准化批量写入与统计链路，支持事务内直写、增量投影、幂等控制、窗口水位和日期切换 - 完善告警处理、配置校验、运行监控及测试覆盖
This commit is contained in:
王绍宏
+3
-2
@@ -40,6 +40,7 @@
|
||||
<result column="manager_name" property="managerName" />
|
||||
<result column="today_parse_count" property="todayParseCount" />
|
||||
<result column="today_non_log_count" property="todayNonLogCount" />
|
||||
<result column="today_stats_date" property="todayStatsDate" />
|
||||
<result column="create_dept" property="createDept" />
|
||||
<result column="device_collect_id" property="deviceCollectId" />
|
||||
</resultMap>
|
||||
@@ -51,7 +52,7 @@
|
||||
control_agent_id, license_start_time, license_end_time, is_monitoring,
|
||||
security_scope_id, owner_id, ssh_config_id, status, created_by_id, decode_type,
|
||||
miss_policy, tenant_id, create_time, update_time, create_by, update_by, del_flag,
|
||||
manager_name, today_parse_count, today_non_log_count, create_dept, device_collect_id
|
||||
manager_name, today_parse_count, today_non_log_count, today_stats_date, create_dept, device_collect_id
|
||||
</sql>
|
||||
|
||||
<!-- 根据ID查询 -->
|
||||
@@ -235,4 +236,4 @@
|
||||
WHERE vendor = #{vendor}
|
||||
</select>
|
||||
|
||||
</mapper>
|
||||
</mapper>
|
||||
|
||||
+10
-5
@@ -13,6 +13,7 @@
|
||||
<result column="receive_time" property="receiveTime" />
|
||||
<result column="receive_time_str" property="receiveTimeStr" />
|
||||
<result column="syslog_message" property="syslogMessage" />
|
||||
<result column="push_success" property="pushSuccess" />
|
||||
</resultMap>
|
||||
|
||||
<!-- 插入单条记录 -->
|
||||
@@ -24,7 +25,8 @@
|
||||
device_ip,
|
||||
receive_time,
|
||||
receive_time_str,
|
||||
syslog_message
|
||||
syslog_message,
|
||||
push_success
|
||||
) VALUES (
|
||||
COALESCE(#{createdAt}, SYSDATE),
|
||||
#{deviceCollectId},
|
||||
@@ -32,7 +34,8 @@
|
||||
#{deviceIp},
|
||||
#{receiveTime},
|
||||
#{receiveTimeStr},
|
||||
#{syslogMessage}
|
||||
#{syslogMessage},
|
||||
#{pushSuccess}
|
||||
)
|
||||
</insert>
|
||||
|
||||
@@ -45,7 +48,8 @@
|
||||
device_ip,
|
||||
receive_time,
|
||||
receive_time_str,
|
||||
syslog_message
|
||||
syslog_message,
|
||||
push_success
|
||||
) VALUES
|
||||
<foreach collection="list" item="item" separator=",">
|
||||
(
|
||||
@@ -55,7 +59,8 @@
|
||||
#{item.deviceIp},
|
||||
#{item.receiveTime},
|
||||
#{item.receiveTimeStr},
|
||||
#{item.syslogMessage}
|
||||
#{item.syslogMessage},
|
||||
#{item.pushSuccess}
|
||||
)
|
||||
</foreach>
|
||||
</insert>
|
||||
@@ -161,4 +166,4 @@
|
||||
ORDER BY log_count DESC
|
||||
</select>
|
||||
|
||||
</mapper>
|
||||
</mapper>
|
||||
|
||||
+7
-1
@@ -90,6 +90,12 @@
|
||||
WHERE id in ( select normalize_rule_id from dm_normalize_rule_device where device_id =#{id}) AND del_flag = '0'
|
||||
</select>
|
||||
|
||||
<select id="selectDeviceIdsByRuleId" parameterType="java.lang.Long" resultType="java.lang.Long">
|
||||
SELECT device_id
|
||||
FROM dm_normalize_rule_device
|
||||
WHERE normalize_rule_id = #{ruleId}
|
||||
</select>
|
||||
|
||||
|
||||
<!-- 根据ID和租户ID查询 -->
|
||||
<select id="selectByIdAndTenant" resultMap="BaseResultMap">
|
||||
@@ -157,4 +163,4 @@
|
||||
WHERE id = #{id} AND tenant_id = #{tenantId} AND del_flag = '0'
|
||||
</update>
|
||||
|
||||
</mapper>
|
||||
</mapper>
|
||||
|
||||
+3
-1
@@ -376,6 +376,7 @@
|
||||
<if test="dataMap.destination_system != null">destination_system,</if>
|
||||
<if test="dataMap.etl_host != null">etl_host,</if>
|
||||
<if test="dataMap.normalize_rule_id != null">normalize_rule_id,</if>
|
||||
<if test="dataMap.rule_version != null">rule_version,</if>
|
||||
<if test="dataMap.normalize_rule_name != null">normalize_rule_name,</if>
|
||||
<if test="dataMap.syslog_uuid != null">syslog_uuid,</if>
|
||||
<if test="dataMap.syslog_topic != null">syslog_topic,</if>
|
||||
@@ -751,6 +752,7 @@
|
||||
<if test="dataMap.destination_system != null">#{dataMap.destination_system},</if>
|
||||
<if test="dataMap.etl_host != null">#{dataMap.etl_host},</if>
|
||||
<if test="dataMap.normalize_rule_id != null">#{dataMap.normalize_rule_id},</if>
|
||||
<if test="dataMap.rule_version != null">#{dataMap.rule_version},</if>
|
||||
<if test="dataMap.normalize_rule_name != null">#{dataMap.normalize_rule_name},</if>
|
||||
<if test="dataMap.syslog_uuid != null">#{dataMap.syslog_uuid},</if>
|
||||
<if test="dataMap.syslog_topic != null">#{dataMap.syslog_topic},</if>
|
||||
@@ -786,4 +788,4 @@
|
||||
</foreach>
|
||||
</insert>
|
||||
|
||||
</mapper>
|
||||
</mapper>
|
||||
|
||||
+3
-1
@@ -513,6 +513,7 @@
|
||||
<if test="dataMap.destination_system != null">destination_system,</if>
|
||||
<if test="dataMap.etl_host != null">etl_host,</if>
|
||||
<if test="dataMap.normalize_rule_id != null">normalize_rule_id,</if>
|
||||
<if test="dataMap.rule_version != null">rule_version,</if>
|
||||
<if test="dataMap.normalize_rule_name != null">normalize_rule_name,</if>
|
||||
<if test="dataMap.syslog_uuid != null">syslog_uuid,</if>
|
||||
<if test="dataMap.syslog_topic != null">syslog_topic,</if>
|
||||
@@ -888,6 +889,7 @@
|
||||
<if test="dataMap.destination_system != null">#{dataMap.destination_system},</if>
|
||||
<if test="dataMap.etl_host != null">#{dataMap.etl_host},</if>
|
||||
<if test="dataMap.normalize_rule_id != null">#{dataMap.normalize_rule_id},</if>
|
||||
<if test="dataMap.rule_version != null">#{dataMap.rule_version},</if>
|
||||
<if test="dataMap.normalize_rule_name != null">#{dataMap.normalize_rule_name},</if>
|
||||
<if test="dataMap.syslog_uuid != null">#{dataMap.syslog_uuid},</if>
|
||||
<if test="dataMap.syslog_topic != null">#{dataMap.syslog_topic},</if>
|
||||
@@ -924,4 +926,4 @@
|
||||
</foreach>
|
||||
</insert>
|
||||
|
||||
</mapper>
|
||||
</mapper>
|
||||
|
||||
Reference in New Issue
Block a user