feat(syslog): 优化大日志通道并增加异常连续消费兜底

- 128 KiB 及以上日志默认进入大日志 Topic - InfluxDB 原文及达梦大字段超过 256 KiB 时截断保存 - 截断内容追加 HB_XDR_TRUNCATED 标记 - InfluxDB 写入失败时继续执行规则解析和达梦写入 - 未知或系统性异常优先写入普通/大日志 DLT - DLT 写入失败时记录特殊告警并提交 offset，避免阻塞后续消息 - DLT 失败等待时间默认限制为 1 秒 - 增加连续性放行、DLT 成功及失败监控指标 - 更新 Docker Compose、v3 升级文档及相关测试
2026-09-12 19:22:52 +08:00
parent 3b6c9e4714
commit 0e9e6656db
64 changed files with 3189 additions and 63 deletions
@@ -42,6 +42,11 @@ class SyslogFailureRouterReliabilityTest {
        ReflectionTestUtils.setField(router, "retry10mTopic", "raw.RETRY.10M");
        ReflectionTestUtils.setField(router, "retry1hTopic", "raw.RETRY.1H");
        ReflectionTestUtils.setField(router, "dltTopic", "raw.DLT");
+        ReflectionTestUtils.setField(router, "largeOriginalTopic", "raw.large");
+        ReflectionTestUtils.setField(router, "largeRetry1mTopic", "raw.large.RETRY.1M");
+        ReflectionTestUtils.setField(router, "largeRetry10mTopic", "raw.large.RETRY.10M");
+        ReflectionTestUtils.setField(router, "largeRetry1hTopic", "raw.large.RETRY.1H");
+        ReflectionTestUtils.setField(router, "largeDltTopic", "raw.large.DLT");
        ReflectionTestUtils.setField(router, "publishTimeoutMs", 1000L);
        ReflectionTestUtils.setField(router, "consumerVersion", "test");
        record = new ConsumerRecord<>("raw", 2, 42L, "device-1", "ciphertext");
@@ -86,6 +91,22 @@ class SyslogFailureRouterReliabilityTest {
        assertFalse(body.getValue().contains("schema_version"));
    }

+    @Test
+    void largeMessageFailureNeverFallsBackIntoNormalRetryTopic() {
+        ConsumerRecord<String, String> largeRecord = new ConsumerRecord<>(
+                "raw.large", 1, 7L, "event", "large-ciphertext");
+        SettableListenableFuture<SendResult<String, String>> acknowledged = new SettableListenableFuture<>();
+        acknowledged.set(null);
+        when(template.send(eq("raw.large.RETRY.1M"), anyString(), anyString()))
+                .thenReturn(acknowledged);
+
+        SyslogFailureRouter.TerminalState state = router.route(
+                largeRecord, router.resolve(largeRecord), new SocketException("reset"));
+
+        assertEquals(SyslogFailureRouter.TerminalState.RETRY_PUBLISHED, state);
+        verify(template).send(eq("raw.large.RETRY.1M"), anyString(), anyString());
+    }
+
    @Test
    void publishFailureNeverBecomesTerminal() {
        SettableListenableFuture<SendResult<String, String>> failed = new SettableListenableFuture<>();
@@ -4,12 +4,17 @@ import org.junit.jupiter.api.Test;
 import org.springframework.kafka.annotation.KafkaListener;

 import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.List;

+import org.apache.kafka.clients.consumer.ConsumerRecord;
+
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import com.common.service.StatsMaterializationConsumer;
+import com.common.service.StatsProjectionDeltaConsumer;

 /** 防止多节点 consumer-rule 的监听器 ID 或显式消费组在后续合并中回退。 */
 class SyslogKafkaTopologyTest {
@@ -50,6 +55,56 @@ class SyslogKafkaTopologyTest {
        }, listener.topics());
    }

+    @Test
+    void largeRawListenerUsesDedicatedTopicGroupAndFactory() throws Exception {
+        Method method = SysLogProcessor.class.getMethod(
+                "consumeLargeMessages", List.class,
+                org.springframework.kafka.support.Acknowledgment.class);
+        KafkaListener listener = method.getAnnotation(KafkaListener.class);
+
+        assertNotNull(listener);
+        assertEquals("syslog-consumer-rule-raw-large", listener.id());
+        assertEquals("${syslog.large-listener.group-id:${spring.kafka.consumer.group-id}-large}",
+                listener.groupId());
+        assertEquals("largeRawKafkaListenerContainerFactory", listener.containerFactory());
+        assertArrayEquals(new String[]{"${spring.kafka.consumer.large-topic:agent-syslog-topic.large}"},
+                listener.topics());
+    }
+
+    @Test
+    void largeRetryReplayListenerIsAlsoIsolated() throws Exception {
+        Method method = SysLogProcessor.class.getMethod(
+                "consumeLargeRetryAndReplay", List.class,
+                org.springframework.kafka.support.Acknowledgment.class);
+        KafkaListener listener = method.getAnnotation(KafkaListener.class);
+
+        assertNotNull(listener);
+        assertEquals("syslog-consumer-rule-large-retry-replay", listener.id());
+        assertEquals("largeRawKafkaListenerContainerFactory", listener.containerFactory());
+        assertArrayEquals(new String[]{
+                "${syslog.large-reliability.retry-1m-topic:${spring.kafka.consumer.large-topic}.RETRY.1M}",
+                "${syslog.large-reliability.retry-10m-topic:${spring.kafka.consumer.large-topic}.RETRY.10M}",
+                "${syslog.large-reliability.retry-1h-topic:${spring.kafka.consumer.large-topic}.RETRY.1H}",
+                "${syslog.large-reliability.replay-topic:${spring.kafka.consumer.large-topic}.REPLAY}"
+        }, listener.topics());
+    }
+
+    @Test
+    void largePollIsPartitionedByUtf8BytesInsteadOfOnlyRecordCount() {
+        List<ConsumerRecord<String, String>> records = Arrays.asList(
+                new ConsumerRecord<>("large", 0, 0L, "k1", "中文"),
+                new ConsumerRecord<>("large", 0, 1L, "k2", "12"),
+                new ConsumerRecord<>("large", 0, 2L, "k3", "🙂"));
+
+        List<List<ConsumerRecord<String, String>>> chunks =
+                SysLogProcessor.partitionRecordsByBytes(records, 7L);
+
+        assertEquals(2, chunks.size());
+        assertEquals(1, chunks.get(0).size());
+        assertEquals(2, chunks.get(1).size());
+        assertEquals(6, chunks.get(0).get(0).value().getBytes(StandardCharsets.UTF_8).length);
+    }
+
    @Test
    void statsMaterializerUsesDedicatedTopicAndGroup() throws Exception {
        Method method = StatsMaterializationConsumer.class.getMethod(
@@ -65,4 +120,20 @@ class SyslogKafkaTopologyTest {
                "${stats.materialization.topic:${spring.kafka.consumer.topic}.STATS.MATERIALIZE}"
        }, listener.topics());
    }
+
+    @Test
+    void statsProjectionUsesSeparateSingleLaneTopicAndGroup() throws Exception {
+        Method method = StatsProjectionDeltaConsumer.class.getMethod(
+                "consume", List.class, org.springframework.kafka.support.Acknowledgment.class);
+        KafkaListener listener = method.getAnnotation(KafkaListener.class);
+
+        assertNotNull(listener);
+        assertEquals("syslog-stats-projection", listener.id());
+        assertEquals("${stats.projection-async.group-id:${spring.kafka.consumer.group-id}-stats-projection}",
+                listener.groupId());
+        assertEquals("statsProjectionKafkaListenerContainerFactory", listener.containerFactory());
+        assertArrayEquals(new String[]{
+                "${stats.projection-async.topic:${spring.kafka.consumer.topic}.STATS.PROJECTION}"
+        }, listener.topics());
+    }
 }
@@ -119,12 +119,14 @@ class DirectStatsProjectionWriterTest {
    }

    @Test
-    void kafkaMaterializationRemovesBusinessTableUpdatesFromFactTransaction() {
+    void durableOutboxRemovesDailyAndBusinessProjectionUpdatesFromFactTransaction() {
        JdbcTemplate jdbc = mock(JdbcTemplate.class);
        StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
        when(publisher.isEnabled()).thenReturn(true);
+        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
+        when(outbox.isEnabled()).thenReturn(true);
        DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(
-                jdbc, new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), publisher);
+                jdbc, new StatsWriteModeProvider("DIRECT"), new DirectStatsWriteMetrics(), publisher, outbox);
        configure(writer, true);
        TransactionSynchronizationManager.setActualTransactionActive(true);

@@ -135,20 +137,19 @@ class DirectStatsProjectionWriterTest {
        row.put("normalize_rule_id", 11L);
        writer.writeStandardizedRow(row);

-        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
-        verify(jdbc, times(2)).batchUpdate(sql.capture(), anyList());
-        assertTrue(sql.getAllValues().get(0).contains("MERGE INTO device_daily_stats"));
-        assertTrue(sql.getAllValues().get(1).contains("MERGE INTO normalize_rule_daily_stats"));
-        verify(publisher).enqueueAfterCommit(anyCollection());
-        assertTrue("KAFKA_DIRTY_KEY".equals(writer.status().get("business_materialization_mode")));
+        verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class));
+        verifyNoInteractions(jdbc);
+        assertTrue("DURABLE_OUTBOX".equals(writer.status().get("daily_projection_write_mode")));
    }

    private DirectStatsProjectionWriter writer(JdbcTemplate jdbc, String mode, boolean readEnabled) {
        StatsWriteModeProvider modeProvider = new StatsWriteModeProvider(mode);
        StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
        when(publisher.isEnabled()).thenReturn(false);
+        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
+        when(outbox.isEnabled()).thenReturn(false);
        DirectStatsProjectionWriter writer = new DirectStatsProjectionWriter(
-                jdbc, modeProvider, new DirectStatsWriteMetrics(), publisher);
+                jdbc, modeProvider, new DirectStatsWriteMetrics(), publisher, outbox);
        configure(writer, readEnabled);
        return writer;
    }
@@ -0,0 +1,62 @@
+package com.common.service;
+
+import com.common.entity.SyslogNonNormalMessage;
+import org.junit.jupiter.api.Test;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class DmLargeTextLimiterTest {
+
+    @Test
+    void normalConfiguredFieldsAreTruncatedButOtherFieldsRemainUntouched() {
+        DmLargeTextLimiter limiter = limiter(96L, "http_req_body,description", 0L);
+        Map<String, Object> row = new HashMap<>();
+        row.put("http_req_body", repeat("中文🙂", 100));
+        row.put("other", repeat("x", 200));
+
+        limiter.truncateNormalRow(row);
+
+        String body = (String) row.get("http_req_body");
+        assertTrue(body.contains("HB_XDR_TRUNCATED"));
+        assertTrue(body.getBytes(StandardCharsets.UTF_8).length <= 96);
+        assertEquals(repeat("x", 200), row.get("other"));
+        assertEquals(1L, limiter.getNormalFieldsTruncated());
+    }
+
+    @Test
+    void allNonNormalLargeColumnsUseTheSameVisibleMarker() {
+        DmLargeTextLimiter limiter = limiter(0L, "", 96L);
+        SyslogNonNormalMessage message = new SyslogNonNormalMessage();
+        message.setSyslogMessage(repeat("原文", 100));
+        message.setHeaderMessage(repeat("头", 100));
+        message.setReasonDetail(repeat("原因", 100));
+
+        limiter.truncateNonNormal(message);
+
+        assertTrue(message.getSyslogMessage().contains("HB_XDR_TRUNCATED"));
+        assertTrue(message.getHeaderMessage().contains("HB_XDR_TRUNCATED"));
+        assertTrue(message.getReasonDetail().contains("HB_XDR_TRUNCATED"));
+        assertEquals(3L, limiter.getNonNormalFieldsTruncated());
+    }
+
+    private DmLargeTextLimiter limiter(long normal, String fields, long nonNormal) {
+        DmLargeTextLimiter limiter = new DmLargeTextLimiter();
+        ReflectionTestUtils.setField(limiter, "normalMaxBytes", normal);
+        ReflectionTestUtils.setField(limiter, "normalFieldsValue", fields);
+        ReflectionTestUtils.setField(limiter, "nonNormalMaxBytes", nonNormal);
+        limiter.validate();
+        return limiter;
+    }
+
+    private static String repeat(String value, int count) {
+        StringBuilder result = new StringBuilder(value.length() * count);
+        for (int i = 0; i < count; i++) result.append(value);
+        return result.toString();
+    }
+}
@@ -2,6 +2,11 @@ package com.common.service;

 import com.common.entity.IngressStatsEvent;
 import org.junit.jupiter.api.Test;
+import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.support.SimpleTransactionStatus;
+import org.springframework.jdbc.core.JdbcTemplate;

 import java.time.LocalDateTime;
 import java.util.Arrays;
@@ -9,6 +14,11 @@ import java.util.ArrayList;
 import java.util.List;

 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;

 class IngressStatsProjectionWriterTest {
    @Test
@@ -41,4 +51,28 @@ class IngressStatsProjectionWriterTest {
        assertEquals(Arrays.asList(8L, 2L, 9L), devices);
        assertEquals(Arrays.asList(3L, 4L, 7L), new ArrayList<>(result.collects.keySet()));
    }
+
+    @Test
+    void directAsyncModeWritesOnlyOneOutboxEvent() {
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        PlatformTransactionManager manager = mock(PlatformTransactionManager.class);
+        when(manager.getTransaction(any(TransactionDefinition.class)))
+                .thenReturn(new SimpleTransactionStatus());
+        StatsProjectionOutboxWriter outbox = mock(StatsProjectionOutboxWriter.class);
+        when(outbox.isEnabled()).thenReturn(true);
+        IngressStatsProjectionWriter writer = new IngressStatsProjectionWriter(
+                jdbc, manager, new StatsWriteModeProvider("DIRECT"),
+                mock(StatsMaterializationPublisher.class), outbox);
+        ReflectionTestUtils.setField(writer, "enabled", true);
+        ReflectionTestUtils.setField(writer, "transactionTimeoutSeconds", 45);
+
+        int groups = writer.writeBatch(Arrays.asList(
+                new IngressStatsEvent("a", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 0)),
+                new IngressStatsEvent("b", 7L, 3L, LocalDateTime.of(2026, 9, 12, 8, 1))));
+
+        assertEquals(2, groups);
+        verify(outbox).append(org.mockito.ArgumentMatchers.any(StatsProjectionOutboxEvent.class));
+        verifyNoInteractions(jdbc);
+        verify(manager).commit(any());
+    }
 }
@@ -0,0 +1,151 @@
+package com.common.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.support.SimpleTransactionStatus;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class StatsProjectionAsyncTest {
+    @AfterEach
+    void clearTransactionState() {
+        TransactionSynchronizationManager.clear();
+    }
+
+    @Test
+    void outboxEventAggregatesOneFactBatchByDimension() {
+        LocalDateTime time = LocalDateTime.of(2026, 9, 12, 8, 0);
+        List<Map<String, Object>> rows = new ArrayList<>();
+        for (long ruleId : Arrays.asList(11L, 12L, 11L)) {
+            Map<String, Object> row = new HashMap<>();
+            row.put("created_at", time);
+            row.put("log_time", time.plusMinutes(ruleId));
+            row.put("device_id", 7L);
+            row.put("normalize_rule_id", ruleId);
+            rows.add(row);
+        }
+
+        StatsProjectionOutboxEvent event = StatsProjectionOutboxEvent.fromAccumulator(
+                new DirectStatsAccumulator().addStandardizedRows(rows));
+
+        event.validate();
+        assertEquals(1, event.getDevices().size());
+        assertEquals(3L, event.getDevices().get(0).getParseDelta());
+        assertEquals(2, event.getRules().size());
+    }
+
+    @Test
+    void outboxWriterRequiresAndUsesFactTransaction() {
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        StatsProjectionOutboxWriter writer = new StatsProjectionOutboxWriter(jdbc, new ObjectMapper());
+        ReflectionTestUtils.setField(writer, "enabled", true);
+        ReflectionTestUtils.setField(writer, "outboxTable", "stats_projection_outbox");
+        StatsProjectionOutboxEvent event = oneDeviceEvent("event-1");
+
+        assertThrows(IllegalStateException.class, () -> writer.append(event));
+
+        TransactionSynchronizationManager.setActualTransactionActive(true);
+        writer.append(event);
+        verify(jdbc).update(org.mockito.ArgumentMatchers.contains("INSERT INTO stats_projection_outbox"),
+                org.mockito.ArgumentMatchers.eq("event-1"),
+                org.mockito.ArgumentMatchers.eq(StatsProjectionOutboxEvent.SCHEMA),
+                org.mockito.ArgumentMatchers.anyString());
+    }
+
+    @Test
+    void projectionConsumerAppliesFreshEventAndRecordsIdempotencyMarker() {
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        when(jdbc.queryForList(org.mockito.ArgumentMatchers.anyString(),
+                org.mockito.ArgumentMatchers.eq(String.class), org.mockito.ArgumentMatchers.<Object>any()))
+                .thenReturn(Collections.emptyList());
+        PlatformTransactionManager manager = mock(PlatformTransactionManager.class);
+        when(manager.getTransaction(any(TransactionDefinition.class))).thenReturn(new SimpleTransactionStatus());
+        StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
+        StatsProjectionDeltaConsumer consumer = new StatsProjectionDeltaConsumer(
+                jdbc, manager, new ObjectMapper(), publisher);
+        configure(consumer);
+
+        StatsProjectionDeltaConsumer.ApplyResult result = consumer.applyWithRetry(
+                Collections.singletonList(oneDeviceEvent("event-1")));
+
+        assertEquals(1, result.appliedEvents);
+        assertEquals(1, result.deviceGroups);
+        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
+        verify(jdbc, times(3)).batchUpdate(sql.capture(), anyList());
+        assertTrue(sql.getAllValues().get(0).contains("MERGE INTO device_daily_stats"));
+        assertTrue(sql.getAllValues().get(1).contains("stats_projection_applied_event"));
+        assertTrue(sql.getAllValues().get(2).contains("SET status='APPLIED'"));
+        verify(publisher).enqueueAfterCommit(org.mockito.ArgumentMatchers.anyCollection());
+        verify(manager).commit(any());
+    }
+
+    @Test
+    void projectionConsumerDoesNotApplyDuplicateDeltaTwice() {
+        JdbcTemplate jdbc = mock(JdbcTemplate.class);
+        when(jdbc.queryForList(org.mockito.ArgumentMatchers.anyString(),
+                org.mockito.ArgumentMatchers.eq(String.class), org.mockito.ArgumentMatchers.<Object>any()))
+                .thenReturn(Collections.singletonList("event-1"));
+        PlatformTransactionManager manager = mock(PlatformTransactionManager.class);
+        when(manager.getTransaction(any(TransactionDefinition.class))).thenReturn(new SimpleTransactionStatus());
+        StatsMaterializationPublisher publisher = mock(StatsMaterializationPublisher.class);
+        StatsProjectionDeltaConsumer consumer = new StatsProjectionDeltaConsumer(
+                jdbc, manager, new ObjectMapper(), publisher);
+        configure(consumer);
+
+        StatsProjectionDeltaConsumer.ApplyResult result = consumer.applyWithRetry(
+                Collections.singletonList(oneDeviceEvent("event-1")));
+
+        assertEquals(0, result.appliedEvents);
+        assertEquals(1, result.duplicates);
+        ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
+        verify(jdbc).batchUpdate(sql.capture(), anyList());
+        assertTrue(sql.getValue().contains("SET status='APPLIED'"));
+    }
+
+    private StatsProjectionOutboxEvent oneDeviceEvent(String eventId) {
+        DirectStatsAccumulator accumulator = new DirectStatsAccumulator();
+        Map<String, Object> row = new HashMap<>();
+        row.put("created_at", LocalDate.of(2026, 9, 12));
+        row.put("log_time", LocalDateTime.of(2026, 9, 12, 8, 0));
+        row.put("device_id", 7L);
+        StatsProjectionOutboxEvent event = StatsProjectionOutboxEvent.fromAccumulator(
+                accumulator.addStandardizedRow(row));
+        event.setEventId(eventId);
+        return event;
+    }
+
+    private void configure(StatsProjectionDeltaConsumer consumer) {
+        ReflectionTestUtils.setField(consumer, "enabled", true);
+        ReflectionTestUtils.setField(consumer, "appliedTable", "stats_projection_applied_event");
+        ReflectionTestUtils.setField(consumer, "outboxTable", "stats_projection_outbox");
+        ReflectionTestUtils.setField(consumer, "deviceTable", "device_daily_stats");
+        ReflectionTestUtils.setField(consumer, "ruleTable", "normalize_rule_daily_stats");
+        ReflectionTestUtils.setField(consumer, "transactionTimeoutSeconds", 45);
+        ReflectionTestUtils.setField(consumer, "maxConcurrencyRetries", 3);
+        ReflectionTestUtils.setField(consumer, "retryBackoffMs", 0L);
+    }
+}
@@ -0,0 +1,43 @@
+package com.common.util;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class Utf8TextTruncatorTest {
+    @Test
+    void truncatesWithinBudgetWithoutSplittingChineseOrEmoji() {
+        String source = repeat("中文🙂", 100);
+
+        Utf8TextTruncator.Result result = Utf8TextTruncator.truncate(source, 96L);
+
+        assertTrue(result.isTruncated());
+        assertTrue(result.getValue().endsWith("]"));
+        assertTrue(result.getValue().contains("HB_XDR_TRUNCATED"));
+        assertTrue(result.getValue().getBytes(StandardCharsets.UTF_8).length <= 96);
+        assertEquals(source.getBytes(StandardCharsets.UTF_8).length, result.getOriginalUtf8Bytes());
+        assertNotNull(result.getOriginalSha256());
+        assertFalse(result.getValue().contains("�"));
+    }
+
+    @Test
+    void zeroMeansUnlimitedAndSmallEnabledLimitFailsClearly() {
+        Utf8TextTruncator.Result result = Utf8TextTruncator.truncate("中文🙂", 0L);
+        assertFalse(result.isTruncated());
+        assertEquals("中文🙂", result.getValue());
+        assertThrows(IllegalArgumentException.class,
+                () -> Utf8TextTruncator.truncate(repeat("x", 100), 8L));
+    }
+
+    private static String repeat(String value, int count) {
+        StringBuilder result = new StringBuilder(value.length() * count);
+        for (int i = 0; i < count; i++) result.append(value);
+        return result.toString();
+    }
+}
@@ -67,6 +67,25 @@ class InfluxRawV2ContractTest {
        String line=raw.getPoint().toLineProtocol(); assertFalse(line.substring(0,line.indexOf(' ')).contains("uuid=")); assertTrue(line.contains("event_bucket=2085")); assertTrue(line.contains("uuid=\"20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5\"")); assertEquals(WritePrecision.NS,raw.getPoint().getPrecision()); assertEquals("yelangbucket_v2",raw.getBucket());
    }

+    @Test void oversizedRawMessageIsMarkedAndCarriesStructuredTruncationMetadata() {
+        RawStorageProperties properties=mock(RawStorageProperties.class);
+        when(properties.getWriteSchema()).thenReturn(RawStorageProperties.WriteSchema.V2);
+        when(properties.getV2Bucket()).thenReturn("yelangbucket_v2");
+        when(properties.getMessageMaxUtf8Bytes()).thenReturn(128L);
+        RawInfluxMetrics metrics=new RawInfluxMetrics();
+        InfluxRawPointFactory factory=new InfluxRawPointFactory(properties,metrics);
+
+        String line=factory.create("20260814-a3d874a0-6f50-4f0c-9e7a-9c20303360f5",
+                repeat("中文🙂",100),"20260814000000000","126","1","raw.large",
+                ReceiveTimeParser.parse("20260814000000000"),1_786_636_800_000_000_123L)
+                .getPoint().toLineProtocol();
+
+        assertTrue(line.contains("HB_XDR_TRUNCATED"));
+        assertTrue(line.contains("message_truncated=true"));
+        assertTrue(line.contains("message_original_sha256="));
+        assertEquals(1L,metrics.snapshot().get("raw_message_truncated_total"));
+    }
+
    @Test void onlyMissingPointTimeFallsBack() {
        RawStorageProperties properties=mock(RawStorageProperties.class); when(properties.isLegacyFallbackEnabled()).thenReturn(true); RawInfluxMetrics metrics=new RawInfluxMetrics(); PointTimeResolver resolver=new PointTimeResolver(properties,metrics); long millis=ReceiveTimeParser.parse("20260814000000000").getEpochMillis();
        assertNotNull(resolver.resolve(false,null,millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,"-1",millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,"not-number",millis,true)); assertThrows(IllegalArgumentException.class,()->resolver.resolve(true,Long.toString(millis*1_000_000L+1_000_000L),millis,true)); assertEquals(1L,metrics.snapshot().get("legacy_missing_point_time_total"));
@@ -108,4 +127,10 @@ class InfluxRawV2ContractTest {
        assertTrue(query.indexOf("|> group()") < query.indexOf("|> sort("));
        assertTrue(query.indexOf("|> sort(") < query.indexOf("|> limit(n: 2)"));
    }
+
+    private static String repeat(String value, int count) {
+        StringBuilder result = new StringBuilder(value.length() * count);
+        for (int i=0;i<count;i++) result.append(value);
+        return result.toString();
+    }
 }