diff --git a/packages/memory-host-sdk/src/host/internal.ts b/packages/memory-host-sdk/src/host/internal.ts index 004d91d23e2..55c12593472 100644 --- a/packages/memory-host-sdk/src/host/internal.ts +++ b/packages/memory-host-sdk/src/host/internal.ts @@ -398,13 +398,20 @@ export function chunkMarkdown( if (line.length === 0) { segments.push(""); } else { - // Use token count (not maxChars) as the split step so that CJK lines - // – where 1 char ≈ 1 token – are sliced into budget-sized segments. - // For Latin text the token count is ≥ maxChars/4, which still produces - // segments well within the char budget after weighting. - const splitStep = Math.max(1, chunking.tokens); - for (let start = 0; start < line.length; start += splitStep) { - segments.push(line.slice(start, start + splitStep)); + // First pass: slice at maxChars (preserves original behaviour for Latin). + // Second pass: if a segment's *weighted* size still exceeds the budget + // (happens for CJK-heavy text where 1 char ≈ 1 token), re-split it at + // chunking.tokens so the chunk stays within the token budget. + for (let start = 0; start < line.length; start += maxChars) { + const coarse = line.slice(start, start + maxChars); + if (estimateStringChars(coarse) > maxChars) { + const fineStep = Math.max(1, chunking.tokens); + for (let j = 0; j < coarse.length; j += fineStep) { + segments.push(coarse.slice(j, j + fineStep)); + } + } else { + segments.push(coarse); + } } } for (const segment of segments) { diff --git a/src/utils/cjk-chars.ts b/src/utils/cjk-chars.ts index 483076749f7..54efd58fae4 100644 --- a/src/utils/cjk-chars.ts +++ b/src/utils/cjk-chars.ts @@ -46,9 +46,9 @@ export function estimateStringChars(text: string): number { /** * Matches surrogate pairs whose code point falls in the CJK Extension B+ - * range (U+20000–U+2FA1F). Only these surrogates need adjustment because + * range (U+20000–U+2FA1F). Only these surrogates need adjustment because * they are matched by {@link NON_LATIN_RE} and already counted in - * `nonLatinCount`. Other surrogates (emoji, symbols) are not matched by + * `nonLatinCount`. Other surrogates (emoji, symbols) are not matched by * that regex, so collapsing them would create an inconsistency. * * High-surrogate range for U+20000–U+2FA1F is D840–D87E. @@ -57,7 +57,7 @@ const CJK_SURROGATE_HIGH_RE = /[\uD840-\uD87E][\uDC00-\uDFFF]/g; /** * Return the code-point-aware length of the string, adjusting only for - * CJK Extension B+ surrogate pairs. For text without such characters + * CJK Extension B+ surrogate pairs. For text without such characters * (the vast majority of inputs) this returns `text.length` unchanged. */ function countCodePoints(text: string, nonLatinCount: number): number {