diff --git a/.prettierignore b/.prettierignore index 5cd9cad0590..3841067e73e 100644 --- a/.prettierignore +++ b/.prettierignore @@ -32,6 +32,8 @@ packages/cua-driver/ # "intentionally non-idiomatic ink input/output", so normalising them to # house style would corrupt the very data the codemod tests assert on. scripts/codemod/fixtures/ +# Preserve the exact Markdown spacing and lengths used by chunking tests. +packages/channels/base/src/fixtures/synthetic-knowledge-search.md packages/live-host/ packages/desktop/runtime/ packages/core/src/skills/bundled/browser-use/runtime/ diff --git a/docs/design/markdown-message-chunks.md b/docs/design/markdown-message-chunks.md new file mode 100644 index 00000000000..c2b70cbb8ca --- /dev/null +++ b/docs/design/markdown-message-chunks.md @@ -0,0 +1,35 @@ +# Markdown message chunking + +[English](markdown-message-chunks.md) | [简体中文](markdown-message-chunks.zh-CN.md) + +## Problem and scope + +DingTalk and WeCom currently split long Markdown inside links, table rows and code fences. Keep the existing ordinary-message delivery paths and add shared structure-aware splitting. Interactive cards, attachments, generic delivery/retry behavior and unrelated refactors are outside this change. + +## Contract + +The normal target remains 3800; a complete table, code block or inline element may exceed it up to 20,000. DingTalk counts UTF-16 units and WeCom counts UTF-8 bytes. Prefixes, mentions, repeated headers, reference definitions and generated fences consume these budgets. Prefer block, paragraph, list-item and sentence boundaries; keep a heading with the first following fragment when it fits. Preserve original URL spelling and Unicode characters. + +Tables above the hard maximum split only between rows and repeat the original two-line header. Oversized code blocks close and reopen fences, retaining language and metadata; prefer line boundaries. An indivisible element that exceeds the maximum uses explicitly labeled plain-text fenced fragments preserving its entire source. This cannot keep an oversized URL clickable across messages. + +Every ordinary delivery batch is planned before its first send, using the text already available for that batch (not future output from the entire task). Candidate chunks must each fit the hard budget, number at most four times `ceil((source size + reservedLength) / (targetLength + reservedLength))` (at least one), and total at most four times `(source size + reservedLength)` plus one full hard-budget allowance. `reservedLength` is the caller prefix/header length already subtracted from content budgets; count it on every candidate fragment when checking aggregate output (conservatively also for first-message-only mentions/headers). These proportional bounds allow boundary slack and lossless fencing; they are not a fixed message-count cap. Never truncate a batch to meet them. If a candidate fails, re-plan from the original source with the hard maximum as the target; if that still fails, use explicitly labeled literal fragments of the original batch without repeated Markdown container context. Nested budget `RangeError`s and candidate-output overflow abort only a structure-preserving attempt; try the next complete plan from source. Errors from the final literal attempt remain explicit. If even literal fragments cannot meet the bounds, fail before sending any fragment. + +When a fence wrapper consumes half the preferred budget, use the hard budget. A large wrapper or table header alone does not justify literal fallback: retain code language/metadata and table structure whenever the complete candidate satisfies the batch limits. Downgrade locally only when the actual hard budget cannot contain a code character with its wrapper or a table row with its header. Bound intermediate candidate arrays by the batch output allowance while generating them, abandoning an overflowing candidate and restarting from the original source rather than constructing thousands of repeated wrappers. Intermediate pieces may later be packed together, so their count is not treated as the final message count. Literal fragments use the hard budget, so nested list prefixes cannot leave only a few body characters at the soft target. + +## Implementation and rationale + +A shared `channel-base/markdown-chunks` subpath uses CommonMark source positions rather than Markdown serialization. Validate GFM table candidates with only their two header lines, avoiding full large-table parsing; parse non-table paragraphs for inline boundaries. A prose fragment that becomes a code fence at its new boundary uses the labeled lossless fallback so its text remains visible. Reopened code fences retain the original info-string spelling, including entities and escapes. Split lists at items and quoted content at block boundaries, retaining container markers and their budget. Unsupported oversized structures use the same labeled lossless fallback. Repeat top-level reference definitions in split messages. Nested multiline reference resolution remains outside this change. + +DingTalk and WeCom subtract their existing source/mention prefixes before splitting. If these prefixes leave less than half the ordinary target for content, use the remaining hard budget as the content target to avoid repeating a large label for tiny fragments. DingTalk proactive messages additionally measure the actual serialized `{title,text}` JSON against 15,000 UTF-8 bytes, including continuation titles. Re-plan the whole candidate batch from its source when lowering the JSON payload budget, rather than splitting already wrapped fragments again. Keep the prepared source in the delivery plan. If no text fragment has been sent, re-plan that entire source after background header/title recomposition; once sending has started, retain the existing unsent-fragment retry behavior. Streaming status cards use their separate update path; card fallbacks and individual background responses use this ordinary batch planning without waiting for task completion. Repacking must not repeat attachment preparation or alter the send/retry loop. + +Affected files are the new shared splitter and tests, channel-base package manifest and lockfile, DingTalk Markdown utilities and adapters/tests, and WeCom adapter/tests. The parser is not imported through the general channel-base barrel. No public configuration is added. + +## Validation and acceptance + +Use synthetic example-domain URLs only. Focused tests must retain complete links and all table rows/headers, preserve code content and fence closure, enforce both counting modes and prefix budgets, and capture serialized proactive payloads after background recomposition. Cover large documents using bounded table detection, fence info lengths 3789 and 19985, budget boundary neighbors, affordable 10.5k–15k code info and 10.5k–11k table headers that must retain structure, early candidate abandonment, repeated table headers, deep ordered-list prefixes, Chinese/emoji, source-label budgets and header/title recomposition. Assert complete source/body recovery as well as bounded count, aggregate output and individual payload size. Run package-local unit tests, build, bundle, typecheck and lint; perform independent read-only test-engineer verification, two clean self-audits and code review. + +Try the global CLI for the baseline and the built CLI for verification. If the CLI cannot reach channel delivery without real platform accounts, use captured adapter payloads and an independent Markdown renderer as a documented test-script fallback. + +## Open platform acceptance + +The 3800/20,000 values are application budgets, not proof of endpoint acceptance. Real DingTalk session replies and proactive messages, and WeCom Markdown delivery, still require recipient-based capacity and rendering validation. Local serialization and renderer tests cannot establish real platform behavior. This gap remains explicit; it does not change the accepted design. diff --git a/docs/design/markdown-message-chunks.zh-CN.md b/docs/design/markdown-message-chunks.zh-CN.md new file mode 100644 index 00000000000..7a2e46ebc5a --- /dev/null +++ b/docs/design/markdown-message-chunks.zh-CN.md @@ -0,0 +1,35 @@ +# Markdown 消息分段 + +[English](markdown-message-chunks.md) | [简体中文](markdown-message-chunks.zh-CN.md) + +## 问题和范围 + +钉钉和企业微信当前会在链接、表格行和代码围栏内部截断长 Markdown。保留现有普通消息发送路径，加入共享的结构感知分段。交互卡片、附件、通用发送/重试行为和无关重构不在本次范围内。 + +## 行为约定 + +常规目标仍为 3800；完整表格、代码块或行内元素可以超过目标，但不得超过 20,000。钉钉按 UTF-16 单位计量，企业微信按 UTF-8 字节计量。前缀、提及、重复表头、引用定义和生成的围栏均占用预算。优先选择块、段落、列表项和句子边界；预算允许时将标题与后续首个片段放在一起。保留 URL 原始拼写和完整 Unicode 字符。 + +超过硬上限的表格仅在行间拆分，并重复原始两行表头。超长代码块关闭并重新打开围栏，保留语言和元数据，优先按行拆分。超过硬上限的不可分元素使用明确标注的纯文本围栏片段，保留完整原文；超长 URL 此时无法在消息间保持可点击。 + +普通消息每批都在发送第一条之前完成分片计划，使用该批已经拿到的文本，不等待整轮任务未来的输出。候选分片必须逐片符合硬预算；数量不得超过 `ceil((原文大小 + reservedLength) / (targetLength + reservedLength))` 的四倍（至少允许一片）；总大小不得超过 `(原文大小 + reservedLength)` 的四倍加一个完整硬预算余量。`reservedLength` 是调用端已经从正文预算扣除的前缀/头部长度；检查总输出时为每片计入此开销（仅首片使用的提及/头部也保守计入）。这些相对边界为结构边界和原文围栏留出余量，不是固定发送条数上限，也不得靠截断满足。候选不合格时从原文重新规划，将硬上限作为目标；仍不合格则将该批完整原文降级为明确标注的纯文本片段，不再重复 Markdown 容器上下文。嵌套预算 `RangeError` 和候选输出超量只中止当前保结构尝试，随后从原文尝试下一完整方案；最终纯文本尝试的错误仍明确抛出。若纯文本片段仍无法满足边界，则在发送任何片段前报错。 + +围栏包装占用一半目标预算时改用硬预算。包装或表头较大本身不构成纯文本降级的理由：只要完整候选满足整批边界，就保留代码语言/metadata 和表格结构。只有实际硬预算无法容纳包装加正文字符，或表头加表格行时，才局部降级。生成过程中用整批输出余量限制中间候选数组，超量即放弃该候选并从原文重新规划，避免先生成数千个重复包装。中间片段可能随后合并，因此不将其中间数量当成最终消息数。纯文本片段使用硬预算，避免深层列表前缀在软目标下仅留下几个正文字符。 + +## 实现及理由 + +共享的 `channel-base/markdown-chunks` 子路径使用 CommonMark 源码位置，不重新序列化 Markdown。GFM 表格候选只用两行表头验证，避免解析完整大表格；非表格段落另行解析行内边界。普通正文片段若在新边界处变成代码围栏，则使用明确标注、保留原文的降级方式，确保文字可见。重新打开的代码围栏保留原始 info 字符串拼写，包括实体和转义。列表按项、引用按块拆分，保留容器标记并计入预算。不支持的超长结构采用相同的明确标注、保留原文的降级方式。拆分后的消息重复顶层引用定义；嵌套多行引用解析不在本次范围内。 + +钉钉和企业微信先扣除现有来源/提及前缀再分段。若前缀使正文剩余目标不足常规目标的一半，则以剩余硬预算作为正文目标，避免为极短正文反复发送长标签。钉钉主动消息还将实际序列化的 `{title,text}` JSON 与 15,000 UTF-8 字节上限比较，包括续篇标题。降低 JSON 载荷预算时，从原文重新规划整批候选，而不是再次拆分已经包装的片段。发送计划保存已经完成预处理的原文。尚未发送任何正文分片时，后台标题/头部重组后对整批原文重新规划；一旦开始发送，则保留现有仅重试未发送片段的行为。流式状态卡使用独立更新路径；卡片回退和单个后台 response 使用此普通消息批次规划，不等待整轮任务完成。重新分段不得重复附件准备，也不得改变发送/重试循环。 + +涉及文件为新增共享分段器及测试、channel-base 清单及锁文件、钉钉 Markdown 工具和适配器/测试、企业微信适配器/测试。解析器不经由 channel-base 通用聚合入口导入。不增加公开配置。 + +## 验证和验收 + +仅使用示例域名的合成 URL。定向测试需保留完整链接、所有表格行及表头，保留代码内容及闭合围栏，验证两种计量和前缀预算，并捕获后台重组后的主动消息序列化载荷。覆盖采用有界表头检测的大文档、3789 和 19985 长度的围栏信息、预算临界点邻居、预算允许时必须保留结构的 10.5k–15k 代码 info 和 10.5k–11k 表头、提前放弃超量候选、重复表头、深层有序列表前缀、中文/emoji、来源标签预算及 header/title 重组。同时断言完整原文/正文恢复、分片数量、总输出及单片载荷有界。执行包内单元测试、build、bundle、typecheck 和 lint；完成独立只读 test-engineer 验证、两轮干净自审和代码评审。 + +基线尝试全局 CLI，修复后尝试构建后的 CLI。如果没有真实平台账户就无法从 CLI 到达通道发送路径，则使用捕获适配器载荷及独立 Markdown 渲染器的测试脚本替代，并记录原因。 + +## 待完成的平台验收 + +3800/20,000 是应用预算，不证明端点接受能力。真实钉钉会话回复、主动消息及企业微信 Markdown 发送仍需接收方实测容量和渲染。本地序列化和渲染器测试不能证明真实平台行为。明确保留这一缺口，不因此改变已接受设计。 diff --git a/integration-tests/tsconfig.json b/integration-tests/tsconfig.json index ec3138717be..8f1de2aacc2 100644 --- a/integration-tests/tsconfig.json +++ b/integration-tests/tsconfig.json @@ -179,6 +179,9 @@ // sites share one declaration — mixing src and dist declarations // produces duplicate-private-class errors. "@qwen-code/channel-base": ["../packages/channels/base/src/index.ts"], + "@qwen-code/channel-base/markdown-chunks": [ + "../packages/channels/base/src/markdown-chunks.ts" + ], // cli's channel-registry.ts imports the builtin channel // adapters and html.ts imports web-templates; without entries they // resolve through their exports maps to dist, leaving the typecheck diff --git a/packages/channels/base/package.json b/packages/channels/base/package.json index 81308b65bad..dac893c4b00 100644 --- a/packages/channels/base/package.json +++ b/packages/channels/base/package.json @@ -14,6 +14,10 @@ ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" + }, + "./markdown-chunks": { + "types": "./dist/markdown-chunks.d.ts", + "default": "./dist/markdown-chunks.js" } }, "files": [ @@ -25,9 +29,13 @@ "test:ci": "vitest run" }, "dependencies": { - "@agentclientprotocol/sdk": "^0.14.1" + "@agentclientprotocol/sdk": "^0.14.1", + "remark-gfm": "^4.0.0", + "remark-parse": "^11.0.0", + "unified": "^11.0.0" }, "devDependencies": { - "typescript": "^5.0.0" + "typescript": "^5.0.0", + "@types/mdast": "^4.0.4" } } diff --git a/packages/channels/base/src/fixtures/synthetic-knowledge-search.md b/packages/channels/base/src/fixtures/synthetic-knowledge-search.md new file mode 100644 index 00000000000..1974854039b --- /dev/null +++ b/packages/channels/base/src/fixtures/synthetic-knowledge-search.md @@ -0,0 +1,26 @@ +此消息完全由合成测试数据构成。“示例”知识检索命中（HAS_DATA），包含演示文档与模型条目。 + +## 📄 示例知识库文档 + +### 示例知识库「演示指标体系」 +| 文档 | 内容摘要 | +|---|---| +| 示例指标一 | 示例人数、比例、金额与计算口径，包含演示 SQL [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-1/wiki?embed=1&pageId=wiki%3Ademo-document-01#__dac_citation=%7B%22title%22%3A%22Example%2001%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%22%7D) | +| 示例指标二 | 示例交易、用户、商品等维度的基础定义 [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-1/wiki?embed=1&pageId=wiki%3Ademo-document-02#__dac_citation=%7B%22title%22%3A%22Example%2002%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8Bxxxxxxx%22%7D) | +| 示例指标三 | 示例分类指标与维度说明 [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-1/wiki?embed=1&pageId=wiki%3Ademo-document-03#__dac_citation=%7B%22title%22%3A%22Example%2003%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8Bxxxxx%22%7D) | +| 示例指标四 | 示例状态及处理规则说明 [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-1/wiki?embed=1&pageId=wiki%3Ademo-document-04#__dac_citation=%7B%22title%22%3A%22Example%2004%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8Bxxxxx%22%7D) | +| 示例指标五 | 示例统计指标与计算方式 [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-1/wiki?embed=1&pageId=wiki%3Ademo-document-05#__dac_citation=%7B%22title%22%3A%22Example%2005%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8Bxxxxx%22%7D) | + +### 示例知识库「演示数据域」 +| 文档 | 内容摘要 | +|---|---| +| 示例汇总指标 | 示例均值、占比、耗时及费用口径 [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-2/wiki?embed=1&pageId=wiki%3Ademo-document-06#__dac_citation=%7B%22title%22%3A%22Example%2006%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8Bxxx%22%7D) | + +## 📊 示例模型对象 + +- **sample_item（示例条目）**：示例记录粒度，包含分类、数值和状态字段 [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-2/wiki?embed=1&pageId=wiki%3Ademo-document-07#__dac_citation=%7B%22title%22%3A%22Example%2007%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8Bxxxxxxx%22%7D) +- **sample_event（示例事件）**：示例事件粒度，包含编号、类型和时间字段 [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-2/wiki?embed=1&pageId=wiki%3Ademo-document-08#__dac_citation=%7B%22title%22%3A%22Example%2008%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8Bxxxxxxx%22%7D) +- 字段语义：示例编号 [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-2/wiki?embed=1&pageId=wiki%3Ademo-document-09#__dac_citation=%7B%22title%22%3A%22Example%2009%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8Bxxxxxxx%22%7D)、示例类型 [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-2/wiki?embed=1&pageId=wiki%3Ademo-document-10#__dac_citation=%7B%22title%22%3A%22Example%2010%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8Bxxxx%22%7D) +- 术语：示例主体 [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-2/wiki?embed=1&pageId=wiki%3Ademo-document-11#__dac_citation=%7B%22title%22%3A%22Example%2011%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8Bx%22%7D)、示例类别 [来源](https://docs.example.com/sample-region/knowledge-bases/demo-library-2/wiki?embed=1&pageId=wiki%3Ademo-document-12#__dac_citation=%7B%22title%22%3A%22Example%2012%22%2C%22summary%22%3A%22Synthetic%20%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8B%E6%B5%8Bx%22%7D) + +> 说明：这是用于验证长引用链接、表格和列表跨消息分段的合成样本。所有域名、区域、文档标识及引用元数据均为虚构内容。 \ No newline at end of file diff --git a/packages/channels/base/src/markdown-chunks.test.ts b/packages/channels/base/src/markdown-chunks.test.ts new file mode 100644 index 00000000000..95f0c96f5b3 --- /dev/null +++ b/packages/channels/base/src/markdown-chunks.test.ts @@ -0,0 +1,632 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it, vi } from 'vitest'; +import type { Nodes } from 'mdast'; +import { unified } from 'unified'; +import remarkParse from 'remark-parse'; +import remarkGfm from 'remark-gfm'; +import { splitMarkdown } from './markdown-chunks.js'; + +const parser = unified().use(remarkParse).use(remarkGfm); +const fixture = readFileSync( + new URL('./fixtures/synthetic-knowledge-search.md', import.meta.url), + 'utf8', +); +function nodes(text: string): Nodes[] { + const result: Nodes[] = []; + const visit = (node: Nodes) => { + result.push(node); + if ('children' in node) node.children.forEach(visit); + }; + visit(parser.parse(text)); + return result; +} +const links = (text: string) => + nodes(text).flatMap((node) => (node.type === 'link' ? [node.url] : [])); +const code = (text: string) => + nodes(text).flatMap((node) => (node.type === 'code' ? [node] : [])); +const rows = (text: string) => + nodes(text).flatMap((node) => + node.type === 'table' + ? node.children + .slice(1) + .map((row) => + row.children + .map((cell) => + cell.children + .map((child) => ('value' in child ? child.value : '')) + .join(''), + ) + .join('|'), + ) + : [], + ); + +describe.each(['utf16', 'utf8'] as const)('Markdown chunks (%s)', (unit) => { + const options = { targetLength: 3800, maxLength: 20_000, unit }; + const measure = (text: string) => + unit === 'utf8' ? Buffer.byteLength(text) : text.length; + const split = (text: string) => { + const parts = splitMarkdown(text, options); + for (const part of parts) { + expect(measure(part)).toBeLessThanOrEqual(20_000); + expect(part.isWellFormed()).toBe(true); + } + return parts; + }; + it('keeps synthetic citation URLs and all rows intact across multiple messages', () => { + const parts = split(fixture); + expect(parts.length).toBeGreaterThan(1); + expect(links(fixture)).toHaveLength(12); + expect( + links(fixture).every( + (url) => new URL(url).hostname === 'docs.example.com', + ), + ).toBe(true); + expect(parts.flatMap(links)).toEqual(links(fixture)); + expect(parts.flatMap(rows)).toEqual(rows(fixture)); + expect(parts.flatMap(rows)).toHaveLength(6); + for (const url of links(fixture)) + expect(parts.filter((part) => part.includes(url))).toHaveLength(1); + }); + it('splits ordinary paragraphs near the target without cutting Unicode', () => { + const text = '合成😀 prose. '.repeat(1500); + const parts = split(text); + expect(parts.join('')).toBe(text); + expect(parts.every((part) => measure(part) <= 3800)).toBe(true); + }); + it('replans from source when nested container budgets cannot fit a fragment', () => { + const text = + Array.from( + { length: 15 }, + (_, i) => ' '.repeat(i) + '- level ' + i, + ).join('\n') + + '\n' + + ' '.repeat(15) + + '\u0001'.repeat(32000); + const parts = splitMarkdown(text, { + targetLength: 1250, + maxLength: 1250, + unit, + }); + expect(parts.length).toBeGreaterThan(1); + expect(parts.every((part) => measure(part) <= 1250)).toBe(true); + expect( + parts + .flatMap(code) + .map((block) => block.value) + .join(''), + ).toBe(text); + }); + it.each([ + '[source](https://docs.example.com/' + 'x'.repeat(6000) + ')', + '`' + 'x'.repeat(6000) + '`', + 'https://docs.example.com/' + 'x'.repeat(6000), + '~~' + 'x'.repeat(6000) + '~~', + '| a | b |\n|---|---|\n| x |' + 'x'.repeat(6000) + '|', + '```ts example\n' + 'x'.repeat(6000) + '\n```', + ])( + 'keeps a complete structure above the target within the hard limit', + (text) => { + expect(split(text)).toEqual([text]); + }, + ); + it('repeats exact table headers and never loses a row in a large document', () => { + const header = '| key | value |\n|:---|---:|\n'; + const data = Array.from( + { length: 900 }, + (_, index) => `| row-${index} | ${'x'.repeat(140)} |`, + ).join('\n'); + const parts = split(header + data); + expect(parts.every((part) => part.startsWith(header))).toBe(true); + expect(parts.flatMap(rows)).toEqual(rows(header + data)); + }); + it('does not misclassify a pipe-free underline as a two-column table', () => { + const text = 'lead '.repeat(900) + '\na | b\n-----\ntail'; + const parts = split(text); + expect(parts.flatMap(rows)).toHaveLength(0); + expect(parts.join('\n')).toContain('a | b'); + }); + it('retains the indentation that defines fenced code content', () => { + const text = ' ```ts\n x\n ```'; + expect( + split(text) + .flatMap(code) + .map((block) => block.value), + ).toEqual(['x']); + expect(split('```ts')[0]).toMatch(/\n```$/u); + }); + it('splits oversized code at lines with language and metadata retained', () => { + const body = Array.from( + { length: 2000 }, + (_, index) => `line${index} 中😀`, + ).join('\n'); + const text = `~~~ts example\n${body}\n~~~`; + const parts = split(text); + const blocks = parts.flatMap(code); + expect(blocks.length).toBe(parts.length); + expect( + blocks.every((block) => block.lang === 'ts' && block.meta === 'example'), + ).toBe(true); + expect(blocks.map((block) => block.value).join('\n')).toBe(body); + }); + it.each(['js` ~~~', 'A example\\* meta'])( + 'preserves source info when re-fencing encoded or escaped metadata: %s', + (info) => { + for (const newline of ['\n', '\r\n', '\r']) { + for (const closed of [false, true]) { + const body = ' synthetic **text** '.repeat(900); + const text = `~~~${info}${newline}${body}${closed ? `${newline}~~~` : ''}`; + const original = code(text)[0]!; + const parts = split(text); + const blocks = parts.flatMap(code); + expect(blocks).toHaveLength(parts.length); + expect(blocks.every((block) => block.lang === original.lang)).toBe( + true, + ); + expect(blocks.every((block) => block.meta === original.meta)).toBe( + true, + ); + expect(blocks.map((block) => block.value).join('')).toBe(body); + expect( + parts.every((part) => part.split('\n')[0]!.endsWith(info)), + ).toBe(true); + expect( + parts.flatMap(nodes).some((node) => node.type === 'html'), + ).toBe(false); + } + } + }, + ); + it('splits an oversized code line and closes an unfinished fence', () => { + const body = '中😀'.repeat(9000); + const parts = split('```ts\n' + body); + expect( + parts + .flatMap(code) + .map((block) => block.value) + .join(''), + ).toBe(body); + expect(parts.every((part) => part.endsWith('```'))).toBe(true); + }); + it.each([ + '[source](https://docs.example.com/' + 'x'.repeat(21000) + ')', + '| a | b |\n|---|---|\n| source |' + 'x'.repeat(21000) + '|', + ])( + 'labels oversized indivisible source and retains it losslessly', + (text) => { + const parts = split(text); + expect( + parts.every((part) => part.includes('original text follows in parts')), + ).toBe(true); + expect( + parts + .flatMap(code) + .map((block) => block.value) + .join(''), + ).toBe(text); + }, + ); + it('retains headings with their first content and repeats reference definitions', () => { + const url = 'https://docs.example.com/ref'; + const text = + '# First\n\n' + 'hello [ref]. '.repeat(800) + '\n\n[ref]: ' + url; + const parts = split(text); + expect(parts[0]).toContain('# First\n\nhello [ref]'); + expect(parts.every((part) => part.includes('[ref]: ' + url))).toBe(true); + expect( + parts.flatMap(nodes).filter((node) => node.type === 'linkReference'), + ).toHaveLength(800); + }); + it('reserves large reference definitions before protecting entities', () => { + const definition = '[ref]: https://docs.example.com/' + 'x'.repeat(19700); + const text = + 'x'.repeat(4000) + + '&' + + '#0000000000000000000000000000065;' + + '\n\n' + + definition; + const parts = split(text); + expect(parts.every((part) => measure(part) <= 20000)).toBe(true); + expect(parts.length).toBeLessThanOrEqual( + 4 * Math.ceil(measure(text) / 3800), + ); + expect( + parts.every((part) => part.includes('original text follows in parts')), + ).toBe(true); + expect( + parts + .flatMap(code) + .map((block) => block.value) + .join(''), + ).toBe(text); + }); + it.each(['> ', '- '])( + 'splits long %s containers and retains their links', + (marker) => { + const text = + marker + 'hello [source](https://docs.example.com/page) '.repeat(900); + const parts = split(text); + expect(parts.length).toBeGreaterThan(1); + expect(parts.every((part) => part.startsWith(marker))).toBe(true); + expect(parts.flatMap(links)).toEqual(links(text)); + }, + ); + it('preserves code content in a list with tab-indented continuation lines', () => { + const text = '- intro\n\t```ts\n\t' + 'x'.repeat(5000) + '\n\t```'; + const blocks = split(text).flatMap(code); + expect(blocks).toHaveLength(1); + expect(blocks[0]!.lang).toBe('ts'); + expect(blocks[0]!.value).toBe('x'.repeat(5000)); + }); + it('splits lists at existing item boundaries', () => { + const text = Array.from( + { length: 1000 }, + (_, i) => `- item ${i} [source](https://docs.example.com/${i})`, + ).join('\n'); + const parts = split(text); + expect(parts.flatMap(links)).toEqual(links(text)); + expect( + parts.every((part) => nodes(part).some((node) => node.type === 'list')), + ).toBe(true); + }); + it('keeps fallback within the hard budget when the soft budget is small', () => { + for (const text of [ + '[source](https://docs.example.com/' + 'x'.repeat(201) + ')', + '\n'.repeat(200) + '[ref]: https://docs.example.com/ref', + ]) { + const parts = splitMarkdown(text, { + targetLength: 10, + maxLength: 200, + unit, + }); + expect(parts.every((part) => measure(part) <= 200)).toBe(true); + expect( + parts + .flatMap(code) + .map((block) => block.value) + .join(''), + ).toBe(text); + } + }); + it('retains deeply nested list content when the depth fallback begins with a newline', () => { + const body = 'tail ' + 'y'.repeat(6000); + const text = + Array.from( + { length: 16 }, + (_, i) => ' '.repeat(2 * i) + '- level' + i, + ).join('\n') + + '\n' + + ' '.repeat(32) + + body; + const parts = split(text); + expect( + parts + .flatMap(code) + .map((block) => block.value) + .join(''), + ).toContain(body); + }); + it.each(['> quoted body', '- item body', 'x'.repeat(3000)])( + 'keeps content after a gap that consumes the target', + (body) => { + for (const length of [3801, 19895, 19900]) { + const text = '\n'.repeat(length) + body; + const parts = split(text); + expect(parts.length).toBeLessThan(20); + expect(parts.join('')).toBe(text); + } + }, + ); + it('keeps bare list items after a nearly full heading', () => { + const text = '# ' + 'h'.repeat(3795) + '\n\n1.\n\nrest'; + expect(split(text).join('')).toBe(text); + }); + it('retains heading references when their reserved budget is exhausted', () => { + const definition = '[ref]: https://docs.example.com/ref'; + const text = '# [ref] ' + 'h'.repeat(3740) + '\n\nbody\n\n' + definition; + const parts = split(text); + expect(parts.every((part) => part.includes(definition))).toBe(true); + expect( + parts.flatMap(nodes).filter((node) => node.type === 'linkReference'), + ).toHaveLength(1); + }); + it('preserves large code metadata without one-character fragments', () => { + const language = 'x'.repeat(3797); + const body = 'y'.repeat(30000); + const blocks = split('```' + language + '\n' + body + '\n```').flatMap( + code, + ); + expect(blocks.length).toBeLessThan(100); + expect(blocks.every((block) => block.lang === language)).toBe(true); + expect(blocks.map((block) => block.value).join('')).toBe(body); + }); + it.each([10500, 11000, 15000])( + 'retains affordable code language and metadata above half the hard budget (%s)', + (length) => { + const metadata = 'x'.repeat(length - 3); + const body = '中😀y'.repeat(7500); + const parts = split('```ts ' + metadata + '\n' + body + '\n```'); + const blocks = parts.flatMap(code); + expect(blocks).toHaveLength(parts.length); + expect( + blocks.every((block) => block.lang === 'ts' && block.meta === metadata), + ).toBe(true); + expect(blocks.map((block) => block.value).join('')).toBe(body); + expect( + parts.every((part) => !part.includes('original text follows in parts')), + ).toBe(true); + }, + ); + it.each([10500, 11000])( + 'retains affordable large table headers and every row (%s)', + (length) => { + const header = '| ' + 'h'.repeat(length) + ' |\n|---|\n'; + const data = Array.from( + { length: 100 }, + (_, i) => '| row-' + i + ' ' + '中😀'.repeat(100) + ' |', + ).join('\n'); + const parts = split(header + data); + expect(parts.every((part) => part.startsWith(header))).toBe(true); + expect( + parts.every((part) => + nodes(part).some((node) => node.type === 'table'), + ), + ).toBe(true); + expect(parts.map((part) => part.slice(header.length)).join('')).toBe( + data, + ); + expect(parts.flatMap(rows)).toHaveLength(100); + }, + ); + it.each([ + 1891, 1892, 3788, 3789, 3790, 3797, 9983, 9984, 19985, 19988, 19989, 19990, + 19991, 19992, + ])( + 'bounds fence repetition around budget boundaries (info length %s)', + (length) => { + const info = 'x'.repeat(length); + const body = '中😀y'.repeat(7500); + const text = '```' + info + '\n' + body + '\n```'; + const parts = split(text); + expect(parts.length).toBeLessThanOrEqual( + 4 * Math.ceil(measure(text) / options.targetLength), + ); + expect( + parts.reduce((sum, part) => sum + measure(part), 0), + ).toBeLessThanOrEqual(4 * measure(text) + options.maxLength); + const blocks = parts.flatMap(code); + if ( + parts.some((part) => part.includes('original text follows in parts')) + ) { + expect(blocks.map((block) => block.value).join('')).toBe(text); + } else { + expect(blocks.every((block) => block.lang === info)).toBe(true); + expect(blocks.map((block) => block.value).join('')).toBe(body); + } + }, + ); + it('does not turn the exact three-character fence budget into 10000 messages', () => { + const info = 'x'.repeat(3789); + const body = 'y'.repeat(30000); + const parts = split('```' + info + '\n' + body + '\n```'); + expect(parts).toHaveLength(2); + expect(parts.flatMap(code).every((block) => block.lang === info)).toBe( + true, + ); + expect( + parts + .flatMap(code) + .map((block) => block.value) + .join(''), + ).toBe(body); + }); + it.each([3780, 9980, 19960])( + 'bounds repeated table headers and preserves all source (%s)', + (length) => { + const header = '| ' + 'h'.repeat(length) + ' |\n|---|\n'; + const data = Array(3000).fill('| 中😀 |').join('\n'); + const text = header + data; + const parts = split(text); + expect(parts.length).toBeLessThanOrEqual( + 4 * Math.ceil(measure(text) / 3800), + ); + expect( + parts.reduce((sum, part) => sum + measure(part), 0), + ).toBeLessThanOrEqual(4 * measure(text) + 20000); + if ( + parts.some((part) => part.includes('original text follows in parts')) + ) { + expect( + parts + .flatMap(code) + .map((block) => block.value) + .join(''), + ).toBe(text); + } else { + expect(parts.every((part) => part.startsWith(header))).toBe(true); + expect(parts.map((part) => part.slice(header.length)).join('')).toBe( + data, + ); + } + }, + ); + it.each(['123456789. ', '123456789. '])( + 'bounds deep repeated list prefixes (%s)', + (marker) => { + const body = '中😀y'.repeat(7500); + const text = marker.repeat(16) + body; + const parts = split(text); + expect(parts.length).toBeLessThanOrEqual( + 4 * Math.ceil(measure(text) / 3800), + ); + expect( + parts.reduce((sum, part) => sum + measure(part), 0), + ).toBeLessThanOrEqual(4 * measure(text) + 20000); + const restored = parts + .flatMap(code) + .map((block) => block.value) + .join(''); + expect(restored).toContain(body); + expect(restored.match(/中😀y/gu)).toHaveLength(7500); + }, + ); + it('preserves nested code after a nearly exhausted prefix budget', () => { + const body = 'x'.repeat(39500); + const text = '\n'.repeat(19883) + '> - ```sh\n> ' + body + '\n> ```'; + const blocks = split(text).flatMap(code); + expect(blocks.every((block) => block.lang === 'sh')).toBe(true); + expect(blocks.map((block) => block.value).join('')).toBe(body); + }); + it('retains an oversized header-only table after its introduction', () => { + const table = + Array.from({ length: 1400 }, (_, i) => 'column-' + i).join('|') + + '\n' + + Array(1400).fill('---').join('|'); + const parts = split('Synthetic lead-in.\n' + table); + expect(parts[0]).toBe('Synthetic lead-in.\n'); + expect( + parts + .flatMap(code) + .map((block) => block.value) + .join(''), + ).toBe(table); + }); + it('preserves task checkboxes with continued fenced code', () => { + const body = 'a'.repeat(25000); + const text = '- [x]\n ```bash\n ' + body + '\n ```'; + const parts = split(text); + expect(parts.every((part) => part.startsWith('- [x]'))).toBe(true); + expect( + parts + .flatMap(code) + .map((block) => block.value) + .join(''), + ).toBe(code(text)[0]!.value); + }); + it('keeps code metadata when a list item and its gap reach the hard limit', () => { + const body = 'x'.repeat(19982); + const parts = split('- ```ts\n ' + body + '\n ```\n- next'); + const blocks = parts.flatMap(code); + expect(blocks.every((block) => block.lang === 'ts')).toBe(true); + expect(blocks.map((block) => block.value).join('')).toBe(body); + expect(parts.at(-1)).toContain('- next'); + }); + it('preserves original list item gaps across chunk boundaries', () => { + for (const separator of ['\n', '\n\n']) { + const text = Array.from( + { length: 60 }, + (_, i) => '- bullet ' + i + ' filler '.repeat(15), + ).join(separator); + expect(split(text).join('')).toBe(text); + } + expect(split(fixture).join('')).toBe(fixture); + }); + it('does not turn a split literal backtick run into inline code', () => { + const stem = '[a](https://docs.example.com/'; + const link = stem + 'x'.repeat(3798 - stem.length - 1) + ')'; + for (const prefix of [ + 'x'.repeat(3798) + '```', + link + '```', + stem + ')' + '`'.repeat(3801 - stem.length - 1), + ]) { + const text = prefix + 'y'.repeat(10) + '`' + 'z'.repeat(10); + const parts = split(text); + expect(parts.join('')).toBe(text); + expect(parts.flatMap(links)).toEqual(links(text)); + expect( + parts + .flatMap(nodes) + .filter((node) => node.type === 'inlineCode' || node.type === 'code'), + ).toEqual([]); + expect(parts.filter((part) => part.includes('```'))).toHaveLength(1); + } + }); + it.each(['```', '~~~'])( + 'keeps prose visible when a continuation begins with %s', + (marker) => { + const stem = '[a](https://docs.example.com/'; + const link = stem + 'x'.repeat(3800 - stem.length - 1) + ')'; + for (const prefix of ['word '.repeat(760), link]) { + const text = + prefix + marker + ' fences like this. ' + 'tail '.repeat(600); + const parts = split(text); + const blocks = parts.flatMap(code); + expect(blocks.length).toBeGreaterThan(0); + expect(blocks.every((block) => block.lang === 'text')).toBe(true); + expect( + parts + .map((part) => + code(part).length + ? code(part) + .map((block) => block.value) + .join('') + : part, + ) + .join(''), + ).toBe(text); + expect(parts.flatMap(links)).toEqual(links(text)); + } + }, + ); + it('reserves reference definitions within the ordinary prose target', () => { + const definition = '[ref]: https://docs.example.com/' + 'a'.repeat(70); + const text = 'Ordinary [ref] sentence. '.repeat(600) + '\n\n' + definition; + const parts = split(text); + expect(parts.every((part) => measure(part) <= 3800)).toBe(true); + expect(parts.every((part) => part.includes(definition))).toBe(true); + expect( + parts.flatMap(nodes).filter((node) => node.type === 'linkReference'), + ).toHaveLength(600); + }); + it('preserves escaped punctuation and character references across cuts', () => { + const text = 'x'.repeat(3799) + '\\* & ' + 'y'.repeat(5000); + const parts = split(text); + expect(parts.join('')).toBe(text); + expect(parts.some((part) => part.includes('\\*'))).toBe(true); + expect(parts.some((part) => part.includes('&'))).toBe(true); + }); +}); + +it('abandons an over-budget candidate before constructing thousands of repeated fences', () => { + const opener = '```' + 'x'.repeat(19985) + '\n'; + const text = opener + 'y'.repeat(30000) + '\n```'; + const byteLength = Buffer.byteLength; + let wrappedMeasurements = 0; + const spy = vi + .spyOn(Buffer, 'byteLength') + .mockImplementation((value, encoding) => { + if (typeof value === 'string' && value.startsWith(opener)) + wrappedMeasurements++; + return byteLength(value, encoding); + }); + let parts: string[]; + try { + parts = splitMarkdown(text, { + targetLength: 3800, + maxLength: 20000, + unit: 'utf8', + }); + } finally { + spy.mockRestore(); + } + expect(wrappedMeasurements).toBeLessThan(100); + expect( + parts + .flatMap(code) + .map((block) => block.value) + .join(''), + ).toBe(text); +}); + +it('reports an impossible literal budget instead of swallowing the final failure', () => { + expect(() => + splitMarkdown( + '[source](https://docs.example.com/' + 'x'.repeat(400) + ')', + { + targetLength: 64, + maxLength: 64, + unit: 'utf8', + }, + ), + ).toThrow('Markdown budget cannot contain a fenced fragment'); +}); diff --git a/packages/channels/base/src/markdown-chunks.ts b/packages/channels/base/src/markdown-chunks.ts new file mode 100644 index 00000000000..a6453b49e57 --- /dev/null +++ b/packages/channels/base/src/markdown-chunks.ts @@ -0,0 +1,513 @@ +import type { Nodes, RootContent } from 'mdast'; +import { unified } from 'unified'; +import remarkParse from 'remark-parse'; +import remarkGfm from 'remark-gfm'; + +export interface MarkdownChunkOptions { + targetLength: number; + maxLength: number; + unit: 'utf16' | 'utf8'; + /** Per-message prefix already subtracted from the content budgets. */ + reservedLength?: number; +} + +const blocks = unified().use(remarkParse); +const gfm = unified().use(remarkParse).use(remarkGfm); +const candidateOverflow = Symbol('Markdown candidate exceeds batch budget'); +const notice = + '> Markdown element exceeds the message limit; original text follows in parts.\n\n'; + +export function splitMarkdown( + text: string, + options: MarkdownChunkOptions, +): string[] { + const { targetLength, maxLength, unit, reservedLength = 0 } = options; + if ( + !Number.isSafeInteger(targetLength) || + !Number.isSafeInteger(maxLength) || + targetLength <= 0 || + maxLength < targetLength || + !Number.isSafeInteger(reservedLength) || + reservedLength < 0 + ) { + throw new RangeError('Invalid Markdown chunk budget'); + } + const size = (value: string) => + unit === 'utf8' ? Buffer.byteLength(value, 'utf8') : value.length; + const length = size(text) + reservedLength; + const maxOutput = 4 * length + maxLength + reservedLength; + // Relative limits bound amplification, never truncate a delivery batch. + const fits = (parts: string[]) => + parts.length <= + Math.max(1, 4 * Math.ceil(length / (targetLength + reservedLength))) && + parts.reduce((total, part) => total + size(part) + reservedLength, 0) <= + maxOutput && + parts.every((part) => size(part) <= maxLength); + const attempt = (target: number, depth: number) => { + try { + const parts = split(text, target, maxLength, size, depth, maxOutput); + return fits(parts) ? parts : undefined; + } catch (error) { + if ( + error !== candidateOverflow && + !(depth === 0 && error instanceof RangeError) + ) + throw error; + return undefined; + } + }; + const preferred = attempt( + reservedLength > targetLength ? maxLength : targetLength, + 0, + ); + if (preferred) return preferred; + const expanded = attempt(maxLength, 0); + if (expanded) return expanded; + // Re-plan from the original source, without repeated container context. + const literal = attempt(maxLength, 16); + if (literal) return literal; + throw new RangeError( + 'Markdown budget cannot contain a bounded lossless batch', + ); +} + +function split( + text: string, + target: number, + maximum: number, + size: (value: string) => number, + depth: number, + maxOutput: number, +): string[] { + // Intermediate arrays can contain pieces later packed together. Bound their + // output size, not their count, without charging the same pieces twice. + const totals = new WeakMap(); + const append = (parts: string[], ...values: string[]) => { + let total = + totals.get(parts) ?? parts.reduce((sum, part) => sum + size(part), 0); + for (const part of values) { + total += size(part); + if (total > maxOutput) throw candidateOverflow; + parts.push(part); + } + totals.set(parts, total); + }; + const source = (node: Nodes) => + text.slice(node.position!.start.offset!, node.position!.end.offset!); + function cut(value: string, budget: number): number { + let lo = 0; + let hi = value.length; + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2); + if (size(value.slice(0, mid)) <= budget) lo = mid; + else hi = mid - 1; + } + if ( + lo < value.length && + /[\uD800-\uDBFF]/u.test(value[lo - 1] ?? '') && + /[\uDC00-\uDFFF]/u.test(value[lo]!) + ) + lo--; + if (!lo && value) + throw new RangeError( + 'Markdown budget cannot contain one Unicode character', + ); + return lo; + } + function fence(value: string, info: string): string { + const char = info.includes('`') ? '~' : '`'; + let length = 3; + for (const run of value.matchAll(new RegExp(`${char}+`, 'g'))) { + length = Math.max(length, run[0].length + 1); + } + return char.repeat(length); + } + function plain(value: string, limit: number): string[] { + const result: string[] = []; + while (value) { + // At most two extra copies of the body length are needed for safe fences. + const budget = Math.max( + size('😀'), + size(String.fromCodePoint(value.codePointAt(0)!)), + Math.floor((limit - size(notice + '```text\n\n```')) / 3), + ); + const end = cut(value, budget); + const part = value.slice(0, end); + const marker = fence(part, 'text'); + const chunk = `${notice}${marker}text\n${part}\n${marker}`; + if (size(chunk) > limit) + throw new RangeError( + 'Markdown budget cannot contain a fenced fragment', + ); + append(result, chunk); + value = value.slice(end); + } + return result; + } + if (size(text) <= target && !/`{3}|~{3}/u.test(text)) return [text]; + if (depth >= 16) return plain(text, maximum); + const tree = blocks.parse(text); + const definitions = tree.children.filter( + (node) => node.type === 'definition', + ); + const suffix = definitions.length + ? '\n\n' + definitions.map(source).join('\n') + : ''; + const hard = maximum - size(suffix); + const minimum = size(notice + '```text\n😀\n```'); + const soft = Math.min( + target - size(suffix) >= minimum ? target - size(suffix) : target, + hard, + ); + if (hard < size(notice + '```text\n😀\n```')) return plain(text, maximum); + const references = new Map( + definitions.map((node) => [node.identifier.toUpperCase(), source(node)]), + ); + + function paragraph(raw: string, preferred: number, limit: number): string[] { + const lines = raw.match(/[^\n]*\n|[^\n]+$/gu) ?? []; + let offset = 0; + for (let i = 1; i < lines.length; i++) { + const header = lines[i - 1]! + lines[i]!; + // GFM's full table parser is quadratic on large bodies; validate two lines only. + if ( + /^[ \t|:-]+\r?\n?$/u.test(lines[i]!) && + gfm.parse(header).children[0]?.type === 'table' + ) { + const before = offset + ? paragraph(raw.slice(0, offset), preferred, limit) + : []; + const table = raw.slice(offset); + if (size(table) <= limit) return [...before, table]; + const result = [...before]; + const tableStart = result.length; + let current = header; + for (const row of lines.slice(i + 1)) { + if (size(header + row) > limit) { + if (current !== header) append(result, current); + append(result, ...plain(header + row, limit)); + current = header; + } else { + if (current !== header && size(current + row) > preferred) { + append(result, current); + current = header; + } + current += row; + } + } + if (current !== header) append(result, current); + if (result.length === tableStart) + append(result, ...plain(table, limit)); + return result; + } + offset += lines[i - 1]!.length; + } + const context = new Set(); + for (const match of raw.matchAll(/\[((?:\\.|[^[\]\\])*)\]/gu)) { + const id = match[1]! + .replace(/[\t\n\r ]+/g, ' ') + .trim() + .toUpperCase(); + const definition = references.get(id); + if (definition) context.add(definition); + } + const node = gfm.parse( + raw + (context.size ? '\n\n' + [...context].join('\n') : ''), + ).children[0]; + if ( + node?.type !== 'paragraph' || + node.children.some((child) => !child.position) + ) { + return size(raw) <= limit ? [raw] : plain(raw, limit); + } + const result: string[] = []; + let current = ''; + const flush = () => { + if (current) { + const becomesCode = + /`{3}|~{3}/u.test(current) && + blocks.parse(current).children.some((node) => node.type === 'code'); + append(result, ...(becomesCode ? plain(current, limit) : [current])); + } + current = ''; + }; + for (const child of node.children) { + let value = raw.slice( + child.position!.start.offset!, + child.position!.end.offset!, + ); + if (child.type !== 'text') { + if (current && size(current + value) > preferred) flush(); + if (size(value) > limit) { + flush(); + append(result, ...plain(value, limit)); + } else current += value; + continue; + } + while (value) { + if ( + current && + size(current + String.fromCodePoint(value.codePointAt(0)!)) > + preferred + ) + flush(); + let end = cut( + value, + Math.max( + size(String.fromCodePoint(value.codePointAt(0)!)), + preferred - size(current), + ), + ); + if (end < value.length) { + const candidate = value.slice(0, end); + const boundary = [ + ...candidate.matchAll(/[。！？]|[.!?](?=\s)|\s+/gu), + ].at(-1); + if (boundary) end = boundary.index + boundary[0].length; + const amp = value.lastIndexOf('&', end - 1); + const entity = + amp >= 0 + ? /^&(?:#\d+|#x[\da-f]+|[a-z][a-z\d]*);/iu.exec(value.slice(amp)) + : null; + if (entity && amp + entity[0].length > end) + end = amp || entity[0].length; + if (/[`~]/u.test(value[end] ?? '') && value[end - 1] === value[end]) { + const run = value.slice(0, end).match(/(`+|~+)$/u)!; + end = run.index || /^(`+|~+)/u.exec(value)![0].length; + } + const escapes = value.slice(0, end).match(/\\+$/u)?.[0].length ?? 0; + if (escapes % 2 && /[!-/:-@[-`{-~]/u.test(value[end]!)) + end = end > 1 ? end - 1 : 2; + } + const piece = value.slice(0, end); + if (size(current + piece) > limit) flush(); + if (size(piece) > limit) append(result, ...plain(piece, limit)); + else current += piece; + value = value.slice(end); + if (value) flush(); + } + } + const tail = raw.slice(node.children.at(-1)?.position?.end.offset ?? 0); + if (size(current + tail) > limit) flush(); + current += tail; + flush(); + return result; + } + + function block( + node: RootContent, + preferred: number, + limit: number, + ): string[] { + const raw = source(node); + if (node.type === 'code') { + const opener = /^ {0,3}(`{3,}|~{3,})([^\r\n]*)(?:\r\n|[\r\n]|$)/u.exec( + raw, + ); + const closed = + !opener || + new RegExp( + `\\n {0,3}${opener[1]![0]}{${opener[1]!.length},}[ \\t]*\\r?$`, + 'u', + ).test(raw); + if (closed && size(raw) <= limit) return [raw]; + const info = opener ? opener[2]! : ''; + const marker = fence(node.value, info); + const opening = marker + (info.startsWith(marker[0]!) ? ' ' : '') + info; + const wrap = (part: string) => + `${opening}\n${part}${part.endsWith('\n') ? '' : '\n'}${marker}`; + if (size(wrap(node.value)) <= limit) return [wrap(node.value)]; + const overhead = size(wrap('')); + if (overhead >= limit) return plain(raw, limit); + const budget = + overhead < preferred / 2 ? preferred - overhead : limit - overhead; + const result: string[] = []; + let current = ''; + for (let line of node.value.match(/[^\n]*\n|[^\n]+$/gu) ?? []) { + if (current && size(wrap(current + line)) > preferred) { + append(result, wrap(current)); + current = ''; + } + while (size(line) > Math.min(budget, limit - overhead)) { + if ( + size(String.fromCodePoint(line.codePointAt(0)!)) > + limit - overhead + ) + return plain(raw, limit); + const end = cut( + line, + Math.min( + Math.max( + budget, + size(String.fromCodePoint(line.codePointAt(0)!)), + ), + limit - overhead, + ), + ); + append(result, wrap(line.slice(0, end))); + line = line.slice(end); + } + current += line; + } + if (current) append(result, wrap(current)); + return result; + } + if (size(raw) <= preferred) return [raw]; + if (node.type === 'paragraph') return paragraph(raw, preferred, limit); + if (node.type === 'blockquote') { + const body = raw.replace(/^ {0,3}> ?/gm, ''); + const wrap = (part: string) => + part + .split('\n') + .map((line) => (line ? `> ${line}` : '>')) + .join('\n'); + return split( + body, + preferred, + limit, + (value) => (value ? size(wrap(value)) : 0), + depth + 1, + maxOutput, + ).map(wrap); + } + if (node.type === 'list') { + const result: string[] = []; + for (const [itemIndex, item] of node.children.entries()) { + const nextItem = node.children[itemIndex + 1]; + const gap = nextItem + ? text.slice( + item.position!.end.offset!, + nextItem.position!.start.offset!, + ) + : ''; + const itemText = source(item); + if (size(itemText) <= preferred) { + append(result, itemText, gap); + continue; + } + const marker = + /^(\s*(?:[-+*]|\d+[.)])\s+)(?:\[[ xX]\](?:[ \t]+|(?=\r?\n|$)))?/u.exec( + itemText, + ); + if (!marker) { + append(result, itemText, gap); + continue; + } + const indent = ' '.repeat(marker[1]!.length); + const body = itemText + .slice(marker[0].length) + .replace(/^[ \t]+/gm, (leading) => { + let column = 0; + let end = 0; + while (end < leading.length && column < indent.length) { + column += leading[end++] === '\t' ? 4 - (column % 4) : 1; + } + return ( + ' '.repeat(Math.max(0, column - indent.length)) + + leading.slice(end) + ); + }); + const wrap = (part: string) => { + const head = + /^(?:`{3,}|~{3,}|#{1,6}(?:\s|$)|>|\||[-+*] |\d+[.)] )/u.test(part) + ? `${marker[0].trimEnd()}\n${indent}` + : marker[0]; + return head + part.replace(/\n/g, `\n${indent}`); + }; + const parts = split( + body, + preferred, + limit, + (value) => (value ? size(wrap(value) + '\n') : 0), + depth + 1, + maxOutput, + ).map( + (part, index, parts) => + wrap(part) + (index < parts.length - 1 ? '\n' : ''), + ); + append(result, ...parts, gap); + } + return result; + } + return size(raw) <= limit ? [raw] : plain(raw, limit); + } + + const result: string[] = []; + let current = ''; + let heading = ''; + let previousEnd = 0; + const flush = () => { + if (current) append(result, current + suffix); + current = ''; + }; + for (const node of tree.children) { + const gap = text.slice(previousEnd, node.position!.start.offset!); + previousEnd = node.position!.end.offset!; + if (node.type === 'definition') continue; + if (node.type === 'heading' && size(heading + gap + source(node)) < soft) { + heading += gap + source(node); + continue; + } + let head = heading ? heading + gap : ''; + let leading = heading ? '' : gap; + heading = ''; + let reserve = size(head + leading); + if (reserve && (reserve >= hard - minimum || soft - reserve < minimum)) { + flush(); + append( + result, + ...split(head + leading, soft, hard, size, depth + 1, maxOutput).map( + (part) => part + suffix, + ), + ); + head = ''; + leading = ''; + reserve = 0; + } + const pieces = block(node, Math.max(1, soft - reserve), hard - reserve); + for (const [index, piece] of pieces.entries()) { + let next = index === 0 ? head + piece : piece; + const separator = + index === 0 ? leading : node.type === 'list' ? '' : '\n\n'; + let prefix = index === 0 ? leading : ''; + if ( + current && + ((index > 0 && node.type !== 'list') || + size(current + separator + next) > soft) + ) { + if ( + index === 0 && + !/[ \t]$/u.test(separator) && + size(current + separator) <= hard + ) { + current += separator; + prefix = ''; + } + flush(); + } + if (!current) next = prefix + next; + if (size(next) > hard) { + flush(); + append(result, ...plain(next + suffix, maximum)); + } else current = current ? current + separator + next : next; + } + } + if (heading) { + if (current && size(current + heading) > soft) flush(); + current += heading; + } + if (!definitions.length) { + const tail = text.slice(previousEnd); + if (size(current + tail) <= hard) current += tail; + else { + flush(); + append(result, ...plain(tail, maximum)); + } + } + flush(); + return result.length + ? result + : size(text) <= maximum + ? [text] + : plain(text, maximum); +} diff --git a/packages/channels/dingtalk/src/DingtalkAdapter.test.ts b/packages/channels/dingtalk/src/DingtalkAdapter.test.ts index 82730e0e322..29178ec0307 100644 --- a/packages/channels/dingtalk/src/DingtalkAdapter.test.ts +++ b/packages/channels/dingtalk/src/DingtalkAdapter.test.ts @@ -16,6 +16,8 @@ import { DWClient } from 'dingtalk-stream-sdk-nodejs'; import type { DWClientDownStream } from 'dingtalk-stream-sdk-nodejs'; import type { BackgroundResponseContext, + BackgroundOutputDelivery, + BackgroundOutputPacket, ChannelOutputSegmentContext, ChannelOutputSegmentEndReason, ChannelPermissionRequestContext, @@ -8563,6 +8565,38 @@ describe('DingtalkChannel reply mentions', () => { expect(body).not.toHaveProperty('at'); }); + it('sends the synthetic knowledge-search fixture in multiple messages with intact citation URLs and tables', async () => { + const channel = createChannel(); + seedWebhook(channel, 'cid123'); + const fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response('{}', { status: 200 })); + const text = readFileSync( + new URL( + '../../base/src/fixtures/synthetic-knowledge-search.md', + import.meta.url, + ), + 'utf8', + ); + await channel.sendMessage('cid123', text); + const contents = fetchSpy.mock.calls.map( + ([, init]) => + JSON.parse(String((init as RequestInit).body)).markdown.text as string, + ); + expect(contents.length).toBeGreaterThan(1); + const urls = [...text.matchAll(/\[来源\]\((https:\/\/[^)]+)\)/gu)].map( + (match) => match[1]!, + ); + expect(urls).toHaveLength(12); + for (const url of urls) + expect(contents.filter((chunk) => chunk.includes(url))).toHaveLength(1); + expect(contents.find((chunk) => chunk.includes('示例指标五'))).toContain( + '| 文档 | 内容摘要 |\n|---|---|', + ); + for (const chunk of contents) + expect(chunk.length).toBeLessThanOrEqual(20_000); + }); + it('reserves the mention prefix within the first markdown chunk limit', async () => { const channel = createChannel({ atSender: true }); seedWebhook(channel, 'cid123'); @@ -8604,7 +8638,7 @@ describe('DingtalkChannel reply mentions', () => { const fetchSpy = vi .spyOn(globalThis, 'fetch') .mockResolvedValue(new Response('{}', { status: 200 })); - const text = `\`\`\`\n${'a'.repeat(3800)}\n\`\`\``; + const text = `\`\`\`\n${'a'.repeat(40_000)}\n\`\`\``; getPromptHook(channel, 'onPromptStart')('cid123', 'session-1', 'm1'); await getResponseHook(channel)('cid123', text, 'session-1'); @@ -8615,6 +8649,7 @@ describe('DingtalkChannel reply mentions', () => { ? body.markdown.text.slice('@staff-1\n\n'.length) : body.markdown.text; }); + expect(contents.length).toBeGreaterThan(1); expect(contents[0]).toMatch(/^```/u); expect(contents.at(-1)).toMatch(/```$/u); expect(contents.join('').replace(/[`\n]/gu, '')).toBe( @@ -8623,7 +8658,7 @@ describe('DingtalkChannel reply mentions', () => { expect( fetchSpy.mock.calls.every(([, init]) => { const body = JSON.parse(String((init as RequestInit).body)); - return body.markdown.text.length <= 3800; + return body.markdown.text.length <= 20_000; }), ).toBe(true); }); @@ -13908,6 +13943,244 @@ describe('DingtalkChannel proactive send', () => { expect(tokenCalls()).toHaveLength(1); }); + it.each([ + ['escaped controls', '\u0001'.repeat(3790)], + ['Chinese code', '```ts\n' + '中'.repeat(6000) + '\n```'], + ['escaped code', '```text\n' + '\\'.repeat(9000) + '\n```'], + [ + 'indivisible URL', + '[source](https://docs.example.com/' + 'a'.repeat(18_000) + ')', + ], + ])( + 'keeps serialized proactive msgParam within 15000 bytes for %s', + async (_name, text) => { + const channel = proactive(createChannel()); + const { sendCalls } = stubProactiveFetch(); + await channel.pushProactive(groupTarget, text); + const sends = sendCalls(); + expect(sends.length).toBeGreaterThan(1); + for (const call of sends) { + expect( + Buffer.byteLength(JSON.parse(String(call[1]?.body)).msgParam), + ).toBeLessThanOrEqual(15_000); + } + if (text.startsWith('[source]')) { + expect( + sends.every((call) => + msgParamOf(call).text.includes('original text follows in parts'), + ), + ).toBe(true); + } + }, + ); + + it.each(['\u0001', '\ud800'])( + 'delivers nested lists with serialization-amplifying content (%j)', + async (character) => { + const channel = proactive(createChannel()); + const { sendCalls } = stubProactiveFetch(); + const text = + Array.from( + { length: 15 }, + (_, i) => ' '.repeat(i) + '- level ' + i, + ).join('\n') + + '\n' + + ' '.repeat(15) + + character.repeat(32000); + + await channel.pushProactive(groupTarget, text); + + const sends = sendCalls(); + expect(sends.length).toBeGreaterThan(1); + for (const call of sends) { + expect( + Buffer.byteLength(JSON.parse(String(call[1]?.body)).msgParam), + ).toBeLessThanOrEqual(15_000); + } + expect( + sends + .map( + (call) => + msgParamOf(call).text.match(/```text\n([\s\S]*)\n```$/u)?.[1], + ) + .join(''), + ).toBe(text); + }, + ); + + it.each([ + ['Worker one', 14_939, 'ts'], + ['S'.repeat(3775), 11_142, 'ts'], + ['Worker one', 30000, 'z'.repeat(3789)], + ] as const)( + 'rechecks serialized payload budgets after an unsent background header changes (%#)', + async (sourceLabel, length, info) => { + const channel = proactive(createChannel({ interactiveCards: undefined })); + const { sendCalls } = stubProactiveFetch( + (index) => new Response('{}', { status: index === 0 ? 429 : 200 }), + ); + const deliver = ( + channel as unknown as { + createBackgroundOutputDelivery( + sessionId: string, + target: { target: SessionTarget; sourceLabel: string }, + ): BackgroundOutputDelivery; + } + ).createBackgroundOutputDelivery('synthetic-session', { + target: groupTarget, + sourceLabel, + }); + const value = 'x'.repeat(length); + const packet: BackgroundOutputPacket = { + kind: 'agent', + status: 'running', + text: `\`\`\`${info}\n${value}\n\`\`\``, + partial: false, + turnComplete: false, + }; + await expect(deliver(packet)).rejects.toThrow(); + await expect( + deliver({ + ...packet, + status: 'completed', + text: '', + turnComplete: true, + }), + ).resolves.toBeDefined(); + const retried = sendCalls().slice(1).map(msgParamOf); + expect(retried.length).toBeGreaterThan(1); + expect(retried.length).toBeLessThanOrEqual( + 4 * Math.ceil(packet.text.length / 3800), + ); + for (const message of retried) { + expect(Buffer.byteLength(JSON.stringify(message))).toBeLessThanOrEqual( + 15_000, + ); + expect(message.text.startsWith(sourceLabel + '\n\n')).toBe(true); + } + expect( + retried + .map((message) => + [...message.text.matchAll(/x+/g)].map((run) => run[0]).join(''), + ) + .join(''), + ).toBe(value); + expect(retried[0]!.text).toContain('✅ Background task completed'); + for (const message of retried.slice(1)) { + expect(message.text).toContain('\n\n```' + info + '\n'); + expect(message.text).toMatch(/x+\n```$/u); + } + }, + ); + + it.each([ + [undefined, 'Worker 1'], + ['synthetic-user', 'Worker 1'], + [undefined, 'S'.repeat(3775)], + ] as const)( + 'rechecks reply budgets after an unsent background header changes (%#)', + async (atUserId, sourceLabel) => { + const channel = createChannel({ + interactiveCards: undefined, + atSender: true, + }); + seedWebhook(channel, 'synthetic-dm'); + if (atUserId) + ( + channel as unknown as { sessionMentionTargets: Map } + ).sessionMentionTargets.set('synthetic-session', atUserId); + const messages: string[] = []; + vi.spyOn(globalThis, 'fetch').mockImplementation((_input, init) => { + const body = JSON.parse(String(init?.body)) as { + markdown: { text: string }; + }; + messages.push(body.markdown.text); + return Promise.resolve( + new Response('{}', { status: messages.length === 1 ? 429 : 200 }), + ); + }); + const deliver = ( + channel as unknown as { + createBackgroundOutputDelivery( + sessionId: string, + target: { + target: SessionTarget; + sourceLabel: string; + }, + ): BackgroundOutputDelivery; + } + ).createBackgroundOutputDelivery('synthetic-session', { + target: { ...directTarget, chatId: 'synthetic-dm' }, + sourceLabel, + }); + const prefix = `${atUserId ? `@${atUserId}\n\n` : ''}${sourceLabel}\n\n`; + const value = 'x'.repeat(20_000 - prefix.length - 10); + const packet: BackgroundOutputPacket = { + kind: 'agent', + status: 'running', + text: `\`\`\`ts\n${value}\n\`\`\``, + partial: false, + turnComplete: false, + }; + await expect(deliver(packet)).rejects.toThrow(); + expect(messages[0]!.length).toBe(20_000); + const receipt = await deliver({ + ...packet, + status: 'completed', + text: '', + turnComplete: true, + }); + expect(receipt?.turnComplete).toBe(true); + const retried = messages.slice(1); + expect(retried.length).toBeGreaterThan(1); + for (const message of retried.slice(1)) { + expect(message).toMatch(/\n\n```ts\nx+\n```$/u); + } + expect(retried[0]).toContain('✅ Background task completed'); + expect(retried[0]!.startsWith(prefix)).toBe(true); + for (const message of retried) { + expect(message.length).toBeLessThanOrEqual(20_000); + expect(message).toContain(sourceLabel + '\n\n'); + } + expect( + retried.map((message) => message.match(/x+/g)?.join('') ?? '').join(''), + ).toBe(value); + if (atUserId) + expect(retried.join('').match(/@synthetic-user/g)).toHaveLength(1); + }, + ); + + it.each([3789, 19985])( + 'bounds the complete proactive batch for large code metadata (%s)', + async (length) => { + const channel = proactive(createChannel()); + const { sendCalls } = stubProactiveFetch(); + const text = + '```' + 'x'.repeat(length) + '\n' + 'y'.repeat(30000) + '\n```'; + await channel.pushProactive(groupTarget, text); + const sends = sendCalls(); + expect(sends.length).toBeLessThanOrEqual( + 4 * Math.ceil(text.length / 3800), + ); + const messages = sends.map(msgParamOf); + expect( + messages.reduce((sum, message) => sum + message.text.length, 0), + ).toBeLessThanOrEqual(4 * text.length + 20000); + expect( + messages + .map((message) => message.text) + .join('') + .match(/y/g), + ).toHaveLength(30000); + for (const message of messages) { + expect(message.text.length).toBeLessThanOrEqual(20000); + expect(Buffer.byteLength(JSON.stringify(message))).toBeLessThanOrEqual( + 15000, + ); + } + }, + ); + it('splits long proactive messages into continuation chunks', async () => { const channel = proactive(createChannel()); const { sendCalls } = stubProactiveFetch(); @@ -13991,8 +14264,11 @@ describe('DingtalkChannel proactive send', () => { }); const sends = sendCalls(); - expect(sends).toHaveLength(4); - expect(msgParamOf(sends[3]!).text).toBe('✅ Background task completed'); + expect(sends.length).toBeGreaterThanOrEqual(4); + expect(msgParamOf(sends[1]!).text).toBe(msgParamOf(sends[2]!).text); + expect(msgParamOf(sends.at(-1)!).text).toBe( + '✅ Background task completed', + ); } finally { vi.useRealTimers(); } diff --git a/packages/channels/dingtalk/src/DingtalkAdapter.ts b/packages/channels/dingtalk/src/DingtalkAdapter.ts index b5561c2490d..9e3c7e49085 100644 --- a/packages/channels/dingtalk/src/DingtalkAdapter.ts +++ b/packages/channels/dingtalk/src/DingtalkAdapter.ts @@ -22,6 +22,7 @@ import { } from '@qwen-code/channel-base'; import { DINGTALK_CHUNK_LIMIT, + DINGTALK_MAX_CHUNK_LENGTH, escapeDingTalkMarkdown, normalizeDingTalkMarkdown, extractTitle, @@ -932,12 +933,14 @@ type DingtalkChannelConfig = ChannelConfig & { }; interface ProactiveTextDelivery { + sourceText: string; title: string; chunks: string[]; nextChunk: number; } interface ReplyTextDelivery { + sourceText: string; title: string; chunks: string[]; nextChunk: number; @@ -1683,17 +1686,24 @@ export class DingtalkChannel extends ChannelBase { sourceLabel && outgoingText.trim().length > 0 ? `${escapeDingTalkMarkdown(sourceLabel)}\n\n` : ''; - const contentLimit = - DINGTALK_CHUNK_LIMIT - mentionPrefix.length - sourcePrefix.length; + const overhead = mentionPrefix.length + sourcePrefix.length; + const contentLimit = DINGTALK_CHUNK_LIMIT - overhead; + const maxLength = DINGTALK_MAX_CHUNK_LENGTH - overhead; if (contentLimit <= 0) { throw new Error('DingTalk source label exceeds the message limit.'); } - const chunks = normalizeDingTalkMarkdown(outgoingText, contentLimit).map( + const chunks = normalizeDingTalkMarkdown( + outgoingText, + contentLimit, + maxLength, + overhead, + ).map( (chunk, index) => `${index === 0 ? mentionPrefix : ''}${sourcePrefix}${chunk}`, ); return { title: extractTitle(outgoingText), + sourceText: outgoingText, chunks, nextChunk: 0, ...(atUserId ? { atUserId } : {}), @@ -1881,14 +1891,60 @@ export class DingtalkChannel extends ChannelBase { const sourcePrefix = sourceLabel ? `${escapeDingTalkMarkdown(sourceLabel)}\n\n` : ''; + const title = extractTitle(outgoingText); + return { + title, + sourceText: outgoingText, + chunks: this.createProactiveTextChunks(outgoingText, title, sourcePrefix), + nextChunk: 0, + }; + } + + private createProactiveTextChunks( + text: string, + title: string, + sourcePrefix: string, + header = '', + ): string[] { const contentLimit = DINGTALK_CHUNK_LIMIT - sourcePrefix.length; + const targetLength = Math.max(1, contentLimit - header.length); + const maxLength = + DINGTALK_MAX_CHUNK_LENGTH - sourcePrefix.length - header.length; if (contentLimit <= 0) { throw new Error('DingTalk source label exceeds the message limit.'); } - const chunks = normalizeDingTalkMarkdown(outgoingText, contentLimit).map( - (chunk) => `${sourcePrefix}${chunk}`, + const fits = (chunk: string) => + Buffer.byteLength( + JSON.stringify({ + title: `${title} (cont.)`, + text: sourcePrefix + header + chunk, + }), + 'utf8', + ) <= 15_000; + if (!fits('')) { + throw new Error( + 'DingTalk source label exceeds the proactive payload limit.', + ); + } + let budget = maxLength; + let pieces = normalizeDingTalkMarkdown( + text, + targetLength, + budget, + sourcePrefix.length + header.length, ); - return { title: extractTitle(outgoingText), chunks, nextChunk: 0 }; + // Re-plan the whole batch from source before sending its first message. + // OpenAPI limits the serialized msgParam, not UTF-16 text length. + while (!pieces.every(fits)) { + budget = Math.floor(budget / 2); + pieces = normalizeDingTalkMarkdown( + text, + Math.min(targetLength, budget), + budget, + sourcePrefix.length + header.length, + ); + } + return pieces.map((piece) => sourcePrefix + header + piece); } private async deliverProactiveText( @@ -2885,8 +2941,39 @@ export class DingtalkChannel extends ChannelBase { (target.sourceLabel ? `${escapeDingTalkMarkdown(target.sourceLabel)}\n\n` : ''); - plan.chunks[0] = - prefix + replaceHeader(plan.chunks[0]!.slice(prefix.length)); + plan.sourceText = replaceHeader(plan.sourceText); + const terminalHeader = + !output.text && plan.sourceText.startsWith(header) ? header : ''; + const source = plan.sourceText.slice(terminalHeader.length); + if (replyPlan) { + const mentionLength = replyPlan.atUserId + ? `@${replyPlan.atUserId}\n\n`.length + : 0; + const reservedLength = prefix.length + terminalHeader.length; + replyPlan.chunks = normalizeDingTalkMarkdown( + source, + Math.max(1, DINGTALK_CHUNK_LIMIT - reservedLength), + DINGTALK_MAX_CHUNK_LENGTH - reservedLength, + reservedLength, + ).map( + (chunk, index) => + (index === 0 + ? prefix + terminalHeader + : prefix.slice(mentionLength)) + chunk, + ); + } + if (proactivePlan) { + proactivePlan.chunks = this.createProactiveTextChunks( + source, + plan.title, + prefix, + terminalHeader, + ).map((piece, index) => + index > 0 + ? prefix + piece.slice(prefix.length + terminalHeader.length) + : piece, + ); + } } if (preparedReplyBody) { preparedReplyBody = replaceHeader(preparedReplyBody); diff --git a/packages/channels/dingtalk/src/markdown.test.ts b/packages/channels/dingtalk/src/markdown.test.ts index 47ab7c5e8ef..6ca70c1276f 100644 --- a/packages/channels/dingtalk/src/markdown.test.ts +++ b/packages/channels/dingtalk/src/markdown.test.ts @@ -18,9 +18,9 @@ describe('DingTalk markdown utilities', () => { it.each([1, 4, 7])( 'rejects code-fence limit %i when it cannot make progress', (chunkLimit) => { - expect(() => splitChunks('```\nabc\n```', chunkLimit)).toThrow( - RangeError, - ); + expect(() => + splitChunks('```\nabc\n```', chunkLimit, chunkLimit), + ).toThrow(RangeError); }, ); @@ -61,7 +61,7 @@ describe('DingTalk markdown utilities', () => { it('preserves surrounding newlines when splitting a long line', () => { const text = ['before', 'b'.repeat(5000), 'after'].join('\n'); const chunks = splitChunks(text); - expect(chunks.length).toBe(2); + expect(chunks.length).toBeGreaterThan(1); expect(chunks.join('')).toBe(text); }); @@ -75,7 +75,7 @@ describe('DingTalk markdown utilities', () => { }); it('does not add fences for long plain text with inline backticks', () => { - const text = 'before ``` inline ``` ' + 'x'.repeat(5000); + const text = 'before ``` inline ``` ' + 'x'.repeat(21000); const chunks = splitChunks(text); expect(chunks.join('')).toBe(text); chunks.forEach((chunk) => { @@ -84,7 +84,7 @@ describe('DingTalk markdown utilities', () => { }); it('closes and reopens code fences across boundaries', () => { - const longCode = '```\n' + 'x\n'.repeat(2000) + '```'; + const longCode = '```\n' + 'x\n'.repeat(11000) + '```'; const chunks = splitChunks(longCode); expect(chunks.length).toBeGreaterThan(1); // First chunk should end with closing fence @@ -96,7 +96,7 @@ describe('DingTalk markdown utilities', () => { }); it('reopens code fences without inserting a blank line', () => { - const longCode = '```\n' + 'x\n'.repeat(2000) + '```'; + const longCode = '```\n' + 'x\n'.repeat(11000) + '```'; const chunks = splitChunks(longCode); expect(chunks.length).toBeGreaterThan(1); // The reopened fence must be followed by the code, not a blank line. @@ -107,7 +107,7 @@ describe('DingTalk markdown utilities', () => { }); it('splits a long code line while preserving fences', () => { - const longCode = '```\n' + 'x'.repeat(5000) + '\n```'; + const longCode = '```\n' + 'x'.repeat(21000) + '\n```'; const chunks = splitChunks(longCode); expect(chunks.length).toBeGreaterThan(1); expect(chunks[0]!.endsWith('\n```')).toBe(true); @@ -118,7 +118,7 @@ describe('DingTalk markdown utilities', () => { }); it('accounts for closing fence overhead when splitting code chunks', () => { - const longCode = '```\n' + 'x'.repeat(3793) + '\n```'; + const longCode = '```\n' + 'x'.repeat(19993) + '\n```'; const chunks = splitChunks(longCode); expect(chunks.length).toBeGreaterThan(1); chunks.forEach((chunk) => { @@ -127,7 +127,7 @@ describe('DingTalk markdown utilities', () => { }); it('keeps chunks within limit when a long code line ends with a fence', () => { - const longCode = '```\n' + 'x'.repeat(5000) + '```'; + const longCode = '```\n' + 'x'.repeat(21000) + '```'; const chunks = splitChunks(longCode); expect(chunks.length).toBeGreaterThan(1); chunks.forEach((chunk) => { @@ -135,14 +135,17 @@ describe('DingTalk markdown utilities', () => { }); }); - it('keeps room for closing fences after a long opening fence line', () => { - const longCode = '```' + 'x'.repeat(3797) + '\ny\n```'; + it('sends an oversized code language marker as plain text with a notice', () => { + const longCode = '```' + 'x'.repeat(20001) + '\ny\n```'; const chunks = splitChunks(longCode); expect(chunks.length).toBeGreaterThan(1); - expect(chunks[0]!.endsWith('\n```')).toBe(true); - expect(chunks[1]!.startsWith('```\n')).toBe(true); + expect( + chunks.every((chunk) => + chunk.includes('original text follows in parts'), + ), + ).toBe(true); chunks.forEach((chunk) => { - expect(chunk.length).toBeLessThanOrEqual(3800); + expect(chunk.length).toBeLessThanOrEqual(20_000); }); }); @@ -152,7 +155,7 @@ describe('DingTalk markdown utilities', () => { const chunks = splitChunks(longCode); expect(chunks.join('')).toBe(longCode); expect(chunks[0]!.endsWith('\n`')).toBe(false); - expect(chunks[1]!.startsWith('\n```')).toBe(true); + expect(chunks[1]!.startsWith('```')).toBe(true); chunks.forEach((chunk) => { expect(chunk.length).toBeLessThanOrEqual(3800); }); @@ -193,6 +196,20 @@ describe('DingTalk markdown utilities', () => { expect(result).toEqual([input]); }); + it('forwards a caller-reserved hard budget with a source prefix', () => { + const prefix = 'source\n\n'; + const chunks = normalizeDingTalkMarkdown( + '[source](https://docs.example.com/' + + 'x'.repeat(20_000 - '[source](https://docs.example.com/)'.length) + + ')', + 3800 - prefix.length, + 20_000 - prefix.length, + ).map((chunk) => prefix + chunk); + expect(chunks.length).toBeGreaterThan(1); + for (const chunk of chunks) + expect(chunk.length).toBeLessThanOrEqual(20_000); + }); + it('passes through plain text', () => { const result = normalizeDingTalkMarkdown('simple text'); expect(result).toEqual(['simple text']); diff --git a/packages/channels/dingtalk/src/markdown.ts b/packages/channels/dingtalk/src/markdown.ts index 34cccf2b01e..859269d207c 100644 --- a/packages/channels/dingtalk/src/markdown.ts +++ b/packages/channels/dingtalk/src/markdown.ts @@ -1,151 +1,24 @@ -/** - * DingTalk markdown normalization. - * - * DingTalk's markdown renderer is a limited subset with quirks: - * - Max message length ~3800 chars — split into chunks - * - Code fences must be closed/reopened across chunk boundaries - */ +import { splitMarkdown } from '@qwen-code/channel-base/markdown-chunks'; export const DINGTALK_CHUNK_LIMIT = 3800; +export const DINGTALK_MAX_CHUNK_LENGTH = 20_000; export function escapeDingTalkMarkdown(value: string): string { return value.replace(/([\\`*_[\]{}()#+.!|>~:-])/gu, '\\$1'); } -// --- Chunk splitting --- - -function safeUtf16SliceEnd(value: string, end: number): number { - if (end <= 0 || end >= value.length) return end; - const previous = value.charCodeAt(end - 1); - const next = value.charCodeAt(end); - return previous >= 0xd800 && - previous <= 0xdbff && - next >= 0xdc00 && - next <= 0xdfff - ? end - 1 - : end; -} - export function splitChunks( text: string, chunkLimit = DINGTALK_CHUNK_LIMIT, + maxLength = DINGTALK_MAX_CHUNK_LENGTH, + reservedLength = 0, ): string[] { - if (!text || text.length <= chunkLimit) { - return [text]; - } - - const chunks: string[] = []; - let buf = ''; - const lines = text.split('\n'); - let inCode = false; - - const flush = (keepCodeOpen = inCode) => { - if (keepCodeOpen) { - buf += '\n```'; - } - chunks.push(buf); - buf = keepCodeOpen ? '```' : ''; - }; - - const appendLine = ( - line: string, - needsLineBreak: boolean, - closesCodeFence: boolean, - leavesCodeFenceOpen: boolean, - ) => { - let remaining = line; - let prefixPending = needsLineBreak; - let lineOpenedFenceInBuffer = false; - - while (remaining.length > 0 || prefixPending) { - const prefix = prefixPending ? '\n' : ''; - const fitsAsFinalPiece = - remaining.length <= chunkLimit - buf.length - prefix.length; - const closeFenceOverhead = - (inCode && !(closesCodeFence && fitsAsFinalPiece)) || - (!inCode && leavesCodeFenceOpen) - ? '\n```'.length - : 0; - const available = - chunkLimit - closeFenceOverhead - buf.length - prefix.length; - - if (available <= 0) { - const keepCodeOpen = inCode || lineOpenedFenceInBuffer; - if (buf === (keepCodeOpen ? '```' : '')) { - throw new RangeError( - 'chunk limit cannot contain one Unicode character', - ); - } - flush(keepCodeOpen); - continue; - } - - let pieceLength = Math.min(available, remaining.length); - if (pieceLength < remaining.length) { - for ( - let fenceStart = Math.max(0, pieceLength - 2); - fenceStart < pieceLength; - fenceStart++ - ) { - if ( - remaining.slice(fenceStart, fenceStart + 3) === '```' && - pieceLength < fenceStart + 3 - ) { - pieceLength = fenceStart; - break; - } - } - } - pieceLength = safeUtf16SliceEnd(remaining, pieceLength); - - if (pieceLength === 0 && remaining.length > 0) { - const keepCodeOpen = inCode || lineOpenedFenceInBuffer; - if (!buf || (keepCodeOpen && buf === '```')) { - throw new RangeError( - 'chunk limit cannot contain one Unicode character', - ); - } - flush(keepCodeOpen); - continue; - } - - const piece = remaining.slice(0, pieceLength); - const appendedText = prefix + piece; - buf += appendedText; - remaining = remaining.slice(piece.length); - prefixPending = false; - lineOpenedFenceInBuffer ||= - !inCode && leavesCodeFenceOpen && appendedText.includes('```'); - - if (remaining.length > 0) { - const keepCodeOpen = inCode || lineOpenedFenceInBuffer; - flush(keepCodeOpen); - prefixPending = keepCodeOpen; - } - } - }; - - for (let i = 0; i < lines.length; i++) { - const line = lines[i] || ''; - const fenceCount = (line.match(/```/g) || []).length; - const togglesCodeFence = fenceCount % 2 === 1; - appendLine( - line, - i > 0, - inCode && togglesCodeFence, - inCode !== togglesCodeFence, - ); - - if (togglesCodeFence) { - inCode = !inCode; - } - } - - if (buf) { - chunks.push(buf); - } - - return chunks; + return splitMarkdown(text, { + targetLength: chunkLimit, + reservedLength, + maxLength, + unit: 'utf16', + }); } /** Extract a short title from the first line of markdown for the webhook payload. */ @@ -155,10 +28,12 @@ export function extractTitle(text: string): string { return cleaned || 'Reply'; } -/** Split long Markdown messages without changing their formatting. */ +/** Split long Markdown messages without cutting inline structures. */ export function normalizeDingTalkMarkdown( text: string, chunkLimit = DINGTALK_CHUNK_LIMIT, + maxLength = DINGTALK_MAX_CHUNK_LENGTH, + reservedLength = 0, ): string[] { - return splitChunks(text, chunkLimit); + return splitChunks(text, chunkLimit, maxLength, reservedLength); } diff --git a/packages/channels/dingtalk/vitest.config.ts b/packages/channels/dingtalk/vitest.config.ts index 080e20966ec..6c962a0afed 100644 --- a/packages/channels/dingtalk/vitest.config.ts +++ b/packages/channels/dingtalk/vitest.config.ts @@ -17,6 +17,10 @@ export default defineConfig({ }, resolve: { alias: { + '@qwen-code/channel-base/markdown-chunks': path.resolve( + __dirname, + '../base/src/markdown-chunks.ts', + ), // Resolve the in-repo channel-base to its live SOURCE so a package-local // test run (e.g. `cd packages/channels/dingtalk && vitest`) doesn't depend // on a prior `tsc --build` of base — its dist may be absent or stale during diff --git a/packages/channels/wecom/src/WeComAdapter.test.ts b/packages/channels/wecom/src/WeComAdapter.test.ts index 477c29d8a6b..3d1be3a8f56 100644 --- a/packages/channels/wecom/src/WeComAdapter.test.ts +++ b/packages/channels/wecom/src/WeComAdapter.test.ts @@ -5,6 +5,7 @@ import { mkdirSync, mkdtempSync, readdirSync, + readFileSync, realpathSync, rmSync, writeFileSync, @@ -4189,6 +4190,36 @@ describe('WeComChannel', () => { }); }); + it('sends the synthetic knowledge-search fixture in multiple messages without splitting citation URLs or table rows', async () => { + const channel = new TestWeComChannel('bot', makeConfig(), makeBridge()); + await channel.connect(); + const text = readFileSync( + new URL( + '../../base/src/fixtures/synthetic-knowledge-search.md', + import.meta.url, + ), + 'utf8', + ); + await channel.sendAttributed('chat-1', text, '[检索😀]'); + const contents = lastClient().sendMessage.mock.calls.map( + (call) => (call[1] as { markdown: { content: string } }).markdown.content, + ); + expect(contents.length).toBeGreaterThan(1); + const urls = [...text.matchAll(/\[来源\]\((https:\/\/[^)]+)\)/gu)].map( + (match) => match[1]!, + ); + expect(urls).toHaveLength(12); + for (const url of urls) + expect(contents.filter((chunk) => chunk.includes(url))).toHaveLength(1); + expect(contents.find((chunk) => chunk.includes('示例指标五'))).toContain( + '| 文档 | 内容摘要 |\n|---|---|', + ); + for (const chunk of contents) { + expect(chunk.startsWith('\\[检索😀\\]\n')).toBe(true); + expect(Buffer.byteLength(chunk)).toBeLessThanOrEqual(20_000); + } + }); + it('splits long markdown responses before sending', async () => { const channel = new WeComChannel('bot', makeConfig(), makeBridge()); await channel.connect(); @@ -4214,6 +4245,54 @@ describe('WeComChannel', () => { ); }); + it.each(['review', 'S'.repeat(1898), 'S'.repeat(1899), 'S'.repeat(3775)])( + 'plans a bounded attributed batch before sending (%#)', + async (sourceLabel) => { + const channel = new TestWeComChannel('bot', makeConfig(), makeBridge()); + await channel.connect(); + const client = lastClient(); + const prefix = sourceLabel + '\n'; + for (const text of [ + '中😀'.repeat(10000), + '|' + + 'h'.repeat(1800) + + '|\n|---|\n' + + Array(100) + .fill('|' + 'y'.repeat(600) + '|') + .join('\n'), + '```' + + 'x'.repeat(3800 - prefix.length - 11) + + '\n' + + 'y'.repeat(30000) + + '\n```', + ]) { + client.sendMessage.mockClear(); + await channel.sendAttributed('chat-1', text, sourceLabel); + const chunks = client.sendMessage.mock.calls.map( + (call) => + (call[1] as { markdown: { content: string } }).markdown.content, + ); + expect(chunks.length).toBeLessThanOrEqual( + 4 * Math.ceil(Buffer.byteLength(text) / 3800), + ); + expect( + chunks.reduce((total, part) => total + Buffer.byteLength(part), 0), + ).toBeLessThanOrEqual(4 * Buffer.byteLength(text) + 20000); + for (const part of chunks) { + expect(part.startsWith(prefix)).toBe(true); + expect(Buffer.byteLength(part)).toBeLessThanOrEqual(20000); + expect(part.isWellFormed()).toBe(true); + } + const bodies = chunks.map((part) => part.slice(prefix.length)); + if (text.startsWith('中')) expect(bodies.join('')).toBe(text); + else + expect(bodies.join('').match(/y/g)).toHaveLength( + text.startsWith('|') ? 60000 : 30000, + ); + } + }, + ); + it('extracts media first and repeats an escaped label within every byte-limited chunk', async () => { const channel = new TestWeComChannel('bot', makeConfig(), makeBridge()); await channel.connect(); @@ -4247,7 +4326,7 @@ describe('WeComChannel', () => { await channel.sendAttributed( 'chat-1', - `\`\`\`text\n${'x'.repeat(3900)}\n\`\`\``, + `\`\`\`text\n${'x'.repeat(40_000)}\n\`\`\``, '[review]', ); @@ -4280,7 +4359,7 @@ describe('WeComChannel', () => { const channel = new WeComChannel('bot', makeConfig(), makeBridge()); await channel.connect(); const client = lastClient(); - const text = `intro\n\`\`\`ts\n${'a'.repeat(3900)}\n\`\`\`\noutro`; + const text = `intro\n\`\`\`ts\n${'a'.repeat(40_000)}\n\`\`\`\noutro`; await channel.sendMessage('chat-1', text); @@ -4290,11 +4369,12 @@ describe('WeComChannel', () => { }); expect(chunks.length).toBeGreaterThan(1); for (const chunk of chunks) { - expect(Buffer.byteLength(chunk, 'utf8')).toBeLessThan(4096); + expect(Buffer.byteLength(chunk, 'utf8')).toBeLessThanOrEqual(3800); expect((chunk.match(/```/g) ?? []).length % 2).toBe(0); } - expect(chunks[0]).toMatch(/^intro\n```ts\n/); - expect(chunks[0]).toMatch(/\n```$/); + expect(chunks[0]).toBe('intro\n'); + expect(chunks[1]).toMatch(/^```ts\n/); + expect(chunks[1]).toMatch(/\n```$/); expect(chunks[1]).toMatch(/^```/); expect(chunks.at(-1)).toContain('outro'); }); diff --git a/packages/channels/wecom/src/WeComAdapter.ts b/packages/channels/wecom/src/WeComAdapter.ts index 7e58d5768bc..7b9c6d96941 100644 --- a/packages/channels/wecom/src/WeComAdapter.ts +++ b/packages/channels/wecom/src/WeComAdapter.ts @@ -10,6 +10,7 @@ import { isIP, type LookupFunction } from 'node:net'; import { lookup } from 'node:dns/promises'; import { WSClient, decryptFile } from '@wecom/aibot-node-sdk'; import { ChannelBase, sanitizeLogText } from '@qwen-code/channel-base'; +import { splitMarkdown } from '@qwen-code/channel-base/markdown-chunks'; import type { Attachment, ChannelAgentBridge, @@ -80,6 +81,7 @@ const SENSITIVE_ERROR_FIELDS = new Set([ const DEDUP_TTL_MS = 5 * 60 * 1000; const MAX_MEDIA_BYTES = 20 * 1024 * 1024; const MARKDOWN_CHUNK_BYTES = 3800; +const MARKDOWN_MAX_CHUNK_BYTES = 20_000; const AUTHENTICATION_TIMEOUT_MS = 30_000; const KICK_RECONNECT_MAX_ATTEMPTS = 3; const KICK_RECONNECT_MAX_RETRY_CYCLES = 3; @@ -1506,105 +1508,26 @@ function isWeComMediaType(value: string | undefined): value is WeComMediaType { ); } -function splitMarkdownChunks(text: string, prefix?: string): string[] { +function splitMarkdownChunks(text: string, prefix = ''): string[] { if (!text) return []; - - const contentLimit = MARKDOWN_CHUNK_BYTES - Buffer.byteLength(prefix ?? ''); + const overhead = Buffer.byteLength(prefix, 'utf8'); + const maxLength = MARKDOWN_MAX_CHUNK_BYTES - overhead; + const contentLimit = MARKDOWN_CHUNK_BYTES - overhead; if (contentLimit <= 0) { throw new Error('WeCom source label exceeds the markdown message limit.'); } - - const chunks: string[] = []; - let current = ''; - let codeFence: FenceToken | undefined; - const fits = (value: string, nextCodeFence = codeFence): boolean => - Buffer.byteLength( - nextCodeFence ? `${value}\n${nextCodeFence}` : value, - 'utf8', - ) <= contentLimit; - const flush = (closeCode = true): void => { - if (!current) return; - chunks.push(closeCode && codeFence ? `${current}\n${codeFence}` : current); - current = closeCode && codeFence ? codeFence : ''; - }; - - for (const line of text.split('\n')) { - const candidate = current ? `${current}\n${line}` : line; - const candidateCodeFence = toggleCodeFenceState(line, codeFence); - if (fits(candidate, candidateCodeFence)) { - current = candidate; - codeFence = candidateCodeFence; - continue; - } - - flush(); - const retried = current ? `${current}\n${line}` : line; - const retriedCodeFence = toggleCodeFenceState(line, codeFence); - if (fits(retried, retriedCodeFence)) { - current = retried; - codeFence = retriedCodeFence; - continue; - } - - let needsLineBreak = Boolean(current); - for (let index = 0; index < line.length; ) { - const codePoint = line.codePointAt(index); - const token = line.startsWith('```', index) - ? '```' - : line.startsWith('~~~', index) - ? '~~~' - : codePoint === undefined - ? '' - : String.fromCodePoint(codePoint); - if (!token) break; - const nextCodeFence = - token === codeFence - ? undefined - : !codeFence && isFenceToken(token) - ? token - : codeFence; - const addition = needsLineBreak && current ? `\n${token}` : token; - const candidate = `${current}${addition}`; - if (!fits(candidate, nextCodeFence)) { - flush(); - current = current ? `${current}\n${token}` : token; - } else { - current = candidate; - } - codeFence = nextCodeFence; - needsLineBreak = false; - index += token.length; - } - } - - flush(); - return prefix ? chunks.map((chunk) => `${prefix}${chunk}`) : chunks; + return splitMarkdown(text, { + targetLength: contentLimit, + reservedLength: overhead, + maxLength, + unit: 'utf8', + }).map((chunk) => prefix + chunk); } function escapeWeComMarkdown(value: string): string { return value.replace(/([\\`*_{}[\]()#+\-.!|>])/gu, '\\$1'); } -function toggleCodeFenceState( - line: string, - codeFence: FenceToken | undefined, -): FenceToken | undefined { - let nextCodeFence = codeFence; - for (const match of line.matchAll(/```|~~~/g)) { - const token = match[0] as FenceToken; - if (nextCodeFence === token) { - nextCodeFence = undefined; - } else if (!nextCodeFence) { - nextCodeFence = token; - } - } - return nextCodeFence; -} - -function isFenceToken(token: string): token is FenceToken { - return token === '```' || token === '~~~'; -} - async function readOutboundMedia( rawPath: string, cwd: string, diff --git a/packages/channels/wecom/vitest.config.ts b/packages/channels/wecom/vitest.config.ts index 2ede0186076..408dd80763c 100644 --- a/packages/channels/wecom/vitest.config.ts +++ b/packages/channels/wecom/vitest.config.ts @@ -17,6 +17,10 @@ export default defineConfig({ }, resolve: { alias: { + '@qwen-code/channel-base/markdown-chunks': path.resolve( + __dirname, + '../base/src/markdown-chunks.ts', + ), '@qwen-code/channel-base': path.resolve( __dirname, '../base/src/index.ts', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce012a2067e..5dd9c2f5171 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -429,7 +429,19 @@ importers: '@agentclientprotocol/sdk': specifier: ^0.14.1 version: 0.14.1(zod@3.25.76) + remark-gfm: + specifier: ^4.0.0 + version: 4.0.1(supports-color@7.2.0) + remark-parse: + specifier: ^11.0.0 + version: 11.0.0(supports-color@7.2.0) + unified: + specifier: ^11.0.0 + version: 11.0.5 devDependencies: + '@types/mdast': + specifier: ^4.0.4 + version: 4.0.4 typescript: specifier: 5.8.3 version: 5.8.3 diff --git a/scripts/dev.js b/scripts/dev.js index 10b4381385b..d4d4b46c95a 100755 --- a/scripts/dev.js +++ b/scripts/dev.js @@ -154,6 +154,11 @@ for (const name of readdirSync(join(root, 'packages', 'channels'))) { coreSubpathSourceUrls[channel.name] = pathToFileURL(entry).href; } +coreSubpathSourceUrls['@qwen-code/channel-base/markdown-chunks'] = + pathToFileURL( + join(root, 'packages/channels/base/src/markdown-chunks.ts'), + ).href; + const loaderCode = ` import { existsSync } from 'node:fs'; import { fileURLToPath } from 'node:url';