fix(dingtalk): handle replied rich-text media (#11759)

Co-authored-by: qqqys <[email protected]>

docs/users/features/channels/dingtalk.md

@@ -205,7 +205,7 @@ The multi-line layout above is what the agent sees in a 1:1 chat. In a group the
 
 - **Use DingTalk markdown-aware instructions** — DingTalk supports headings, bold text, links, code blocks, and tables. Keep tables compact because narrow screens may scroll horizontally.
 - **Restrict access** — In an organization context, `senderPolicy: "open"` may be acceptable. For tighter control, use `"allowlist"` or `"pairing"`. See [DM Pairing](./overview#dm-pairing) for details.
-- **Referenced messages** — Quoting (replying to) a user message includes the quoted text as context for the agent. If the quoted message is a picture, file, audio, or video message, the bot downloads and attaches it the same way as when sent directly. Quoting bot responses is not yet supported.
+- **Referenced messages** — Quoting (replying to) a user message includes the quoted text as context for the agent. Rich-text quotes preserve their text order and attach embedded pictures. If the quoted message is a picture, file, audio, or video message, the bot downloads and attaches it the same way as when sent directly. Quoting bot responses is not yet supported.
 
 ## Troubleshooting
 

packages/channels/dingtalk/src/DingtalkAdapter.test.ts

@@ -5737,7 +5737,10 @@ describe('DingtalkChannel quoted media', () => {
     vi.restoreAllMocks();
   });
 
-  function mockMediaDownload(mimeType: string, bytes: Uint8Array): string[] {
+  function mockMediaDownload(
+    mimeType: string,
+    bytes: Uint8Array | Record<string, Uint8Array>,
+  ): string[] {
     const downloadCodes: string[] = [];
     vi.spyOn(globalThis, 'fetch').mockImplementation(
       (input: RequestInfo | URL, init?: RequestInit) => {
@@ -5759,16 +5762,22 @@ describe('DingtalkChannel quoted media', () => {
           downloadCodes.push(request.downloadCode);
           return Promise.resolve(
             new Response(
-              JSON.stringify({ downloadUrl: 'https://example.com/media' }),
+              JSON.stringify({
+                downloadUrl: `https://example.com/${request.downloadCode}`,
+              }),
               { status: 200 },
             ),
           );
         }
+        const downloadCode = url.slice(url.lastIndexOf('/') + 1);
         return Promise.resolve(
-          new Response(bytes, {
-            status: 200,
-            headers: { 'content-type': mimeType },
-          }),
+          new Response(
+            bytes instanceof Uint8Array ? bytes : bytes[downloadCode],
+            {
+              status: 200,
+              headers: { 'content-type': mimeType },
+            },
+          ),
         );
       },
     );
@@ -6039,6 +6048,48 @@ describe('DingtalkChannel quoted media', () => {
     ]);
   });
 
+  it('summarizes and downloads nested pictures from replied rich text', async () => {
+    const downloadCodes = mockMediaDownload('image/png', {
+      'quoted-rich-picture-1': new Uint8Array([1]),
+      'quoted-rich-picture-2': new Uint8Array([2]),
+    });
+    const channel = createChannel();
+
+    replyToMedia(channel, 'richText', {
+      richText: [
+        { msgType: 'picture', downloadCode: 'quoted-rich-picture-1' },
+        { msgType: 'text', content: '输出123' },
+        { msgType: 'picture', downloadCode: 'quoted-rich-picture-2' },
+      ],
+    });
+
+    await vi.waitFor(() => {
+      expect(channel.handleInbound).toHaveBeenCalledOnce();
+    });
+    expect(downloadCodes).toEqual([
+      'quoted-rich-picture-1',
+      'quoted-rich-picture-2',
+    ]);
+    expect(channel.handleInbound).toHaveBeenCalledWith(
+      expect.objectContaining({
+        text: 'inspect this',
+        referencedText: '[image]输出123[image]',
+        attachments: [
+          {
+            type: 'image',
+            data: Buffer.from([1]).toString('base64'),
+            mimeType: 'image/png',
+          },
+          {
+            type: 'image',
+            data: Buffer.from([2]).toString('base64'),
+            mimeType: 'image/png',
+          },
+        ],
+      }),
+    );
+  });
+
   it('downloads a replied picture and attaches it to the prompt', async () => {
     const downloadCodes = mockMediaDownload(
       'image/png',

packages/channels/dingtalk/src/DingtalkAdapter.ts

@@ -89,7 +89,9 @@ import type {
 
 interface DingTalkRichTextPart {
   type?: string;
+  msgType?: string;
   text?: string;
+  content?: string;
   downloadCode?: string;
   atName?: string;
 }
@@ -114,6 +116,12 @@ interface DingTalkRepliedMsg {
   content?: DingTalkMessageContent;
 }
 
+interface DingTalkQuotedMedia {
+  downloadCode: string;
+  mediaType: 'image' | 'file' | 'audio' | 'video';
+  fileName?: string;
+}
+
 interface DingTalkAtUser {
   dingtalkId?: string;
   staffId?: string;
@@ -152,6 +160,10 @@ function nonEmptyString(value: unknown): string | undefined {
   return trimmed || undefined;
 }
 
+function richTextPartType(part: DingTalkRichTextPart): string {
+  return part.type || part.msgType || 'text';
+}
+
 function parseJsonArray(value: unknown): unknown[] | undefined {
   if (Array.isArray(value)) return value;
   if (typeof value !== 'string') return undefined;
@@ -2865,11 +2877,7 @@ export class DingtalkChannel extends ChannelBase {
   private extractQuotedContext(data: DingTalkMessageData): {
     referencedText?: string;
     isReplyToBot: boolean;
-    media?: {
-      downloadCode: string;
-      mediaType: 'image' | 'file' | 'audio' | 'video';
-      fileName?: string;
-    };
+    media: DingTalkQuotedMedia[];
   } {
     // Newer format: text.repliedMsg
     if (data.text?.isReplyMsg && data.text.repliedMsg) {
@@ -2880,20 +2888,30 @@ export class DingtalkChannel extends ChannelBase {
       // Note: DingTalk doesn't include content for interactiveCard replies
       // (bot responses sent via webhook). Only user message quotes have text.
       const text = this.summarizeRepliedContent(replied);
-      const downloadCode = replied.content?.downloadCode;
-      const mediaType = this.mediaTypeFromMsgType(replied.msgType);
+      const media: DingTalkQuotedMedia[] = [];
+      const richText = replied.content?.richText;
+      if (Array.isArray(richText)) {
+        for (const part of richText) {
+          const mediaType = this.mediaTypeFromMsgType(richTextPartType(part));
+          if (part.downloadCode && mediaType) {
+            media.push({ downloadCode: part.downloadCode, mediaType });
+          }
+        }
+      } else {
+        const downloadCode = replied.content?.downloadCode;
+        const mediaType = this.mediaTypeFromMsgType(replied.msgType);
+        if (downloadCode && mediaType) {
+          media.push({
+            downloadCode,
+            mediaType,
+            fileName: replied.content?.fileName,
+          });
+        }
+      }
       return {
         referencedText: text || undefined,
         isReplyToBot,
-        ...(downloadCode && mediaType
-          ? {
-              media: {
-                downloadCode,
-                mediaType,
-                fileName: replied.content?.fileName,
-              },
-            }
-          : {}),
+        media,
       };
     }
 
@@ -2903,10 +2921,10 @@ export class DingtalkChannel extends ChannelBase {
       const isReplyToBot =
         !!data.chatbotUserId && quote.senderId === data.chatbotUserId;
       const text = quote.text?.content?.trim();
-      return { referencedText: text || undefined, isReplyToBot };
+      return { referencedText: text || undefined, isReplyToBot, media: [] };
     }
 
-    return { isReplyToBot: false };
+    return { isReplyToBot: false, media: [] };
   }
 
   /**
@@ -2958,9 +2976,11 @@ export class DingtalkChannel extends ChannelBase {
     if (content?.richText && Array.isArray(content.richText)) {
       const parts: string[] = [];
       for (const part of content.richText) {
-        const partType = part.type || 'text';
-        if (partType === 'text' && part.text) {
-          parts.push(part.text);
+        const partType = richTextPartType(part);
+        const partText =
+          typeof part.text === 'string' ? part.text : part.content;
+        if (partType === 'text' && partText) {
+          parts.push(partText);
         } else if (partType === 'picture') {
           parts.push('[image]');
         } else if (partType === 'at' && part.atName) {
@@ -3370,7 +3390,7 @@ export class DingtalkChannel extends ChannelBase {
       }
 
       const processMessage =
-        content.downloadCodes.length > 0 || quoted.media
+        content.downloadCodes.length > 0 || quoted.media.length > 0
           ? this.prepareThenHandleInbound(envelope, async () => {
               // Download media in callback order.
               if (content.downloadCodes.length > 0 && content.mediaType) {
@@ -3384,12 +3404,12 @@ export class DingtalkChannel extends ChannelBase {
                   );
                 }
               }
-              if (quoted.media) {
+              for (const media of quoted.media) {
                 await this.attachMedia(
                   envelope,
-                  quoted.media.downloadCode,
-                  quoted.media.mediaType,
-                  quoted.media.fileName,
+                  media.downloadCode,
+                  media.mediaType,
+                  media.fileName,
                 );
               }
             })