Fix/agentflow Add support for XML tag in markdown input and render markdown conditionally (#6145)

* Changes to limit markdown

* Remove extra content

* Test changes

* Add support to escape XML tags

* FIx lint errors

* Revert changes to server file

* Fix Gemini comments

* Fix gemini comments

* Refactor to remove duplicated logic
This commit is contained in:
j-sanaa
2026-04-06 11:52:25 -07:00
committed by GitHub
parent 833f911220
commit e2ecc0b0a6
11 changed files with 897 additions and 187 deletions
@@ -1,5 +1,12 @@
import { createElement, forwardRef } from 'react'
/** Extract all text from a ProseMirror JSON node recursively. */
function extractPmText(node: { text?: string; content?: unknown[] }): string {
if (node.text) return node.text
if (node.content) return (node.content as { text?: string; content?: unknown[] }[]).map(extractPmText).join('')
return ''
}
export const useEditor = (config?: Record<string, unknown>) => {
// Track the current content so getMarkdown/getHTML reflect setContent() calls,
// mirroring real TipTap behaviour where setContent updates the editor state.
@@ -9,12 +16,22 @@ export const useEditor = (config?: Record<string, unknown>) => {
getMarkdown: () => currentContent,
isEmpty: !currentContent,
setEditable: jest.fn(),
// Returns a minimal ProseMirror JSON for the current string content.
// Allows unescapeXmlEntities() to walk and mutate the text node.
getJSON: () => ({
type: 'doc',
content: [{ type: 'paragraph', content: [{ type: 'text', text: currentContent }] }]
}),
commands: {
focus: jest.fn(),
// Capture the first argument (the content string) so getMarkdown/getHTML
// return the value that was last loaded into the editor.
setContent: jest.fn((content: string) => {
currentContent = content
// Accepts both string content and ProseMirror JSON objects (second setContent call
// in the two-step XML escape/unescape load sequence).
setContent: jest.fn((content: string | { text?: string; content?: unknown[] }) => {
if (typeof content === 'string') {
currentContent = content
} else if (content && typeof content === 'object') {
currentContent = extractPmText(content)
}
})
},
_onUpdate: config?.onUpdate
@@ -4,7 +4,7 @@ import { Box, Button, Dialog, DialogActions, DialogContent, TextField, ToggleBut
import { IconCode, IconPencil } from '@tabler/icons-react'
import type { Editor } from '@tiptap/react'
import { getEditorMarkdown } from '@/atoms/utils/'
import { getEditorMarkdown, unescapeXmlTags } from '@/atoms/utils/'
import { CodeInput } from './CodeInput'
import { RichTextEditor } from './RichTextEditor.lazy'
@@ -73,7 +73,7 @@ export function ExpandTextDialog({
// When switching to Source, flush the editor's current state to markdown so the
// textarea shows markdown rather than a raw HTML string (Gap 3 fix — mirrors PR #6021).
if (newMode === 'source' && editorRef.current) {
setLocalValue(getEditorMarkdown(editorRef.current))
setLocalValue(unescapeXmlTags(getEditorMarkdown(editorRef.current)))
}
setMode(newMode)
}
@@ -1,4 +1,6 @@
import { render, screen } from '@testing-library/react'
import { Markdown } from '@tiptap/markdown'
import StarterKit from '@tiptap/starter-kit'
import { RichTextEditor } from './RichTextEditor'
@@ -6,15 +8,18 @@ import { RichTextEditor } from './RichTextEditor'
// editor.getMarkdown() is added directly on the Editor interface by @tiptap/markdown
// via module augmentation — it is NOT nested under storage.markdown.
let capturedOnUpdate: ((args: { editor: { getMarkdown: () => string } }) => void) | undefined
let capturedExtensions: unknown[] | undefined
jest.mock('@tiptap/react', () => ({
useEditor: (config: Record<string, unknown>) => {
// Capture the onUpdate callback so tests can simulate edits
capturedOnUpdate = config.onUpdate as typeof capturedOnUpdate
capturedExtensions = config.extensions as unknown[]
return {
setEditable: jest.fn(),
commands: { focus: jest.fn(), setContent: jest.fn() },
getMarkdown: () => 'mock markdown'
getMarkdown: () => 'mock markdown',
getJSON: () => ({ type: 'doc', content: [] })
}
},
EditorContent: ({ editor, ...rest }: { editor: unknown; [key: string]: unknown }) => (
@@ -49,14 +54,16 @@ jest.mock('lowlight', () => ({
createLowlight: jest.fn(() => ({ register: jest.fn() }))
}))
const mockOnChange = jest.fn()
beforeEach(() => {
jest.clearAllMocks()
capturedOnUpdate = undefined
})
describe('RichTextEditor', () => {
let mockOnChange: jest.Mock
beforeEach(() => {
jest.clearAllMocks()
capturedOnUpdate = undefined
capturedExtensions = undefined
mockOnChange = jest.fn()
})
it('should render the editor container', () => {
render(<RichTextEditor value='Hello' onChange={mockOnChange} />)
@@ -100,4 +107,70 @@ describe('RichTextEditor', () => {
expect(screen.getByTestId('rich-text-editor')).toBeInTheDocument()
})
describe('buildExtensions — useMarkdown flag', () => {
it('includes Markdown extension when useMarkdown is true (default)', () => {
render(<RichTextEditor value='' onChange={mockOnChange} useMarkdown={true} />)
expect(capturedExtensions).toContain(Markdown)
})
it('excludes Markdown extension when useMarkdown is false', () => {
render(<RichTextEditor value='' onChange={mockOnChange} useMarkdown={false} />)
expect(capturedExtensions).not.toContain(Markdown)
})
it('passes link:false to StarterKit when useMarkdown is false', () => {
render(<RichTextEditor value='' onChange={mockOnChange} useMarkdown={false} />)
expect(StarterKit.configure).toHaveBeenCalledWith(expect.objectContaining({ link: false }))
})
it('does not pass link:false to StarterKit when useMarkdown is true (default)', () => {
render(<RichTextEditor value='' onChange={mockOnChange} useMarkdown={true} />)
expect(StarterKit.configure).not.toHaveBeenCalledWith(expect.objectContaining({ link: false }))
})
})
describe('XML tag preservation in onUpdate', () => {
it('should unescape entity-escaped XML tags before calling onChange', () => {
render(<RichTextEditor value='' onChange={mockOnChange} />)
// Simulate getMarkdown() returning entity-escaped tags (safety-net path)
capturedOnUpdate!({
editor: { getMarkdown: () => '&lt;instructions&gt;Be helpful&lt;/instructions&gt;' }
})
expect(mockOnChange).toHaveBeenCalledWith('<instructions>Be helpful</instructions>')
})
it('should pass through raw XML tags in markdown unchanged', () => {
render(<RichTextEditor value='' onChange={mockOnChange} />)
capturedOnUpdate!({
editor: { getMarkdown: () => '<instructions>Be helpful</instructions>' }
})
expect(mockOnChange).toHaveBeenCalledWith('<instructions>Be helpful</instructions>')
})
it('should preserve XML tags mixed with markdown', () => {
render(<RichTextEditor value='' onChange={mockOnChange} />)
capturedOnUpdate!({
editor: { getMarkdown: () => '# Title\n&lt;question&gt;{{input}}&lt;/question&gt;\n**bold**' }
})
expect(mockOnChange).toHaveBeenCalledWith('# Title\n<question>{{input}}</question>\n**bold**')
})
it('should render without error when useMarkdown is false', () => {
// HTML mode is exercised by other tests; this guards the component mounts cleanly.
render(<RichTextEditor value='' onChange={mockOnChange} useMarkdown={false} />)
expect(screen.getByTestId('rich-text-editor')).toBeInTheDocument()
})
})
})
+22 -12
View File
@@ -16,7 +16,7 @@ import python from 'highlight.js/lib/languages/python'
import typescript from 'highlight.js/lib/languages/typescript'
import { createLowlight } from 'lowlight'
import { getEditorMarkdown, isHtmlContent } from '@/atoms/utils/'
import { escapeXmlTags, getEditorMarkdown, isHtmlContent, unescapeXmlEntities, unescapeXmlTags } from '@/atoms/utils/'
import { tokens } from '@/core/theme/tokens'
const lowlight = createLowlight()
@@ -42,11 +42,22 @@ export interface RichTextEditorProps {
useMarkdown?: boolean
}
/* ── Helpers ── */
function loadContent(editor: Editor, value: string, markdown: boolean) {
if (!markdown || isHtmlContent(value)) {
editor.commands.setContent(value, { emitUpdate: false, contentType: 'html' })
} else {
editor.commands.setContent(escapeXmlTags(value), { emitUpdate: false, contentType: 'markdown' })
editor.commands.setContent(unescapeXmlEntities(editor.getJSON()), { emitUpdate: false })
}
}
/* ── TipTap extensions (no mention/variable support — that belongs in features/) ── */
const buildExtensions = (placeholder?: string) => [
Markdown,
StarterKit.configure({ codeBlock: false }),
const buildExtensions = (placeholder?: string, useMarkdown = true) => [
...(useMarkdown ? [Markdown] : []),
StarterKit.configure({ codeBlock: false, ...(!useMarkdown && { link: false }) }),
CodeBlockLowlight.configure({ lowlight, enableTabIndentation: true, tabSize: 2 }),
...(placeholder ? [Placeholder.configure({ placeholder })] : [])
]
@@ -174,7 +185,7 @@ export function RichTextEditor({
const initialValueRef = useRef(value)
const useMarkdownRef = useRef(useMarkdown)
const extensions = useMemo(() => buildExtensions(placeholder), [placeholder])
const extensions = useMemo(() => buildExtensions(placeholder, useMarkdown), [placeholder, useMarkdown])
const editor = useEditor({
extensions,
@@ -182,9 +193,10 @@ export function RichTextEditor({
editable: !disabled,
autofocus: autoFocus ? 'end' : false,
onUpdate: ({ editor: ed }) => {
const value = useMarkdown ? getEditorMarkdown(ed) : ed.getHTML()
lastEmittedRef.current = value
onChangeRef.current(value)
const raw = useMarkdown ? getEditorMarkdown(ed) : ed.getHTML()
const emitted = useMarkdown ? unescapeXmlTags(raw) : raw
lastEmittedRef.current = emitted
onChangeRef.current(emitted)
}
})
@@ -199,16 +211,14 @@ export function RichTextEditor({
// Reads from refs so only `editor` needs to be in the dep array.
useEffect(() => {
if (!editor || !initialValueRef.current) return
const contentType = !useMarkdownRef.current || isHtmlContent(initialValueRef.current) ? 'html' : 'markdown'
editor.commands.setContent(initialValueRef.current, { emitUpdate: false, contentType })
loadContent(editor, initialValueRef.current, useMarkdownRef.current)
lastEmittedRef.current = initialValueRef.current
}, [editor])
// Sync genuine external value changes (e.g. parent resets the field programmatically).
useEffect(() => {
if (editor && value !== lastEmittedRef.current) {
const contentType = !useMarkdown || isHtmlContent(value) ? 'html' : 'markdown'
editor.commands.setContent(value, { emitUpdate: false, contentType })
loadContent(editor, value, useMarkdown)
lastEmittedRef.current = value
}
}, [editor, value, useMarkdown])
@@ -1,6 +1,7 @@
import { createTheme, ThemeProvider } from '@mui/material/styles'
import { render, screen } from '@testing-library/react'
import * as TiptapReact from '@tiptap/react'
import StarterKit from '@tiptap/starter-kit'
import { VariableInput, type VariableInputProps } from './VariableInput'
@@ -98,4 +99,92 @@ describe('VariableInput', () => {
// The mock editor has getHTML/getMarkdown/commands etc.
expect(onEditorReady).toHaveBeenCalledWith(expect.objectContaining({ getHTML: expect.any(Function) }))
})
describe('useMarkdown derived from rows — StarterKit link extension', () => {
beforeEach(() => {
jest.clearAllMocks()
})
it('passes link:false to StarterKit when rows is not set (single-line field)', () => {
renderVariableInput({ rows: undefined })
expect(StarterKit.configure).toHaveBeenCalledWith(expect.objectContaining({ link: false }))
})
it('does not pass link:false to StarterKit when rows is set (multiline/markdown field)', () => {
renderVariableInput({ rows: 4 })
expect(StarterKit.configure).not.toHaveBeenCalledWith(expect.objectContaining({ link: false }))
})
})
describe('onUpdate always emits markdown regardless of rows', () => {
it('calls onChange with getEditorMarkdown output when rows is not set', () => {
const onChange = jest.fn()
const useEditorSpy = jest.spyOn(TiptapReact, 'useEditor')
renderVariableInput({ onChange, rows: undefined })
const config = useEditorSpy.mock.calls[0][0] as { onUpdate: (args: { editor: unknown }) => void }
const mockEditor = { getMarkdown: () => 'plain url text', getHTML: jest.fn() }
config.onUpdate({ editor: mockEditor })
expect(onChange).toHaveBeenCalledWith('plain url text')
expect(mockEditor.getHTML).not.toHaveBeenCalled()
useEditorSpy.mockRestore()
})
it('calls onChange with getEditorMarkdown output when rows is set', () => {
const onChange = jest.fn()
const useEditorSpy = jest.spyOn(TiptapReact, 'useEditor')
renderVariableInput({ onChange, rows: 4 })
const config = useEditorSpy.mock.calls[0][0] as { onUpdate: (args: { editor: unknown }) => void }
const mockEditor = { getMarkdown: () => '**bold**', getHTML: jest.fn() }
config.onUpdate({ editor: mockEditor })
expect(onChange).toHaveBeenCalledWith('**bold**')
expect(mockEditor.getHTML).not.toHaveBeenCalled()
useEditorSpy.mockRestore()
})
})
describe('XML tag preservation in onUpdate', () => {
it('should unescape entity-escaped XML tags before calling onChange', () => {
const onChange = jest.fn()
const useEditorSpy = jest.spyOn(TiptapReact, 'useEditor')
renderVariableInput({ onChange })
const config = useEditorSpy.mock.calls[0][0] as { onUpdate: (args: { editor: unknown }) => void }
config.onUpdate({ editor: { getMarkdown: () => '&lt;instructions&gt;Be helpful&lt;/instructions&gt;', getHTML: jest.fn() } })
expect(onChange).toHaveBeenCalledWith('<instructions>Be helpful</instructions>')
useEditorSpy.mockRestore()
})
it('should pass through raw XML tags unchanged', () => {
const onChange = jest.fn()
const useEditorSpy = jest.spyOn(TiptapReact, 'useEditor')
renderVariableInput({ onChange })
const config = useEditorSpy.mock.calls[0][0] as { onUpdate: (args: { editor: unknown }) => void }
config.onUpdate({ editor: { getMarkdown: () => '<question>What?</question>', getHTML: jest.fn() } })
expect(onChange).toHaveBeenCalledWith('<question>What?</question>')
useEditorSpy.mockRestore()
})
it('should preserve XML tags mixed with variables', () => {
const onChange = jest.fn()
const useEditorSpy = jest.spyOn(TiptapReact, 'useEditor')
renderVariableInput({ onChange })
const config = useEditorSpy.mock.calls[0][0] as { onUpdate: (args: { editor: unknown }) => void }
config.onUpdate({
editor: { getMarkdown: () => '&lt;context&gt;{{question}}&lt;/context&gt;', getHTML: jest.fn() }
})
expect(onChange).toHaveBeenCalledWith('<context>{{question}}</context>')
useEditorSpy.mockRestore()
})
})
})
+25 -10
View File
@@ -15,7 +15,8 @@ import python from 'highlight.js/lib/languages/python'
import typescript from 'highlight.js/lib/languages/typescript'
import { createLowlight } from 'lowlight'
import { getEditorMarkdown, isHtmlContent } from '@/atoms/utils/'
import type { JsonNode } from '@/atoms/utils/'
import { escapeXmlTags, getEditorMarkdown, isHtmlContent, restoreTextMentions, unescapeXmlTags } from '@/atoms/utils/'
import { CustomMention } from '@/core/primitives/customMention'
import { tokens } from '@/core/theme/tokens'
@@ -133,6 +134,15 @@ const StyledEditorContent = styled(EditorContent, {
}
})
function loadContent(ed: Editor, content: string, hasMentions: boolean): void {
if (isHtmlContent(content)) {
ed.commands.setContent(content, { emitUpdate: false, contentType: 'html' })
} else {
ed.commands.setContent(escapeXmlTags(content), { emitUpdate: false, contentType: 'markdown' })
ed.commands.setContent(restoreTextMentions(ed.getJSON() as JsonNode, hasMentions)[0], { emitUpdate: false })
}
}
/**
* A TipTap-based text input with `{{ variable }}` mention support.
*
@@ -170,16 +180,23 @@ export function VariableInput({
// `editor` without suppressing the exhaustive-deps rule.
const initialValueRef = useRef(value)
const useMarkdown = !!rows
const suggestionConfig = useMemo(
() => (suggestionItems?.length ? createSuggestionConfig(suggestionItems) : undefined),
[suggestionItems]
)
const suggestionConfigRef = useRef(suggestionConfig)
useEffect(() => {
suggestionConfigRef.current = suggestionConfig
}, [suggestionConfig])
const extensions = useMemo(
() => [
Markdown,
StarterKit.configure({
codeBlock: false
codeBlock: false,
...(!useMarkdown && { link: false })
}),
CodeBlockLowlight.configure({ lowlight, enableTabIndentation: true, tabSize: 2 }),
...(placeholder ? [Placeholder.configure({ placeholder })] : []),
@@ -200,7 +217,7 @@ export function VariableInput({
]
: [])
],
[placeholder, suggestionConfig]
[placeholder, suggestionConfig, useMarkdown]
)
const editor = useEditor({
@@ -209,9 +226,9 @@ export function VariableInput({
editable: !disabled,
autofocus: autoFocus ? 'end' : false,
onUpdate: ({ editor: ed }) => {
const value = getEditorMarkdown(ed)
lastEmittedRef.current = value
onChangeRef.current(value)
const emitted = unescapeXmlTags(getEditorMarkdown(ed))
lastEmittedRef.current = emitted
onChangeRef.current(emitted)
}
})
@@ -219,16 +236,14 @@ export function VariableInput({
// Reads from a ref so only `editor` needs to be in the dep array.
useEffect(() => {
if (!editor || !initialValueRef.current) return
const contentType = isHtmlContent(initialValueRef.current) ? 'html' : 'markdown'
editor.commands.setContent(initialValueRef.current, { emitUpdate: false, contentType })
loadContent(editor, initialValueRef.current, !!suggestionConfigRef.current)
lastEmittedRef.current = initialValueRef.current
}, [editor])
// Sync genuine external value changes (e.g. parent resets the field programmatically).
useEffect(() => {
if (editor && value !== lastEmittedRef.current) {
const contentType = isHtmlContent(value) ? 'html' : 'markdown'
editor.commands.setContent(value, { emitUpdate: false, contentType })
loadContent(editor, value, !!suggestionConfigRef.current)
lastEmittedRef.current = value
}
}, [editor, value])
@@ -1,120 +0,0 @@
import { getEditorMarkdown, isHtmlContent } from '../../atoms/utils/editorUtils'
describe('editorUtils', () => {
describe('isHtmlContent', () => {
// Falsy / non-string inputs
it('returns false for empty string', () => {
expect(isHtmlContent('')).toBe(false)
})
it('returns false for null', () => {
expect(isHtmlContent(null)).toBe(false)
})
it('returns false for undefined', () => {
expect(isHtmlContent(undefined)).toBe(false)
})
it('returns false for a number', () => {
expect(isHtmlContent(42)).toBe(false)
})
// Markdown content (must NOT be detected as HTML)
it('returns false for plain text', () => {
expect(isHtmlContent('You are a helpful assistant.')).toBe(false)
})
it('returns false for markdown heading', () => {
expect(isHtmlContent('## My heading')).toBe(false)
})
it('returns false for markdown bullet list', () => {
expect(isHtmlContent('- item 1\n- item 2')).toBe(false)
})
it('returns false for markdown with variable syntax', () => {
expect(isHtmlContent('Hello {{question}}, today is {{current_date_time}}')).toBe(false)
})
it('returns false for markdown code fence', () => {
expect(isHtmlContent('```js\nconsole.log("hello")\n```')).toBe(false)
})
it('returns false for markdown bold/italic', () => {
expect(isHtmlContent('**bold** and *italic* text')).toBe(false)
})
// Legacy HTML content (MUST be detected)
it('returns true for <p> tag', () => {
expect(isHtmlContent('<p>Hello world</p>')).toBe(true)
})
it('returns true for <p> with variable mention span', () => {
expect(isHtmlContent('<p>Hello <span data-type="mention" data-id="question">{{question}}</span></p>')).toBe(true)
})
it('returns true for <h1> heading tag', () => {
expect(isHtmlContent('<h1>Title</h1>')).toBe(true)
})
it('returns true for <h2> heading tag', () => {
expect(isHtmlContent('<h2>Section</h2>')).toBe(true)
})
it('returns true for <ul> list tag', () => {
expect(isHtmlContent('<ul><li>item</li></ul>')).toBe(true)
})
it('returns true for <pre><code> block', () => {
expect(isHtmlContent('<pre><code>const x = 1</code></pre>')).toBe(true)
})
it('returns true for <strong> tag', () => {
expect(isHtmlContent('<strong>bold</strong>')).toBe(true)
})
it('returns true for <em> tag', () => {
expect(isHtmlContent('<em>italic</em>')).toBe(true)
})
it('returns true for <blockquote> tag', () => {
expect(isHtmlContent('<blockquote>quote</blockquote>')).toBe(true)
})
it('returns true for multiline HTML with mixed content', () => {
expect(isHtmlContent('<p>First paragraph</p>\n<p>Second paragraph</p>')).toBe(true)
})
it('returns true for uppercase tag like <P>', () => {
expect(isHtmlContent('<P>uppercase</P>')).toBe(true)
})
})
describe('getEditorMarkdown', () => {
it('returns markdown when getMarkdown() returns a non-empty string', () => {
const editor = { getMarkdown: () => '## heading', getHTML: () => '<h2>heading</h2>', isEmpty: false }
expect(getEditorMarkdown(editor)).toBe('## heading')
})
it('returns empty string when getMarkdown() returns "" and editor is empty', () => {
const editor = { getMarkdown: () => '', getHTML: () => '', isEmpty: true }
expect(getEditorMarkdown(editor)).toBe('')
})
it('falls back to HTML when getMarkdown() returns "" but editor is not empty', () => {
const editor = { getMarkdown: () => '', getHTML: () => '<p>hello</p>', isEmpty: false }
expect(getEditorMarkdown(editor)).toBe('<p>hello</p>')
})
it('falls back to HTML when getMarkdown() throws', () => {
const editor = {
getMarkdown: () => {
throw new Error('serialization failed')
},
getHTML: () => '<p>fallback</p>',
isEmpty: false
}
expect(getEditorMarkdown(editor)).toBe('<p>fallback</p>')
})
})
})
@@ -1,30 +0,0 @@
/**
* Returns true when `content` looks like legacy HTML (saved by the old getHTML()
* serializer), so callers can choose the right TipTap `setContent` contentType.
*
* Ported from packages/ui/src/ui-component/input/RichInput.jsx (PR #6021).
* A simple tag-presence check is intentionally used here — it is fast, has no
* dependencies, and the false-positive/negative risk for agentflow node inputs
* is negligible (users don't normally write raw HTML into prompt fields).
*/
export function isHtmlContent(content: unknown): boolean {
if (!content || typeof content !== 'string') return false
return /<(?:p|div|span|h[1-6]|ul|ol|li|br|code|pre|blockquote|table|strong|em)\b/i.test(content)
}
/**
* Safely serialise a TipTap editor to markdown, falling back to HTML when
* `getMarkdown()` returns an empty string for a non-empty document — a known
* issue in `@tiptap/markdown` v3 where the MarkdownManager may silently fail
* to serialise certain node types.
*/
export function getEditorMarkdown(editor: { getMarkdown(): string; getHTML(): string; isEmpty: boolean }): string {
try {
const markdown = editor.getMarkdown()
if (markdown) return markdown
if (editor.isEmpty) return ''
return editor.getHTML()
} catch {
return editor.getHTML()
}
}
+2 -1
View File
@@ -1 +1,2 @@
export { getEditorMarkdown, isHtmlContent } from './editorUtils'
export type { JsonNode } from './xmlTagUtils'
export { escapeXmlTags, getEditorMarkdown, isHtmlContent, restoreTextMentions, unescapeXmlEntities, unescapeXmlTags } from './xmlTagUtils'
@@ -0,0 +1,482 @@
import type { JsonNode } from './xmlTagUtils'
import { escapeXmlTags, getEditorMarkdown, isHtmlContent, restoreTextMentions, unescapeXmlEntities, unescapeXmlTags } from './xmlTagUtils'
// ── isHtmlContent ──────────────────────────────────────────────────────────────
describe('isHtmlContent', () => {
// Falsy / non-string inputs
it('returns false for empty string', () => {
expect(isHtmlContent('')).toBe(false)
})
it('returns false for null', () => {
expect(isHtmlContent(null)).toBe(false)
})
it('returns false for undefined', () => {
expect(isHtmlContent(undefined)).toBe(false)
})
it('returns false for a number', () => {
expect(isHtmlContent(42)).toBe(false)
})
// Markdown content (must NOT be detected as HTML)
it('returns false for plain text', () => {
expect(isHtmlContent('You are a helpful assistant.')).toBe(false)
})
it('returns false for markdown heading', () => {
expect(isHtmlContent('## My heading')).toBe(false)
})
it('returns false for markdown bullet list', () => {
expect(isHtmlContent('- item 1\n- item 2')).toBe(false)
})
it('returns false for markdown with variable syntax', () => {
expect(isHtmlContent('Hello {{question}}, today is {{current_date_time}}')).toBe(false)
})
it('returns false for markdown code fence', () => {
expect(isHtmlContent('```js\nconsole.log("hello")\n```')).toBe(false)
})
it('returns false for markdown bold/italic', () => {
expect(isHtmlContent('**bold** and *italic* text')).toBe(false)
})
// Legacy HTML content (MUST be detected)
it('returns true for <p> tag', () => {
expect(isHtmlContent('<p>Hello world</p>')).toBe(true)
})
it('returns true for <p> with variable mention span', () => {
expect(isHtmlContent('<p>Hello <span data-type="mention" data-id="question">{{question}}</span></p>')).toBe(true)
})
it('returns true for <h1> heading tag', () => {
expect(isHtmlContent('<h1>Title</h1>')).toBe(true)
})
it('returns true for <h2> heading tag', () => {
expect(isHtmlContent('<h2>Section</h2>')).toBe(true)
})
it('returns true for <ul> list tag', () => {
expect(isHtmlContent('<ul><li>item</li></ul>')).toBe(true)
})
it('returns true for <pre><code> block', () => {
expect(isHtmlContent('<pre><code>const x = 1</code></pre>')).toBe(true)
})
it('returns true for <strong> tag', () => {
expect(isHtmlContent('<strong>bold</strong>')).toBe(true)
})
it('returns true for <em> tag', () => {
expect(isHtmlContent('<em>italic</em>')).toBe(true)
})
it('returns true for <blockquote> tag', () => {
expect(isHtmlContent('<blockquote>quote</blockquote>')).toBe(true)
})
it('returns true for multiline HTML with mixed content', () => {
expect(isHtmlContent('<p>First paragraph</p>\n<p>Second paragraph</p>')).toBe(true)
})
it('returns true for uppercase tag like <P>', () => {
expect(isHtmlContent('<P>uppercase</P>')).toBe(true)
})
// Anchor fix — a user prompt that *contains* a standard HTML tag but does NOT start with one
it('returns false for a prompt starting with a custom tag that contains HTML inside', () => {
expect(isHtmlContent('<instructions><div>Test</div></instructions>')).toBe(false)
})
it('returns false for a custom XML tag like <instructions>', () => {
expect(isHtmlContent('<instructions>Be helpful</instructions>')).toBe(false)
})
})
// ── getEditorMarkdown ──────────────────────────────────────────────────────────
describe('getEditorMarkdown', () => {
it('returns markdown when getMarkdown() returns a non-empty string', () => {
const editor = { getMarkdown: () => '## heading', getHTML: () => '<h2>heading</h2>', isEmpty: false }
expect(getEditorMarkdown(editor)).toBe('## heading')
})
it('returns empty string when getMarkdown() returns "" and editor is empty', () => {
const editor = { getMarkdown: () => '', getHTML: () => '', isEmpty: true }
expect(getEditorMarkdown(editor)).toBe('')
})
it('falls back to HTML when getMarkdown() returns "" but editor is not empty', () => {
const editor = { getMarkdown: () => '', getHTML: () => '<p>hello</p>', isEmpty: false }
expect(getEditorMarkdown(editor)).toBe('<p>hello</p>')
})
it('falls back to HTML when getMarkdown() throws', () => {
const editor = {
getMarkdown: () => {
throw new Error('serialization failed')
},
getHTML: () => '<p>fallback</p>',
isEmpty: false
}
expect(getEditorMarkdown(editor)).toBe('<p>fallback</p>')
})
})
// ── escapeXmlTags ──────────────────────────────────────────────────────────────
describe('escapeXmlTags', () => {
it('should escape opening and closing tags to entities', () => {
expect(escapeXmlTags('<question>text</question>')).toBe('&lt;question&gt;text&lt;/question&gt;')
})
it('should escape self-closing tags', () => {
expect(escapeXmlTags('<my-separator />')).toBe('&lt;my-separator /&gt;')
})
it('should escape tags with attributes', () => {
expect(escapeXmlTags('<context type="user">hello</context>')).toBe('&lt;context type="user"&gt;hello&lt;/context&gt;')
})
it('should escape standard HTML tags too', () => {
expect(escapeXmlTags('<div><p>text</p></div>')).toBe('&lt;div&gt;&lt;p&gt;text&lt;/p&gt;&lt;/div&gt;')
})
it('should escape all tags in mixed content', () => {
const input = '# Heading\n<question>What is {{name}}?</question>\n<p>paragraph</p>'
const expected = '# Heading\n&lt;question&gt;What is {{name}}?&lt;/question&gt;\n&lt;p&gt;paragraph&lt;/p&gt;'
expect(escapeXmlTags(input)).toBe(expected)
})
it('should handle nested tags', () => {
const input = '<outer><inner>text</inner></outer>'
const expected = '&lt;outer&gt;&lt;inner&gt;text&lt;/inner&gt;&lt;/outer&gt;'
expect(escapeXmlTags(input)).toBe(expected)
})
it('should return empty string as-is', () => {
expect(escapeXmlTags('')).toBe('')
})
it('should handle text with no tags', () => {
expect(escapeXmlTags('just plain text')).toBe('just plain text')
})
it('should handle tags with dots and hyphens in names', () => {
expect(escapeXmlTags('<my.tag>text</my.tag>')).toBe('&lt;my.tag&gt;text&lt;/my.tag&gt;')
expect(escapeXmlTags('<my-tag>text</my-tag>')).toBe('&lt;my-tag&gt;text&lt;/my-tag&gt;')
})
it('should not double-escape already-escaped content', () => {
const alreadyEscaped = '&lt;question&gt;text&lt;/question&gt;'
expect(escapeXmlTags(alreadyEscaped)).toBe(alreadyEscaped)
})
})
// ── unescapeXmlEntities ────────────────────────────────────────────────────────
describe('unescapeXmlEntities', () => {
it('should unescape entities in text nodes', () => {
const json = {
type: 'doc',
content: [
{
type: 'paragraph',
content: [{ type: 'text', text: '&lt;question&gt;What?&lt;/question&gt;' }]
}
]
}
unescapeXmlEntities(json)
expect(json.content[0].content[0].text).toBe('<question>What?</question>')
})
it('should unescape standard HTML tag entities too', () => {
const json = {
type: 'doc',
content: [{ type: 'paragraph', content: [{ type: 'text', text: '&lt;div&gt;content&lt;/div&gt;' }] }]
}
unescapeXmlEntities(json)
expect(json.content[0].content[0].text).toBe('<div>content</div>')
})
it('should handle nodes across multiple paragraphs', () => {
const json = {
type: 'doc',
content: [
{ type: 'paragraph', content: [{ type: 'text', text: '&lt;outer&gt;' }] },
{ type: 'paragraph', content: [{ type: 'text', text: '&lt;inner&gt;text&lt;/inner&gt;' }] },
{ type: 'paragraph', content: [{ type: 'text', text: '&lt;/outer&gt;' }] }
]
}
unescapeXmlEntities(json)
expect(json.content[0].content[0].text).toBe('<outer>')
expect(json.content[1].content[0].text).toBe('<inner>text</inner>')
expect(json.content[2].content[0].text).toBe('</outer>')
})
it('should not modify nodes without entities', () => {
const json = {
type: 'doc',
content: [{ type: 'paragraph', content: [{ type: 'text', text: 'plain text' }] }]
}
unescapeXmlEntities(json)
expect(json.content[0].content[0].text).toBe('plain text')
})
it('should return the same object for chaining', () => {
const json = { type: 'doc', content: [] }
expect(unescapeXmlEntities(json)).toBe(json)
})
})
// ── unescapeXmlTags ────────────────────────────────────────────────────────────
describe('unescapeXmlTags', () => {
it('should unescape entity-escaped tags', () => {
expect(unescapeXmlTags('&lt;question&gt;text&lt;/question&gt;')).toBe('<question>text</question>')
})
it('should unescape standard HTML tags too', () => {
expect(unescapeXmlTags('&lt;div&gt;text&lt;/div&gt;')).toBe('<div>text</div>')
})
it('should unescape tags with attributes', () => {
expect(unescapeXmlTags('&lt;context type="user"&gt;hello&lt;/context&gt;')).toBe('<context type="user">hello</context>')
})
it('should handle mixed content', () => {
const input = '# Heading\n&lt;question&gt;text&lt;/question&gt;\nsome markdown'
const expected = '# Heading\n<question>text</question>\nsome markdown'
expect(unescapeXmlTags(input)).toBe(expected)
})
it('should return empty string as-is', () => {
expect(unescapeXmlTags('')).toBe('')
})
it('should pass through raw (unescaped) tags unchanged', () => {
expect(unescapeXmlTags('<question>text</question>')).toBe('<question>text</question>')
})
})
// ── roundtrip — escape → unescape ─────────────────────────────────────────────
describe('roundtrip — escape → unescape', () => {
const cases = [
'<question>What is the answer?</question>',
'<context type="system">You are helpful</context>',
'<instructions>\n- Step 1\n- Step 2\n</instructions>',
'<outer><inner>nested</inner></outer>',
'# Title\n<question>{{input}}</question>\n**bold** text',
'<my-component />',
'No tags here, just **markdown**',
'<div>standard html preserved</div>'
]
cases.forEach((input) => {
it(`should roundtrip: ${input.substring(0, 50)}`, () => {
expect(unescapeXmlTags(escapeXmlTags(input))).toBe(input)
})
})
})
// ── markdown formatting preservation ──────────────────────────────────────────
describe('markdown formatting preservation', () => {
it('should preserve heading syntax', () => {
const input = '## My Task\nDo something useful'
expect(escapeXmlTags(input)).toBe(input)
expect(unescapeXmlTags(input)).toBe(input)
})
it('should preserve bold and italic syntax', () => {
const input = 'This is **bold** and *italic* and ***both***'
expect(escapeXmlTags(input)).toBe(input)
expect(unescapeXmlTags(input)).toBe(input)
})
it('should preserve code blocks', () => {
const input = '```python\nprint("hello")\n```'
expect(escapeXmlTags(input)).toBe(input)
expect(unescapeXmlTags(input)).toBe(input)
})
it('should preserve markdown links', () => {
const input = 'See [the docs](https://example.com/path?q=1) for details'
expect(escapeXmlTags(input)).toBe(input)
expect(unescapeXmlTags(input)).toBe(input)
})
it('should roundtrip HTML anchor links', () => {
const input = 'Visit <a href="https://flowiseai.com">Flowise</a> for docs'
expect(unescapeXmlTags(escapeXmlTags(input))).toBe(input)
})
})
// ── full editor save/reload cycle simulation ───────────────────────────────────
describe('full editor save/reload cycle simulation', () => {
/**
* Simulates the load → display → save cycle:
* LOAD: escapeXmlTags(value) → setContent(markdown) → unescapeXmlEntities(json) → setContent(json)
* SAVE: getMarkdown() → unescapeXmlTags(markdown)
*/
function simulateEditorCycle(userInput: string): string {
// LOAD — Step 1: escape tags so marked treats them as text
const escaped = escapeXmlTags(userInput)
// LOAD — Step 2: marked creates text nodes with entities; unescapeXmlEntities fixes them
const mockJson = {
type: 'doc',
content: [{ type: 'paragraph', content: [{ type: 'text', text: escaped }] }]
}
unescapeXmlEntities(mockJson)
// Verify editor displays raw angle brackets (no entities remain)
const displayText = mockJson.content[0].content[0].text
expect(displayText).not.toMatch(/&lt;|&gt;/)
// SAVE — getMarkdown() outputs text as-is; unescapeXmlTags is the safety net
return unescapeXmlTags(displayText)
}
it('should preserve XML-tagged prompt', () => {
expect(simulateEditorCycle('<question>What is the answer?</question>')).toBe('<question>What is the answer?</question>')
})
it('should preserve tags with attributes', () => {
expect(simulateEditorCycle('<context type="system">You are helpful</context>')).toBe(
'<context type="system">You are helpful</context>'
)
})
it('should preserve multiline structured prompt', () => {
expect(simulateEditorCycle('<instructions>\n- Step 1\n- Step 2\n</instructions>')).toBe(
'<instructions>\n- Step 1\n- Step 2\n</instructions>'
)
})
it('should preserve XML tags mixed with markdown and variables', () => {
expect(simulateEditorCycle('# Title\n<question>{{input}}</question>\n**bold** text')).toBe(
'# Title\n<question>{{input}}</question>\n**bold** text'
)
})
it('should leave plain markdown unchanged', () => {
expect(simulateEditorCycle('No tags here, just **markdown**')).toBe('No tags here, just **markdown**')
})
})
// ── restoreTextMentions ────────────────────────────────────────────────────────
describe('restoreTextMentions', () => {
describe('hasMentions: false — XML unescape only', () => {
it('unescapes XML entities in a text node', () => {
const node: JsonNode = { type: 'text', text: '&lt;instructions&gt;hello&lt;/instructions&gt;' }
const result = restoreTextMentions(node, false)
expect(result).toEqual([{ type: 'text', text: '<instructions>hello</instructions>' }])
})
it('returns plain text unchanged', () => {
const node: JsonNode = { type: 'text', text: 'hello world' }
expect(restoreTextMentions(node, false)).toEqual([{ type: 'text', text: 'hello world' }])
})
it('does not split {{variable}} patterns when hasMentions is false', () => {
const node: JsonNode = { type: 'text', text: 'Hello {{question}}' }
expect(restoreTextMentions(node, false)).toEqual([{ type: 'text', text: 'Hello {{question}}' }])
})
})
describe('hasMentions: true — splits {{variable}} into mention nodes', () => {
it('converts a lone {{variable}} into a mention node', () => {
const node: JsonNode = { type: 'text', text: '{{question}}' }
const result = restoreTextMentions(node, true)
expect(result).toEqual([{ type: 'mention', attrs: { id: 'question', label: 'question' } }])
})
it('splits text before a {{variable}} into a text node + mention', () => {
const node: JsonNode = { type: 'text', text: 'Hello {{name}}' }
const result = restoreTextMentions(node, true)
expect(result).toEqual([
{ type: 'text', text: 'Hello ' },
{ type: 'mention', attrs: { id: 'name', label: 'name' } }
])
})
it('splits text after a {{variable}} into mention + text node', () => {
const node: JsonNode = { type: 'text', text: '{{name}} is here' }
const result = restoreTextMentions(node, true)
expect(result).toEqual([
{ type: 'mention', attrs: { id: 'name', label: 'name' } },
{ type: 'text', text: ' is here' }
])
})
it('handles multiple {{variables}} in one text node', () => {
const node: JsonNode = { type: 'text', text: '{{a}} and {{b}}' }
const result = restoreTextMentions(node, true)
expect(result).toEqual([
{ type: 'mention', attrs: { id: 'a', label: 'a' } },
{ type: 'text', text: ' and ' },
{ type: 'mention', attrs: { id: 'b', label: 'b' } }
])
})
it('trims whitespace inside {{ variable }}', () => {
const node: JsonNode = { type: 'text', text: '{{ question }}' }
const result = restoreTextMentions(node, true)
expect(result).toEqual([{ type: 'mention', attrs: { id: 'question', label: 'question' } }])
})
it('restores mention inside a URL text node', () => {
// This is the core bug fix — MarkedJS URL tokenizer swallows {{var}} inside URLs
const node: JsonNode = { type: 'text', text: 'https://example.com/{{question}}' }
const result = restoreTextMentions(node, true)
expect(result).toEqual([
{ type: 'text', text: 'https://example.com/' },
{ type: 'mention', attrs: { id: 'question', label: 'question' } }
])
})
it('unescapes XML entities AND restores mentions together', () => {
const node: JsonNode = { type: 'text', text: '&lt;context&gt;{{question}}&lt;/context&gt;' }
const result = restoreTextMentions(node, true)
expect(result).toEqual([
{ type: 'text', text: '<context>' },
{ type: 'mention', attrs: { id: 'question', label: 'question' } },
{ type: 'text', text: '</context>' }
])
})
})
describe('recursive tree walking', () => {
it('walks content arrays and processes nested text nodes', () => {
const doc: JsonNode = {
type: 'doc',
content: [
{
type: 'paragraph',
content: [{ type: 'text', text: 'Visit https://example.com/{{question}}' }]
}
]
}
const [result] = restoreTextMentions(doc, true)
const para = (result.content as JsonNode[])[0]
expect(para.content).toEqual([
{ type: 'text', text: 'Visit https://example.com/' },
{ type: 'mention', attrs: { id: 'question', label: 'question' } }
])
})
it('passes through non-text, non-content nodes unchanged', () => {
const node: JsonNode = { type: 'hardBreak' }
expect(restoreTextMentions(node, true)).toEqual([{ type: 'hardBreak' }])
})
})
})
@@ -0,0 +1,173 @@
/**
* Utilities for preserving XML/HTML tags in prompt text through TipTap's markdown round-trip,
* plus editor helpers for content-type detection and markdown serialisation.
*
* Problem: When content like `<instructions>text</instructions>` is parsed by marked
* (via @tiptap/markdown), the lexer tokenises tags as HTML tokens. TipTap's parseHTMLToken
* then calls generateJSON which creates DOM elements — unrecognised tags are stripped and
* only inner text survives.
*
* Solution: Three-step process:
* 1. escapeXmlTags — Convert all tags to HTML entities before setContent so marked
* treats them as plain text, not HTML tokens.
* 2. unescapeXmlEntities — After TipTap builds the ProseMirror document, walk the JSON tree
* and decode &lt;/&gt; back to </> in text nodes for proper display.
* 3. unescapeXmlTags — After getMarkdown(), reverse any remaining entity-escaped tags
* in the serialised output (safety net — typically a no-op).
*
* Ported from packages/ui/src/utils/xmlTagUtils.js (PR #6095).
*/
// ── Content-type detection ─────────────────────────────────────────────────────
/**
* Returns true when `content` looks like legacy HTML (saved by the old getHTML()
* serialiser), so callers can choose the right TipTap `setContent` contentType.
*
* The regex is anchored with `^\s*` so a user prompt that merely *contains* a
* standard HTML tag (e.g. `<instructions><div>…</div></instructions>`) is not
* misclassified as legacy HTML.
*
* Ported from packages/ui/src/ui-component/input/RichInput.jsx (PR #6021).
*/
export function isHtmlContent(content: unknown): boolean {
if (!content || typeof content !== 'string') return false
return /^\s*<(?:p|div|span|h[1-6]|ul|ol|li|br|code|pre|blockquote|table|strong|em)\b/i.test(content)
}
// ── Markdown serialisation ─────────────────────────────────────────────────────
/**
* Safely serialise a TipTap editor to markdown, falling back to HTML when
* `getMarkdown()` returns an empty string for a non-empty document — a known
* issue in `@tiptap/markdown` v3 where the MarkdownManager may silently fail
* to serialise certain node types.
*/
export function getEditorMarkdown(editor: { getMarkdown(): string; getHTML(): string; isEmpty: boolean }): string {
try {
const markdown = editor.getMarkdown()
if (markdown) return markdown
if (editor.isEmpty) return ''
return editor.getHTML()
} catch {
return editor.getHTML()
}
}
// ── XML tag escape / unescape ──────────────────────────────────────────────────
/** Regex matching opening, closing, and self-closing XML/HTML tags. */
const XML_TAG_REGEX = /<(\/?)([a-zA-Z][a-zA-Z0-9_.-]*)(\s[^>]*)?(\/?)>/g
/**
* Escape all XML/HTML tags to HTML entities so marked doesn't parse them as HTML.
* In prompt editing context, users want tags preserved literally, not rendered.
*
* @example
* escapeXmlTags('<instructions>Be helpful</instructions>')
* // → '&lt;instructions&gt;Be helpful&lt;/instructions&gt;'
*/
export function escapeXmlTags(text: string): string {
if (!text || typeof text !== 'string') return text
return text.replace(XML_TAG_REGEX, (_, slash, tagName, attrs, selfClose) => {
return `&lt;${slash}${tagName}${attrs ?? ''}${selfClose}&gt;`
})
}
/** Minimal interface for walking a ProseMirror JSON tree. */
interface PmNode {
text?: string
content?: PmNode[]
[key: string]: unknown
}
/**
* ProseMirror JSON node type used when mention nodes (with `attrs`) may be present.
* Superset of PmNode — the index signature covers any additional fields.
*/
export type JsonNode = { type?: string; text?: string; content?: JsonNode[]; attrs?: Record<string, unknown>; [k: string]: unknown }
/**
* Unescape XML tag entities in ProseMirror JSON text nodes.
* Call this after setContent() to fix the visual display in the editor.
* Mutates the JSON in-place and returns it.
*
* @example
* const json = { type: 'doc', content: [
* { type: 'paragraph', content: [{ type: 'text', text: '&lt;question&gt;What?&lt;/question&gt;' }] }
* ]}
* unescapeXmlEntities(json)
* // json.content[0].content[0].text → '<question>What?</question>'
*/
export function unescapeXmlEntities(json: PmNode): PmNode {
if (json.text) {
json.text = unescapeXmlTags(json.text)
}
if (json.content) {
json.content.forEach(unescapeXmlEntities)
}
return json
}
/**
* Unescape all entity-escaped XML/HTML tags after markdown serialisation.
* Typically a no-op — used as a safety net for any entities TipTap did not
* convert back to angle brackets during getMarkdown().
*
* @example
* unescapeXmlTags('&lt;question&gt;text&lt;/question&gt;')
* // → '<question>text</question>'
*
* unescapeXmlTags('<question>text</question>')
* // → '<question>text</question>' (raw tags pass through unchanged)
*/
export function unescapeXmlTags(text: string): string {
if (!text || typeof text !== 'string') return text
return text.replace(/&lt;(\/?)([a-zA-Z][a-zA-Z0-9_.-]*)(\s.*?)?(\/?)&gt;/g, (_, slash, tagName, attrs, selfClose) => {
return `<${slash}${tagName}${attrs ?? ''}${selfClose}>`
})
}
// ── Mention restoration ────────────────────────────────────────────────────────
/**
* Post-process the ProseMirror JSON after markdown setContent:
* 1. Unescape XML tag entities in text nodes.
* 2. When `hasMentions` is true, split any remaining `{{label}}` text patterns
* into proper mention nodes. This is needed because MarkedJS's URL tokenizer
* can swallow `{{variable}}` when it appears inside a URL
* (e.g. `https://example.com/{{question}}`), preventing the mention tokenizer
* from running. Walking the resulting JSON and splitting inline fixes the chip.
*
* Returns an array because one text node may expand into [text, mention, text, …].
*/
export function restoreTextMentions(node: JsonNode, hasMentions: boolean): JsonNode[] {
if (node.type === 'text' && typeof node.text === 'string') {
const unescaped = unescapeXmlTags(node.text)
if (!hasMentions) return [{ ...node, text: unescaped }]
const regex = /\{\{([^{}]+)\}\}/g
const parts: JsonNode[] = []
let lastIndex = 0
let match: RegExpExecArray | null
while ((match = regex.exec(unescaped)) !== null) {
if (match.index > lastIndex) parts.push({ ...node, text: unescaped.slice(lastIndex, match.index) })
const label = match[1].trim()
parts.push({ type: 'mention', attrs: { id: label, label } })
lastIndex = regex.lastIndex
}
if (parts.length === 0) return [{ ...node, text: unescaped }]
if (lastIndex < unescaped.length) parts.push({ ...node, text: unescaped.slice(lastIndex) })
return parts
}
if (Array.isArray(node.content)) {
const newContent: JsonNode[] = []
for (const child of node.content) newContent.push(...restoreTextMentions(child, hasMentions))
return [{ ...node, content: newContent }]
}
return [node]
}