Python: Improve prompt-template msg serialize and sample usage (#13738)

### Motivation and Context

This PR updates the Jinja2 and Handlebars prompt-template helpers to
serialize chat messages through the existing XML/message serializer
instead of assembling message XML manually. It also aligns the
prompt-template samples with the serializer-backed helper and adds
regression coverage for common message content.

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

Prompt template message serialize improvements.

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [X] The code builds clean without any errors or warnings
- [X] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [X] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄

---------

Co-authored-by: MAF Dashboard Bot <[email protected]>
Co-authored-by: Copilot <[email protected]>

python/samples/concepts/prompt_templates/azure_chat_gpt_api_handlebars.py

@@ -39,7 +39,7 @@ req_settings.function_choice_behavior = FunctionChoiceBehavior.Auto()
 chat_function = kernel.add_function(
     prompt_template_config=PromptTemplateConfig(
         template="""{{system_message}}{{#each chat_history}}
-        {{#message role=role}}{{~content~}}{{/message}} {{/each}}""",
+        {{message_to_prompt}} {{/each}}""",
         template_format="handlebars",
         allow_dangerously_set_content=True,
     ),

python/samples/concepts/prompt_templates/azure_chat_gpt_api_jinja2.py

@@ -38,7 +38,7 @@ req_settings.function_choice_behavior = FunctionChoiceBehavior.Auto()
 
 chat_function = kernel.add_function(
     prompt_template_config=PromptTemplateConfig(
-        template="""{{system_message}}{% for item in chat_history %}{{ message(item) }}{% endfor %}""",
+        template="""{{system_message}}{% for item in chat_history %}{{ message_to_prompt(item) }}{% endfor %}""",
         template_format="jinja2",
         allow_dangerously_set_content=True,
     ),

python/semantic_kernel/prompt_template/utils/handlebars_system_helpers.py

@@ -5,6 +5,7 @@ import logging
 import re
 from collections.abc import Callable
 from enum import Enum
+from xml.etree.ElementTree import Element, SubElement, tostring  # nosec B405
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -26,23 +27,32 @@ def _message_to_prompt(this, *args, **kwargs):
 
 
 def _message(this, options, *args, **kwargs):
+    from semantic_kernel.contents.chat_message_content import ChatMessageContent
     from semantic_kernel.contents.const import CHAT_MESSAGE_CONTENT_TAG
 
-    # everything in kwargs, goes to <ROOT_KEY_MESSAGE kwargs_key="kwargs_value">
-    # everything in options, goes in between <ROOT_KEY_MESSAGE>options</ROOT_KEY_MESSAGE>
-    start = f"<{CHAT_MESSAGE_CONTENT_TAG}"
+    # When the context is a ChatMessageContent, delegate to to_element() so that
+    # the XML contract is consistent with the Jinja2 path.
+    if isinstance(this.context, ChatMessageContent):
+        message = this.context.to_element()
+        return tostring(message, encoding="unicode", short_empty_elements=False)
+
+    # Fallback: build the element manually from kwargs and block content.
+    from semantic_kernel.contents.const import TEXT_CONTENT_TAG
+
+    message = Element(CHAT_MESSAGE_CONTENT_TAG)
     for key, value in kwargs.items():
         if isinstance(value, Enum):
             value = value.value
         if value is not None:
-            start += f' {key}="{value}"'
-    start += ">"
-    end = f"</{CHAT_MESSAGE_CONTENT_TAG}>"
+            message.set(key, str(value))
     try:
-        content = options["fn"](this)
+        content = str(options["fn"](this))
     except Exception:  # pragma: no cover
         content = ""
-    return f"{start}{content}{end}"
+    if content:
+        text_elem = SubElement(message, TEXT_CONTENT_TAG)
+        text_elem.text = content
+    return tostring(message, encoding="unicode", short_empty_elements=False)
 
 
 def _set(this, *args, **kwargs):

python/semantic_kernel/prompt_template/utils/jinja2_system_helpers.py

@@ -3,7 +3,7 @@
 import logging
 import re
 from collections.abc import Callable
-from enum import Enum
+from xml.etree.ElementTree import tostring  # nosec B405
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -25,17 +25,8 @@ def _message_to_prompt(context):
 
 
 def _message(item):
-    from semantic_kernel.contents.const import CHAT_MESSAGE_CONTENT_TAG
-
-    start = f"<{CHAT_MESSAGE_CONTENT_TAG}"
-    role = item.role
-    content = item.content
-    if isinstance(role, Enum):
-        role = role.value
-    start += f' role="{role}"'
-    start += ">"
-    end = f"</{CHAT_MESSAGE_CONTENT_TAG}>"
-    return f"{start}{content}{end}"
+    message = item.to_element()
+    return tostring(message, encoding="unicode", short_empty_elements=False)
 
 
 # Wrap the _get function to safely handle calls without arguments

python/tests/unit/prompt_template/test_handlebars_prompt_template.py

@@ -261,6 +261,108 @@ async def test_helpers_message(kernel: Kernel):
     assert "Assistant message" in rendered
 
 
+async def test_helpers_message_escapes_xml_metacharacters(kernel: Kernel):
+    template = """
+{{#each chat_history}}
+    {{#message role=role}}
+    {{~content~}}
+    {{/message}}
+{{/each}}
+"""
+    target = create_handlebars_prompt_template(template, allow_dangerously_set_content=True)
+    chat_history = ChatHistory()
+    chat_history.add_user_message('What does a < b & a > c & "d" mean?')
+
+    rendered = await target.render(kernel, KernelArguments(chat_history=chat_history))
+
+    assert "&lt;" in rendered
+    assert "&amp;" in rendered
+    # ElementTree does not escape > in text content (valid XML); verify round-trip works regardless.
+    assert "a > c" in rendered or "a &gt; c" in rendered
+    assert '"d"' in rendered
+    assert ChatHistory.from_rendered_prompt(rendered) == chat_history
+
+
+async def test_helpers_message_uses_text_element(kernel: Kernel):
+    """Verify handlebars {{#message}} wraps content in <text> like the Jinja2 path."""
+    template = """
+{{#each chat_history}}
+    {{#message role=role}}
+    {{~content~}}
+    {{/message}}
+{{/each}}
+"""
+    target = create_handlebars_prompt_template(template, allow_dangerously_set_content=True)
+    chat_history = ChatHistory()
+    chat_history.add_user_message("User message")
+    chat_history.add_assistant_message("Assistant message")
+    rendered = await target.render(kernel, KernelArguments(chat_history=chat_history))
+    assert '<message role="user"><text>User message</text></message>' in rendered
+    assert '<message role="assistant"><text>Assistant message</text></message>' in rendered
+    chat_history2 = ChatHistory.from_rendered_prompt(rendered)
+    assert chat_history2 == chat_history
+
+
+async def test_helpers_message_empty_content(kernel: Kernel):
+    """Empty message content should produce <message role="..."></message>, not self-closing."""
+    template = """
+{{#each chat_history}}
+    {{#message role=role}}
+    {{~content~}}
+    {{/message}}
+{{/each}}
+"""
+    target = create_handlebars_prompt_template(template, allow_dangerously_set_content=True)
+    chat_history = ChatHistory()
+    chat_history.add_user_message("")
+    rendered = await target.render(kernel, KernelArguments(chat_history=chat_history))
+    assert "<message" in rendered
+    assert "/>" not in rendered
+    assert ChatHistory.from_rendered_prompt(rendered) is not None
+
+
+async def test_helpers_message_fallback_empty_content(kernel: Kernel):
+    """Fallback path (non-ChatMessageContent context) with empty block content.
+
+    Should produce <message role="..."></message>, not self-closing.
+    """
+    template = '{{#message role="user"}}{{/message}}'
+    target = create_handlebars_prompt_template(template, allow_dangerously_set_content=True)
+    rendered = await target.render(kernel, KernelArguments())
+    assert '<message role="user"></message>' in rendered
+    assert "/>" not in rendered
+    assert ChatHistory.from_rendered_prompt(rendered) is not None
+
+
+async def test_helpers_message_fallback_with_content(kernel: Kernel):
+    """Fallback path wraps block content in a <text> child element."""
+    template = '{{#message role="user"}}Hello world{{/message}}'
+    target = create_handlebars_prompt_template(template, allow_dangerously_set_content=True)
+    rendered = await target.render(kernel, KernelArguments())
+    assert '<message role="user"><text>Hello world</text></message>' in rendered
+    chat_history = ChatHistory.from_rendered_prompt(rendered)
+    assert chat_history is not None
+    assert len(chat_history) == 1
+    assert chat_history[0].content == "Hello world"
+
+
+async def test_helpers_message_escapes_greater_than(kernel: Kernel):
+    """ElementTree does not escape > in text; verify round-trip still works."""
+    template = """
+{{#each chat_history}}
+    {{#message role=role}}
+    {{~content~}}
+    {{/message}}
+{{/each}}
+"""
+    target = create_handlebars_prompt_template(template, allow_dangerously_set_content=True)
+    chat_history = ChatHistory()
+    chat_history.add_user_message("Is a > b true?")
+    rendered = await target.render(kernel, KernelArguments(chat_history=chat_history))
+    assert "a > b" in rendered or "a &gt; b" in rendered
+    assert ChatHistory.from_rendered_prompt(rendered) == chat_history
+
+
 async def test_helpers_message_to_prompt(kernel: Kernel):
     template = """{{#each chat_history}}{{message_to_prompt}} {{/each}}"""
     target = create_handlebars_prompt_template(template, allow_dangerously_set_content=True)

python/tests/unit/prompt_template/test_handlebars_prompt_template_e2e.py

@@ -2,6 +2,7 @@
 
 
 from semantic_kernel import Kernel
+from semantic_kernel.contents import AuthorRole
 from semantic_kernel.contents.chat_history import ChatHistory
 from semantic_kernel.functions import kernel_function
 from semantic_kernel.functions.kernel_arguments import KernelArguments
@@ -94,9 +95,46 @@ class TestHandlebarsPromptTemplateEngine:
         chat_history.add_user_message("User message")
         chat_history.add_assistant_message("Assistant message")
         rendered = await target.render(kernel, KernelArguments(chat_history=chat_history))
-        assert (
-            rendered.strip()
-            == """<message role="user">User message</message> <message role="assistant">Assistant message</message>"""
+        expected = (
+            '<message role="user"><text>User message</text></message>'
+            ' <message role="assistant"><text>Assistant message</text></message>'
         )
+        assert rendered.strip() == expected
         chat_history2 = ChatHistory.from_rendered_prompt(rendered)
         assert chat_history2 == chat_history
+
+    async def test_chat_history_round_trip_with_xml_metacharacters(self, kernel: Kernel):
+        # Arrange
+        template = """{{#each chat_history}}{{#message role=role}}{{~content~}}{{/message}} {{/each}}"""
+        target = create_handlebars_prompt_template(template)
+        chat_history = ChatHistory()
+        chat_history.add_user_message("What does a < b mean in Python?")
+        chat_history.add_assistant_message('Use "&" carefully in XML and HTML.')
+
+        rendered = await target.render(kernel, KernelArguments(chat_history=chat_history))
+
+        assert "&lt;" in rendered
+        assert "&amp;" in rendered
+        assert '"&amp;"' in rendered
+        assert ChatHistory.from_rendered_prompt(rendered) == chat_history
+
+    async def test_message_helper_preserves_system_role_with_xml_metacharacters(self, kernel: Kernel):
+        # Arrange
+        template = (
+            """{{system_message}}{{#each chat_history}}{{#message role=role}}{{~content~}}{{/message}} {{/each}}"""
+        )
+        target = create_handlebars_prompt_template(template)
+        system_message = "You are a helpful assistant."
+        chat_history = ChatHistory()
+        chat_history.add_user_message("What does a < b mean in Python?")
+
+        rendered = await target.render(
+            kernel,
+            KernelArguments(system_message=system_message, chat_history=chat_history),
+        )
+
+        parsed = ChatHistory.from_rendered_prompt(rendered)
+        assert parsed.messages[0].role == AuthorRole.SYSTEM
+        assert parsed.messages[0].content == system_message
+        assert parsed.messages[1].role == AuthorRole.USER
+        assert parsed.messages[1].content == "What does a < b mean in Python?"

python/tests/unit/prompt_template/test_jinja2_prompt_template.py

@@ -264,6 +264,20 @@ async def test_helpers_message(kernel: Kernel):
     assert "Assistant message" in rendered
 
 
+async def test_helpers_message_escapes_xml_metacharacters(kernel: Kernel):
+    template = """{% for item in chat_history %}{{ message(item) }}{% endfor %}"""
+    target = create_jinja2_prompt_template(template, allow_dangerously_set_content=True)
+    chat_history = ChatHistory()
+    chat_history.add_user_message('What does a < b & "c" mean?')
+
+    rendered = await target.render(kernel, KernelArguments(chat_history=chat_history))
+
+    assert "&lt;" in rendered
+    assert "&amp;" in rendered
+    assert '"c"' in rendered
+    assert ChatHistory.from_rendered_prompt(rendered) == chat_history
+
+
 async def test_helpers_message_to_prompt(kernel: Kernel):
     template = """
     {% for chat in chat_history %}

python/tests/unit/prompt_template/test_jinja2_prompt_template_e2e.py

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft. All rights reserved.
 
 
+from semantic_kernel.contents import AuthorRole
 from semantic_kernel.contents.chat_history import ChatHistory
 from semantic_kernel.functions import kernel_function
 from semantic_kernel.functions.kernel_arguments import KernelArguments
@@ -98,9 +99,81 @@ async def test_chat_history_round_trip(kernel: Kernel):
     chat_history.add_user_message("User message")
     chat_history.add_assistant_message("Assistant message")
     rendered = await target.render(kernel, KernelArguments(chat_history=chat_history))
-    assert (
-        rendered.strip()
-        == """<message role="user">User message</message><message role="assistant">Assistant message</message>"""
+    expected = (
+        '<message role="user"><text>User message</text></message>'
+        '<message role="assistant"><text>Assistant message</text></message>'
     )
+    assert rendered.strip() == expected
     chat_history2 = ChatHistory.from_rendered_prompt(rendered)
     assert chat_history2 == chat_history
+
+
+async def test_from_rendered_prompt_backward_compat_old_format(kernel: Kernel):
+    """from_rendered_prompt handles the old format without <text> wrapper for backward compatibility."""
+    old_format = '<message role="user">User message</message><message role="assistant">Assistant message</message>'
+    parsed = ChatHistory.from_rendered_prompt(old_format)
+    assert len(parsed) == 2
+    assert parsed[0].role == AuthorRole.USER
+    assert parsed[0].content == "User message"
+    assert parsed[1].role == AuthorRole.ASSISTANT
+    assert parsed[1].content == "Assistant message"
+
+
+async def test_chat_history_round_trip_with_xml_metacharacters(kernel: Kernel):
+    template = """{% for item in chat_history %}{{ message(item) }}{% endfor %}"""
+    target = create_jinja2_prompt_template(template)
+    chat_history = ChatHistory()
+    chat_history.add_user_message("What does a < b mean in Python?")
+    chat_history.add_assistant_message('Use "&" carefully in XML and HTML.')
+
+    rendered = await target.render(kernel, KernelArguments(chat_history=chat_history))
+
+    assert "&lt;" in rendered
+    assert "&amp;" in rendered
+    assert '"&amp;"' in rendered
+    assert ChatHistory.from_rendered_prompt(rendered) == chat_history
+
+
+async def test_message_helper_preserves_system_role_with_xml_metacharacters(kernel: Kernel):
+    template = """{{system_message}}{% for item in chat_history %}{{ message(item) }}{% endfor %}"""
+    target = create_jinja2_prompt_template(template)
+    system_message = "You are a helpful assistant."
+    chat_history = ChatHistory()
+    chat_history.add_user_message("What does a < b mean in Python?")
+
+    rendered = await target.render(
+        kernel,
+        KernelArguments(system_message=system_message, chat_history=chat_history),
+    )
+
+    parsed = ChatHistory.from_rendered_prompt(rendered)
+    assert parsed.messages[0].role == AuthorRole.SYSTEM
+    assert parsed.messages[0].content == system_message
+    assert parsed.messages[1].role == AuthorRole.USER
+
+    assert parsed.messages[1].content == "What does a < b mean in Python?"
+
+
+def test_from_rendered_prompt_backward_compat_old_format_no_text_wrapper():
+    """from_rendered_prompt must handle the old format without <text> wrapper."""
+    old_format = '<message role="user">User message</message><message role="assistant">Assistant message</message>'
+    parsed = ChatHistory.from_rendered_prompt(old_format)
+    assert len(parsed.messages) == 2
+    assert parsed.messages[0].role == AuthorRole.USER
+    assert parsed.messages[0].content == "User message"
+    assert parsed.messages[1].role == AuthorRole.ASSISTANT
+    assert parsed.messages[1].content == "Assistant message"
+
+
+def test_from_rendered_prompt_new_text_element_format():
+    """from_rendered_prompt must handle the new format with <text> wrapper."""
+    new_format = (
+        '<message role="user"><text>User message</text></message>'
+        '<message role="assistant"><text>Assistant message</text></message>'
+    )
+    parsed = ChatHistory.from_rendered_prompt(new_format)
+    assert len(parsed.messages) == 2
+    assert parsed.messages[0].role == AuthorRole.USER
+    assert parsed.messages[0].content == "User message"
+    assert parsed.messages[1].role == AuthorRole.ASSISTANT
+    assert parsed.messages[1].content == "Assistant message"