Python: updates to chat_prompt and related (#4720)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->
Fixes #4711 and #4705 and addresses some inconsistencies in the way the
chat_prompt_template is created, now has more control.

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->
Added parse flags to chatPromptTemplate to allow the user to control is
something is loaded into messages or not.
Added better docstrings and tests for this as well.

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

python/semantic_kernel/connectors/ai/open_ai/request_settings/open_ai_request_settings.py

@@ -4,7 +4,6 @@ from typing import Any, Dict, List, Literal, Optional, Union
 from pydantic import Field, field_validator, model_validator
 
 from semantic_kernel.connectors.ai.ai_request_settings import AIRequestSettings
-from semantic_kernel.connectors.ai.open_ai.const import DEFAULT_CHAT_SYSTEM_PROMPT
 
 logger = logging.getLogger(__name__)
 
@@ -59,7 +58,7 @@ class OpenAIChatRequestSettings(OpenAIRequestSettings):
     tool_choice: Optional[str] = None
     function_call: Optional[str] = None
     functions: Optional[List[Dict[str, Any]]] = None
-    messages: Optional[List[Dict[str, Any]]] = [{"role": "system", "content": DEFAULT_CHAT_SYSTEM_PROMPT}]
+    messages: Optional[List[Dict[str, Any]]] = None
 
     @field_validator("functions", "function_call", mode="after")
     @classmethod

python/semantic_kernel/connectors/ai/open_ai/semantic_functions/open_ai_chat_prompt_template.py

@@ -1,19 +1,13 @@
 # Copyright (c) Microsoft. All rights reserved.
 
 import logging
-from typing import Any, Dict, List, Optional
+from typing import Any, Optional
 
 from semantic_kernel.connectors.ai.open_ai.models.chat.open_ai_chat_message import (
     OpenAIChatMessage,
 )
 from semantic_kernel.semantic_functions.chat_prompt_template import ChatPromptTemplate
 from semantic_kernel.semantic_functions.prompt_template import PromptTemplate
-from semantic_kernel.semantic_functions.prompt_template_config import (
-    PromptTemplateConfig,
-)
-from semantic_kernel.template_engine.protocols.prompt_templating_engine import (
-    PromptTemplatingEngine,
-)
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -49,7 +43,7 @@ class OpenAIChatPromptTemplate(ChatPromptTemplate[OpenAIChatMessage]):
                     )
                 )
                 return
-            self._log.warning("function_call is only used with role: assistant, ignoring")
+            logger.warning("function_call is only used with role: assistant, ignoring")
             function_call = None
         self.messages.append(
             OpenAIChatMessage(
@@ -59,39 +53,3 @@ class OpenAIChatPromptTemplate(ChatPromptTemplate[OpenAIChatMessage]):
                 function_call=function_call,
             )
         )
-
-    @classmethod
-    def restore(
-        cls,
-        messages: List[Dict[str, str]],
-        template: str,
-        template_engine: PromptTemplatingEngine,
-        prompt_config: PromptTemplateConfig,
-        log: Optional[Any] = None,
-    ) -> "OpenAIChatPromptTemplate":
-        """Restore a ChatPromptTemplate from a list of role and message pairs.
-
-        If there is a chat_system_prompt in the prompt_config.execution_settings settings,
-        that takes precedence over the first message in the list of messages,
-        if that is a system message.
-        """
-        if log:
-            logger.warning("The `log` parameter is deprecated. Please use the `logging` module instead.")
-        chat_template = cls(template, template_engine, prompt_config)
-        if prompt_config.execution_settings.chat_system_prompt and messages[0]["role"] == "system":
-            existing_system_message = messages.pop(0)
-            if existing_system_message["message"] != prompt_config.execution_settings.chat_system_prompt:
-                logger.info(
-                    "Overriding system prompt with chat_system_prompt, old system message: %s, new system message: %s",
-                    existing_system_message["message"],
-                    prompt_config.execution_settings.chat_system_prompt,
-                )
-        for message in messages:
-            chat_template.add_message(
-                message["role"],
-                message["message"],
-                name=message["name"],
-                function_call=message["function_call"],
-            )
-
-        return chat_template

python/semantic_kernel/semantic_functions/chat_prompt_template.py

@@ -31,20 +31,43 @@ class ChatPromptTemplate(PromptTemplate, Generic[ChatMessageT]):
         template: str,
         template_engine: PromptTemplatingEngine,
         prompt_config: PromptTemplateConfig,
-        log: Optional[Any] = None,
+        parse_chat_system_prompt: bool = False,
+        parse_messages: bool = False,
+        **kwargs: Any,
     ) -> None:
+        """Initialize a chat prompt template.
+
+        if there is a field 'chat_system_prompt' in the prompt_config.execution_settings.extension_data,
+        that value is added to the messages list as a system message,
+        can be controlled by setting the parse_chat_system_prompt parameter to True.
+
+        After that any messages that are in messages in the prompt_config.execution_settings
+        are added to the messages list.
+        Can be controlled by setting the parse_messages parameter to True.
+
+        Arguments:
+            template {str} -- The template to use for the chat prompt.
+            template_engine {PromptTemplatingEngine} -- The templating engine to use.
+            prompt_config {PromptTemplateConfig} -- The prompt config to use.
+            parse_chat_system_prompt {bool} -- Whether to parse the chat_system_prompt from
+                the prompt_config.execution_settings.extension_data.
+            parse_messages {bool} -- Whether to parse the messages from the prompt_config.execution_settings.
+
+        """
         super().__init__(template, template_engine, prompt_config)
-        if log:
+        if "log" in kwargs:
             logger.warning("The `log` parameter is deprecated. Please use the `logging` module instead.")
-        self._messages = []
-        if self.prompt_config.execution_settings.extension_data.get("chat_system_prompt"):
+
+        if parse_chat_system_prompt and "chat_system_prompt" in self.prompt_config.execution_settings.extension_data:
             self.add_system_message(self.prompt_config.execution_settings.extension_data["chat_system_prompt"])
+
         if (
-            hasattr(self.prompt_config.execution_settings, "messages")
+            parse_messages
+            and hasattr(self.prompt_config.execution_settings, "messages")
             and self.prompt_config.execution_settings.messages
         ):
             for message in self.prompt_config.execution_settings.messages:
-                self.add_message(message["role"], message["content"])
+                self.add_message(**message)
 
     async def render_async(self, context: "KernelContext") -> str:
         raise NotImplementedError(
@@ -75,6 +98,9 @@ class ChatPromptTemplate(PromptTemplate, Generic[ChatMessageT]):
         # When the type is not explicitly set, it is still the typevar, replace with generic ChatMessage
         if isinstance(concrete_message, TypeVar):
             concrete_message = ChatMessage
+        assert issubclass(concrete_message, ChatMessage)
+        if not message and "content" in kwargs:
+            message = kwargs["content"]
         self.messages.append(
             concrete_message(
                 role=role,
@@ -104,26 +130,35 @@ class ChatPromptTemplate(PromptTemplate, Generic[ChatMessageT]):
         template: str,
         template_engine: PromptTemplatingEngine,
         prompt_config: PromptTemplateConfig,
-        log: Optional[Any] = None,
+        parse_chat_system_prompt: bool = False,
+        parse_messages: bool = False,
+        **kwargs: Any,
     ) -> "ChatPromptTemplate":
         """Restore a ChatPromptTemplate from a list of role and message pairs.
 
-        If there is a chat_system_prompt in the prompt_config.execution_settings settings,
-        that takes precedence over the first message in the list of messages,
-        if that is a system message.
-        """
-        if log:
-            logger.warning("The `log` parameter is deprecated. Please use the `logging` module instead.")
-        chat_template = cls(template, template_engine, prompt_config)
-        if prompt_config.execution_settings.chat_system_prompt and messages[0]["role"] == "system":
-            existing_system_message = messages.pop(0)
-            if existing_system_message["message"] != prompt_config.execution_settings.chat_system_prompt:
-                logger.info(
-                    "Overriding system prompt with chat_system_prompt, old system message: %s, new system message: %s",
-                    existing_system_message["message"],
-                    prompt_config.execution_settings.chat_system_prompt,
-                )
-        for message in messages:
-            chat_template.add_message(message["role"], message["message"])
+        The parse_messages and parse_chat_system_prompt parameters control whether
+        the messages and chat_system_prompt from the prompt_config.execution_settings
+        are parsed and added to the messages list, not whether or not the 'messages'
+        from the messages parameter are parsed, those are always parsed.
 
+        Arguments:
+            messages {List[Dict[str, str]]} -- The messages to restore,
+                the default format is [{"role": "user", "message": "Hi there"}].
+                if the ChatPromptTemplate is created with a different message type,
+                the messages should contain any fields that are relevant to the message,
+                for instance: ChatPromptTemplate[OpenAIChatMessage].restore can be used with a format:
+                [{"role": "assistant", "function_call": FunctionCall()}].
+            template {str} -- The template to use for the chat prompt.
+            template_engine {PromptTemplatingEngine} -- The templating engine to use.
+            prompt_config {PromptTemplateConfig} -- The prompt config to use.
+            parse_chat_system_prompt {bool} -- Whether to parse the chat_system_prompt from the
+                prompt_config.execution_settings.extension_data.
+            parse_messages {bool} -- Whether to parse the messages from the prompt_config.execution_settings.
+
+        """
+        chat_template = cls(
+            template, template_engine, prompt_config, parse_chat_system_prompt, parse_messages, **kwargs
+        )
+        for message in messages:
+            chat_template.add_message(**message)
         return chat_template

python/tests/unit/ai/open_ai/test_openai_request_settings.py

@@ -26,7 +26,7 @@ def test_default_openai_chat_request_settings():
     assert settings.stop is None
     assert settings.number_of_responses == 1
     assert settings.logit_bias == {}
-    assert settings.messages[0]["content"] == "Assistant is a large language model."
+    assert settings.messages is None
 
 
 def test_custom_openai_chat_request_settings():

python/tests/unit/plugin_definition/test_prompt_templates.py

@@ -151,15 +151,31 @@ def test_chat_prompt_template():
     assert chat_prompt_template.messages == []
 
 
-def test_chat_prompt_template_with_system_prompt():
+def test_chat_prompt_template_with_messages():
     prompt_template_config = PromptTemplateConfig[OpenAIChatRequestSettings].from_execution_settings(
         messages=[{"role": "system", "content": "Custom system prompt."}],
     )
-
     chat_prompt_template = ChatPromptTemplate[OpenAIChatMessage](
         "{{$user_input}}",
         PromptTemplateEngine(),
         prompt_config=prompt_template_config,
+        parse_messages=True,
+    )
+    print(chat_prompt_template.messages)
+    assert len(chat_prompt_template.messages) == 1
+    assert chat_prompt_template.messages[0].role == "system"
+    assert chat_prompt_template.messages[0].content_template.template == "Custom system prompt."
+
+
+def test_chat_prompt_template_with_system_prompt():
+    prompt_template_config = PromptTemplateConfig[OpenAIChatRequestSettings].from_execution_settings(
+        chat_system_prompt="Custom system prompt.",
+    )
+    chat_prompt_template = ChatPromptTemplate[OpenAIChatMessage](
+        "{{$user_input}}",
+        PromptTemplateEngine(),
+        prompt_config=prompt_template_config,
+        parse_chat_system_prompt=True,
     )
     print(chat_prompt_template.messages)
     assert len(chat_prompt_template.messages) == 1