mirror of
https://github.com/microsoft/semantic-kernel.git
synced 2026-10-12 11:19:56 +00:00
Jordan Henkel
016cd3334f
### Motivation and Context The C# Semantic Kernel has recently undergone an upgrade to its `PromptTemplateEngine`. This brings Python back in line with the semantics of Prompt Templates in C#. ### Description Here, unlike the original port, I've tried to make things more pythonic/idiomatic (instead of trying to directly mirror the C# codebase). I've also brought over the corresponding unit tests (and added some of my own as I was building/validating).
167 lines
6.4 KiB
Python
167 lines
6.4 KiB
Python
# Copyright (c) Microsoft. All rights reserved.
|
|
|
|
from logging import Logger
|
|
from typing import List
|
|
|
|
from semantic_kernel.template_engine.blocks.block import Block
|
|
from semantic_kernel.template_engine.blocks.block_types import BlockTypes
|
|
from semantic_kernel.template_engine.blocks.function_id_block import FunctionIdBlock
|
|
from semantic_kernel.template_engine.blocks.symbols import Symbols
|
|
from semantic_kernel.template_engine.blocks.val_block import ValBlock
|
|
from semantic_kernel.template_engine.blocks.var_block import VarBlock
|
|
from semantic_kernel.utils.null_logger import NullLogger
|
|
|
|
|
|
# BNF parsed by CodeTokenizer:
|
|
# [template] ::= "" | [variable] " " [template]
|
|
# | [value] " " [template]
|
|
# | [function-call] " " [template]
|
|
# [variable] ::= "$" [valid-name]
|
|
# [value] ::= "'" [text] "'" | '"' [text] '"'
|
|
# [function-call] ::= [function-id] | [function-id] [parameter]
|
|
# [parameter] ::= [variable] | [value]
|
|
class CodeTokenizer:
|
|
def __init__(self, log: Logger = None):
|
|
self.log = log or NullLogger()
|
|
|
|
def tokenize(self, text: str) -> List[Block]:
|
|
# Remove spaces, which are ignored anyway
|
|
text = text.strip() if text else ""
|
|
|
|
# Render None/empty to []
|
|
if not text or text == "":
|
|
return []
|
|
|
|
# Track what type of token we're reading
|
|
current_token_type = None
|
|
|
|
# Track the content of the current token
|
|
current_token_content = []
|
|
|
|
# Other state we need to track
|
|
text_value_delimiter = None
|
|
blocks = []
|
|
next_char = text[0]
|
|
space_separator_found = False
|
|
skip_next_char = False
|
|
|
|
# 1 char only edge case
|
|
if len(text) == 1:
|
|
if next_char == Symbols.VAR_PREFIX:
|
|
blocks.append(VarBlock(text, self.log))
|
|
elif next_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE):
|
|
blocks.append(ValBlock(text, self.log))
|
|
else:
|
|
blocks.append(FunctionIdBlock(text, self.log))
|
|
|
|
return blocks
|
|
|
|
for next_char_cursor in range(1, len(text)):
|
|
current_char = next_char
|
|
next_char = text[next_char_cursor]
|
|
|
|
if skip_next_char:
|
|
skip_next_char = False
|
|
continue
|
|
|
|
# First char is easy
|
|
if next_char_cursor == 1:
|
|
if current_char == Symbols.VAR_PREFIX:
|
|
current_token_type = BlockTypes.VARIABLE
|
|
elif current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE):
|
|
current_token_type = BlockTypes.VALUE
|
|
text_value_delimiter = current_char
|
|
else:
|
|
current_token_type = BlockTypes.FUNCTION_ID
|
|
|
|
current_token_content.append(current_char)
|
|
continue
|
|
|
|
# While reading values between quotes
|
|
if current_token_type == BlockTypes.VALUE:
|
|
# If the current char is escaping the next special char we:
|
|
# - skip the current char (escape char)
|
|
# - add the next char (special char)
|
|
# - jump to the one after (to handle "\\" properly)
|
|
if current_char == Symbols.ESCAPE_CHAR and self._can_be_escaped(
|
|
next_char
|
|
):
|
|
current_token_content.append(next_char)
|
|
skip_next_char = True
|
|
continue
|
|
|
|
current_token_content.append(current_char)
|
|
|
|
# When we reach the end of the value, we add the block
|
|
if current_char == text_value_delimiter:
|
|
blocks.append(ValBlock("".join(current_token_content), self.log))
|
|
current_token_content.clear()
|
|
current_token_type = None
|
|
space_separator_found = False
|
|
|
|
continue
|
|
|
|
# If we're not between quotes, a space signals the end of the current token
|
|
# Note: there might be multiple consecutive spaces
|
|
if self._is_blank_space(current_char):
|
|
if current_token_type == BlockTypes.VARIABLE:
|
|
blocks.append(VarBlock("".join(current_token_content), self.log))
|
|
current_token_content.clear()
|
|
elif current_token_type == BlockTypes.FUNCTION_ID:
|
|
blocks.append(
|
|
FunctionIdBlock("".join(current_token_content), self.log)
|
|
)
|
|
current_token_content.clear()
|
|
|
|
space_separator_found = True
|
|
current_token_type = None
|
|
|
|
continue
|
|
|
|
# If we're not inside a quoted value and we're not processing a space
|
|
current_token_content.append(current_char)
|
|
|
|
if current_token_type is None:
|
|
if not space_separator_found:
|
|
raise ValueError("Tokens must be separated by one space least")
|
|
|
|
if current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE):
|
|
# A quoted value starts here
|
|
current_token_type = BlockTypes.VALUE
|
|
text_value_delimiter = current_char
|
|
elif current_char == Symbols.VAR_PREFIX:
|
|
# A variable starts here
|
|
current_token_type = BlockTypes.VARIABLE
|
|
else:
|
|
# A function id starts here
|
|
current_token_type = BlockTypes.FUNCTION_ID
|
|
|
|
# Capture last token
|
|
current_token_content.append(next_char)
|
|
|
|
if current_token_type == BlockTypes.VALUE:
|
|
blocks.append(ValBlock("".join(current_token_content), self.log))
|
|
elif current_token_type == BlockTypes.VARIABLE:
|
|
blocks.append(VarBlock("".join(current_token_content), self.log))
|
|
elif current_token_type == BlockTypes.FUNCTION_ID:
|
|
blocks.append(FunctionIdBlock("".join(current_token_content), self.log))
|
|
else:
|
|
raise ValueError("Tokens must be separated by one space least")
|
|
|
|
return blocks
|
|
|
|
def _is_blank_space(self, c: str) -> bool:
|
|
return c in (
|
|
Symbols.SPACE,
|
|
Symbols.NEW_LINE,
|
|
Symbols.CARRIAGE_RETURN,
|
|
Symbols.TAB,
|
|
)
|
|
|
|
def _can_be_escaped(self, c: str) -> bool:
|
|
return c in (
|
|
Symbols.DBL_QUOTE,
|
|
Symbols.SGL_QUOTE,
|
|
Symbols.ESCAPE_CHAR,
|
|
)
|