diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..25314849 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,192 @@ +# Agent Guidelines + +Code conventions and patterns for this project, learned from review feedback. + +## Code structure + +### Break up complex functions with helpers + +When a function has deeply nested logic or multiple concerns, extract helpers. Use `flatMap` + small named functions instead of imperative loops with nested `when`/`if`: + +```kotlin +// Good +fun format(variables: Map): PromptMessages { + val formatted = messages.flatMap { msg -> + if (msg.isPlaceholder()) expandPlaceholder(msg, variables) + else listOf(PromptMessage.withTemplate(msg, msg.format(variables))) + } + return PromptMessages(formatted, inputVariables, outputSchema) +} + +private fun expandPlaceholder(msg: PromptMessage, variables: Map): List { + val items = variables[msg.template] as? List<*> ?: return emptyList() + return items.mapNotNull(::toPromptMessage) +} + +// Bad — deeply nested imperative loop +fun format(variables: Map): PromptMessages { + val formatted = mutableListOf() + for (msg in messages) { + if (msg.isPlaceholder()) { + val value = variables[msg.template] + if (value is List<*>) { + for (item in value) { + when (item) { + is PromptMessage -> formatted.add(item) + is Map<*, *> -> { /* 15 more lines */ } + } + } + } + } else { ... } + } +} +``` + +## Kotlin idioms + +### Use `buildMap` / `buildList` instead of mutable + convert + +```kotlin +// Good +val messages = items.map { msg -> + buildMap { + put("role", msg.role) + put("content", msg.content) + msg.toolCallId?.let { put("tool_call_id", it) } + } +} + +// Bad — unnecessary mutable/immutable conversion +val messages = items.map { msg -> + val base = mutableMapOf( + "role" to msg.role, + "content" to msg.content, + ) + if (msg.toolCallId != null) { + base["tool_call_id"] = msg.toolCallId + } + base.toMap() +} +``` + +### Use `buildList` for conditional `toString()` parts + +```kotlin +// Good +override fun toString(): String { + val parts = buildList { + add("messages=[${messages.joinToString(", ")}]") + if (inputVariables.isNotEmpty()) add("inputVariables=$inputVariables") + commitHash?.let { add("commitHash=$it") } + outputSchema?.let { add("outputSchema=${it["title"] ?: "..."}") } + } + return "Prompt{${parts.joinToString(", ")}}" +} + +// Bad — chained ternary string concatenation +override fun toString(): String = + "Prompt{messages=[...]" + + (if (commitHash != null) ", commitHash=$commitHash" else "") + + (if (hasOutputSchema()) ", outputSchema=..." else "") + + "}" +``` + +### Use `partition` instead of double `filter` + +```kotlin +// Good — single pass +val (systemMessages, nonSystemMessages) = + messages.partition { it.role == Role.SYSTEM } + +// Bad — iterates the list twice +val systemMessages = messages.filter { it.role == Role.SYSTEM } +val nonSystemMessages = messages.filter { it.role != Role.SYSTEM } +``` + +### Use parameterized tests for table-driven cases + +When multiple tests share the exact same structure (input → assert same fields), use `@ParameterizedTest` with `@MethodSource`: + +```kotlin +data class Case(val input: String, val expected: String) + +@ParameterizedTest(name = "{index}: \"{0}\"") +@MethodSource("cases") +fun myTest(case: Case) { + assertThat(transform(case.input)).isEqualTo(case.expected) +} + +companion object { + @JvmStatic + fun cases(): Stream = Stream.of( + Case("input1", "expected1"), + Case("input2", "expected2"), + ) +} +``` + +Only do this when every test has the same assertion shape. If tests have different setup or assertions, keep them as individual `@Test` methods. + +### Extract test assertion helpers to reduce repetition + +When the same assertion pattern appears across many tests, extract a helper: + +```kotlin +// Good — readable, DRY +private fun assertMessage(msg: Map, role: String, content: String) { + assertThat(msg["role"]).isEqualTo(role) + assertThat(msg["content"]).isEqualTo(content) +} + +assertMessage(result.messages[0], "system", "You are helpful.") +assertMessage(result.messages[1], "user", "Hello") + +// Bad — verbose, repetitive +assertThat(result.messages[0]).isEqualTo(mapOf("role" to "system", "content" to "You are helpful.")) +assertThat(result.messages[1]).isEqualTo(mapOf("role" to "user", "content" to "Hello")) +``` + +## Formatting and linting + +```bash +./gradlew :langsmith-java-core:formatKotlin +./gradlew lintKotlin +``` + +The project uses ktfmt with `--kotlinlang-style`. + +## Testing + +### Running tests + +```bash +# All prompt tests (unit + integration) +./gradlew :langsmith-java-core:test --tests "com.langchain.smith.prompts.*" + +# Just integration tests (requires API keys) +./gradlew :langsmith-java-core:test --tests "com.langchain.smith.prompts.PromptIntegrationTest" + +# Force re-run (skip Gradle cache) +./gradlew :langsmith-java-core:test --tests "..." --rerun + +# See println output +./gradlew :langsmith-java-core:test --tests "..." --rerun --info +``` + +### Integration tests + +Integration tests require environment variables: + +```bash +export LANGSMITH_API_KEY="lsv2_pt_..." +export OPENAI_API_KEY="sk-..." +export ANTHROPIC_API_KEY="sk-ant-..." +``` + +Tests skip gracefully via `assumeTrue` if keys are missing. + +## Code style + +- `toString()` should be single-line, following the `ClassName{field=value, field=value}` convention used by the rest of the SDK. +- Avoid `@Suppress("UNCHECKED_CAST")` — restructure code to use safe patterns (`as? String`, `is Map<*, *>` with `entries.associate`, etc). +- Anthropic SDK is a `compileOnly` dependency — users must add it themselves. Methods that use Anthropic types should catch `NoClassDefFoundError` and throw `IllegalStateException` with a clear message. diff --git a/langsmith-java-core/build.gradle.kts b/langsmith-java-core/build.gradle.kts index 523a1c4c..777d7a9c 100644 --- a/langsmith-java-core/build.gradle.kts +++ b/langsmith-java-core/build.gradle.kts @@ -39,9 +39,17 @@ dependencies { // OpenAI SDK (for OpenTelemetry wrappers) api("com.openai:openai-java:4.6.1") + // Mustache template engine (for prompt template formatting) + implementation("com.samskivert:jmustache:1.16") + // SLF4J for logging (API only - consumers choose implementation) api("org.slf4j:slf4j-api:2.0.17") + // Anthropic SDK — optional peer dependency for prompt conversion. + // Users who call AnthropicPayload.toAnthropicParams() must add this to their own dependencies. + compileOnly("com.anthropic:anthropic-java:2.18.0") + testImplementation("com.anthropic:anthropic-java:2.18.0") + testImplementation(kotlin("test")) // Simple logging for tests only testImplementation("org.slf4j:slf4j-simple:2.0.17") diff --git a/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/Converters.kt b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/Converters.kt new file mode 100644 index 00000000..5850eafe --- /dev/null +++ b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/Converters.kt @@ -0,0 +1,222 @@ +@file:JvmName("PromptConverters") + +package com.langchain.smith.prompts + +/** + * Converts a formatted [PromptValue] to a typed OpenAI [ChatCompletionCreateParams.Builder]. + * + * The returned builder has messages (and structured output response format, if applicable) already + * set. Call `.model(...)` and any other options, then `.build()`. + * + * ## Example (Java) + * + * ```java + * import static com.langchain.smith.prompts.PromptConverters.convertToOpenAIParams; + * + * Prompt prompt = promptClient.pull("my-org/joke-generator"); + * PromptValue formattedPrompt = prompt.invoke(Map.of("topic", "cats")); + * + * ChatCompletion completion = openai.chat().completions().create( + * convertToOpenAIParams(formattedPrompt) + * .model(ChatModel.GPT_4_1_MINI) + * .build()); + * ``` + * + * ## Example (Kotlin) + * + * ```kotlin + * import com.langchain.smith.prompts.convertToOpenAIParams + * + * val completion = openai.chat().completions().create( + * convertToOpenAIParams(formattedPrompt) + * .model(ChatModel.GPT_4_1_MINI) + * .build()) + * ``` + * + * @param promptValue the formatted prompt value from [Prompt.invoke] + * @param strictStructuredOutput whether to enable strict schema validation for structured outputs. + * Defaults to `true`, which guarantees the response matches the schema exactly but requires + * `additionalProperties: false` on all object nodes (injected automatically). + * @return a [ChatCompletionCreateParams.Builder] with messages and response format set + * @see Prompt.invoke + */ +@JvmOverloads +fun convertToOpenAIParams( + promptValue: PromptValue, + strictStructuredOutput: Boolean = true, +): com.openai.models.chat.completions.ChatCompletionCreateParams.Builder { + val pm = promptValue.promptMessages + val builder = com.openai.models.chat.completions.ChatCompletionCreateParams.builder() + + for (msg in pm.messages) { + when (msg.role) { + PromptMessage.Role.SYSTEM -> + builder.addMessage( + com.openai.models.chat.completions.ChatCompletionMessageParam.ofSystem( + com.openai.models.chat.completions.ChatCompletionSystemMessageParam + .builder() + .content(msg.template) + .build() + ) + ) + PromptMessage.Role.AI -> + builder.addMessage( + com.openai.models.chat.completions.ChatCompletionMessageParam.ofAssistant( + com.openai.models.chat.completions.ChatCompletionAssistantMessageParam + .builder() + .content(msg.template) + .build() + ) + ) + PromptMessage.Role.TOOL -> { + builder.addMessage( + com.openai.models.chat.completions.ChatCompletionMessageParam.ofTool( + com.openai.models.chat.completions.ChatCompletionToolMessageParam.builder() + .toolCallId(msg.toolCallId ?: "") + .content(msg.template) + .build() + ) + ) + } + else -> + builder.addMessage( + com.openai.models.chat.completions.ChatCompletionMessageParam.ofUser( + com.openai.models.chat.completions.ChatCompletionUserMessageParam.builder() + .content(msg.template) + .build() + ) + ) + } + } + + val outputSchema = pm.outputSchema + if (outputSchema != null) { + val schemaName = (outputSchema["title"] as? String) ?: "structured_output" + val resolvedSchema = + if (strictStructuredOutput) strictSchemaForStructuredOutput(outputSchema) + else outputSchema + val schemaBuilder = com.openai.models.ResponseFormatJsonSchema.JsonSchema.Schema.builder() + resolvedSchema.forEach { (k, v) -> + schemaBuilder.putAdditionalProperty(k, com.openai.core.JsonValue.from(v)) + } + builder.responseFormat( + com.openai.models.chat.completions.ChatCompletionCreateParams.ResponseFormat + .ofJsonSchema( + com.openai.models.ResponseFormatJsonSchema.builder() + .jsonSchema( + com.openai.models.ResponseFormatJsonSchema.JsonSchema.builder() + .name(schemaName) + .strict(strictStructuredOutput) + .schema(schemaBuilder.build()) + .build() + ) + .build() + ) + ) + } + + return builder +} + +/** + * Converts a formatted [PromptValue] to a typed Anthropic [MessageCreateParams.Builder]. + * + * The returned builder has system message, messages, and structured output config (if applicable) + * already set. Call `.model(...)`, `.maxTokens(...)`, and any other options, then `.build()`. + * + * Requires `com.anthropic:anthropic-java` (>= 2.18.0) as a dependency. + * + * ## Example (Java) + * + * ```java + * import static com.langchain.smith.prompts.PromptConverters.convertToAnthropicParams; + * + * Prompt prompt = promptClient.pull("my-org/joke-generator"); + * PromptValue formattedPrompt = prompt.invoke(Map.of("topic", "cats")); + * + * Message message = anthropic.messages().create( + * convertToAnthropicParams(formattedPrompt) + * .model(Model.CLAUDE_SONNET_4_6) + * .maxTokens(1024) + * .build()); + * ``` + * + * ## Example (Kotlin) + * + * ```kotlin + * import com.langchain.smith.prompts.convertToAnthropicParams + * + * val message = anthropic.messages().create( + * convertToAnthropicParams(formattedPrompt) + * .model(Model.CLAUDE_SONNET_4_6) + * .maxTokens(1024) + * .build()) + * ``` + * + * @param promptValue the formatted prompt value from [Prompt.invoke] + * @param strictStructuredOutput whether to inject `additionalProperties: false` into the schema. + * Defaults to `true`, which is required by Anthropic's `output_config` schemas. + * @return a [MessageCreateParams.Builder] with system, messages, and output config set + * @see Prompt.invoke + */ +@JvmOverloads +fun convertToAnthropicParams( + promptValue: PromptValue, + strictStructuredOutput: Boolean = true, +): com.anthropic.models.messages.MessageCreateParams.Builder { + val pm = promptValue.promptMessages + val builder = + try { + com.anthropic.models.messages.MessageCreateParams.builder() + } catch (e: NoClassDefFoundError) { + throw IllegalStateException( + "Anthropic SDK not found. Add com.anthropic:anthropic-java to your dependencies.", + e, + ) + } + + val (systemMessages, nonSystemMessages) = + pm.messages.partition { it.role == PromptMessage.Role.SYSTEM } + + val system = systemMessages.joinToString("\n") { it.template } + if (system.isNotEmpty()) { + builder.system(system) + } + + for (msg in nonSystemMessages) { + val role = + when (msg.role) { + PromptMessage.Role.HUMAN -> com.anthropic.models.messages.MessageParam.Role.USER + PromptMessage.Role.AI -> com.anthropic.models.messages.MessageParam.Role.ASSISTANT + else -> com.anthropic.models.messages.MessageParam.Role.USER + } + builder.addMessage( + com.anthropic.models.messages.MessageParam.builder() + .role(role) + .content(msg.template) + .build() + ) + } + + val outputSchema = pm.outputSchema + if (outputSchema != null) { + val resolvedSchema = + if (strictStructuredOutput) strictSchemaForStructuredOutput(outputSchema) + else outputSchema + val schemaBuilder = com.anthropic.models.messages.JsonOutputFormat.Schema.builder() + resolvedSchema.forEach { (k, v) -> + schemaBuilder.putAdditionalProperty(k, com.anthropic.core.JsonValue.from(v)) + } + builder.outputConfig( + com.anthropic.models.messages.OutputConfig.builder() + .format( + com.anthropic.models.messages.JsonOutputFormat.builder() + .schema(schemaBuilder.build()) + .build() + ) + .build() + ) + } + + return builder +} diff --git a/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/ManifestParser.kt b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/ManifestParser.kt new file mode 100644 index 00000000..09f21e56 --- /dev/null +++ b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/ManifestParser.kt @@ -0,0 +1,231 @@ +package com.langchain.smith.prompts + +import com.langchain.smith.core.JsonValue + +/** + * Parses LangChain-serialized prompt manifests into typed [PromptMessages] objects. + * + * The LangSmith hub stores prompts in LangChain's serialization format, which uses a nested + * structure with `lc`, `type`, `id`, and `kwargs` fields. This parser understands that format and + * extracts the messages, input variables, and (for structured prompts) the output schema. + * + * Supported manifest types: + * - `ChatPromptTemplate` — a chat prompt with multiple message templates + * - `PromptTemplate` — a single-template prompt (converted to a single human message) + * - `StructuredPrompt` — a chat prompt with an associated JSON Schema for structured output + * + * @see PromptCommit.toMessages + */ +internal object ManifestParser { + + /** + * Parses a LangChain-serialized manifest [JsonValue] into [PromptMessages]. + * + * @param manifest the raw JSON manifest from the LangSmith API + * @return the parsed prompt messages + * @throws IllegalArgumentException if the manifest format is not recognized + */ + fun parse(manifest: JsonValue): PromptMessages { + val obj = + manifest.asObject().orElseThrow { + IllegalArgumentException("Manifest must be a JSON object") + } + return parseNode(obj) + } + + private fun parseNode(obj: Map): PromptMessages { + val id = extractId(obj) + val kwargs = extractKwargs(obj) + + return when { + id.any { it.contains("StructuredPrompt") } -> parseStructuredPrompt(kwargs) + + id.any { it.contains("ChatPromptTemplate") } -> + if (hasSchemaField(kwargs)) parseStructuredPrompt(kwargs) + else parseChatPromptTemplate(kwargs) + + id.any { it.contains("PromptTemplate") } -> parsePromptTemplate(kwargs) + + else -> + // Try to detect type from kwargs structure + if (kwargs.containsKey("messages") && hasSchemaField(kwargs)) { + parseStructuredPrompt(kwargs) + } else if (kwargs.containsKey("messages")) { + parseChatPromptTemplate(kwargs) + } else if (kwargs.containsKey("template")) { + parsePromptTemplate(kwargs) + } else { + throw IllegalArgumentException( + "Unrecognized manifest type. Expected ChatPromptTemplate, " + + "StructuredPrompt, or PromptTemplate, got id=$id" + ) + } + } + } + + private fun parseChatPromptTemplate( + kwargs: Map, + outputSchema: Map? = null, + ): PromptMessages { + val messagesValue = + kwargs["messages"] + ?: throw IllegalArgumentException("ChatPromptTemplate missing 'messages' in kwargs") + + val messagesList = + messagesValue.asArray().orElseThrow { + IllegalArgumentException("'messages' must be a JSON array") + } + + val inputVariables = extractInputVariables(kwargs) + + val messages = + messagesList.mapNotNull { msgValue -> + val msgObj = msgValue.asObject().orElse(null) ?: return@mapNotNull null + parseMessageTemplate(msgObj) + } + + return PromptMessages(messages, inputVariables, outputSchema) + } + + /** + * Parses a `StructuredPrompt` manifest. This is like a `ChatPromptTemplate` but with an + * additional `schema_` (or `schema`) field containing a JSON Schema object that defines the + * expected structured output. + */ + private fun parseStructuredPrompt(kwargs: Map): PromptMessages { + val outputSchema = extractOutputSchema(kwargs) + return parseChatPromptTemplate(kwargs, outputSchema) + } + + private fun parsePromptTemplate(kwargs: Map): PromptMessages { + val template = + kwargs["template"]?.asString()?.orElse(null) + ?: throw IllegalArgumentException("PromptTemplate missing 'template' in kwargs") + + val inputVariables = extractInputVariables(kwargs) + + return PromptMessages(listOf(PromptMessage.human(template)), inputVariables) + } + + /** Holds a parsed template string and its format. */ + private data class TemplateInfo(val template: String, val templateFormat: String = "f-string") + + private fun parseMessageTemplate(msgObj: Map): PromptMessage? { + val id = extractId(msgObj) + val kwargs = extractKwargs(msgObj) + val className = id.lastOrNull() ?: "" + + // Handle MessagesPlaceholder — a slot for runtime message injection + if (className.contains("MessagesPlaceholder") || className.contains("Placeholder")) { + val variableName = kwargs["variable_name"]?.asString()?.orElse(null) ?: return null + return PromptMessage.placeholder(variableName) + } + + val role = PromptMessage.Role.fromLangchainClassName(className) + + // The template can be: + // 1. Nested in a "prompt" sub-object (PromptTemplate pattern) + // 2. Direct "template" field in kwargs + // 3. Direct "content" field in kwargs (raw message objects like ToolMessage) + val info = extractTemplate(kwargs) + val content = info?.template ?: kwargs["content"]?.asString()?.orElse(null) ?: return null + val templateFormat = info?.templateFormat ?: "f-string" + + // Handle ToolMessage / ToolMessagePromptTemplate — has a tool_call_id + if (role == PromptMessage.Role.TOOL) { + val toolCallId = kwargs["tool_call_id"]?.asString()?.orElse(null) + return PromptMessage( + role, + content, + toolCallId = toolCallId, + templateFormat = templateFormat, + ) + } + + // Handle ChatMessagePromptTemplate — has a custom role string + if (role == PromptMessage.Role.CHAT) { + val customRole = kwargs["role"]?.asString()?.orElse(null) + return PromptMessage( + role, + content, + customRole = customRole, + templateFormat = templateFormat, + ) + } + + return PromptMessage(role, content, templateFormat = templateFormat) + } + + /** + * Extracts the template string and format from kwargs. Handles two layouts: + * 1. Direct: `kwargs.template` + `kwargs.template_format` + * 2. Nested: `kwargs.prompt.kwargs.template` + `kwargs.prompt.kwargs.template_format` + */ + private fun extractTemplate(kwargs: Map): TemplateInfo? { + // Check for direct template + kwargs["template"]?.asString()?.orElse(null)?.let { tmpl -> + val fmt = kwargs["template_format"]?.asString()?.orElse(null) ?: "f-string" + return TemplateInfo(tmpl, fmt) + } + + // Check for nested prompt object + val prompt = kwargs["prompt"]?.asObject()?.orElse(null) ?: return null + val promptKwargs = extractKwargs(prompt) + val tmpl = promptKwargs["template"]?.asString()?.orElse(null) ?: return null + val fmt = promptKwargs["template_format"]?.asString()?.orElse(null) ?: "f-string" + return TemplateInfo(tmpl, fmt) + } + + /** Extracts the `id` field as a list of strings. */ + private fun extractId(obj: Map): List { + val idValue = obj["id"] ?: return emptyList() + return idValue.asArray().orElse(emptyList()).mapNotNull { it.asString().orElse(null) } + } + + /** Extracts the `kwargs` field as a map. */ + private fun extractKwargs(obj: Map): Map = + obj["kwargs"]?.asObject()?.orElse(emptyMap()) ?: emptyMap() + + /** Extracts `input_variables` from kwargs as a list of strings. */ + private fun extractInputVariables(kwargs: Map): List { + val vars = kwargs["input_variables"] ?: return emptyList() + return vars.asArray().orElse(emptyList()).mapNotNull { it.asString().orElse(null) } + } + + /** Returns `true` if kwargs contains a `schema_` or `schema` field. */ + private fun hasSchemaField(kwargs: Map): Boolean = + kwargs.containsKey("schema_") || kwargs.containsKey("schema") + + /** + * Extracts the output schema from kwargs. + * + * StructuredPrompt manifests store the schema under `schema_` (Python-style, to avoid collision + * with the `schema` JSON Schema keyword) or `schema`. The value is a JSON Schema object, which + * may itself be a LangChain-serialized object or a plain JSON Schema. + * + * @return the output schema as a `Map`, or `null` if not present + */ + private fun extractOutputSchema(kwargs: Map): Map? { + val schemaValue = kwargs["schema_"] ?: kwargs["schema"] ?: return null + return jsonValueToMap(schemaValue) + } + + /** + * Recursively converts a [JsonValue] to a plain `Map` / `List` / scalar structure + * suitable for inclusion in API payloads. + */ + private fun jsonValueToMap(value: JsonValue): Map? { + val obj = value.asObject().orElse(null) ?: return null + return obj.mapValues { (_, v) -> jsonValueToPlain(v) } + } + + private fun jsonValueToPlain(value: JsonValue): Any? = + value.asString().map { it as Any }.orElse(null) + ?: value.asNumber().map { it as Any }.orElse(null) + ?: value.asBoolean().map { it as Any }.orElse(null) + ?: value.asArray().map { list -> list.map { jsonValueToPlain(it) } as Any }.orElse(null) + ?: value + .asObject() + .map { map -> map.mapValues { (_, v) -> jsonValueToPlain(v) } as Any } + .orElse(null) +} diff --git a/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/Prompt.kt b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/Prompt.kt new file mode 100644 index 00000000..f0871992 --- /dev/null +++ b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/Prompt.kt @@ -0,0 +1,137 @@ +package com.langchain.smith.prompts + +import com.langchain.smith.core.JsonValue + +/** + * A prompt pulled from the LangSmith hub that can be invoked with input variables. + * + * This is the primary object returned by [PromptClient.pull]. It wraps a parsed prompt manifest and + * provides an [invoke] method to format the prompt with variable values, producing a [PromptValue] + * that can then be converted to provider-specific formats using [convertToOpenAIParams] or + * [convertToAnthropicParams]. + * + * ## Example (Java) + * + * ```java + * PromptClient promptClient = PromptClient.create(langsmithClient); + * Prompt prompt = promptClient.pull("my-org/joke-generator"); + * PromptValue formattedPrompt = prompt.invoke(Map.of("topic", "cats")); + * + * // OpenAI + * ChatCompletion completion = openai.chat().completions().create( + * convertToOpenAIParams(formattedPrompt) + * .model(ChatModel.GPT_4_1_MINI) + * .build()); + * + * // Anthropic + * Message message = anthropic.messages().create( + * convertToAnthropicParams(formattedPrompt) + * .model(Model.CLAUDE_SONNET_4_6) + * .maxTokens(1024) + * .build()); + * ``` + * + * @see PromptClient.pull + * @see PromptValue + * @see convertToOpenAIParams + * @see convertToAnthropicParams + */ +class Prompt +internal constructor( + /** The parsed prompt messages (unformatted templates). */ + private val promptMessages: PromptMessages, + /** The commit metadata, if available. */ + private val commit: PromptCommit?, +) { + + /** + * The input variable names expected by this prompt. + * + * These are the `{variable}` placeholders found in the message templates. + */ + val inputVariables: List + get() = promptMessages.inputVariables + + /** The structured output JSON Schema, or `null` if this is not a structured prompt. */ + val outputSchema: Map? + get() = promptMessages.outputSchema + + /** Returns `true` if this prompt includes a structured output schema. */ + fun hasOutputSchema(): Boolean = promptMessages.hasOutputSchema() + + /** The commit hash of the pulled prompt, if available. */ + val commitHash: String? + get() = commit?.commitHash + + /** The raw manifest JSON, if available. */ + val manifest: JsonValue? + get() = commit?.manifest + + /** + * Formats this prompt with the given input variables, producing a [PromptValue]. + * + * Each `{variable_name}` placeholder in the message templates is replaced with the + * corresponding value from [variables]. Variables not present in the map are left as-is. + * + * @param variables a map from variable name to its substitution value. Values are converted to + * strings via [Any.toString]. + * @return a [PromptValue] containing the formatted messages + */ + fun invoke(variables: Map): PromptValue { + val formatted = promptMessages.format(variables) + return PromptValue(formatted) + } + + /** + * Produces a [PromptValue] with no variable substitution. + * + * Use this when the prompt has no input variables. Placeholders (if any) will be dropped since + * no variables are provided. + * + * @return a [PromptValue] containing the messages + */ + fun invoke(): PromptValue = invoke(emptyMap()) + + override fun toString(): String { + val parts = buildList { + add("messages=[${promptMessages.messages.joinToString(", ")}]") + if (inputVariables.isNotEmpty()) add("inputVariables=$inputVariables") + commitHash?.let { add("commitHash=$it") } + outputSchema?.let { add("outputSchema=${it["title"] ?: "..."}") } + } + return "Prompt{${parts.joinToString(", ")}}" + } + + companion object { + + /** + * Creates a [Prompt] from a [PromptCommit]. + * + * The commit's manifest is parsed into typed messages. This is called internally by + * [PromptClient.pull]. + */ + @JvmStatic + internal fun fromCommit(commit: PromptCommit): Prompt { + val messages = commit.toMessages() + return Prompt(messages, commit) + } + + /** + * Creates a [Prompt] directly from a list of messages. + * + * Useful for testing or when constructing prompts programmatically without pulling from the + * hub. + * + * @param messages the prompt messages + * @param inputVariables the input variable names (optional) + * @param outputSchema the JSON Schema for structured output (optional) + */ + @JvmStatic + @JvmOverloads + fun of( + messages: List, + inputVariables: List = emptyList(), + outputSchema: Map? = null, + ): Prompt = Prompt(PromptMessages(messages, inputVariables, outputSchema), null) + } +} diff --git a/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptClient.kt b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptClient.kt new file mode 100644 index 00000000..c387e0f3 --- /dev/null +++ b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptClient.kt @@ -0,0 +1,182 @@ +package com.langchain.smith.prompts + +import com.langchain.smith.client.LangsmithClient +import com.langchain.smith.models.commits.CommitRetrieveParams + +/** + * A high-level client for pulling prompts from the LangSmith hub. + * + * This wraps the lower-level [LangsmithClient.commits] service and provides a convenient interface + * that mirrors the Python and TypeScript SDKs' prompt pulling experience. + * + * ## Quick start (Java) + * + * ```java + * import com.langchain.smith.client.okhttp.LangsmithOkHttpClient; + * import com.langchain.smith.prompts.PromptClient; + * import static com.langchain.smith.prompts.PromptConverters.convertToOpenAIParams; + * import static com.langchain.smith.prompts.PromptConverters.convertToAnthropicParams; + * + * LangsmithClient client = LangsmithOkHttpClient.fromEnv(); + * PromptClient promptClient = PromptClient.create(client); + * Prompt prompt = promptClient.pull("my-org/joke-generator"); + * PromptValue formattedPrompt = prompt.invoke(Map.of("topic", "cats")); + * + * ChatCompletion completion = openai.chat().completions().create( + * convertToOpenAIParams(formattedPrompt) + * .model(ChatModel.GPT_4_1_MINI) + * .build()); + * ``` + * + * ## Prompt identifier format + * + * The prompt identifier can be in any of these formats: + * - `"name"` — uses the default owner (`"-"`) and latest commit + * - `"owner/name"` — specifies the owner + * - `"name:commit_or_tag"` — specifies a commit hash or tag + * - `"owner/name:commit_or_tag"` — specifies both owner and commit/tag + * + * @see Prompt + * @see PromptValue + * @see convertToOpenAIParams + * @see convertToAnthropicParams + */ +class PromptClient private constructor(private val client: LangsmithClient) { + + /** + * Pulls a prompt from the LangSmith hub and returns a [Prompt] that can be invoked with input + * variables. + * + * This is the primary method for working with prompts. The returned [Prompt] has an + * [invoke][Prompt.invoke] method that formats the prompt with variable values, producing a + * [PromptValue] that can be converted to provider-specific formats using + * [convertToOpenAIParams] or [convertToAnthropicParams]. + * + * ```java + * Prompt prompt = promptClient.pull("my-org/joke-generator"); + * PromptValue formatted = prompt.invoke(Map.of("topic", "cats")); + * ChatCompletion completion = openai.chat().completions().create( + * convertToOpenAIParams(formatted).model(ChatModel.GPT_4_1_MINI).build()); + * ``` + * + * @param promptIdentifier the prompt identifier (e.g., `"owner/name"`, `"name"`, + * `"owner/name:commit_hash"`) + * @param includeModel whether to include model configuration in the response + * @return a [Prompt] that can be invoked with variables + */ + @JvmOverloads + fun pull(promptIdentifier: String, includeModel: Boolean = false): Prompt { + val commit = pullPromptCommit(promptIdentifier, includeModel) + return Prompt.fromCommit(commit) + } + + /** + * Pulls the raw prompt commit from the LangSmith hub. + * + * This is the lower-level method that returns the raw [PromptCommit] with the manifest JSON and + * commit metadata. Most users should prefer [pull] instead. + * + * @param promptIdentifier the prompt identifier (e.g., `"owner/name"`, `"name"`, + * `"owner/name:commit_hash"`) + * @param includeModel whether to include model configuration in the response + * @return the raw prompt commit + */ + @JvmOverloads + fun pullPromptCommit(promptIdentifier: String, includeModel: Boolean = false): PromptCommit { + val (owner, repo, commit) = parsePromptIdentifier(promptIdentifier) + + val params = + CommitRetrieveParams.builder() + .owner(owner) + .repo(repo) + .commit(commit) + .includeModel(includeModel) + .build() + + val response = client.commits().retrieve(params) + + return PromptCommit.of( + owner = owner, + repo = repo, + commitHash = response.commitHash().orElse(commit), + manifest = response._manifest(), + examples = response.examples().orElse(emptyList()), + ) + } + + companion object { + + /** + * Creates a [PromptClient] wrapping the given [LangsmithClient]. + * + * @param client the LangSmith API client + * @return a new prompt client + */ + @JvmStatic fun create(client: LangsmithClient): PromptClient = PromptClient(client) + + /** + * Parses a prompt identifier string into its component parts. + * + * Supported formats: + * - `"name"` → `("-", "name", "latest")` + * - `"owner/name"` → `("owner", "name", "latest")` + * - `"name:commit"` → `("-", "name", "commit")` + * - `"owner/name:commit"` → `("owner", "name", "commit")` + * + * @param identifier the prompt identifier string + * @return a [PromptIdentifier] with owner, repo, and commit fields + * @throws IllegalArgumentException if the identifier is blank or has an invalid format + */ + @JvmStatic + fun parsePromptIdentifier(identifier: String): PromptIdentifier { + require(identifier.isNotBlank()) { "Prompt identifier must not be blank" } + + val trimmed = identifier.trim() + + // Split off commit/tag from the end (after ':') + val (nameAndOwner, commit) = + if (':' in trimmed) { + val colonIdx = trimmed.lastIndexOf(':') + val namePart = trimmed.substring(0, colonIdx) + val commitPart = trimmed.substring(colonIdx + 1) + require(commitPart.isNotBlank()) { + "Commit/tag after ':' must not be blank in identifier: $identifier" + } + namePart to commitPart + } else { + trimmed to "latest" + } + + // Split owner and repo name (on '/') + val (owner, repo) = + if ('/' in nameAndOwner) { + val slashIdx = nameAndOwner.indexOf('/') + val ownerPart = nameAndOwner.substring(0, slashIdx) + val repoPart = nameAndOwner.substring(slashIdx + 1) + require(ownerPart.isNotBlank()) { + "Owner must not be blank in identifier: $identifier" + } + require(repoPart.isNotBlank()) { + "Repo name must not be blank in identifier: $identifier" + } + require('/' !in repoPart) { + "Identifier must have at most one '/' separator: $identifier" + } + ownerPart to repoPart + } else { + "-" to nameAndOwner + } + + return PromptIdentifier(owner, repo, commit) + } + } +} + +/** + * The parsed components of a prompt identifier string. + * + * @property owner the repository owner (tenant handle), or `"-"` for private repos + * @property repo the repository handle (prompt name) + * @property commit the commit hash, tag, or `"latest"` + */ +data class PromptIdentifier(val owner: String, val repo: String, val commit: String) diff --git a/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptCommit.kt b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptCommit.kt new file mode 100644 index 00000000..b0662512 --- /dev/null +++ b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptCommit.kt @@ -0,0 +1,71 @@ +package com.langchain.smith.prompts + +import com.langchain.smith.core.JsonValue +import com.langchain.smith.models.commits.CommitRetrieveResponse + +/** + * Represents a pulled prompt commit from the LangSmith hub. + * + * This is the low-level result of calling [PromptClient.pullPromptCommit]. It contains the raw + * manifest (the LangChain-serialized prompt definition) along with metadata about the commit. + * + * Most users should prefer [PromptClient.pull], which returns a [Prompt] that can be invoked + * directly. + * + * @see PromptClient.pull + * @see PromptClient.pullPromptCommit + */ +class PromptCommit +private constructor( + /** The repository owner (tenant handle), or "-" for authenticated private repos. */ + val owner: String, + /** The repository handle (prompt name). */ + val repo: String, + /** The resolved commit hash. */ + val commitHash: String, + /** The raw LangChain-serialized prompt manifest. */ + val manifest: JsonValue, + /** Optional examples associated with this commit. */ + val examples: List, +) { + + /** + * Parses the manifest into a [PromptMessages] object. + * + * This handles the LangChain serialization format used by LangSmith, supporting + * `ChatPromptTemplate`, `StructuredPrompt`, and `PromptTemplate` manifests. + * + * @return the parsed prompt messages (with optional output schema) + * @throws IllegalArgumentException if the manifest format is not recognized + */ + internal fun toMessages(): PromptMessages = ManifestParser.parse(manifest) + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is PromptCommit) return false + return owner == other.owner && repo == other.repo && commitHash == other.commitHash + } + + override fun hashCode(): Int { + var result = owner.hashCode() + result = 31 * result + repo.hashCode() + result = 31 * result + commitHash.hashCode() + return result + } + + override fun toString(): String = + "PromptCommit{owner=$owner, repo=$repo, commitHash=$commitHash}" + + companion object { + + @JvmStatic + @JvmOverloads + fun of( + owner: String, + repo: String, + commitHash: String, + manifest: JsonValue, + examples: List = emptyList(), + ): PromptCommit = PromptCommit(owner, repo, commitHash, manifest, examples) + } +} diff --git a/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptMessage.kt b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptMessage.kt new file mode 100644 index 00000000..d127bfb2 --- /dev/null +++ b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptMessage.kt @@ -0,0 +1,223 @@ +package com.langchain.smith.prompts + +/** + * Represents a single message in a prompt template. + * + * Each message has a [role] (system, human/user, ai/assistant, tool, or a custom chat role) and a + * [template] string that may contain `{variable}` placeholders for formatting. + * + * For tool messages, [toolCallId] contains the ID of the tool call this is responding to. For chat + * messages with custom roles, [customRole] contains the role string. + * + * @see Prompt + */ +class PromptMessage +internal constructor( + /** The role of this message. */ + val role: Role, + /** + * The template string for this message. + * + * May contain variable placeholders that can be substituted via [Prompt.invoke]. The syntax + * depends on [templateFormat]: f-string uses `{variable}`, mustache uses `{{variable}}`. + */ + val template: String, + /** + * The tool call ID this message is responding to, or `null` for non-tool messages. + * + * Only set when [role] is [Role.TOOL]. + */ + val toolCallId: String? = null, + /** + * A custom role string for chat messages, or `null` for standard roles. + * + * Only set when [role] is [Role.CHAT]. This allows arbitrary role names beyond the standard + * system/human/ai/tool set. + */ + val customRole: String? = null, + /** + * The template format — `"f-string"` (default) or `"mustache"`. + * + * f-string templates use `{variable}` syntax. Mustache templates use `{{variable}}` syntax. + */ + val templateFormat: String = "f-string", +) { + + /** The role of a prompt message. */ + enum class Role( + /** The LangChain class name suffix (e.g., "SystemMessagePromptTemplate"). */ + internal val langchainClassName: String, + /** The role name used by OpenAI (e.g., "system", "user", "assistant"). */ + val openAiRole: String, + ) { + /** System message — sets the behavior/context for the assistant. */ + SYSTEM("SystemMessagePromptTemplate", "system"), + + /** Human/user message. */ + HUMAN("HumanMessagePromptTemplate", "user"), + + /** AI/assistant message. */ + AI("AIMessagePromptTemplate", "assistant"), + + /** Tool/function result message. */ + TOOL("ToolMessagePromptTemplate", "tool"), + + /** + * A chat message with a custom role string. + * + * The actual role name is in [PromptMessage.customRole]. + */ + CHAT("ChatMessagePromptTemplate", "user"), + + /** + * A placeholder for runtime message injection (e.g., chat history). + * + * This corresponds to LangChain's `MessagesPlaceholder`. The [PromptMessage.template] field + * contains the variable name. At invoke time, the variable is looked up and replaced with a + * `List`. + * + * Placeholders are not sent to providers — they are expanded before conversion. + */ + PLACEHOLDER("MessagesPlaceholder", ""); + + companion object { + + /** + * Resolves a [Role] from a LangChain class name (e.g., "SystemMessagePromptTemplate"). + */ + @JvmStatic + fun fromLangchainClassName(className: String): Role = + entries.find { className.contains(it.langchainClassName) } + ?: entries.find { className.contains(it.name, ignoreCase = true) } + ?: HUMAN + } + } + + /** + * Returns the effective OpenAI role string for this message. + * + * For [Role.CHAT] messages, returns the [customRole] if set, otherwise falls back to the enum's + * [Role.openAiRole]. + */ + fun effectiveOpenAiRole(): String = customRole ?: role.openAiRole + + /** + * Returns `true` if this is a placeholder message (a [MessagesPlaceholder] slot). + * + * Placeholder messages are expanded at invoke time — the variable named [template] is looked up + * and replaced with a list of messages. + */ + fun isPlaceholder(): Boolean = role == Role.PLACEHOLDER + + /** + * Formats this message template by substituting variable placeholders with the given values. + * Values are converted to strings via [Any.toString]. + * + * The substitution syntax depends on [templateFormat]: + * - `"f-string"` (default): `{variable}` placeholders + * - `"mustache"`: `{{variable}}` placeholders + * + * Placeholder messages are not formatted (they are expanded separately). + * + * @param variables a map from variable name to its value + * @return the formatted message text + */ + fun format(variables: Map): String { + if (isPlaceholder()) return template + return TemplateFormatter.format(template, variables, templateFormat) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is PromptMessage) return false + return role == other.role && + template == other.template && + toolCallId == other.toolCallId && + customRole == other.customRole && + templateFormat == other.templateFormat + } + + override fun hashCode(): Int { + var result = role.hashCode() + result = 31 * result + template.hashCode() + result = 31 * result + (toolCallId?.hashCode() ?: 0) + result = 31 * result + (customRole?.hashCode() ?: 0) + result = 31 * result + templateFormat.hashCode() + return result + } + + override fun toString(): String { + val roleName = customRole ?: role.name.lowercase() + val extras = buildList { + if (toolCallId != null) add("toolCallId=$toolCallId") + if (templateFormat != "f-string") add("templateFormat=$templateFormat") + } + val suffix = if (extras.isNotEmpty()) " (${extras.joinToString()})" else "" + return "$roleName: \"$template\"$suffix" + } + + companion object { + + /** Creates a system message with the given template. */ + @JvmStatic + fun system(template: String): PromptMessage = PromptMessage(Role.SYSTEM, template) + + /** Creates a human/user message with the given template. */ + @JvmStatic fun human(template: String): PromptMessage = PromptMessage(Role.HUMAN, template) + + /** Creates an AI/assistant message with the given template. */ + @JvmStatic fun ai(template: String): PromptMessage = PromptMessage(Role.AI, template) + + /** + * Creates a tool result message. + * + * @param template the tool result content (or template with variables) + * @param toolCallId the ID of the tool call this is responding to + */ + @JvmStatic + fun tool(template: String, toolCallId: String): PromptMessage = + PromptMessage(Role.TOOL, template, toolCallId = toolCallId) + + /** + * Creates a chat message with a custom role string. + * + * Use this for roles beyond the standard system/human/ai/tool set. + * + * @param template the message content (or template with variables) + * @param role the custom role string (e.g., "moderator", "narrator") + */ + @JvmStatic + fun chat(template: String, role: String): PromptMessage = + PromptMessage(Role.CHAT, template, customRole = role) + + /** + * Creates a messages placeholder. + * + * At invoke time, the variable named [variableName] should be a `List` that + * replaces this placeholder in the message list. + * + * @param variableName the variable name to look up at invoke time + */ + @JvmStatic + fun placeholder(variableName: String): PromptMessage = + PromptMessage(Role.PLACEHOLDER, variableName) + + /** Creates a message with the given role and template. */ + @JvmStatic + fun of(role: Role, template: String): PromptMessage = PromptMessage(role, template) + + /** + * Creates a copy of the given message with a new template string, preserving [toolCallId], + * [customRole], and [templateFormat]. + */ + @JvmStatic + internal fun withTemplate(source: PromptMessage, newTemplate: String): PromptMessage = + PromptMessage( + source.role, + newTemplate, + source.toolCallId, + source.customRole, + source.templateFormat, + ) + } +} diff --git a/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptMessages.kt b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptMessages.kt new file mode 100644 index 00000000..50d00ff5 --- /dev/null +++ b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptMessages.kt @@ -0,0 +1,138 @@ +package com.langchain.smith.prompts + +/** + * Injects `"additionalProperties": false` into every object node in a JSON Schema tree. This is + * required when using strict mode for structured outputs — OpenAI requires it when `strict: true` + * is set, and Anthropic requires it for `output_config` schemas. + * + * Recursion covers all schema locations where objects can appear: `properties`, `items`, `anyOf`, + * `oneOf`, `allOf`, `$defs`, and any other nested map or list-of-maps structure. + * + * We enable strict mode by default because it guarantees the response matches the schema exactly. + * + * See: + * https://community.openai.com/t/schema-additionalproperties-must-be-false-when-strict-is-true/929996 + * + * Returns a new map — the input is not modified. + */ +internal fun strictSchemaForStructuredOutput(schema: Map): Map = + buildMap { + for ((key, value) in schema) { + put(key, strictifyValue(value)) + } + if (isObjectType(schema["type"])) { + put("additionalProperties", false) + } + } + +/** + * Checks if a schema type is or includes "object". Handles both `"object"` and `["string", + * "object"]`. + */ +private fun isObjectType(type: Any?): Boolean = + when (type) { + is String -> type == "object" + is List<*> -> type.any { it == "object" } + else -> false + } + +/** + * Recursively processes a value within a JSON Schema, applying strict mode to any nested schemas. + */ +private fun strictifyValue(value: Any?): Any? = + when (value) { + is Map<*, *> -> { + val mapWithStringKeys = + buildMap { + for ((k, v) in value) { + put(k?.toString() ?: continue, v) + } + } + strictSchemaForStructuredOutput(mapWithStringKeys) + } + is List<*> -> value.map { strictifyValue(it) } + else -> value + } + +/** + * Internal representation of parsed prompt messages from a LangChain manifest. + * + * This is not part of the public API. Users interact with [Prompt] (before formatting) and + * [PromptValue] (after formatting), and use [convertToOpenAIParams] / [convertToAnthropicParams] + * for provider conversion. + * + * @see ManifestParser + */ +internal class PromptMessages( + val messages: List, + val inputVariables: List, + val outputSchema: Map? = null, +) { + + fun hasOutputSchema(): Boolean = outputSchema != null + + fun format(variables: Map): PromptMessages { + val formatted = + messages.flatMap { msg -> + if (msg.isPlaceholder()) { + expandPlaceholder(msg, variables) + } else { + listOf(PromptMessage.withTemplate(msg, msg.format(variables))) + } + } + return PromptMessages(formatted, inputVariables, outputSchema) + } + + private fun expandPlaceholder( + msg: PromptMessage, + variables: Map, + ): List { + val items = variables[msg.template] as? List<*> ?: return emptyList() + return items.mapNotNull(::toPromptMessage) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is PromptMessages) return false + return messages == other.messages && + inputVariables == other.inputVariables && + outputSchema == other.outputSchema + } + + override fun hashCode(): Int { + var result = messages.hashCode() + result = 31 * result + inputVariables.hashCode() + result = 31 * result + (outputSchema?.hashCode() ?: 0) + return result + } + + override fun toString(): String { + val parts = buildList { + add("messages=$messages") + if (inputVariables.isNotEmpty()) add("inputVariables=$inputVariables") + outputSchema?.let { add("outputSchema=${it["title"] ?: "..."}") } + } + return "PromptMessages{${parts.joinToString(", ")}}" + } +} + +private fun toPromptMessage(value: Any?): PromptMessage? = + when (value) { + is PromptMessage -> value + is Map<*, *> -> value.toPromptMessage() + else -> null + } + +private fun Map<*, *>.toPromptMessage(): PromptMessage? { + val role = this["role"] as? String ?: return null + val content = this["content"] as? String ?: return null + return when (role) { + "system" -> PromptMessage.system(content) + "user", + "human" -> PromptMessage.human(content) + "assistant", + "ai" -> PromptMessage.ai(content) + "tool" -> PromptMessage.tool(content, this["tool_call_id"] as? String ?: "") + else -> PromptMessage.chat(content, role) + } +} diff --git a/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptValue.kt b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptValue.kt new file mode 100644 index 00000000..439e71df --- /dev/null +++ b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/PromptValue.kt @@ -0,0 +1,66 @@ +package com.langchain.smith.prompts + +/** + * The result of invoking a [Prompt] with input variables. + * + * A `PromptValue` holds the formatted messages (with variables substituted) and any structured + * output schema from the original prompt. It is the input to the conversion functions + * [convertToOpenAIParams] and [convertToAnthropicParams]. + * + * This is analogous to LangChain's `ChatPromptValue` — it represents a fully-resolved prompt ready + * to be sent to a model provider. + * + * ## Example (Java) + * + * ```java + * Prompt prompt = promptClient.pull("my-org/joke-generator"); + * PromptValue formatted = prompt.invoke(Map.of("topic", "cats")); + * + * // Inspect the messages + * for (PromptMessage msg : formatted.getMessages()) { + * System.out.println(msg.getRole() + ": " + msg.getTemplate()); + * } + * + * // Convert to provider format + * ChatCompletion completion = openai.chat().completions().create( + * convertToOpenAIParams(formatted).model(ChatModel.GPT_4_1_MINI).build()); + * ``` + * + * @see Prompt.invoke + * @see convertToOpenAIParams + * @see convertToAnthropicParams + */ +class PromptValue internal constructor(internal val promptMessages: PromptMessages) { + + /** + * The formatted messages in this prompt value. + * + * Each message has a [role][PromptMessage.role] and the formatted + * [content][PromptMessage.template] (with variables already substituted). + */ + val messages: List + get() = promptMessages.messages + + /** The structured output JSON Schema, or `null` if this is not a structured prompt. */ + val outputSchema: Map? + get() = promptMessages.outputSchema + + /** Returns `true` if this prompt value includes a structured output schema. */ + fun hasOutputSchema(): Boolean = promptMessages.hasOutputSchema() + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is PromptValue) return false + return promptMessages == other.promptMessages + } + + override fun hashCode(): Int = promptMessages.hashCode() + + override fun toString(): String { + val parts = buildList { + add("messages=[${messages.joinToString(", ")}]") + outputSchema?.let { add("outputSchema=${it["title"] ?: "..."}") } + } + return "PromptValue{${parts.joinToString(", ")}}" + } +} diff --git a/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/TemplateFormatter.kt b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/TemplateFormatter.kt new file mode 100644 index 00000000..5d330612 --- /dev/null +++ b/langsmith-java-core/src/main/kotlin/com/langchain/smith/prompts/TemplateFormatter.kt @@ -0,0 +1,67 @@ +package com.langchain.smith.prompts + +import com.samskivert.mustache.Mustache + +/** + * Formats prompt template strings using either f-string or mustache syntax. + * + * LangChain prompts support two template formats: + * - **f-string** (default): Uses `{variable}` placeholders — e.g., `"Hello {name}"` + * - **mustache**: Uses `{{variable}}` placeholders — e.g., `"Hello {{name}}"` + * + * The format is specified per-template in the LangChain manifest via the `template_format` field. + */ +internal object TemplateFormatter { + + /** + * Matches f-string tokens: escaped braces (`{{`, `}}`) or variable placeholders (`{name}`). + * Escaped braces are matched first so they aren't treated as variables. + */ + private val F_STRING_PATTERN = Regex("\\{\\{|\\}\\}|\\{([^}]+)\\}") + + /** The jmustache compiler, configured to not HTML-escape values. */ + private val MUSTACHE_COMPILER: Mustache.Compiler = + Mustache.compiler().escapeHTML(false).defaultValue("") + + /** + * Formats a template string by substituting variables. + * + * @param template the template string + * @param variables the variable values to substitute + * @param templateFormat the template format — `"f-string"` (default) or `"mustache"` + * @return the formatted string + */ + fun format( + template: String, + variables: Map, + templateFormat: String = "f-string", + ): String = + when (templateFormat) { + "mustache" -> formatMustache(template, variables) + else -> formatFString(template, variables) + } + + /** + * Formats an f-string template. Handles: + * - `{variable}` — substituted from the variables map + * - `{{` — literal `{` + * - `}}` — literal `}` + * + * Uses single-pass regex replacement to avoid cascading substitutions. + */ + private fun formatFString(template: String, variables: Map): String = + F_STRING_PATTERN.replace(template) { match -> + when (match.value) { + "{{" -> "{" + "}}" -> "}" + else -> { + val key = match.groupValues[1] + variables[key]?.toString() ?: match.value + } + } + } + + /** Formats a mustache template using jmustache. */ + private fun formatMustache(template: String, variables: Map): String = + MUSTACHE_COMPILER.compile(template).execute(variables) +} diff --git a/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/ConvertersTest.kt b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/ConvertersTest.kt new file mode 100644 index 00000000..fdf647f5 --- /dev/null +++ b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/ConvertersTest.kt @@ -0,0 +1,178 @@ +package com.langchain.smith.prompts + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +internal class ConvertersTest { + + private fun makePromptValue( + messages: List, + outputSchema: Map? = null, + ): PromptValue = Prompt.of(messages, emptyList(), outputSchema).invoke() + + // --- convertToOpenAIParams --- + + @Test + fun convertToOpenAI_basicMessages() { + val pv = + makePromptValue( + listOf(PromptMessage.system("You are helpful."), PromptMessage.human("Hello")) + ) + + val params = convertToOpenAIParams(pv).model("gpt-4.1-mini").build() + + assertThat(params.messages()).hasSize(2) + assertThat(params.messages()[0].isSystem()).isTrue() + assertThat(params.messages()[1].isUser()).isTrue() + assertThat(params.responseFormat()).isEmpty + } + + @Test + fun convertToOpenAI_withStructuredOutput() { + val schema = + mapOf( + "title" to "MySchema", + "type" to "object", + "properties" to mapOf("name" to mapOf("type" to "string")), + ) + val pv = + makePromptValue( + listOf(PromptMessage.human("Extract from: text")), + outputSchema = schema, + ) + + val params = convertToOpenAIParams(pv).model("gpt-4.1-mini").build() + + assertThat(params.messages()).hasSize(1) + assertThat(params.responseFormat()).isPresent + } + + // --- convertToAnthropicParams --- + + @Test + fun convertToAnthropic_extractsSystem() { + val pv = + makePromptValue( + listOf( + PromptMessage.system("Be helpful."), + PromptMessage.human("Hello"), + PromptMessage.ai("Hi!"), + PromptMessage.human("Question"), + ) + ) + + val params = + convertToAnthropicParams(pv) + .model(com.anthropic.models.messages.Model.CLAUDE_HAIKU_4_5_20251001) + .maxTokens(256) + .build() + + assertThat(params.system()).isPresent + assertThat(params.messages()).hasSize(3) + assertThat(params.messages()[0].role()) + .isEqualTo(com.anthropic.models.messages.MessageParam.Role.USER) + assertThat(params.messages()[1].role()) + .isEqualTo(com.anthropic.models.messages.MessageParam.Role.ASSISTANT) + assertThat(params.messages()[2].role()) + .isEqualTo(com.anthropic.models.messages.MessageParam.Role.USER) + assertThat(params.outputConfig()).isEmpty + } + + @Test + fun convertToAnthropic_withStructuredOutput() { + val schema = + mapOf( + "title" to "MyTool", + "description" to "Extracts info.", + "type" to "object", + "properties" to mapOf("name" to mapOf("type" to "string")), + ) + val pv = + makePromptValue( + listOf( + PromptMessage.system("Extract data."), + PromptMessage.human("Extract from: text"), + ), + outputSchema = schema, + ) + + val params = + convertToAnthropicParams(pv) + .model(com.anthropic.models.messages.Model.CLAUDE_HAIKU_4_5_20251001) + .maxTokens(256) + .build() + + assertThat(params.system()).isPresent + assertThat(params.messages()).hasSize(1) + assertThat(params.outputConfig()).isPresent + } + + // --- End-to-end flow test --- + + @Test + fun endToEnd_pullInvokeConvert() { + val schema = + mapOf( + "title" to "JokeResponse", + "type" to "object", + "properties" to + mapOf( + "setup" to mapOf("type" to "string"), + "punchline" to mapOf("type" to "string"), + ), + ) + val prompt = + Prompt.of( + listOf( + PromptMessage.system("You tell jokes."), + PromptMessage.human("Tell me a joke about {topic}"), + ), + listOf("topic"), + schema, + ) + + val formattedPrompt = prompt.invoke(mapOf("topic" to "cats")) + + // OpenAI + val openAiParams = convertToOpenAIParams(formattedPrompt).model("gpt-4.1-mini").build() + assertThat(openAiParams.messages()).hasSize(2) + assertThat(openAiParams.responseFormat()).isPresent + + // Anthropic + val anthropicParams = + convertToAnthropicParams(formattedPrompt) + .model(com.anthropic.models.messages.Model.CLAUDE_HAIKU_4_5_20251001) + .maxTokens(256) + .build() + assertThat(anthropicParams.system()).isPresent + assertThat(anthropicParams.messages()).hasSize(1) + assertThat(anthropicParams.outputConfig()).isPresent + } + + @Test + fun endToEnd_regularPromptNoSchema() { + val prompt = + Prompt.of( + listOf( + PromptMessage.system("You are helpful."), + PromptMessage.human("Tell me about {topic}"), + ), + listOf("topic"), + ) + + val formattedPrompt = prompt.invoke(mapOf("topic" to "dogs")) + + val openAiParams = convertToOpenAIParams(formattedPrompt).model("gpt-4.1-mini").build() + assertThat(openAiParams.messages()).hasSize(2) + assertThat(openAiParams.responseFormat()).isEmpty + + val anthropicParams = + convertToAnthropicParams(formattedPrompt) + .model(com.anthropic.models.messages.Model.CLAUDE_HAIKU_4_5_20251001) + .maxTokens(256) + .build() + assertThat(anthropicParams.system()).isPresent + assertThat(anthropicParams.messages()).hasSize(1) + assertThat(anthropicParams.outputConfig()).isEmpty + } +} diff --git a/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/ManifestParserTest.kt b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/ManifestParserTest.kt new file mode 100644 index 00000000..9c5257e4 --- /dev/null +++ b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/ManifestParserTest.kt @@ -0,0 +1,747 @@ +package com.langchain.smith.prompts + +import com.langchain.smith.core.JsonValue +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test + +internal class ManifestParserTest { + + /** Builds a typical LangChain ChatPromptTemplate manifest as a map structure. */ + private fun buildChatPromptManifest( + messages: List>, + inputVariables: List = emptyList(), + ): Map = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "prompts", "chat", "ChatPromptTemplate"), + "kwargs" to mapOf("input_variables" to inputVariables, "messages" to messages), + ) + + /** Builds a message prompt template entry with a nested prompt object. */ + private fun buildMessageTemplate(className: String, template: String): Map = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "prompts", "chat", className), + "kwargs" to + mapOf( + "prompt" to + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "prompts", "prompt", "PromptTemplate"), + "kwargs" to mapOf("template" to template), + ) + ), + ) + + @Test + fun parseChatPromptTemplate() { + val manifest = + buildChatPromptManifest( + messages = + listOf( + buildMessageTemplate( + "SystemMessagePromptTemplate", + "You are a helpful assistant.", + ), + buildMessageTemplate("HumanMessagePromptTemplate", "Tell me about {topic}"), + ), + inputVariables = listOf("topic"), + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(2) + assertThat(result.messages[0].role).isEqualTo(PromptMessage.Role.SYSTEM) + assertThat(result.messages[0].template).isEqualTo("You are a helpful assistant.") + assertThat(result.messages[1].role).isEqualTo(PromptMessage.Role.HUMAN) + assertThat(result.messages[1].template).isEqualTo("Tell me about {topic}") + assertThat(result.inputVariables).containsExactly("topic") + } + + @Test + fun parseChatPromptTemplateWithAiMessage() { + val manifest = + buildChatPromptManifest( + messages = + listOf( + buildMessageTemplate( + "SystemMessagePromptTemplate", + "You are an assistant.", + ), + buildMessageTemplate("HumanMessagePromptTemplate", "Hello"), + buildMessageTemplate( + "AIMessagePromptTemplate", + "Hi there! How can I help?", + ), + buildMessageTemplate("HumanMessagePromptTemplate", "Tell me about {topic}"), + ), + inputVariables = listOf("topic"), + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(4) + assertThat(result.messages[0].role).isEqualTo(PromptMessage.Role.SYSTEM) + assertThat(result.messages[1].role).isEqualTo(PromptMessage.Role.HUMAN) + assertThat(result.messages[2].role).isEqualTo(PromptMessage.Role.AI) + assertThat(result.messages[2].template).isEqualTo("Hi there! How can I help?") + assertThat(result.messages[3].role).isEqualTo(PromptMessage.Role.HUMAN) + } + + @Test + fun parsePromptTemplate() { + val manifest = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "prompts", "prompt", "PromptTemplate"), + "kwargs" to + mapOf( + "template" to "Tell me a joke about {topic}", + "input_variables" to listOf("topic"), + ), + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(1) + assertThat(result.messages[0].role).isEqualTo(PromptMessage.Role.HUMAN) + assertThat(result.messages[0].template).isEqualTo("Tell me a joke about {topic}") + assertThat(result.inputVariables).containsExactly("topic") + } + + @Test + fun parseDirectTemplateInKwargs() { + // Some manifests have the template directly in the message kwargs, + // not nested in a "prompt" sub-object + val manifest = + buildChatPromptManifest( + messages = + listOf( + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain_core", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ), + "kwargs" to mapOf("template" to "Direct template text"), + ) + ) + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(1) + assertThat(result.messages[0].role).isEqualTo(PromptMessage.Role.SYSTEM) + assertThat(result.messages[0].template).isEqualTo("Direct template text") + } + + @Test + fun parseManifestWithoutExplicitId() { + // Falls back to detecting type from kwargs structure + val manifest = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf(), // empty id + "kwargs" to + mapOf( + "messages" to + listOf( + buildMessageTemplate("HumanMessagePromptTemplate", "Hello {name}") + ), + "input_variables" to listOf("name"), + ), + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(1) + assertThat(result.messages[0].template).isEqualTo("Hello {name}") + } + + /** Builds a StructuredPrompt manifest with messages and an output schema. */ + private fun buildStructuredPromptManifest( + messages: List>, + inputVariables: List = emptyList(), + schema: Map, + ): Map = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain", "prompts", "structured", "StructuredPrompt"), + "kwargs" to + mapOf( + "input_variables" to inputVariables, + "messages" to messages, + "schema_" to schema, + ), + ) + + private val sampleSchema = + mapOf( + "title" to "JokeResponse", + "description" to "A structured joke response.", + "type" to "object", + "properties" to + mapOf( + "setup" to mapOf("type" to "string", "description" to "The joke setup"), + "punchline" to mapOf("type" to "string", "description" to "The punchline"), + ), + "required" to listOf("setup", "punchline"), + ) + + @Test + fun parseStructuredPrompt() { + val manifest = + buildStructuredPromptManifest( + messages = + listOf( + buildMessageTemplate("SystemMessagePromptTemplate", "You tell jokes."), + buildMessageTemplate( + "HumanMessagePromptTemplate", + "Tell me a joke about {topic}", + ), + ), + inputVariables = listOf("topic"), + schema = sampleSchema, + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(2) + assertThat(result.messages[0].role).isEqualTo(PromptMessage.Role.SYSTEM) + assertThat(result.messages[0].template).isEqualTo("You tell jokes.") + assertThat(result.messages[1].role).isEqualTo(PromptMessage.Role.HUMAN) + assertThat(result.messages[1].template).isEqualTo("Tell me a joke about {topic}") + assertThat(result.inputVariables).containsExactly("topic") + assertThat(result.hasOutputSchema()).isTrue() + assertThat(result.outputSchema).isNotNull() + assertThat(result.outputSchema!!["title"]).isEqualTo("JokeResponse") + assertThat(result.outputSchema!!["type"]).isEqualTo("object") + @Suppress("UNCHECKED_CAST") + val properties = result.outputSchema!!["properties"] as Map + assertThat(properties).containsKey("setup") + assertThat(properties).containsKey("punchline") + } + + @Test + fun parseStructuredPromptWithSchemaKeyword() { + // Some manifests use "schema" instead of "schema_" + val manifest = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain", "prompts", "structured", "StructuredPrompt"), + "kwargs" to + mapOf( + "input_variables" to listOf("topic"), + "messages" to + listOf( + buildMessageTemplate( + "HumanMessagePromptTemplate", + "Tell me about {topic}", + ) + ), + "schema" to sampleSchema, + ), + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.hasOutputSchema()).isTrue() + assertThat(result.outputSchema!!["title"]).isEqualTo("JokeResponse") + } + + @Test + fun parseStructuredPromptDetectedFromKwargs() { + // Fallback: no StructuredPrompt in id, but has messages + schema_ + val manifest = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf(), // empty id + "kwargs" to + mapOf( + "messages" to + listOf( + buildMessageTemplate( + "HumanMessagePromptTemplate", + "Extract info from {text}", + ) + ), + "input_variables" to listOf("text"), + "schema_" to sampleSchema, + ), + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(1) + assertThat(result.hasOutputSchema()).isTrue() + assertThat(result.outputSchema!!["title"]).isEqualTo("JokeResponse") + } + + @Test + fun parseRegularChatPromptHasNoSchema() { + val manifest = + buildChatPromptManifest( + messages = listOf(buildMessageTemplate("HumanMessagePromptTemplate", "Hello")) + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.hasOutputSchema()).isFalse() + assertThat(result.outputSchema).isNull() + } + + @Test + fun parseStructuredPromptPreservesNestedSchema() { + // Schema with nested objects and arrays + val nestedSchema = + mapOf( + "title" to "PersonInfo", + "type" to "object", + "properties" to + mapOf( + "name" to mapOf("type" to "string"), + "age" to mapOf("type" to "integer"), + "hobbies" to mapOf("type" to "array", "items" to mapOf("type" to "string")), + "address" to + mapOf( + "type" to "object", + "properties" to + mapOf( + "street" to mapOf("type" to "string"), + "city" to mapOf("type" to "string"), + ), + ), + ), + "required" to listOf("name", "age"), + ) + + val manifest = + buildStructuredPromptManifest( + messages = + listOf(buildMessageTemplate("HumanMessagePromptTemplate", "Extract: {text}")), + inputVariables = listOf("text"), + schema = nestedSchema, + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.hasOutputSchema()).isTrue() + assertThat(result.outputSchema!!["title"]).isEqualTo("PersonInfo") + @Suppress("UNCHECKED_CAST") + val properties = result.outputSchema!!["properties"] as Map + assertThat(properties).containsKeys("name", "age", "hobbies", "address") + @Suppress("UNCHECKED_CAST") val hobbies = properties["hobbies"] as Map + assertThat(hobbies["type"]).isEqualTo("array") + } + + // --- Tool, MessagesPlaceholder, and ChatMessage tests --- + + @Test + fun parseToolMessagePromptTemplate() { + val manifest = + buildChatPromptManifest( + messages = + listOf( + buildMessageTemplate("HumanMessagePromptTemplate", "Use the tool"), + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain_core", + "prompts", + "chat", + "ToolMessagePromptTemplate", + ), + "kwargs" to + mapOf( + "template" to "Tool result: {result}", + "tool_call_id" to "call_abc123", + ), + ), + ), + inputVariables = listOf("result"), + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(2) + assertThat(result.messages[0].role).isEqualTo(PromptMessage.Role.HUMAN) + assertThat(result.messages[1].role).isEqualTo(PromptMessage.Role.TOOL) + assertThat(result.messages[1].template).isEqualTo("Tool result: {result}") + assertThat(result.messages[1].toolCallId).isEqualTo("call_abc123") + } + + @Test + fun parseToolMessageWithoutToolCallId() { + val manifest = + buildChatPromptManifest( + messages = + listOf( + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain_core", + "prompts", + "chat", + "ToolMessagePromptTemplate", + ), + "kwargs" to mapOf("template" to "Result text"), + ) + ) + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(1) + assertThat(result.messages[0].role).isEqualTo(PromptMessage.Role.TOOL) + assertThat(result.messages[0].toolCallId).isNull() + } + + @Test + fun parseMessagesPlaceholder() { + val manifest = + buildChatPromptManifest( + messages = + listOf( + buildMessageTemplate("SystemMessagePromptTemplate", "You are helpful."), + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf("langchain_core", "prompts", "chat", "MessagesPlaceholder"), + "kwargs" to mapOf("variable_name" to "chat_history"), + ), + buildMessageTemplate("HumanMessagePromptTemplate", "{input}"), + ), + inputVariables = listOf("chat_history", "input"), + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(3) + assertThat(result.messages[0].role).isEqualTo(PromptMessage.Role.SYSTEM) + assertThat(result.messages[1].role).isEqualTo(PromptMessage.Role.PLACEHOLDER) + assertThat(result.messages[1].template).isEqualTo("chat_history") + assertThat(result.messages[1].isPlaceholder()).isTrue() + assertThat(result.messages[2].role).isEqualTo(PromptMessage.Role.HUMAN) + } + + @Test + fun parseChatMessagePromptTemplate() { + val manifest = + buildChatPromptManifest( + messages = + listOf( + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain_core", + "prompts", + "chat", + "ChatMessagePromptTemplate", + ), + "kwargs" to + mapOf("template" to "Narrating: {scene}", "role" to "narrator"), + ) + ), + inputVariables = listOf("scene"), + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(1) + assertThat(result.messages[0].role).isEqualTo(PromptMessage.Role.CHAT) + assertThat(result.messages[0].template).isEqualTo("Narrating: {scene}") + assertThat(result.messages[0].customRole).isEqualTo("narrator") + } + + @Test + fun parseChatMessageWithoutRole() { + val manifest = + buildChatPromptManifest( + messages = + listOf( + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain_core", + "prompts", + "chat", + "ChatMessagePromptTemplate", + ), + "kwargs" to mapOf("template" to "Just text"), + ) + ) + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + assertThat(result.messages).hasSize(1) + assertThat(result.messages[0].role).isEqualTo(PromptMessage.Role.CHAT) + assertThat(result.messages[0].customRole).isNull() + } + + // --- Real-world playground manifest --- + + @Test + fun parsePlaygroundManifestWithAllFeatures() { + // Exact manifest from LangSmith playground: StructuredPrompt with mustache, + // AI message, MessagesPlaceholder, and a raw ToolMessage (not a template) + val manifest = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "prompts", "structured", "StructuredPrompt"), + "kwargs" to + mapOf( + "messages" to + listOf( + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ), + "kwargs" to + mapOf( + "prompt" to + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ), + "kwargs" to + mapOf( + "input_variables" to + emptyList(), + "template_format" to "mustache", + "template" to "You are a chatbot.", + ), + ) + ), + ), + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ), + "kwargs" to + mapOf( + "prompt" to + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ), + "kwargs" to + mapOf( + "input_variables" to listOf("question"), + "template_format" to "mustache", + "template" to "{{question}}", + ), + ) + ), + ), + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain", + "prompts", + "chat", + "AIMessagePromptTemplate", + ), + "kwargs" to + mapOf( + "prompt" to + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ), + "kwargs" to + mapOf( + "input_variables" to + emptyList(), + "template_format" to "mustache", + "template" to "ee", + ), + ) + ), + ), + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain_core", + "prompts", + "chat", + "MessagesPlaceholder", + ), + "kwargs" to mapOf("variable_name" to "foo"), + ), + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "messages", "ToolMessage"), + "kwargs" to + mapOf( + "content" to """{"haha":"ha"}""", + "tool_call_id" to "333", + "name" to "112", + "additional_kwargs" to emptyMap(), + "response_metadata" to emptyMap(), + ), + ), + ), + "input_variables" to listOf("question", "foo"), + "template_format" to "mustache", + "schema_" to + mapOf( + "title" to "extract", + "description" to "Extract information from the user's response.", + "type" to "object", + "properties" to + mapOf( + "correctness" to + mapOf( + "type" to "boolean", + "description" to + "Is the submission correct, accurate, and factual?", + ) + ), + "required" to listOf("correctness"), + ), + ), + ) + + val result = ManifestParser.parse(JsonValue.from(manifest)) + + // 5 messages: system, human, ai, placeholder, tool + assertThat(result.messages).hasSize(5) + + // System message (mustache, no variables) + assertThat(result.messages[0].role).isEqualTo(PromptMessage.Role.SYSTEM) + assertThat(result.messages[0].template).isEqualTo("You are a chatbot.") + assertThat(result.messages[0].templateFormat).isEqualTo("mustache") + + // Human message (mustache with variable) + assertThat(result.messages[1].role).isEqualTo(PromptMessage.Role.HUMAN) + assertThat(result.messages[1].template).isEqualTo("{{question}}") + assertThat(result.messages[1].templateFormat).isEqualTo("mustache") + + // AI message + assertThat(result.messages[2].role).isEqualTo(PromptMessage.Role.AI) + assertThat(result.messages[2].template).isEqualTo("ee") + + // MessagesPlaceholder + assertThat(result.messages[3].isPlaceholder()).isTrue() + assertThat(result.messages[3].template).isEqualTo("foo") + + // Raw ToolMessage (not a prompt template — uses "content" field) + assertThat(result.messages[4].role).isEqualTo(PromptMessage.Role.TOOL) + assertThat(result.messages[4].template).isEqualTo("""{"haha":"ha"}""") + assertThat(result.messages[4].toolCallId).isEqualTo("333") + + // Structured output schema + assertThat(result.hasOutputSchema()).isTrue() + assertThat(result.outputSchema!!["title"]).isEqualTo("extract") + + // Input variables + assertThat(result.inputVariables).containsExactly("question", "foo") + + // End-to-end: invoke with variables and expand placeholder + val prompt = Prompt.of(result.messages, result.inputVariables, result.outputSchema) + val formatted = + prompt.invoke( + mapOf( + "question" to "Is 2+2=4?", + "foo" to + listOf( + PromptMessage.human("prior message"), + PromptMessage.ai("prior response"), + ), + ) + ) + + // Placeholder expanded: system, human, ai, human(prior), ai(prior), tool = 6 messages + assertThat(formatted.messages).hasSize(6) + assertThat(formatted.messages[0].template).isEqualTo("You are a chatbot.") + assertThat(formatted.messages[1].template).isEqualTo("Is 2+2=4?") // mustache substituted + assertThat(formatted.messages[2].template).isEqualTo("ee") + assertThat(formatted.messages[3].template) + .isEqualTo("prior message") // expanded from placeholder + assertThat(formatted.messages[4].template).isEqualTo("prior response") + assertThat(formatted.messages[5].template) + .isEqualTo("""{"haha":"ha"}""") // tool message preserved + assertThat(formatted.hasOutputSchema()).isTrue() + } + + @Test + fun parseInvalidManifestThrows() { + assertThatThrownBy { ManifestParser.parse(JsonValue.from("not an object")) } + .isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("must be a JSON object") + } + + @Test + fun parseUnrecognizedTypeThrows() { + val manifest = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "something", "Unknown"), + "kwargs" to mapOf(), + ) + + assertThatThrownBy { ManifestParser.parse(JsonValue.from(manifest)) } + .isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("Unrecognized manifest type") + } +} diff --git a/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptClientTest.kt b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptClientTest.kt new file mode 100644 index 00000000..023a329b --- /dev/null +++ b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptClientTest.kt @@ -0,0 +1,61 @@ +package com.langchain.smith.prompts + +import java.util.stream.Stream +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource + +internal class PromptClientTest { + + data class Case( + val identifier: String, + val owner: String, + val repo: String, + val commit: String, + ) + + data class ErrorCase(val identifier: String, val expectedMessage: String) + + @ParameterizedTest(name = "{index}: \"{0}\"") + @MethodSource("parsePromptIdentifierCases") + fun parsePromptIdentifier(case: Case) { + val result = PromptClient.parsePromptIdentifier(case.identifier) + + assertThat(result.owner).isEqualTo(case.owner) + assertThat(result.repo).isEqualTo(case.repo) + assertThat(result.commit).isEqualTo(case.commit) + } + + @ParameterizedTest(name = "{index}: \"{0}\"") + @MethodSource("parsePromptIdentifierErrorCases") + fun parsePromptIdentifier_throws(case: ErrorCase) { + assertThatThrownBy { PromptClient.parsePromptIdentifier(case.identifier) } + .isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining(case.expectedMessage) + } + + companion object { + + @JvmStatic + fun parsePromptIdentifierCases(): Stream = + Stream.of( + Case("joke-generator", "-", "joke-generator", "latest"), + Case("my-org/joke-generator", "my-org", "joke-generator", "latest"), + Case("joke-generator:abc123", "-", "joke-generator", "abc123"), + Case("my-org/joke-generator:abc123def", "my-org", "joke-generator", "abc123def"), + Case("my-org/joke-generator:latest", "my-org", "joke-generator", "latest"), + Case(" my-org/joke-generator ", "my-org", "joke-generator", "latest"), + ) + + @JvmStatic + fun parsePromptIdentifierErrorCases(): Stream = + Stream.of( + ErrorCase("", "must not be blank"), + ErrorCase("name:", "must not be blank"), + ErrorCase("/name", "Owner must not be blank"), + ErrorCase("owner/", "Repo name must not be blank"), + ErrorCase("a/b/c", "at most one '/'"), + ) + } +} diff --git a/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptIntegrationTest.kt b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptIntegrationTest.kt new file mode 100644 index 00000000..d41e33d9 --- /dev/null +++ b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptIntegrationTest.kt @@ -0,0 +1,424 @@ +package com.langchain.smith.prompts + +import com.anthropic.client.AnthropicClient +import com.anthropic.client.okhttp.AnthropicOkHttpClient +import com.anthropic.models.messages.Message +import com.anthropic.models.messages.Model +import com.fasterxml.jackson.databind.ObjectMapper +import com.langchain.smith.client.LangsmithClient +import com.langchain.smith.client.okhttp.LangsmithOkHttpClient +import com.langchain.smith.core.JsonValue +import com.langchain.smith.models.commits.CommitCreateParams +import com.langchain.smith.models.repos.RepoCreateParams +import com.langchain.smith.models.repos.RepoListParams +import com.openai.client.OpenAIClient +import com.openai.client.okhttp.OpenAIOkHttpClient +import com.openai.models.ChatModel +import com.openai.models.chat.completions.ChatCompletion +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance + +/** + * Integration tests that pull real prompts from LangSmith and send them to the OpenAI and Anthropic + * APIs using their official Java SDKs. + * + * These tests require the following environment variables: + * - `LANGSMITH_API_KEY` — LangSmith API key + * - `OPENAI_API_KEY` — OpenAI API key (for OpenAI tests) + * - `ANTHROPIC_API_KEY` — Anthropic API key (for Anthropic tests) + * + * The tests are skipped if the required API keys are not set. + * + * Run with: + * ```bash + * ./gradlew :langsmith-java-core:test --tests "com.langchain.smith.prompts.PromptIntegrationTest" + * ``` + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +internal class PromptIntegrationTest { + + private val langsmithApiKey = System.getenv("LANGSMITH_API_KEY") + private val openaiApiKey = System.getenv("OPENAI_API_KEY") + private val anthropicApiKey = System.getenv("ANTHROPIC_API_KEY") + private val objectMapper = ObjectMapper() + + /** + * Exact manifest from a StructuredPrompt created in the LangSmith playground. Uses mustache + * template format and the real StructuredPrompt id path. + */ + private val STRUCTURED_EXTRACT_MANIFEST: Map = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "prompts", "structured", "StructuredPrompt"), + "kwargs" to + mapOf( + "messages" to + listOf( + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ), + "kwargs" to + mapOf( + "prompt" to + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ), + "kwargs" to + mapOf( + "input_variables" to emptyList(), + "template_format" to "mustache", + "template" to "You are a chatbot.", + ), + ) + ), + ), + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ), + "kwargs" to + mapOf( + "prompt" to + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ), + "kwargs" to + mapOf( + "input_variables" to listOf("question"), + "template_format" to "mustache", + "template" to "{{question}}", + ), + ) + ), + ), + ), + "input_variables" to listOf("question"), + "template_format" to "mustache", + "schema_" to + mapOf( + "title" to "extract", + "description" to "Extract information from the user's response.", + "type" to "object", + "properties" to + mapOf( + "correctness" to + mapOf( + "type" to "boolean", + "description" to + "Is the submission correct, accurate, and factual?", + ) + ), + "required" to listOf("correctness"), + ), + ), + ) + + private fun getOwner(): String = + System.getenv("LANGSMITH_OWNER")?.takeIf { it.isNotEmpty() } ?: "-" + + /** Ensures a prompt repo exists with a latest commit, creating both if needed. */ + private fun ensurePrompt(langsmith: LangsmithClient, name: String, manifest: Map) { + val repoExists = + listOf(RepoListParams.IsPublic.FALSE, RepoListParams.IsPublic.TRUE).any { isPublic -> + langsmith + .repos() + .list(RepoListParams.builder().query(name).isPublic(isPublic).build()) + .repos() + .any { it.repoHandle() == name } + } + + if (!repoExists) { + langsmith + .repos() + .create(RepoCreateParams.builder().repoHandle(name).isPublic(false).build()) + } + + // Check if there's a latest commit; if not, push one + val hasCommit = + runCatching { + langsmith + .commits() + .retrieve( + com.langchain.smith.models.commits.CommitRetrieveParams.builder() + .owner(getOwner()) + .repo(name) + .commit("latest") + .build() + ) + } + .isSuccess + + if (!hasCommit) { + langsmith + .commits() + .create( + CommitCreateParams.builder() + .owner(getOwner()) + .repo(name) + .manifest(JsonValue.from(manifest)) + .build() + ) + } + } + + // -- Manifest builders -- + + /** Builds a LangChain ChatPromptTemplate manifest. */ + private fun chatPromptManifest( + messages: List>, + inputVariables: List, + ): Map = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "prompts", "chat", "ChatPromptTemplate"), + "kwargs" to mapOf("input_variables" to inputVariables, "messages" to messages), + ) + + /** Builds a message template entry. */ + private fun messageTemplate(className: String, template: String): Map = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "prompts", "chat", className), + "kwargs" to + mapOf( + "prompt" to + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "prompts", "prompt", "PromptTemplate"), + "kwargs" to mapOf("template" to template), + ) + ), + ) + + // -- Setup: ensure prompts exist -- + + @BeforeAll + fun setUp() { + assumeTrue(!langsmithApiKey.isNullOrBlank(), "Skipping: LANGSMITH_API_KEY must be set") + + val langsmith = LangsmithOkHttpClient.fromEnv() + + // Regular prompt + ensurePrompt( + langsmith, + "joke-generator", + chatPromptManifest( + messages = + listOf( + messageTemplate( + "SystemMessagePromptTemplate", + "You are a helpful assistant that tells jokes.", + ), + messageTemplate( + "HumanMessagePromptTemplate", + "Tell me a joke about {topic}", + ), + ), + inputVariables = listOf("topic"), + ), + ) + + // Structured prompt — exact manifest format from LangSmith playground + ensurePrompt(langsmith, "structured-extract", STRUCTURED_EXTRACT_MANIFEST) + + langsmith.close() + } + + // ------------------------------------------------------- + // OpenAI — regular prompt + // ------------------------------------------------------- + + @Test + fun openAi_pullAndInvokePrompt() { + assumeTrue( + !langsmithApiKey.isNullOrBlank() && !openaiApiKey.isNullOrBlank(), + "Skipping: LANGSMITH_API_KEY and OPENAI_API_KEY must be set", + ) + + val langsmith = LangsmithOkHttpClient.fromEnv() + val promptClient = PromptClient.create(langsmith) + + val prompt = promptClient.pull("joke-generator") + val formattedPrompt = prompt.invoke(mapOf("topic" to "cats")) + + val openai: OpenAIClient = OpenAIOkHttpClient.fromEnv() + val completion: ChatCompletion = + openai + .chat() + .completions() + .create( + convertToOpenAIParams(formattedPrompt) + .model(ChatModel.GPT_4_1_MINI) + .maxCompletionTokens(256) + .build() + ) + + val responseText = completion.choices()[0].message().content().orElse("") + assertThat(responseText).isNotBlank() + println("[OpenAI] Response: $responseText") + + langsmith.close() + } + + // ------------------------------------------------------- + // OpenAI — structured prompt with output schema + // ------------------------------------------------------- + + @Test + fun openAi_structuredPrompt() { + assumeTrue( + !langsmithApiKey.isNullOrBlank() && !openaiApiKey.isNullOrBlank(), + "Skipping: LANGSMITH_API_KEY and OPENAI_API_KEY must be set", + ) + + val langsmith = LangsmithOkHttpClient.fromEnv() + val promptClient = PromptClient.create(langsmith) + + val prompt = promptClient.pull("structured-extract") + assertThat(prompt.hasOutputSchema()).isTrue() + assertThat(prompt.outputSchema!!["title"]).isEqualTo("extract") + + // Uses mustache template format — {{question}} + val formattedPrompt = + prompt.invoke(mapOf("question" to "Is the sky blue? Answer: yes, it is blue.")) + + val openai: OpenAIClient = OpenAIOkHttpClient.fromEnv() + val completion: ChatCompletion = + openai + .chat() + .completions() + .create( + convertToOpenAIParams(formattedPrompt) + .model(ChatModel.GPT_4_1_MINI) + .maxCompletionTokens(256) + .build() + ) + + val responseText = completion.choices()[0].message().content().orElse("") + assertThat(responseText).isNotBlank() + println("[OpenAI Structured] Raw response: $responseText") + + // The response should be valid JSON with a "correctness" boolean field + val json = objectMapper.readTree(responseText) + assertThat(json.has("correctness")).isTrue() + assertThat(json.get("correctness").isBoolean).isTrue() + println("[OpenAI Structured] correctness: ${json.get("correctness").asBoolean()}") + + langsmith.close() + } + + // ------------------------------------------------------- + // Anthropic — regular prompt + // ------------------------------------------------------- + + @Test + fun anthropic_pullAndInvokePrompt() { + assumeTrue( + !langsmithApiKey.isNullOrBlank() && !anthropicApiKey.isNullOrBlank(), + "Skipping: LANGSMITH_API_KEY and ANTHROPIC_API_KEY must be set", + ) + + val langsmith = LangsmithOkHttpClient.fromEnv() + val promptClient = PromptClient.create(langsmith) + + val prompt = promptClient.pull("joke-generator") + val formattedPrompt = prompt.invoke(mapOf("topic" to "dogs")) + + val anthropicClient: AnthropicClient = AnthropicOkHttpClient.fromEnv() + val message: Message = + anthropicClient + .messages() + .create( + convertToAnthropicParams(formattedPrompt) + .model(Model.CLAUDE_HAIKU_4_5_20251001) + .maxTokens(256) + .build() + ) + + val responseText = + message.content().filter { it.isText() }.joinToString("") { it.asText().text() } + assertThat(responseText).isNotBlank() + println("[Anthropic] Response: $responseText") + + langsmith.close() + } + + // ------------------------------------------------------- + // Anthropic — structured prompt with output schema + // ------------------------------------------------------- + + @Test + fun anthropic_structuredPrompt() { + assumeTrue( + !langsmithApiKey.isNullOrBlank() && !anthropicApiKey.isNullOrBlank(), + "Skipping: LANGSMITH_API_KEY and ANTHROPIC_API_KEY must be set", + ) + + val langsmith = LangsmithOkHttpClient.fromEnv() + val promptClient = PromptClient.create(langsmith) + + val prompt = promptClient.pull("structured-extract") + assertThat(prompt.hasOutputSchema()).isTrue() + assertThat(prompt.outputSchema!!["title"]).isEqualTo("extract") + + val formattedPrompt = + prompt.invoke(mapOf("question" to "Is the sky blue? Answer: yes, it is blue.")) + + val anthropicClient: AnthropicClient = AnthropicOkHttpClient.fromEnv() + val message: Message = + anthropicClient + .messages() + .create( + convertToAnthropicParams(formattedPrompt) + .model(Model.CLAUDE_SONNET_4_6) + .maxTokens(256) + .build() + ) + + val responseText = + message.content().filter { it.isText() }.joinToString("") { it.asText().text() } + assertThat(responseText).isNotBlank() + println("[Anthropic Structured] Raw response: $responseText") + + val json = objectMapper.readTree(responseText) + assertThat(json.has("correctness")).isTrue() + assertThat(json.get("correctness").isBoolean).isTrue() + println("[Anthropic Structured] correctness: ${json.get("correctness").asBoolean()}") + + langsmith.close() + } +} diff --git a/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptMessagesTest.kt b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptMessagesTest.kt new file mode 100644 index 00000000..e74faf41 --- /dev/null +++ b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptMessagesTest.kt @@ -0,0 +1,417 @@ +package com.langchain.smith.prompts + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +/** + * Tests the public prompt API: [Prompt.of] → [Prompt.invoke] → [convertToOpenAIParams] / + * [convertToAnthropicParams]. + * + * Covers formatting, OpenAI conversion, Anthropic conversion, and structured output handling. + */ +internal class PromptMessagesTest { + + // --- Formatting via Prompt.invoke --- + + @Test + fun formatSubstitutesVariables() { + val prompt = + Prompt.of( + listOf( + PromptMessage.system("You are a {personality} assistant."), + PromptMessage.human("Tell me about {topic}"), + ), + listOf("personality", "topic"), + ) + + val result = prompt.invoke(mapOf("personality" to "funny", "topic" to "cats")) + + assertThat(result.messages[0].template).isEqualTo("You are a funny assistant.") + assertThat(result.messages[1].template).isEqualTo("Tell me about cats") + } + + @Test + fun formatPreservesUnusedVariables() { + val prompt = + Prompt.of( + listOf(PromptMessage.human("Hello {name}, topic is {topic}")), + listOf("name", "topic"), + ) + + val result = prompt.invoke(mapOf("name" to "Alice")) + + assertThat(result.messages[0].template).isEqualTo("Hello Alice, topic is {topic}") + } + + @Test + fun formatMustacheTemplate() { + // Mustache templates use {{variable}} syntax + val prompt = + Prompt.of( + listOf( + PromptMessage( + PromptMessage.Role.SYSTEM, + "You are a {{personality}} assistant.", + templateFormat = "mustache", + ), + PromptMessage( + PromptMessage.Role.HUMAN, + "Tell me about {{topic}}", + templateFormat = "mustache", + ), + ), + listOf("personality", "topic"), + ) + + val result = prompt.invoke(mapOf("personality" to "funny", "topic" to "cats")) + + assertThat(result.messages[0].template).isEqualTo("You are a funny assistant.") + assertThat(result.messages[1].template).isEqualTo("Tell me about cats") + } + + @Test + fun formatFStringDoesNotCascade() { + // If a value contains {braces}, it should NOT trigger another substitution + val prompt = Prompt.of(listOf(PromptMessage.human("Value is: {x}")), listOf("x")) + + val result = prompt.invoke(mapOf("x" to "{y}", "y" to "WRONG")) + + assertThat(result.messages[0].template).isEqualTo("Value is: {y}") + } + + @Test + fun formatFStringEscapedBraces() { + // {{ and }} are literal braces, not variable references + val prompt = + Prompt.of(listOf(PromptMessage.human("Use {{braces}} and {name} here")), listOf("name")) + + val result = prompt.invoke(mapOf("name" to "Alice")) + + assertThat(result.messages[0].template).isEqualTo("Use {braces} and Alice here") + } + + @Test + fun parseMustacheManifest() { + // A manifest with template_format = "mustache" should be parsed and formatted correctly + val manifest = + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to listOf("langchain_core", "prompts", "chat", "ChatPromptTemplate"), + "kwargs" to + mapOf( + "input_variables" to listOf("name"), + "messages" to + listOf( + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain_core", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ), + "kwargs" to + mapOf( + "prompt" to + mapOf( + "lc" to 1, + "type" to "constructor", + "id" to + listOf( + "langchain_core", + "prompts", + "prompt", + "PromptTemplate", + ), + "kwargs" to + mapOf( + "template" to "Hello {{name}}!", + "template_format" to "mustache", + ), + ) + ), + ) + ), + ), + ) + + val parsed = ManifestParser.parse(com.langchain.smith.core.JsonValue.from(manifest)) + val prompt = Prompt.of(parsed.messages, parsed.inputVariables) + + val result = prompt.invoke(mapOf("name" to "Alice")) + assertThat(result.messages[0].template).isEqualTo("Hello Alice!") + } + + // --- Structured output tests --- + + private val sampleSchema = + mapOf( + "title" to "JokeResponse", + "description" to "A structured joke response.", + "type" to "object", + "properties" to + mapOf( + "setup" to mapOf("type" to "string"), + "punchline" to mapOf("type" to "string"), + ), + "required" to listOf("setup", "punchline"), + ) + + @Test + fun hasOutputSchema_true() { + val prompt = + Prompt.of(listOf(PromptMessage.human("Tell me a joke")), emptyList(), sampleSchema) + assertThat(prompt.hasOutputSchema()).isTrue() + } + + @Test + fun hasOutputSchema_false() { + val prompt = Prompt.of(listOf(PromptMessage.human("Tell me a joke"))) + assertThat(prompt.hasOutputSchema()).isFalse() + } + + @Test + fun formatPreservesOutputSchema() { + val prompt = + Prompt.of( + listOf(PromptMessage.human("Tell me about {topic}")), + listOf("topic"), + sampleSchema, + ) + + val result = prompt.invoke(mapOf("topic" to "cats")) + + assertThat(result.hasOutputSchema()).isTrue() + assertThat(result.outputSchema).isEqualTo(sampleSchema) + assertThat(result.messages[0].template).isEqualTo("Tell me about cats") + } + + // --- strictSchemaForStructuredOutput --- + + @Test + @Suppress("UNCHECKED_CAST") + fun strictSchema_addsAdditionalPropertiesToNestedObjects() { + val schema = + mapOf( + "type" to "object", + "properties" to + mapOf( + "name" to mapOf("type" to "string"), + "address" to + mapOf( + "type" to "object", + "properties" to mapOf("street" to mapOf("type" to "string")), + ), + ), + ) + + val result = strictSchemaForStructuredOutput(schema) + + // Top-level object + assertThat(result["additionalProperties"]).isEqualTo(false) + // Nested object in properties + val address = (result["properties"] as Map)["address"] as Map + assertThat(address["additionalProperties"]).isEqualTo(false) + } + + @Test + @Suppress("UNCHECKED_CAST") + fun strictSchema_recursesIntoArrayItems() { + val schema = + mapOf( + "type" to "object", + "properties" to + mapOf( + "people" to + mapOf( + "type" to "array", + "items" to + mapOf( + "type" to "object", + "properties" to mapOf("name" to mapOf("type" to "string")), + ), + ) + ), + ) + + val result = strictSchemaForStructuredOutput(schema) + + // The object inside array items should also get additionalProperties: false + val people = (result["properties"] as Map)["people"] as Map + val items = people["items"] as Map + assertThat(items["additionalProperties"]).isEqualTo(false) + } + + @Test + @Suppress("UNCHECKED_CAST") + fun strictSchema_recursesIntoAnyOf() { + val schema = + mapOf( + "anyOf" to + listOf( + mapOf( + "type" to "object", + "properties" to mapOf("a" to mapOf("type" to "string")), + ), + mapOf("type" to "string"), + ) + ) + + val result = strictSchemaForStructuredOutput(schema) + + val anyOf = result["anyOf"] as List> + assertThat(anyOf[0]["additionalProperties"]).isEqualTo(false) + // String type should not have additionalProperties + assertThat(anyOf[1]).doesNotContainKey("additionalProperties") + } + + @Test + @Suppress("UNCHECKED_CAST") + fun strictSchema_handlesUnionTypeWithObject() { + // type can be a list like ["string", "object"] + val schema = + mapOf( + "type" to "object", + "properties" to + mapOf( + "items" to + mapOf( + "type" to "array", + "items" to + mapOf( + "type" to listOf("string", "object"), + "properties" to mapOf("foo" to mapOf("type" to "string")), + "required" to listOf("foo"), + ), + ) + ), + ) + + val result = strictSchemaForStructuredOutput(schema) + + val items = (result["properties"] as Map)["items"] as Map + val arrayItems = items["items"] as Map + assertThat(arrayItems["additionalProperties"]).isEqualTo(false) + } + + @Test + fun strictSchema_handlesUnionTypeAtTopLevel() { + val schema = + mapOf( + "type" to listOf("string", "object"), + "properties" to mapOf("name" to mapOf("type" to "string")), + ) + + val result = strictSchemaForStructuredOutput(schema) + + assertThat(result["additionalProperties"]).isEqualTo(false) + } + + @Test + fun strictSchema_ignoresUnionTypeWithoutObject() { + val schema = mapOf("type" to listOf("string", "number")) + + val result = strictSchemaForStructuredOutput(schema) + + assertThat(result).doesNotContainKey("additionalProperties") + } + + @Test + fun strictSchema_leavesNonObjectSchemasAlone() { + val schema = mapOf("type" to "string") + + val result = strictSchemaForStructuredOutput(schema) + + assertThat(result).doesNotContainKey("additionalProperties") + assertThat(result["type"]).isEqualTo("string") + } + + // --- Tool, MessagesPlaceholder, and ChatMessage tests --- + + @Test + fun placeholder_expandsWithPromptMessages() { + val prompt = + Prompt.of( + listOf( + PromptMessage.system("You are helpful."), + PromptMessage.placeholder("chat_history"), + PromptMessage.human("{input}"), + ), + listOf("chat_history", "input"), + ) + + val chatHistory = + listOf(PromptMessage.human("Hi"), PromptMessage.ai("Hello! How can I help?")) + val pv = prompt.invoke(mapOf("chat_history" to chatHistory, "input" to "What is 2+2?")) + + assertThat(pv.messages).hasSize(4) + assertThat(pv.messages[0].role).isEqualTo(PromptMessage.Role.SYSTEM) + assertThat(pv.messages[0].template).isEqualTo("You are helpful.") + assertThat(pv.messages[1].role).isEqualTo(PromptMessage.Role.HUMAN) + assertThat(pv.messages[1].template).isEqualTo("Hi") + assertThat(pv.messages[2].role).isEqualTo(PromptMessage.Role.AI) + assertThat(pv.messages[2].template).isEqualTo("Hello! How can I help?") + assertThat(pv.messages[3].role).isEqualTo(PromptMessage.Role.HUMAN) + assertThat(pv.messages[3].template).isEqualTo("What is 2+2?") + } + + @Test + fun placeholder_expandsWithMaps() { + val prompt = + Prompt.of( + listOf(PromptMessage.placeholder("history"), PromptMessage.human("Next question")), + listOf("history"), + ) + + val history = + listOf( + mapOf("role" to "user", "content" to "Hello"), + mapOf("role" to "assistant", "content" to "Hi there"), + ) + val pv = prompt.invoke(mapOf("history" to history)) + + assertThat(pv.messages).hasSize(3) + assertThat(pv.messages[0].role).isEqualTo(PromptMessage.Role.HUMAN) + assertThat(pv.messages[0].template).isEqualTo("Hello") + assertThat(pv.messages[1].role).isEqualTo(PromptMessage.Role.AI) + assertThat(pv.messages[1].template).isEqualTo("Hi there") + assertThat(pv.messages[2].role).isEqualTo(PromptMessage.Role.HUMAN) + assertThat(pv.messages[2].template).isEqualTo("Next question") + } + + @Test + fun placeholder_missingVariableDropsSilently() { + val prompt = + Prompt.of( + listOf( + PromptMessage.system("System"), + PromptMessage.placeholder("missing_var"), + PromptMessage.human("Hello"), + ) + ) + + val pv = prompt.invoke() + + // Placeholder is dropped, other messages remain + assertThat(pv.messages).hasSize(2) + assertThat(pv.messages[0].role).isEqualTo(PromptMessage.Role.SYSTEM) + assertThat(pv.messages[1].role).isEqualTo(PromptMessage.Role.HUMAN) + } + + @Test + fun placeholder_emptyListExpandsToNothing() { + val prompt = + Prompt.of( + listOf(PromptMessage.placeholder("history"), PromptMessage.human("Hello")), + listOf("history"), + ) + + val pv = prompt.invoke(mapOf("history" to emptyList())) + + assertThat(pv.messages).hasSize(1) + assertThat(pv.messages[0].template).isEqualTo("Hello") + } +} diff --git a/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptTest.kt b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptTest.kt new file mode 100644 index 00000000..3c9559e0 --- /dev/null +++ b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/PromptTest.kt @@ -0,0 +1,69 @@ +package com.langchain.smith.prompts + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +internal class PromptTest { + + @Test + fun invokeFormatsVariables() { + val prompt = + Prompt.of( + listOf( + PromptMessage.system("You are a {personality} assistant."), + PromptMessage.human("Tell me about {topic}"), + ), + listOf("personality", "topic"), + ) + + val result = prompt.invoke(mapOf("personality" to "funny", "topic" to "cats")) + + assertThat(result.messages).hasSize(2) + assertThat(result.messages[0].template).isEqualTo("You are a funny assistant.") + assertThat(result.messages[1].template).isEqualTo("Tell me about cats") + } + + @Test + fun invokeNoArgs() { + val prompt = Prompt.of(listOf(PromptMessage.human("Hello world"))) + + val result = prompt.invoke() + + assertThat(result.messages).hasSize(1) + assertThat(result.messages[0].template).isEqualTo("Hello world") + } + + @Test + fun inputVariables() { + val prompt = Prompt.of(listOf(PromptMessage.human("Hello {name}")), listOf("name")) + + assertThat(prompt.inputVariables).containsExactly("name") + } + + @Test + fun outputSchemaPreserved() { + val schema = + mapOf( + "title" to "Response", + "type" to "object", + "properties" to mapOf("answer" to mapOf("type" to "string")), + ) + val prompt = + Prompt.of(listOf(PromptMessage.human("Extract: {text}")), listOf("text"), schema) + + assertThat(prompt.hasOutputSchema()).isTrue() + assertThat(prompt.outputSchema).isEqualTo(schema) + + val result = prompt.invoke(mapOf("text" to "hello")) + assertThat(result.hasOutputSchema()).isTrue() + assertThat(result.outputSchema).isEqualTo(schema) + } + + @Test + fun noOutputSchema() { + val prompt = Prompt.of(listOf(PromptMessage.human("Hello"))) + + assertThat(prompt.hasOutputSchema()).isFalse() + assertThat(prompt.outputSchema).isNull() + } +} diff --git a/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/TemplateFormatterTest.kt b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/TemplateFormatterTest.kt new file mode 100644 index 00000000..1c1dbc42 --- /dev/null +++ b/langsmith-java-core/src/test/kotlin/com/langchain/smith/prompts/TemplateFormatterTest.kt @@ -0,0 +1,101 @@ +package com.langchain.smith.prompts + +import java.util.stream.Stream +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource + +internal class TemplateFormatterTest { + + data class Case( + val description: String, + val template: String, + val variables: Map, + val format: String = "f-string", + val expected: String, + ) { + override fun toString(): String = description + } + + @ParameterizedTest(name = "{index}: {0}") + @MethodSource("cases") + fun format(case: Case) { + val result = TemplateFormatter.format(case.template, case.variables, case.format) + assertThat(result).isEqualTo(case.expected) + } + + companion object { + + @JvmStatic + fun cases(): Stream = + Stream.of( + // f-string + Case( + "f-string: basic substitution", + "Hello {name}!", + mapOf("name" to "Alice"), + expected = "Hello Alice!", + ), + Case( + "f-string: multiple variables", + "{a} and {b}", + mapOf("a" to "X", "b" to "Y"), + expected = "X and Y", + ), + Case( + "f-string: missing variable left as-is", + "Hello {name}!", + mapOf(), + expected = "Hello {name}!", + ), + Case( + "f-string: escaped braces", + "Use {{braces}} here", + mapOf(), + expected = "Use {braces} here", + ), + Case( + "f-string: escaped and variable mixed", + "{{literal}} and {var}", + mapOf("var" to "value"), + expected = "{literal} and value", + ), + Case( + "f-string: no cascading substitution", + "{a}", + mapOf("a" to "{b}", "b" to "WRONG"), + expected = "{b}", + ), + Case("f-string: empty template", "", mapOf("x" to "y"), expected = ""), + Case( + "f-string: non-string values", + "{n} is {b}", + mapOf("n" to 42, "b" to true), + expected = "42 is true", + ), + // mustache + Case( + "mustache: basic substitution", + "Hello {{name}}!", + mapOf("name" to "Alice"), + "mustache", + "Hello Alice!", + ), + Case( + "mustache: missing variable becomes empty", + "Hello {{name}}!", + mapOf(), + "mustache", + "Hello !", + ), + Case( + "mustache: no HTML escaping", + "{{html}}", + mapOf("html" to "bold"), + "mustache", + "bold", + ), + Case("mustache: empty template", "", mapOf("x" to "y"), "mustache", ""), + ) + } +} diff --git a/langsmith-java-example/src/main/kotlin/com/langchain/smith/example/PromptPullExample.kt b/langsmith-java-example/src/main/kotlin/com/langchain/smith/example/PromptPullExample.kt new file mode 100644 index 00000000..ec4ada43 --- /dev/null +++ b/langsmith-java-example/src/main/kotlin/com/langchain/smith/example/PromptPullExample.kt @@ -0,0 +1,103 @@ +package com.langchain.smith.example + +import com.langchain.smith.client.okhttp.LangsmithOkHttpClient +import com.langchain.smith.prompts.PromptClient +import com.langchain.smith.prompts.convertToAnthropicParams +import com.langchain.smith.prompts.convertToOpenAIParams + +/** + * Demonstrates pulling prompts from the LangSmith hub and converting them + * to OpenAI and Anthropic API formats. + * + * This example shows the simplified high-level prompt API, which mirrors + * the experience in the Python and TypeScript SDKs: + * + * ```java + * Prompt prompt = promptClient.pull("jacob/joke-generator"); + * PromptValue formattedPrompt = prompt.invoke(Map.of("topic", "cats")); + * ChatCompletion completion = openai.chat().completions().create( + * convertToOpenAIParams(formattedPrompt).model(ChatModel.GPT_4_1_MINI).build()); + * ``` + * + * Prerequisites: + * - `LANGSMITH_API_KEY`: Your LangSmith API key + * - A prompt must already exist in your LangSmith hub (e.g., "joke-generator"). + * You can create one via the UI or using the PromptManagementExample. + * + * Running: + * ```bash + * ./gradlew :langsmith-java-example:run -Pexample=PromptPull + * ``` + */ +fun main() { + println("=== LangSmith Prompt Pull Example ===\n") + + val client = LangsmithOkHttpClient.fromEnv() + val promptClient = PromptClient.create(client) + + try { + // ------------------------------------------------------- + // 1. Pull a prompt (like hub.pull() in TS/Python) + // ------------------------------------------------------- + val owner = System.getenv("LANGSMITH_OWNER")?.takeIf { it.isNotEmpty() } ?: "-" + val promptIdentifier = "$owner/joke-generator" + + println("1. Pulling prompt: '$promptIdentifier'") + val prompt = promptClient.pull(promptIdentifier) + println(" ✓ Pulled prompt (commit: ${prompt.commitHash})") + println(" ✓ Input variables: ${prompt.inputVariables}") + if (prompt.hasOutputSchema()) { + println(" ✓ Has structured output schema") + } + println() + + // ------------------------------------------------------- + // 2. Invoke with variables (like prompt.invoke() in TS/Python) + // ------------------------------------------------------- + println("2. Invoking prompt with topic='cats'...") + val formattedPrompt = prompt.invoke(mapOf("topic" to "cats")) + println(" ✓ Formatted ${formattedPrompt.messages.size} message(s):") + for (msg in formattedPrompt.messages) { + println(" [${msg.role}] ${msg.template}") + } + println() + + // ------------------------------------------------------- + // 3. Convert to OpenAI format + // ------------------------------------------------------- + println("3. Use with OpenAI:") + println(" // ChatCompletion completion = openai.chat().completions().create(") + println(" // convertToOpenAIParams(formattedPrompt)") + println(" // .model(ChatModel.GPT_4_1_MINI)") + println(" // .build());") + println() + + // ------------------------------------------------------- + // 4. Convert to Anthropic format + // ------------------------------------------------------- + println("4. Use with Anthropic:") + println(" // Message message = anthropic.messages().create(") + println(" // convertToAnthropicParams(formattedPrompt)") + println(" // .model(Model.CLAUDE_SONNET_4_6)") + println(" // .maxTokens(1024)") + println(" // .build());") + println() + + // ------------------------------------------------------- + // Summary + // ------------------------------------------------------- + println("=== Summary ===") + println("✓ Pulled prompt from hub") + println("✓ Invoked with variables (${formattedPrompt.messages.size} messages)") + if (formattedPrompt.hasOutputSchema()) { + println("✓ Includes structured output schema") + } + + } catch (e: Exception) { + System.err.println("\nError: ${e.message}") + e.printStackTrace() + System.exit(1) + } finally { + client.close() + } +}