fix: validate UUID inputs and add metadata flags to trace command

- Add UUID validation to traces/threads commands to prevent confusion
  when users mistakenly pass trace/thread IDs instead of directory paths
- Add --include-metadata and --include-feedback flags to trace command
  for consistency with traces command (both now default to off)
- Update tests to cover new validation and flag behavior
This commit is contained in:
Lance Martin
2025-12-12 11:39:45 -08:00
parent 4737526a86
commit 73a99d6469
2 changed files with 137 additions and 14 deletions
+72 -13
View File
@@ -208,7 +208,19 @@ def thread(thread_id, project_uuid, format_type, output_file):
metavar="PATH",
help="Save output to file instead of printing to stdout",
)
def trace(trace_id, format_type, output_file):
@click.option(
"--include-metadata",
is_flag=True,
default=False,
help="Include run metadata (status, timing, tokens, costs) in output",
)
@click.option(
"--include-feedback",
is_flag=True,
default=False,
help="Include feedback data in output (requires extra API call)",
)
def trace(trace_id, format_type, output_file, include_metadata, include_feedback):
"""Fetch messages for a single trace by trace ID.
A trace represents a single execution path containing multiple runs (LLM calls,
@@ -220,19 +232,23 @@ def trace(trace_id, format_type, output_file):
\b
RETURNS:
List of messages with role, content, tool calls, and metadata.
List of messages with role, content, tool calls (default).
With --include-metadata: Dictionary with messages, metadata, and feedback.
\b
EXAMPLES:
# Fetch trace with default format (pretty)
# Fetch trace messages only (default)
langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe
# Fetch trace with metadata (status, timing, tokens, costs)
langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe --include-metadata
# Fetch trace with both metadata and feedback
langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe --include-metadata --include-feedback
# Fetch trace as JSON for parsing
langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe --format json
# Fetch trace as raw JSON for piping
langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe --format raw
\b
PREREQUISITES:
- LANGSMITH_API_KEY environment variable must be set, or
@@ -253,13 +269,22 @@ def trace(trace_id, format_type, output_file):
format_type = config.get_default_format()
try:
# Fetch trace with metadata and feedback
trace_data = fetchers.fetch_trace_with_metadata(
trace_id, base_url=base_url, api_key=api_key
)
# Output with metadata and feedback
formatters.print_formatted_trace(trace_data, format_type, output_file)
# Fetch trace with or without metadata/feedback
if include_metadata or include_feedback:
# Fetch with metadata and/or feedback
trace_data = fetchers.fetch_trace_with_metadata(
trace_id,
base_url=base_url,
api_key=api_key,
include_feedback=include_feedback,
)
# Output with metadata and feedback
formatters.print_formatted_trace(trace_data, format_type, output_file)
else:
# Fetch messages only (no metadata/feedback)
messages = fetchers.fetch_trace(trace_id, base_url=base_url, api_key=api_key)
# Output just messages
formatters.print_formatted(messages, format_type, output_file)
except Exception as e:
click.echo(f"Error fetching trace: {e}", err=True)
@@ -401,6 +426,23 @@ def threads(
# DIRECTORY MODE: output_dir provided
if output_dir:
# Check if user mistakenly passed a thread ID (UUID) instead of directory
uuid_pattern = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
if re.match(uuid_pattern, output_dir, re.IGNORECASE):
click.echo(
f"Error: '{output_dir}' looks like a UUID, not a directory path.",
err=True,
)
click.echo(
"To fetch a specific thread by ID, use: langsmith-fetch thread <thread-id>",
err=True,
)
click.echo(
"To fetch multiple threads to a directory, use: langsmith-fetch threads <directory-path>",
err=True,
)
sys.exit(1)
# Validate incompatible options
if format_type:
click.echo(
@@ -673,6 +715,23 @@ def traces(
# DIRECTORY MODE: output_dir provided
if output_dir:
# Check if user mistakenly passed a trace ID instead of directory
uuid_pattern = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
if re.match(uuid_pattern, output_dir, re.IGNORECASE):
click.echo(
f"Error: '{output_dir}' looks like a trace ID, not a directory path.",
err=True,
)
click.echo(
"To fetch a specific trace by ID, use: langsmith-fetch trace <trace-id>",
err=True,
)
click.echo(
"To fetch multiple traces to a directory, use: langsmith-fetch traces <directory-path>",
err=True,
)
sys.exit(1)
# Validate incompatible options
if format_type:
click.echo(
+65 -1
View File
@@ -33,7 +33,7 @@ class TestTraceCommand:
assert result.exit_code == 0
# Check for Rich panel indicators
assert "Message 1:" in result.output
assert "human" in result.output or "user" in result.output
assert "human" in result.output.lower() or "user" in result.output.lower()
@responses.activate
def test_trace_pretty_format(self, sample_trace_response, mock_env_api_key):
@@ -115,6 +115,46 @@ class TestTraceCommand:
assert result.exit_code == 1
assert "Error fetching trace" in result.output
@responses.activate
def test_trace_with_metadata_flag(self, sample_trace_response, mock_env_api_key):
"""Test trace command with --include-metadata flag."""
responses.add(
responses.GET,
f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}?include_messages=true",
json=sample_trace_response,
status=200,
)
runner = CliRunner()
result = runner.invoke(
main, ["trace", TEST_TRACE_ID, "--include-metadata", "--format", "json"]
)
assert result.exit_code == 0
# When metadata is included, output should contain metadata structure
assert "metadata" in result.output or "trace_id" in result.output
@responses.activate
def test_trace_without_metadata_default(
self, sample_trace_response, mock_env_api_key
):
"""Test trace command defaults to no metadata."""
responses.add(
responses.GET,
f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}?include_messages=true",
json=sample_trace_response,
status=200,
)
runner = CliRunner()
result = runner.invoke(main, ["trace", TEST_TRACE_ID, "--format", "json"])
assert result.exit_code == 0
# Without flags, should just return messages array
output_lower = result.output.lower()
# Check that it contains message content but not metadata wrapper
assert "jane" in output_lower
class TestThreadCommand:
"""Tests for thread command."""
@@ -491,6 +531,18 @@ class TestThreadsCommand:
assert (output_dir / "thread_001.json").exists()
assert (output_dir / "thread_002.json").exists()
def test_threads_rejects_uuid_as_directory(self, mock_env_api_key):
"""Test threads command rejects UUID passed as directory."""
runner = CliRunner()
# Pass a valid UUID instead of a directory path
fake_uuid = "3a12d0b2-bda5-4500-8732-c1984f647df5"
result = runner.invoke(main, ["threads", fake_uuid])
assert result.exit_code == 1
assert "looks like a UUID" in result.output
assert "langsmith-fetch thread <thread-id>" in result.output
assert "langsmith-fetch threads <directory-path>" in result.output
class TestTracesCommand:
"""Tests for traces command."""
@@ -802,3 +854,15 @@ class TestTracesCommand:
assert result.exit_code == 0
assert "Found 1 trace(s)" in result.output
def test_traces_rejects_uuid_as_directory(self, mock_env_api_key):
"""Test traces command rejects UUID passed as directory."""
runner = CliRunner()
# Pass a valid UUID instead of a directory path
fake_uuid = "3a12d0b2-bda5-4500-8732-c1984f647df5"
result = runner.invoke(main, ["traces", fake_uuid, "--include-metadata"])
assert result.exit_code == 1
assert "looks like a trace ID" in result.output
assert "langsmith-fetch trace <trace-id>" in result.output
assert "langsmith-fetch traces <directory-path>" in result.output