Merge pull request #23 from langchain-ai/rlm/test-id-w-traces

fix: validate UUID inputs and add metadata flags to trace command

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "langsmith-fetch"
-version = "0.3.0"
+version = "0.3.1"
 description = "LangSmith Fetch - Minimal CLI for fetching LangSmith threads and traces"
 readme = "README.md"
 requires-python = ">=3.10"

src/langsmith_cli/cli.py

@@ -208,7 +208,19 @@ def thread(thread_id, project_uuid, format_type, output_file):
     metavar="PATH",
     help="Save output to file instead of printing to stdout",
 )
-def trace(trace_id, format_type, output_file):
+@click.option(
+    "--include-metadata",
+    is_flag=True,
+    default=False,
+    help="Include run metadata (status, timing, tokens, costs) in output",
+)
+@click.option(
+    "--include-feedback",
+    is_flag=True,
+    default=False,
+    help="Include feedback data in output (requires extra API call)",
+)
+def trace(trace_id, format_type, output_file, include_metadata, include_feedback):
     """Fetch messages for a single trace by trace ID.
 
     A trace represents a single execution path containing multiple runs (LLM calls,
@@ -220,19 +232,23 @@ def trace(trace_id, format_type, output_file):
 
     \b
     RETURNS:
-      List of messages with role, content, tool calls, and metadata.
+      List of messages with role, content, tool calls (default).
+      With --include-metadata: Dictionary with messages, metadata, and feedback.
 
     \b
     EXAMPLES:
-      # Fetch trace with default format (pretty)
+      # Fetch trace messages only (default)
       langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe
 
+      # Fetch trace with metadata (status, timing, tokens, costs)
+      langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe --include-metadata
+
+      # Fetch trace with both metadata and feedback
+      langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe --include-metadata --include-feedback
+
       # Fetch trace as JSON for parsing
       langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe --format json
 
-      # Fetch trace as raw JSON for piping
-      langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe --format raw
-
     \b
     PREREQUISITES:
       - LANGSMITH_API_KEY environment variable must be set, or
@@ -253,13 +269,22 @@ def trace(trace_id, format_type, output_file):
         format_type = config.get_default_format()
 
     try:
-        # Fetch trace with metadata and feedback
-        trace_data = fetchers.fetch_trace_with_metadata(
-            trace_id, base_url=base_url, api_key=api_key
-        )
-
-        # Output with metadata and feedback
-        formatters.print_formatted_trace(trace_data, format_type, output_file)
+        # Fetch trace with or without metadata/feedback
+        if include_metadata or include_feedback:
+            # Fetch with metadata and/or feedback
+            trace_data = fetchers.fetch_trace_with_metadata(
+                trace_id,
+                base_url=base_url,
+                api_key=api_key,
+                include_feedback=include_feedback,
+            )
+            # Output with metadata and feedback
+            formatters.print_formatted_trace(trace_data, format_type, output_file)
+        else:
+            # Fetch messages only (no metadata/feedback)
+            messages = fetchers.fetch_trace(trace_id, base_url=base_url, api_key=api_key)
+            # Output just messages
+            formatters.print_formatted(messages, format_type, output_file)
 
     except Exception as e:
         click.echo(f"Error fetching trace: {e}", err=True)
@@ -401,6 +426,23 @@ def threads(
 
     # DIRECTORY MODE: output_dir provided
     if output_dir:
+        # Check if user mistakenly passed a thread ID (UUID) instead of directory
+        uuid_pattern = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
+        if re.match(uuid_pattern, output_dir, re.IGNORECASE):
+            click.echo(
+                f"Error: '{output_dir}' looks like a UUID, not a directory path.",
+                err=True,
+            )
+            click.echo(
+                "To fetch a specific thread by ID, use: langsmith-fetch thread <thread-id>",
+                err=True,
+            )
+            click.echo(
+                "To fetch multiple threads to a directory, use: langsmith-fetch threads <directory-path>",
+                err=True,
+            )
+            sys.exit(1)
+
         # Validate incompatible options
         if format_type:
             click.echo(
@@ -673,6 +715,23 @@ def traces(
 
     # DIRECTORY MODE: output_dir provided
     if output_dir:
+        # Check if user mistakenly passed a trace ID instead of directory
+        uuid_pattern = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
+        if re.match(uuid_pattern, output_dir, re.IGNORECASE):
+            click.echo(
+                f"Error: '{output_dir}' looks like a trace ID, not a directory path.",
+                err=True,
+            )
+            click.echo(
+                "To fetch a specific trace by ID, use: langsmith-fetch trace <trace-id>",
+                err=True,
+            )
+            click.echo(
+                "To fetch multiple traces to a directory, use: langsmith-fetch traces <directory-path>",
+                err=True,
+            )
+            sys.exit(1)
+
         # Validate incompatible options
         if format_type:
             click.echo(

tests/test_cli.py

@@ -33,7 +33,7 @@ class TestTraceCommand:
         assert result.exit_code == 0
         # Check for Rich panel indicators
         assert "Message 1:" in result.output
-        assert "human" in result.output or "user" in result.output
+        assert "human" in result.output.lower() or "user" in result.output.lower()
 
     @responses.activate
     def test_trace_pretty_format(self, sample_trace_response, mock_env_api_key):
@@ -115,6 +115,46 @@ class TestTraceCommand:
         assert result.exit_code == 1
         assert "Error fetching trace" in result.output
 
+    @responses.activate
+    def test_trace_with_metadata_flag(self, sample_trace_response, mock_env_api_key):
+        """Test trace command with --include-metadata flag."""
+        responses.add(
+            responses.GET,
+            f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}?include_messages=true",
+            json=sample_trace_response,
+            status=200,
+        )
+
+        runner = CliRunner()
+        result = runner.invoke(
+            main, ["trace", TEST_TRACE_ID, "--include-metadata", "--format", "json"]
+        )
+
+        assert result.exit_code == 0
+        # When metadata is included, output should contain metadata structure
+        assert "metadata" in result.output or "trace_id" in result.output
+
+    @responses.activate
+    def test_trace_without_metadata_default(
+        self, sample_trace_response, mock_env_api_key
+    ):
+        """Test trace command defaults to no metadata."""
+        responses.add(
+            responses.GET,
+            f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}?include_messages=true",
+            json=sample_trace_response,
+            status=200,
+        )
+
+        runner = CliRunner()
+        result = runner.invoke(main, ["trace", TEST_TRACE_ID, "--format", "json"])
+
+        assert result.exit_code == 0
+        # Without flags, should just return messages array
+        output_lower = result.output.lower()
+        # Check that it contains message content but not metadata wrapper
+        assert "jane" in output_lower
+
 
 class TestThreadCommand:
     """Tests for thread command."""
@@ -491,6 +531,20 @@ class TestThreadsCommand:
                 assert (output_dir / "thread_001.json").exists()
                 assert (output_dir / "thread_002.json").exists()
 
+    def test_threads_rejects_uuid_as_directory(self, mock_env_api_key):
+        """Test threads command rejects UUID passed as directory."""
+        runner = CliRunner()
+        # Pass a valid UUID instead of a directory path
+        fake_uuid = "3a12d0b2-bda5-4500-8732-c1984f647df5"
+        result = runner.invoke(
+            main, ["threads", fake_uuid, "--project-uuid", TEST_PROJECT_UUID]
+        )
+
+        assert result.exit_code == 1
+        assert "looks like a UUID" in result.output
+        assert "langsmith-fetch thread <thread-id>" in result.output
+        assert "langsmith-fetch threads <directory-path>" in result.output
+
 
 class TestTracesCommand:
     """Tests for traces command."""
@@ -802,3 +856,15 @@ class TestTracesCommand:
 
             assert result.exit_code == 0
             assert "Found 1 trace(s)" in result.output
+
+    def test_traces_rejects_uuid_as_directory(self, mock_env_api_key):
+        """Test traces command rejects UUID passed as directory."""
+        runner = CliRunner()
+        # Pass a valid UUID instead of a directory path
+        fake_uuid = "3a12d0b2-bda5-4500-8732-c1984f647df5"
+        result = runner.invoke(main, ["traces", fake_uuid, "--include-metadata"])
+
+        assert result.exit_code == 1
+        assert "looks like a trace ID" in result.output
+        assert "langsmith-fetch trace <trace-id>" in result.output
+        assert "langsmith-fetch traces <directory-path>" in result.output