fix(code): handle LangSmith project-not-found and default tracing project (#4153)

`/trace` no longer shows a scary "project not found" error before the
first run is traced, and the default `deepagents-code` tracing project
is now actually used when tracing is enabled with no project configured.

---

`/trace` surfaced LangSmith's raw `Project deepagents-code not found`
404 as an error, even though it only means no trace has been ingested
into that project yet (projects are auto-created lazily on first
ingest). We now classify that 404 as a dedicated
`LangSmithProjectNotFoundError` and show a friendly "created on first
trace" hint instead. We also default `LANGSMITH_PROJECT` to
`deepagents-code` at bootstrap when tracing is enabled but no project is
configured, so ingestion matches the name `get_langsmith_project_name()`
displays and looks up. We deliberately do **not** call `create_project`.

Made by [Open
SWE](https://openswe.vercel.app/agents/f2a5f732-0697-777e-0bb2-7f2fb0704f00)

---------

Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
This commit is contained in:
Mason Daugherty
2026-06-22 23:47:06 -04:00
committed by GitHub
parent 91c0dae3fe
commit e303ce986a
7 changed files with 425 additions and 16 deletions
+8 -3
View File
@@ -1435,11 +1435,16 @@ def create_cli_agent(
# ========== LOCAL MODE ==========
root_dir = effective_cwd if effective_cwd is not None else Path.cwd()
if enable_shell:
# Create environment for shell commands
# Restore user's original LANGSMITH_PROJECT so their code traces separately
# Create environment for shell commands.
# Restore the user's original LANGSMITH_PROJECT so their code traces
# separately. When they had none, drop the agent's override (the
# `deepagents-code` default applied at bootstrap) entirely so shell
# commands don't inherit it.
shell_env = os.environ.copy()
if settings.user_langchain_project:
if settings.user_langchain_project is not None:
shell_env["LANGSMITH_PROJECT"] = settings.user_langchain_project
else:
shell_env.pop("LANGSMITH_PROJECT", None)
# Re-apply a launch-time PYTHONPATH that was stripped from the server
# interpreter but relayed for approval-gated `execute` commands.
_apply_inherited_pythonpath(shell_env)
+17
View File
@@ -6489,6 +6489,7 @@ class DeepAgentsApp(App):
LangSmithApiError,
LangSmithImportError,
LangSmithLookupTimeoutError,
LangSmithProjectNotFoundError,
_assemble_langsmith_thread_url,
fetch_langsmith_project_url_or_raise,
get_langsmith_project_name,
@@ -6552,6 +6553,22 @@ class DeepAgentsApp(App):
),
)
return
except LangSmithProjectNotFoundError:
logger.debug(
"LangSmith project %r not found yet for thread %s",
project_name,
thread_id,
)
await self._mount_message(UserMessage(command))
await self._mount_message(
AppMessage(
f"No traces have been recorded in LangSmith project "
f"{project_name!r} yet. The project is created automatically "
"the first time a run is traced — try `/trace` again after "
"your first message.",
),
)
return
except LangSmithApiError as exc:
logger.warning(
"LangSmith API call failed while resolving thread URL for %s: %s",
+81 -10
View File
@@ -408,6 +408,28 @@ def consume_orphaned_tracing_disabled_notice() -> str | None:
return notice
def _tracing_enabled() -> bool:
"""Whether any LangSmith/LangChain tracing flag is truthy in the environment.
Reads the canonical tracing-enable vars (`_TRACING_ENABLE_ENV_VARS`) and
classifies each present value with `classify_env_bool`, mirroring how the
LangChain/LangSmith SDKs decide whether to start tracing. Shared by
`_disable_orphaned_tracing` and `_apply_default_langsmith_project` so both
read the flags identically.
Returns:
`True` if at least one tracing flag is set to a truthy value,
else `False`.
"""
from deepagents_code._env_vars import classify_env_bool
return any(
classify_env_bool(os.environ[var])
for var in _TRACING_ENABLE_ENV_VARS
if var in os.environ
)
def _disable_orphaned_tracing() -> None:
"""Disable LangSmith tracing when enabled without a usable API key.
@@ -426,14 +448,7 @@ def _disable_orphaned_tracing() -> None:
"""
global _orphaned_tracing_disabled_notice # noqa: PLW0603
from deepagents_code._env_vars import classify_env_bool
tracing_on = any(
classify_env_bool(os.environ[var])
for var in _TRACING_ENABLE_ENV_VARS
if var in os.environ
)
if not tracing_on:
if not _tracing_enabled():
return
has_custom_endpoint = any(
@@ -460,6 +475,26 @@ def _disable_orphaned_tracing() -> None:
)
def _apply_default_langsmith_project() -> None:
"""Route agent traces to the default project when none is configured.
When tracing is active but neither the prefixed override nor a base
`LANGSMITH_PROJECT` is set, ingestion would land in the SDK's `default`
project while `get_langsmith_project_name` advertises `deepagents-code`.
Set the default explicitly so the displayed/looked-up name matches where
traces are actually ingested (and `/trace` resolves once a run flushes).
"""
if os.environ.get("LANGSMITH_PROJECT"):
return
if not _tracing_enabled():
return
from deepagents_code.config_manifest import LANGSMITH_PROJECT_DEFAULT
os.environ["LANGSMITH_PROJECT"] = LANGSMITH_PROJECT_DEFAULT
def _ensure_bootstrap() -> None:
"""Run one-time bootstrap: dotenv loading and `LANGSMITH_PROJECT` override.
@@ -550,6 +585,12 @@ def _ensure_bootstrap() -> None:
# errors; disable it before any traced run starts.
_disable_orphaned_tracing()
# If tracing is still active but no project is configured, default
# to `deepagents-code` so ingestion matches the name we display and
# look up. Runs after `_disable_orphaned_tracing` so a keyless setup
# (tracing already turned off) is left untouched.
_apply_default_langsmith_project()
# Bridge stored service keys (e.g. Tavily web search, entered via
# `/auth`) onto their canonical env vars before settings detection,
# so a stored key activates the feature without exporting the var.
@@ -1933,11 +1974,16 @@ class Settings:
if new_project:
os.environ["LANGSMITH_PROJECT"] = str(new_project)
elif previous["deepagents_langchain_project"]:
# Override was previously active but new value is unset; restore.
# Override was previously active but new value is unset; restore the
# user's original project. With no original, drop the override and
# re-apply the default so ingestion keeps matching the name
# `get_langsmith_project_name` displays (the default is a no-op when
# tracing is off, so a disabled setup is left unset).
if _original_langsmith_project:
os.environ["LANGSMITH_PROJECT"] = _original_langsmith_project
else:
os.environ.pop("LANGSMITH_PROJECT", None)
_apply_default_langsmith_project()
return self._format_reload_changes(previous, refreshed)
@@ -2482,6 +2528,28 @@ class LangSmithApiError(LangSmithLookupError):
"""
class LangSmithProjectNotFoundError(LangSmithApiError):
"""The LangSmith project does not exist yet (lookup returned 404).
Projects are created lazily on the first ingested trace, so this is
expected before any run has flushed and should be surfaced as an
informational message rather than an error.
"""
def _is_langsmith_not_found(exc: Exception) -> bool:
"""Whether a LangSmith SDK error indicates the project does not exist.
Returns:
`True` for a `LangSmithNotFoundError` (404), `False` otherwise.
"""
try:
from langsmith.utils import LangSmithNotFoundError
except ImportError:
return False
return isinstance(exc, LangSmithNotFoundError)
def _assemble_langsmith_thread_url(project_url: str, thread_id: str) -> str:
"""Format a LangSmith thread URL from a project URL prefix.
@@ -2515,7 +2583,8 @@ def fetch_langsmith_project_url_or_raise(project_name: str) -> str:
Raises:
LangSmithImportError: `langsmith` is not installed.
LangSmithLookupTimeoutError: lookup exceeded the hard timeout.
LangSmithApiError: the SDK call raised (auth, 404, network, etc.);
LangSmithProjectNotFoundError: the project does not exist yet (404).
LangSmithApiError: the SDK call raised (auth, network, etc.);
wraps the original exception in `__cause__`.
"""
global _langsmith_url_cache # noqa: PLW0603 # Module-level cache requires global statement
@@ -2584,6 +2653,8 @@ def fetch_langsmith_project_url_or_raise(project_name: str) -> str:
),
)
msg = str(lookup_error) or repr(lookup_error)
if _is_langsmith_not_found(lookup_error):
raise LangSmithProjectNotFoundError(msg) from lookup_error
raise LangSmithApiError(msg) from lookup_error
if not result:
+29
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import contextlib
import os
from typing import TYPE_CHECKING
import pytest
@@ -35,6 +36,34 @@ def _warm_model_caches() -> None:
get_model_profiles()
@pytest.fixture(autouse=True)
def _restore_os_environ() -> Generator[None, None, None]:
"""Snapshot and restore `os.environ` around every test.
Production code under test (`_ensure_bootstrap`, `_load_dotenv`,
`_apply_default_langsmith_project`) writes to `os.environ` directly. When a
test clears a variable with `monkeypatch.delenv(name, raising=False)` that
was already absent, monkeypatch records no undo entry — so a later direct
write by that code survives teardown and leaks into subsequent tests (e.g.
a dotenv-reload test leaking `DEEPAGENTS_CODE_OPENAI_API_KEY` into a gateway
key-mismatch test). Defined before the other autouse fixtures so it tears
down last, leaving `os.environ` pristine no matter how a key was set.
Restores by diffing against the snapshot rather than a blanket
`clear()`/`update()`, so a test that never touches `os.environ` (the vast
majority) triggers zero `putenv` calls on teardown.
"""
snapshot = dict(os.environ)
try:
yield
finally:
for key in [key for key in os.environ if key not in snapshot]:
del os.environ[key]
for key, value in snapshot.items():
if os.environ.get(key) != value:
os.environ[key] = value
@pytest.fixture(autouse=True)
def _clear_langsmith_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Prevent LangSmith env vars loaded from .env from leaking into tests.
+52 -3
View File
@@ -1561,8 +1561,23 @@ class TestCreateCliAgentProjectContext:
assert sources[0] == str(agent_dir / "AGENTS.md")
assert sources[1:] == [str(deepagents_md), str(root_md)]
def test_project_context_sets_local_shell_root_dir(self, tmp_path: Path) -> None:
"""Shell backend root should follow the explicit user working directory."""
@staticmethod
def _build_shell_agent(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
*,
user_langchain_project: str | None,
) -> tuple[Mock, Path]:
"""Build a shell-enabled CLI agent and return the `LocalShellBackend` mock.
The agent's `deepagents-code` override is placed in `os.environ` so the
returned `call_args` reflect how the user's original `LANGSMITH_PROJECT`
is restored or dropped for shell commands.
Returns:
The `LocalShellBackend` mock (for `call_args` assertions) and the
resolved user working directory.
"""
project_root = tmp_path / "project"
project_root.mkdir()
(project_root / ".git").mkdir()
@@ -1591,11 +1606,12 @@ class TestCreateCliAgentProjectContext:
mock_settings.model_unsupported_modalities = frozenset()
mock_settings.model_context_limit = None
mock_settings.project_root = None
mock_settings.user_langchain_project = None
mock_settings.user_langchain_project = user_langchain_project
mock_agent = Mock()
mock_agent.with_config.return_value = mock_agent
mock_backend = Mock()
monkeypatch.setenv("LANGSMITH_PROJECT", "deepagents-code")
fake_model = _make_fake_chat_model()
with (
@@ -1617,7 +1633,40 @@ class TestCreateCliAgentProjectContext:
project_context=project_context,
)
return mock_shell, user_cwd
def test_project_context_sets_local_shell_root_dir(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""Shell backend root follows the cwd; agent override is dropped.
With no user `LANGSMITH_PROJECT` (`user_langchain_project is None`),
the agent's `deepagents-code` override must not leak into the shell
env — it is popped so the user's code does not trace into the agent's
project.
"""
mock_shell, user_cwd = self._build_shell_agent(
monkeypatch, tmp_path, user_langchain_project=None
)
assert mock_shell.call_args.kwargs["root_dir"] == user_cwd
assert "LANGSMITH_PROJECT" not in mock_shell.call_args.kwargs["env"]
@pytest.mark.parametrize("user_project", ["user-project", ""])
def test_project_context_restores_user_shell_langchain_project(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, user_project: str
) -> None:
"""A non-None original project is restored into the shell env verbatim.
The guard is `is not None`, so an empty-string original is restored as
`""` (not popped) — the user explicitly cleared their project and that
intent is preserved for shell commands.
"""
mock_shell, _ = self._build_shell_agent(
monkeypatch, tmp_path, user_langchain_project=user_project
)
assert mock_shell.call_args.kwargs["env"]["LANGSMITH_PROJECT"] == user_project
def test_cwd_sets_local_filesystem_root_dir_without_shell(
self, tmp_path: Path
+30
View File
@@ -3458,6 +3458,36 @@ class TestTraceCommand:
assert "LANGSMITH_API_KEY" in rendered
assert "Check your network" not in rendered
async def test_trace_shows_friendly_message_when_project_not_found(self) -> None:
"""A not-found project shows a 'created on first trace' hint, not an error."""
from deepagents_code.config import LangSmithProjectNotFoundError
app = DeepAgentsApp()
async with app.run_test() as pilot:
await pilot.pause()
app._session_state = TextualSessionState(thread_id="test-thread-123")
with (
patch(
"deepagents_code.config.get_langsmith_project_name",
return_value="deepagents-code",
),
patch(
"deepagents_code.config.fetch_langsmith_project_url_or_raise",
side_effect=LangSmithProjectNotFoundError(
"Project deepagents-code not found"
),
),
):
await app._handle_trace_command("/trace")
await pilot.pause()
app_msgs = app.query(AppMessage)
rendered = " ".join(str(w._content) for w in app_msgs)
assert "No traces have been recorded" in rendered
assert "first time a run is traced" in rendered
assert "rejected the project lookup" not in rendered
async def test_trace_shows_error_when_project_name_raises(self) -> None:
"""Should surface a friendly error if `get_langsmith_project_name` raises."""
app = DeepAgentsApp()
+208
View File
@@ -15,8 +15,11 @@ from deepagents_code.config import (
CLI_MAX_RETRIES_KEY,
RECOMMENDED_SAFE_SHELL_COMMANDS,
SHELL_ALLOW_ALL,
LangSmithApiError,
LangSmithProjectNotFoundError,
ModelResult,
Settings,
_apply_default_langsmith_project,
_create_model_from_class,
_create_model_via_init,
_disable_orphaned_tracing,
@@ -31,6 +34,7 @@ from deepagents_code.config import (
detect_mode_prefix,
detect_provider,
fetch_langsmith_project_url,
fetch_langsmith_project_url_or_raise,
get_langsmith_project_name,
newline_shortcut,
parse_shell_allow_list,
@@ -96,6 +100,56 @@ class TestRuntimeDotenvReload:
finally:
config_mod._dotenv_loaded_values.clear()
def test_reload_redefaults_project_when_override_cleared_and_tracing_on(
self,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Clearing the agent project on reload re-applies the default.
Regression: when a cwd switch unsets `DEEPAGENTS_CODE_LANGSMITH_PROJECT`
and the user has no original `LANGSMITH_PROJECT`, the reload must fall
back to `deepagents-code` (not leave the var unset) so trace ingestion
keeps matching the name `get_langsmith_project_name` displays.
"""
import os
import deepagents_code.config as config_mod
from deepagents_code.config_manifest import LANGSMITH_PROJECT_DEFAULT
current = tmp_path / "current"
target = tmp_path / "target"
current.mkdir()
target.mkdir()
monkeypatch.setattr(
config_mod,
"_GLOBAL_DOTENV_PATH",
tmp_path / "missing-global.env",
)
config_mod._dotenv_loaded_values.clear()
original_ls = config_mod._original_langsmith_project
try:
# User never set LANGSMITH_PROJECT; tracing is active with a key.
config_mod._original_langsmith_project = None
monkeypatch.setenv("LANGSMITH_TRACING", "true")
monkeypatch.setenv("LANGSMITH_API_KEY", "lsv2_test")
monkeypatch.delenv("LANGSMITH_PROJECT", raising=False)
# Agent-project override is active before the reload, cleared after.
monkeypatch.setenv("DEEPAGENTS_CODE_LANGSMITH_PROJECT", "agent-project")
runtime = Settings.from_environment(start_path=current)
assert runtime.deepagents_langchain_project == "agent-project"
monkeypatch.delenv("DEEPAGENTS_CODE_LANGSMITH_PROJECT", raising=False)
runtime.reload_from_environment(start_path=target)
assert os.environ["LANGSMITH_PROJECT"] == LANGSMITH_PROJECT_DEFAULT
finally:
config_mod._original_langsmith_project = original_ls
config_mod._dotenv_loaded_values.clear()
class TestProjectRootDetection:
"""Test project root detection via .git directory."""
@@ -2229,6 +2283,25 @@ class TestFetchLangsmithProjectUrl:
assert second == "https://smith.langchain.com/o/org/projects/p/b"
assert mock_client_cls.return_value.read_project.call_count == 2
def test_or_raise_raises_project_not_found(self) -> None:
"""A 404 from read_project raises LangSmithProjectNotFoundError."""
from langsmith.utils import LangSmithNotFoundError
with patch("langsmith.Client") as mock_client_cls:
mock_client_cls.return_value.read_project.side_effect = (
LangSmithNotFoundError("Project deepagents-code not found")
)
with pytest.raises(LangSmithProjectNotFoundError):
fetch_langsmith_project_url_or_raise("deepagents-code")
def test_or_raise_raises_api_error_on_other_failure(self) -> None:
"""A non-404 SDK error raises the generic LangSmithApiError."""
with patch("langsmith.Client") as mock_client_cls:
mock_client_cls.return_value.read_project.side_effect = OSError("boom")
with pytest.raises(LangSmithApiError) as exc_info:
fetch_langsmith_project_url_or_raise("my-project")
assert not isinstance(exc_info.value, LangSmithProjectNotFoundError)
class TestBuildLangsmithThreadUrl:
"""Tests for build_langsmith_thread_url()."""
@@ -3814,6 +3887,141 @@ class TestLazyModuleAttributes:
config_mod._bootstrap_done = original_done
config_mod._original_langsmith_project = original_ls
def test_bootstrap_defaults_project_when_tracing_and_key(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Tracing on with a key but no project defaults to deepagents-code.
Exercises `_apply_default_langsmith_project` wired into the real
`_ensure_bootstrap` flow (after the override and orphaned-tracing
steps) — coverage the helper-level tests cannot provide.
"""
import os
import deepagents_code.config as config_mod
from deepagents_code.config import _ensure_bootstrap
from deepagents_code.config_manifest import LANGSMITH_PROJECT_DEFAULT
original_done = config_mod._bootstrap_done
original_ls = config_mod._original_langsmith_project
config_mod._bootstrap_done = False
try:
monkeypatch.setenv("LANGSMITH_TRACING", "true")
monkeypatch.setenv("LANGSMITH_API_KEY", "lsv2_test")
monkeypatch.delenv("LANGSMITH_PROJECT", raising=False)
monkeypatch.delenv("DEEPAGENTS_CODE_LANGSMITH_PROJECT", raising=False)
with (
patch("deepagents_code.config._load_dotenv"),
patch(
"deepagents_code.project_utils.get_server_project_context",
return_value=None,
),
):
_ensure_bootstrap()
assert os.environ["LANGSMITH_PROJECT"] == LANGSMITH_PROJECT_DEFAULT
finally:
config_mod._bootstrap_done = original_done
config_mod._original_langsmith_project = original_ls
def test_bootstrap_keyless_tracing_leaves_project_unset(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Keyless tracing is disabled first, so no default project is applied.
Regression guard for the ordering between `_disable_orphaned_tracing`
and `_apply_default_langsmith_project`: a tracing flag with no
resolvable key must be flipped off *before* the default runs, so
`LANGSMITH_PROJECT` is left unset (tracing never starts) rather than
pointed at `deepagents-code`.
"""
import os
import deepagents_code.config as config_mod
from deepagents_code.config import _ensure_bootstrap
original_done = config_mod._bootstrap_done
original_ls = config_mod._original_langsmith_project
config_mod._bootstrap_done = False
try:
monkeypatch.setenv("LANGSMITH_TRACING", "true")
monkeypatch.delenv("LANGSMITH_API_KEY", raising=False)
monkeypatch.delenv("LANGCHAIN_API_KEY", raising=False)
monkeypatch.delenv("LANGSMITH_ENDPOINT", raising=False)
monkeypatch.delenv("LANGCHAIN_ENDPOINT", raising=False)
monkeypatch.delenv("LANGSMITH_PROJECT", raising=False)
monkeypatch.delenv("DEEPAGENTS_CODE_LANGSMITH_PROJECT", raising=False)
with (
patch("deepagents_code.config._load_dotenv"),
patch(
"deepagents_code.project_utils.get_server_project_context",
return_value=None,
),
patch(
"deepagents_code.config._has_langsmith_profile_credentials",
return_value=False,
),
patch(
"deepagents_code.config._has_langsmith_profile_custom_endpoint",
return_value=False,
),
):
_ensure_bootstrap()
assert "LANGSMITH_PROJECT" not in os.environ
assert os.environ["LANGSMITH_TRACING"] == "false"
finally:
config_mod._bootstrap_done = original_done
config_mod._original_langsmith_project = original_ls
class TestApplyDefaultLangsmithProject:
"""Tests for _apply_default_langsmith_project()."""
def test_defaults_when_tracing_on_and_unset(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Tracing on with no project set routes to the default project."""
import os
from deepagents_code.config_manifest import LANGSMITH_PROJECT_DEFAULT
monkeypatch.setenv("LANGSMITH_TRACING", "true")
monkeypatch.delenv("LANGSMITH_PROJECT", raising=False)
_apply_default_langsmith_project()
assert os.environ["LANGSMITH_PROJECT"] == LANGSMITH_PROJECT_DEFAULT
def test_noop_when_project_already_set(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""An existing LANGSMITH_PROJECT is never overwritten."""
import os
monkeypatch.setenv("LANGSMITH_TRACING", "true")
monkeypatch.setenv("LANGSMITH_PROJECT", "user-project")
_apply_default_langsmith_project()
assert os.environ["LANGSMITH_PROJECT"] == "user-project"
def test_noop_when_tracing_off(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""No default is applied when tracing is not enabled."""
import os
monkeypatch.delenv("LANGSMITH_TRACING", raising=False)
monkeypatch.delenv("LANGCHAIN_TRACING_V2", raising=False)
monkeypatch.delenv("LANGSMITH_PROJECT", raising=False)
_apply_default_langsmith_project()
assert "LANGSMITH_PROJECT" not in os.environ
class TestFindDotenvFromStartPath:
"""Tests for _find_dotenv_from_start_path."""