mirror of
https://github.com/langchain-ai/deepagents.git
synced 2026-07-21 17:25:26 -04:00
Nick Hollon
671c578006
## What Replace the serialized per-provider / per-category eval jobs with one dynamically-drained **per-model pool spanning all three categories**. A shard is a 1-task unit carrying its own `(category, dataset, agent_impl, task)`; `unified_prep` emits a per-model flat matrix and one `eval` job drains it. `max_parallel`/`model_parallel` are **derived** from `concurrency`+`rollouts`; `MAX_SHARDS` 64→200 with proportional per-category packing above the cap. Per-category aggregation keeps the tier-2 scorecard (`aggregate_unified.py`) unchanged. `harbor.yml`'s single-dataset path decouples `shard_parallel` from `n_shards` for dynamic dispatch (no-op at the default `n_shards=10`). ## Validated (CI, docker sandbox, bare harness) - **Lite single-model** (`gpt-5.6-terra`): complete scorecard across all 3 categories, macro pass@3 0.479 (context 0.0 → 0.625 after the fixes below). - **Multi-model** (`gpt-5.6-terra` + `anthropic:claude-sonnet-5`): both per-model pools run concurrently under `model_parallel`, total-job guard holds (68 shards), and the non-OpenAI agent runs with the OpenAI judge. ## Fixes folded in (found during the dry-runs) - Context corpus populate + judge `OPENAI_API_KEY` now read the per-shard `matrix.*` values (they were gated on the leaf inputs, which the flat design leaves empty). - LangSmith experiment name scoped per category (fixes the cross-dataset run double-count). ## Dependency Needs the harbor build-lock fix ([harbor-framework/harbor#2340](https://github.com/harbor-framework/harbor/pull/2340)) via `harbor_package_override` until it's released — pin `nick-hollon-lc/harbor@27a6eac8` meanwhile. --------- Co-authored-by: Mason Daugherty <github@mdrxy.com>
90 lines
2.9 KiB
Python
90 lines
2.9 KiB
Python
"""Enumerate a category's task names for the flat shard matrix.
|
|
|
|
Reuses the exact task-name resolution the run step uses:
|
|
- local (--path) datasets: subdir basenames with a task.toml (sorted, stable),
|
|
- synthetic datasets (e.g. "tau3-subset"): a committed task list, since the
|
|
name is not a registry package and would fail a registry lookup,
|
|
- registry datasets: PackageDatasetClient manifest task_ids -> get_name().
|
|
|
|
Registry lookups need harbor installed and network; the local path is pure
|
|
stdlib. Run as a script, main() emits newline-joined names.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import shard_matrix # noqa: E402
|
|
|
|
|
|
def local_task_names(local_dir: str) -> list[str]:
|
|
names = sorted(
|
|
entry.name
|
|
for entry in os.scandir(local_dir)
|
|
if entry.is_dir() and os.path.isfile(os.path.join(entry.path, "task.toml"))
|
|
)
|
|
if not names:
|
|
raise SystemExit(f"No local Harbor tasks (dirs with task.toml) under {local_dir}")
|
|
return names
|
|
|
|
|
|
def registry_task_names(dataset_ref: str) -> list[str]:
|
|
import asyncio
|
|
|
|
from harbor.registry.client.package import PackageDatasetClient
|
|
|
|
md = asyncio.run(
|
|
PackageDatasetClient().get_dataset_metadata(f"{dataset_ref}@latest")
|
|
)
|
|
names = [x for x in (shard_matrix.task_display_name(t) for t in md.task_ids) if x]
|
|
if not names:
|
|
raise SystemExit(
|
|
f"Resolved 0 usable task names from {dataset_ref}@latest "
|
|
f"({len(md.task_ids)} task ids); unexpected task-id shape."
|
|
)
|
|
return names
|
|
|
|
|
|
def tau3_subset_task_names() -> list[str]:
|
|
# "tau3-subset" is a synthetic curated view, not a registry package, so it
|
|
# cannot be resolved via the registry client. Pull its committed 30-task
|
|
# list from the same module the harbor leaf uses (single source of truth).
|
|
from deepagents_evals.tau3_subset import INCLUDE_TASKS
|
|
|
|
names = INCLUDE_TASKS.split()
|
|
if not names:
|
|
raise SystemExit(
|
|
"tau3-subset resolved to 0 tasks; "
|
|
"deepagents_evals.tau3_subset.INCLUDE_TASKS is empty."
|
|
)
|
|
return names
|
|
|
|
|
|
# Synthetic datasets resolve to a committed task list instead of a registry
|
|
# lookup, because their names are not `org/name` registry packages.
|
|
SYNTHETIC_DATASETS = {"tau3-subset": tau3_subset_task_names}
|
|
|
|
|
|
def main() -> int:
|
|
dataset_path = os.environ.get("ENUM_DATASET_PATH", "").strip()
|
|
dataset = os.environ.get("ENUM_DATASET", "").strip()
|
|
if dataset_path:
|
|
names = local_task_names(dataset_path)
|
|
elif dataset in SYNTHETIC_DATASETS:
|
|
names = SYNTHETIC_DATASETS[dataset]()
|
|
else:
|
|
names = registry_task_names(dataset)
|
|
out = os.environ.get("ENUM_OUTPUT")
|
|
text = "\n".join(names) + "\n"
|
|
if out:
|
|
with open(out, "w") as f:
|
|
f.write(text)
|
|
else:
|
|
sys.stdout.write(text)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|