mirror of
https://github.com/langchain-ai/deepagents.git
synced 2026-07-21 17:25:26 -04:00
Nick Hollon
671c578006
## What Replace the serialized per-provider / per-category eval jobs with one dynamically-drained **per-model pool spanning all three categories**. A shard is a 1-task unit carrying its own `(category, dataset, agent_impl, task)`; `unified_prep` emits a per-model flat matrix and one `eval` job drains it. `max_parallel`/`model_parallel` are **derived** from `concurrency`+`rollouts`; `MAX_SHARDS` 64→200 with proportional per-category packing above the cap. Per-category aggregation keeps the tier-2 scorecard (`aggregate_unified.py`) unchanged. `harbor.yml`'s single-dataset path decouples `shard_parallel` from `n_shards` for dynamic dispatch (no-op at the default `n_shards=10`). ## Validated (CI, docker sandbox, bare harness) - **Lite single-model** (`gpt-5.6-terra`): complete scorecard across all 3 categories, macro pass@3 0.479 (context 0.0 → 0.625 after the fixes below). - **Multi-model** (`gpt-5.6-terra` + `anthropic:claude-sonnet-5`): both per-model pools run concurrently under `model_parallel`, total-job guard holds (68 shards), and the non-OpenAI agent runs with the OpenAI judge. ## Fixes folded in (found during the dry-runs) - Context corpus populate + judge `OPENAI_API_KEY` now read the per-shard `matrix.*` values (they were gated on the leaf inputs, which the flat design leaves empty). - LangSmith experiment name scoped per category (fixes the cross-dataset run double-count). ## Dependency Needs the harbor build-lock fix ([harbor-framework/harbor#2340](https://github.com/harbor-framework/harbor/pull/2340)) via `harbor_package_override` until it's released — pin `nick-hollon-lc/harbor@27a6eac8` meanwhile. --------- Co-authored-by: Mason Daugherty <github@mdrxy.com>
76 lines
3.9 KiB
Python
76 lines
3.9 KiB
Python
"""Frozen 'lite' task subsets per category for the unified evals `profile=lite`.
|
|
|
|
A high-signal, low-cost slice: fewer tasks, FULL rollouts. Tasks are biased to the
|
|
difficulty frontier (partial-pass / hard-but-solvable) measured on a weaker model
|
|
(gpt-5.6-luna); saturated (all-pass) and verifier-unstable tasks are excluded.
|
|
|
|
Names are the exact harbor `--include-task-name` filters per category:
|
|
autonomous -> registry ref `harbor-index/<task>`
|
|
conversation -> `sierra-research/tau3-bench__<task_id>` (same form as tau3_subset)
|
|
context -> local task dir basename `cb-cloud-<n>`
|
|
|
|
`include_tasks(category)` returns the space-separated string the workflow passes to
|
|
`_harbor_run.yml`. Keep this list under review; re-calibrate as models/tasks change.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
LITE_TASKS: dict[str, list[str]] = {
|
|
# 15 — luna is weak here, so a rich frontier: partials + hard-but-solvable.
|
|
# Excludes the bix* bioinformatics tasks: their ~6 GB `chenzizhao/bixbench`
|
|
# image exhausts the Docker-sandbox runner disk (and fails the LangSmith
|
|
# builder). Re-add once lite runs on a sandbox that builds big images.
|
|
# `gpqadiamond-cope-rearrangement-products` and `swesmith-fix-oauth1-header-params`
|
|
# replaced `replicationbench-find-galactic-vz-peaks` and `usaco-assign-cows-to-barns`:
|
|
# both originals could run ~30-65 min (replicationbench's naive big-data script
|
|
# hits the 1h per-command timeout), defeating lite's low-cost goal. The swaps are
|
|
# <5 min and still frontier (partial-pass), re-picked on gpt-5.6-terra timing+signal.
|
|
"autonomous": [
|
|
"harbor-index/gpqadiamond-cope-rearrangement-products",
|
|
"harbor-index/swebenchverified-fix-span-selector-axes-limits",
|
|
"harbor-index/omnimath-find-perfect-square-functions",
|
|
"harbor-index/swesmith-fix-oauth1-header-params",
|
|
"harbor-index/featurebench-add-feature-mlflow-bedrock-autolog",
|
|
"harbor-index/build-word2vec-pipeline",
|
|
"harbor-index/tb-dna-insert",
|
|
"harbor-index/swebenchverified-fix-django-mti-parent-link",
|
|
"harbor-index/arcagi2-grid-transform-8b7b",
|
|
"harbor-index/labbench-habenula-fluorescence-change",
|
|
"harbor-index/labbench-read-asap2f-step-response",
|
|
"harbor-index/gso-speedup-pydantic-enum",
|
|
"harbor-index/swebenchpro-fix-file-suffix-chooser",
|
|
"harbor-index/spider2-dbt-airport-arrivals",
|
|
"harbor-index/arcagi2-grid-transform-a32d",
|
|
],
|
|
# 11 — luna is weak on banking (rich frontier); telecom is saturated (1 kept).
|
|
"conversation": [
|
|
"sierra-research/tau3-bench__tau3-banking_knowledge-task-043",
|
|
"sierra-research/tau3-bench__tau3-banking_knowledge-task-056",
|
|
"sierra-research/tau3-bench__tau3-banking_knowledge-task-093",
|
|
"sierra-research/tau3-bench__tau3-banking_knowledge-task-018",
|
|
"sierra-research/tau3-bench__tau3-banking_knowledge-task-029",
|
|
"sierra-research/tau3-bench__tau3-banking_knowledge-task-040",
|
|
"sierra-research/tau3-bench__tau3-banking_knowledge-task-048",
|
|
"sierra-research/tau3-bench__tau3-banking_knowledge-task-061",
|
|
"sierra-research/tau3-bench__tau3-banking_knowledge-task-072",
|
|
"sierra-research/tau3-bench__tau3-banking_knowledge-task-073",
|
|
"sierra-research/tau3-bench__tau3-telecom-service-issue-airplane-mode-on-break-apn-settings-lock-sim-card-pin-overdue-bill-suspension-unseat-sim-card-persona-none",
|
|
],
|
|
# 8 — luna is strong on context, so the frontier is thin: its partials + fails.
|
|
"context": [
|
|
"cb-cloud-10",
|
|
"cb-cloud-41",
|
|
"cb-cloud-47",
|
|
"cb-cloud-0",
|
|
"cb-cloud-26",
|
|
"cb-cloud-49",
|
|
"cb-cloud-15",
|
|
"cb-cloud-5",
|
|
],
|
|
}
|
|
|
|
|
|
def include_tasks(category: str) -> str:
|
|
"""Space-separated include-task filter string for a category, or '' if none."""
|
|
return " ".join(LITE_TASKS.get(category, []))
|