mirror of
https://github.com/langchain-ai/deepagentsjs.git
synced 2026-07-25 12:55:56 -04:00
Hunter Lovell
d49b175dbd
## Summary This PR introduces a first-class aggregate eval entrypoint (`evals/all`) that runs every existing eval suite through one top-level LangSmith/Vitest describe block. To support that cleanly, eval suites were refactored into a reusable two-file shape: `index.ts` defines suite registration logic (`define...Suite(runner)`), and `eval.test.ts` owns LangSmith attribution (`ls.describe(...)`). This removes reliance on ad-hoc global override wiring and gives the all-evals package a single explicit integration surface. ## Changes ### Add aggregate all-evals package - Added `@deepagents/eval-all` package: - `evals/all/eval.test.ts` imports and invokes every suite registration function. - `evals/all/vitest.config.ts` runs only the aggregate `eval.test.ts` entrypoint. - Added package README and package metadata. ### Refactor all eval suites to reusable registration modules - Standardized suite implementation files to `index.ts`. - Standardized eval wrappers to `eval.test.ts`. - For Oolong datasets, standardized to `<dataset>.ts` + `<dataset>.eval.test.ts`. - Moved `ls.describe` dataset/project attribution into wrapper test files so registration modules focus on test definitions only. - Ensured suite registration functions accept `runner` as an argument across suites. ### Ensure all existing evals are included in aggregate execution - `evals/all/eval.test.ts` now imports and calls all suite registration functions for: - core eval suites under `evals/*` - oolong dataset suites under `evals/oolong/datasets/*` ### Documentation updates - Updated `evals/README.md` examples and structure references for `index.ts` + `eval.test.ts` layout. - Updated `evals/all/README.md` to document aggregate-run architecture. ### Lockfile updates - Updated `pnpm-lock.yaml` to include the new `evals/all` workspace package and dependencies.
41 lines
1.0 KiB
TypeScript
41 lines
1.0 KiB
TypeScript
import * as ls from "langsmith/vitest";
|
|
import { expect } from "vitest";
|
|
import type { EvalRunner } from "@deepagents/evals";
|
|
|
|
export function basicSuite(runner: EvalRunner): void {
|
|
ls.test(
|
|
"system prompt: custom system prompt",
|
|
{
|
|
inputs: { query: "what is your name" },
|
|
referenceOutputs: { expectedText: "Foo Bar" },
|
|
},
|
|
async ({ inputs }) => {
|
|
const result = await runner
|
|
.extend({ systemPrompt: "Your name is Foo Bar." })
|
|
.run({ query: inputs.query });
|
|
|
|
expect(result).toHaveFinalTextContaining("Foo Bar");
|
|
ls.logFeedback({
|
|
key: "agent_steps",
|
|
score: result.steps.length,
|
|
});
|
|
},
|
|
);
|
|
|
|
ls.test(
|
|
"avoid unnecessary tool calls",
|
|
{
|
|
inputs: { query: "What is 2+2? Answer with just the number." },
|
|
},
|
|
async ({ inputs }) => {
|
|
const result = await runner.run({ query: inputs.query });
|
|
|
|
expect(result).toHaveFinalTextContaining("4");
|
|
ls.logFeedback({
|
|
key: "agent_steps",
|
|
score: result.steps.length,
|
|
});
|
|
},
|
|
);
|
|
}
|