diff --git a/deepeval/sqlite_store/__init__.py b/deepeval/sqlite_store/__init__.py index 101b0a620..8dd231e59 100644 --- a/deepeval/sqlite_store/__init__.py +++ b/deepeval/sqlite_store/__init__.py @@ -6,6 +6,7 @@ See `deepeval.sqlite_store.store` for the schema and implementation notes. from deepeval.sqlite_store.store import ( DB_FILENAME, INCLUDE_ROW_JSON_ENV_VAR, + MIGRATIONS, SCHEMA_SQL, SCHEMA_VERSION, connect, @@ -25,6 +26,7 @@ from deepeval.sqlite_store.store import ( __all__ = [ "DB_FILENAME", "INCLUDE_ROW_JSON_ENV_VAR", + "MIGRATIONS", "SCHEMA_SQL", "SCHEMA_VERSION", "connect", diff --git a/deepeval/sqlite_store/store.py b/deepeval/sqlite_store/store.py index 300aac7e8..92b1f6815 100644 --- a/deepeval/sqlite_store/store.py +++ b/deepeval/sqlite_store/store.py @@ -34,8 +34,9 @@ import datetime import json import os import sqlite3 +from types import MappingProxyType from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple from deepeval.constants import HIDDEN_DIR from deepeval.test_run.test_run import TestRun, TestRunEncoder @@ -160,6 +161,22 @@ CREATE INDEX IF NOT EXISTS idx_metric_data_owner ON metric_data(owner_type, owne # tests/test_core/test_sqlite_store.py fails if this string drifts from it. SCHEMA_SQL = _SCHEMA +# Ordered schema migrations, keyed by the `PRAGMA user_version` each one +# brings the database *to*. `_ensure_schema` applies every entry above the +# file's current version, in order, inside one transaction. Each value is the +# embedded copy of `sqlite/migrations/_*.sql`; parity tests keep them in +# sync. Never edit a shipped entry: add the next one and bump SCHEMA_VERSION. +_MIGRATIONS: Dict[int, str] = { + 1: _SCHEMA, # 0001_initial.sql +} +MIGRATIONS: Mapping[int, str] = MappingProxyType(_MIGRATIONS) + +if set(_MIGRATIONS) != set(range(1, SCHEMA_VERSION + 1)): # pragma: no cover + raise RuntimeError( + "deepeval.sqlite_store: _MIGRATIONS must have exactly one entry for " + f"every version 1..{SCHEMA_VERSION}, got {sorted(_MIGRATIONS)}." + ) + INCLUDE_ROW_JSON_ENV_VAR = "DEEPEVAL_SQLITE_INCLUDE_ROW_JSON" @@ -264,10 +281,30 @@ def _ensure_schema(conn: sqlite3.Connection) -> None: f"deepeval.db schema version {version} is newer than this " f"deepeval supports ({SCHEMA_VERSION}); please upgrade deepeval." ) - with conn: - conn.executescript(_SCHEMA) - # Future migrations: `if version < 2: ...` blocks go here. + # `executescript` issues a COMMIT first, so drive the transaction by hand: + # every pending step plus the version stamp land atomically, and a crash + # midway leaves the file at its old version for the next open to retry. + conn.execute("BEGIN") + try: + for step in range(version + 1, SCHEMA_VERSION + 1): + for statement in _split_statements(_MIGRATIONS[step]): + conn.execute(statement) conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + conn.execute("COMMIT") + except Exception: + conn.execute("ROLLBACK") + raise + + +def _split_statements(script: str) -> List[str]: + """Split a migration script into statements (`--` comments stripped). + + Migration files are plain DDL without string literals containing `;`, + so splitting on `;` is sufficient and keeps everything in one transaction + (`sqlite3.Cursor.executescript` would commit first). + """ + body = "\n".join(line.split("--", 1)[0] for line in script.splitlines()) + return [stmt.strip() for stmt in body.split(";") if stmt.strip()] # --------------------------------------------------------------------------- # @@ -328,7 +365,9 @@ def set_confident_test_run_id( (str(confident_test_run_id), int(run_id)), ) if cur.rowcount == 0: - raise LookupError(f"{db_path} has no test run with id {run_id}.") + raise LookupError( + f"{db_path} has no test run with id {run_id}." + ) finally: conn.close() diff --git a/docs/content/docs/command-line-interface.mdx b/docs/content/docs/command-line-interface.mdx index 4fa98d2ff..efc1c1e3e 100644 --- a/docs/content/docs/command-line-interface.mdx +++ b/docs/content/docs/command-line-interface.mdx @@ -74,7 +74,7 @@ deepeval reads settings from dotenv files in the current working directory, with deepeval also uses a legacy JSON keystore at `.deepeval/.deepeval` for **non-secret** keys. This keystore is treated as a fallback (dotenv/process env take precedence). Secrets are never written to the JSON keystore. :::tip -Point deepeval at a different directory with `ENV_DIR_PATH=/path/to/project`, or disable dotenv autoloading entirely with `DEEPEVAL_DISABLE_DOTENV=1` (useful in CI to avoid loading local `.env*` files on import). +Point `deepeval` at a different directory with `ENV_DIR_PATH=/path/to/project`, or disable dotenv autoloading entirely with `DEEPEVAL_DISABLE_DOTENV=1` (useful in CI to avoid loading local `.env*` files on import). ::: ## Core Commands @@ -174,14 +174,14 @@ Useful options: | Option | Description | | -------------------------------- | -------------------------------------------------------------- | -| `--verbose`, `-v` | Show verbose pytest output and turn on deepeval verbose mode. | +| `--verbose`, `-v` | Show verbose pytest output and turn on `deepeval` verbose mode.| | `--exit-on-first-failure`, `-x` | Stop after the first failed test. | | `--show-warnings`, `-w` | Show pytest warnings instead of disabling them. | | `--identifier`, `-id` | Attach an identifier to the test run. | | `--num-processes`, `-n` | Run tests with multiple pytest-xdist processes. | | `--repeat`, `-r` | Rerun each test case the specified number of times. | | `--use-cache`, `-c` | Use cached evaluation results when `--repeat` is not set. | -| `--ignore-errors`, `-i` | Continue when deepeval evaluation errors occur. | +| `--ignore-errors`, `-i` | Continue when `deepeval` evaluation errors occur. | | `--skip-on-missing-params`, `-s` | Skip test cases with missing metric parameters. | | `--display`, `-d` | Control final result display. Defaults to showing all results. | | `--mark`, `-m` | Run tests matching a pytest marker expression. | @@ -226,7 +226,7 @@ Useful options: | `--verbose`, `-v` | Turn on verbose logs for every metric. | | `--identifier`, `-i` | Attach an identifier to the test run. | | `--use-cache`, `-c` | Reuse cached metric results where possible. | -| `--ignore-errors` | Continue when deepeval evaluation errors occur. | +| `--ignore-errors` | Continue when `deepeval` evaluation errors occur. | | `--skip-on-missing-params`, `-s` | Skip test cases with missing metric parameters. | | `--display`, `-d` | Control final result display (`all`, `passing`, `failing`). | | `--max-concurrent` | Maximum number of metrics evaluated at once (default: 100). | @@ -405,8 +405,8 @@ npx deepeval diagnose The report shows: -- **Versions**: deepeval and the runtime it's running on. -- **Evaluation model**: the model deepeval will use when a metric (or synthesizer, simulator, etc.) is constructed without an explicit model, plus the reason it won. The default embedding model is listed alongside it, and the selected text-to-speech and speech-to-text models below those, with what each is used by. +- **Versions**: `deepeval` and the runtime it's running on. +- **Evaluation model**: the model `deepeval` will use when a metric (or synthesizer, simulator, etc.) is constructed without an explicit model, plus the reason it won. The default embedding model is listed alongside it, and the selected text-to-speech and speech-to-text models below those, with what each is used by. - **Configuration sources**: the precedence order every variable is resolved against, independently (process environment → `.env.local` → `.env.` → `.env` → JSON keystore → built-in defaults), which dotenv files were loaded, and where the keystore lives. - **Configured settings**: every deepeval setting that is explicitly set, with secrets masked (only the last 6 characters shown) and the **winning source** for each — the report uses the same resolvers as the runtime, so what you see is what your evals get. Storage settings are included here too: `DEEPEVAL_RESULTS_FOLDER` and [`DEEPEVAL_LOCAL_STORE`](/docs/environment-variables#sqlite-local-store), so if runs are landing in `deepeval.db` instead of `test_run_*.json` files (or vice versa), this is where to confirm which value won and where it came from. An unrecognised `DEEPEVAL_LOCAL_STORE` value falls back to `json` at run time with a warning on stderr, so a typo shows up here as the value that lost.A `DEEPEVAL_LOCAL_STORE=sqlite` that this Node version cannot honour (anything below Node 24) is reported as an ignored invalid value with the reason, rather than silently dropped. - **Confident AI**: login status, masked API key, data region, and the API endpoint. The OTEL endpoint is shown too, with warnings if either endpoint looks like it belongs to a different region than your API key. @@ -433,7 +433,7 @@ Use these commands to connect `deepeval` to **Confident AI** (`deepeval` Cloud) /> : Log in to Confident AI. The interactive flow opens a browser for authentication, then returns to the terminal to create an organization and - first project or select an existing project. DeepEval creates a dedicated + first project or select an existing project. `deepeval` creates a dedicated project API key and saves it as `CONFIDENT_API_KEY`. Use `--api-key` for CI, headless automation, or manual fallback. - : Remove your Confident AI credentials from local persistence — the JSON keystore and every dotenv file deepeval auto-loads (`.env`, `.env.`, `.env.local`). If `CONFIDENT_API_KEY` is exported by your shell itself, deepeval cannot unset it and will tell you to run `unset CONFIDENT_API_KEY`. @@ -475,7 +475,7 @@ npx deepeval set-confident-region EU --save=dotenv | `--quiet`, `-q` | Suppress printing to the terminal (useful for CI). | :::caution -Your API key is issued for a single region. Pointing deepeval at a region your key doesn't belong to will fail to authenticate. Run `diagnose` to see the region and endpoint currently in effect. +Your API key is issued for a single region. Pointing `deepeval` at a region your key doesn't belong to will fail to authenticate. Run `diagnose` to see the region and endpoint currently in effect. ::: ### `gate` @@ -517,7 +517,7 @@ All `set-*` / `unset-*` commands follow the same rules: - Non-secrets (model name, endpoint, deployment, etc.) may be mirrored into `.deepeval/.deepeval`. - Secrets (API keys) are never written to `.deepeval/.deepeval`. - Pass `--save=dotenv[:path]` to write settings (including secrets) to a dotenv file (default: `.env.local`). -- If `--save` is omitted, deepeval will use `DEEPEVAL_DEFAULT_SAVE` if set; otherwise it won’t write a dotenv file (some commands like `login` still default to `.env.local`). +- If `--save` is omitted, `deepeval` will use `DEEPEVAL_DEFAULT_SAVE` if set; otherwise it won’t write a dotenv file (some commands like `login` still default to `.env.local`). - Unsetting one provider only removes that provider’s keys. Leftover credentials from other providers (e.g. `OPENAI_API_KEY`) don’t select anything on their own — the `USE_*` flags decide which provider is active. :::tip @@ -820,7 +820,7 @@ For provider-specific flags, run any of the `set-*` commands above with `--help` ## Common Issues - **Nothing printed?** For `set-*` / `unset-*` / `set-debug`, a clean exit with no output often means you are passing the `--quiet` / `-q` flag. -- **Provider still active after unsetting?** Unsetting turns off that provider's `USE_*` flag only; if another provider in the same family still has its flag on, it becomes the active one for that family. With no flag enabled in a family, deepeval falls back to OpenAI for it. To force a provider, run the corresponding `set-` command. -- **Dotenv edits not picked up?** deepeval loads dotenv files from the current working directory by default, or `ENV_DIR_PATH` if set. Ensure your process runs in that context. +- **Provider still active after unsetting?** Unsetting turns off that provider's `USE_*` flag only; if another provider in the same family still has its flag on, it becomes the active one for that family. With no flag enabled in a family, `deepeval` falls back to OpenAI for it. To force a provider, run the corresponding `set-` command. +- **Dotenv edits not picked up?** `deepeval` loads dotenv files from the current working directory by default, or `ENV_DIR_PATH` if set. Ensure your process runs in that context. If you’re still stuck, the dedicated [Troubleshooting](/docs/troubleshooting) page covers deeper debugging (TLS errors, logging, timeouts, dotenv loading, and config caching). diff --git a/docs/content/docs/environment-variables.mdx b/docs/content/docs/environment-variables.mdx index cdcca32c4..a1b49ad2e 100644 --- a/docs/content/docs/environment-variables.mdx +++ b/docs/content/docs/environment-variables.mdx @@ -108,7 +108,7 @@ npx deepeval test run llm_app.test.ts The database lives at `deepeval.db` inside `DEEPEVAL_RESULTS_FOLDER` when set, otherwise inside the `.deepeval` cache folder. The timestamped `test_run_*.json` export is skipped in this mode; `.deepeval/.latest_test_run.json` is still written because `npx deepeval view` uploads from it. :::caution[Node.js 24 or newer] -The TypeScript store is built on the runtime's built-in `node:sqlite` module, which is only stable from Node 24. On an older Node, setting `DEEPEVAL_LOCAL_STORE=sqlite` is rejected with an error telling you the version you are on and to either upgrade or switch back to `json` — deepeval never silently falls back, so your results are not quietly written somewhere else. +The TypeScript store is built on the runtime's built-in `node:sqlite` module, which is only stable from Node 24. On an older Node, setting `DEEPEVAL_LOCAL_STORE=sqlite` is rejected with an error telling you the version you are on and to either upgrade or switch back to `json` — `deepeval` never silently falls back, so your results are not quietly written somewhere else. ::: diff --git a/docs/content/docs/evaluation-local-backend-storage.mdx b/docs/content/docs/evaluation-local-backend-storage.mdx index 4f4a361ac..3e7e7cfba 100644 --- a/docs/content/docs/evaluation-local-backend-storage.mdx +++ b/docs/content/docs/evaluation-local-backend-storage.mdx @@ -24,10 +24,10 @@ The local backend gives you the raw runs. Comparing them across commits, sharing Finished test runs — hyperparameters, every test case, metric scores and reasons, and the traces and spans captured under them — are saved by one of two backends, chosen with the `DEEPEVAL_LOCAL_STORE` environment variable: -| Backend | `DEEPEVAL_LOCAL_STORE` | What you get | Best for | -| ------------------ | ---------------------- | ------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | -| JSON **(default)** | `json` or unset | One `test_run_.json` file per run in a folder you choose, plus a rolling copy of the latest run. | Diffing runs in git, pointing an AI agent at a folder, zero setup. | -| SQLite | `sqlite` | Every run appended to a single `deepeval.db`, with test cases, traces, spans and metrics broken out into tables. | Querying across runs, long histories, exporting to pandas / BI tools / other platforms. | +| Backend | `DEEPEVAL_LOCAL_STORE` | What you get | Best for | +| ------------------ | ---------------------- | ---------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| JSON **(default)** | `json` or unset | One `test_run_.json` file per run in a folder you choose, plus a rolling copy of the latest run. | Diffing runs in git, pointing an AI agent at a folder, zero setup. | +| SQLite | `sqlite` | Every run appended to a single `deepeval.db`, with test cases, traces, spans and metrics broken out into tables. | Querying across runs, long histories, exporting to pandas / BI tools / other platforms. | The default is **JSON**. Nothing changes for you unless you opt in to SQLite. Both backends store the same data, serialized with the same schema Confident AI uses, so switching later loses nothing. @@ -39,25 +39,25 @@ All of `deepeval`'s own files live under a hidden `.deepeval/` folder in your wo -| Path | What it is | -| -------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -| `.deepeval/.latest_run_full.json` | Rolling snapshot of the most recent run (JSON mode). Overwritten every run; this is what zero-arg `deepeval inspect` opens. | -| `/test_run_.json` | One file per run when `results_folder` / `DEEPEVAL_RESULTS_FOLDER` is set (JSON mode). | -| `/deepeval.db` or `.deepeval/deepeval.db` | The SQLite database (SQLite mode). Falls back to the cache folder when no results folder is set. | -| `.deepeval/.deepeval-cache.json` | Metric result cache used by `deepeval test run -c`. Not a record of runs. | -| `.deepeval/.deepeval` | Legacy JSON keystore for **non-secret** settings (model name, endpoint). API keys are never written here. | +| Path | What it is | +| --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| `.deepeval/.latest_run_full.json` | Rolling snapshot of the most recent run (JSON mode). Overwritten every run; this is what zero-arg `deepeval inspect` opens. | +| `/test_run_.json` | One file per run when `results_folder` / `DEEPEVAL_RESULTS_FOLDER` is set (JSON mode). | +| `/deepeval.db` or `.deepeval/deepeval.db` | The SQLite database (SQLite mode). Falls back to the cache folder when no results folder is set. | +| `.deepeval/.deepeval-cache.json` | Metric result cache used by `deepeval test run -c`. Not a record of runs. | +| `.deepeval/.deepeval` | Legacy JSON keystore for **non-secret** settings (model name, endpoint). API keys are never written here. | -| Path | What it is | -| -------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -| `.deepeval/.latest_test_run.json` | The most recent run, written in both modes. `npx deepeval view` uploads from it and zero-arg `npx deepeval inspect` opens it. | -| `/test_run_.json` | One file per run when `DEEPEVAL_RESULTS_FOLDER` is set (JSON mode). | -| `/deepeval.db` or `.deepeval/deepeval.db` | The SQLite database (SQLite mode). Falls back to the cache folder when no results folder is set. | -| `.deepeval/.deepeval-cache.json` | Metric result cache used by `npx deepeval test run -c`. Not a record of runs. | -| `.deepeval/.deepeval` | Legacy JSON keystore for **non-secret** settings (model name, endpoint). API keys are never written here. | +| Path | What it is | +| ------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `.deepeval/.latest_test_run.json` | The most recent run, written in both modes. `npx deepeval view` uploads from it and zero-arg `npx deepeval inspect` opens it. | +| `/test_run_.json` | One file per run when `DEEPEVAL_RESULTS_FOLDER` is set (JSON mode). | +| `/deepeval.db` or `.deepeval/deepeval.db` | The SQLite database (SQLite mode). Falls back to the cache folder when no results folder is set. | +| `.deepeval/.deepeval-cache.json` | Metric result cache used by `npx deepeval test run -c`. Not a record of runs. | +| `.deepeval/.deepeval` | Legacy JSON keystore for **non-secret** settings (model name, endpoint). API keys are never written here. | @@ -150,17 +150,17 @@ The database is created on first write using Node's built-in `node:sqlite` modul ### Schema -`deepeval.db` holds five tables, created the first time the file is opened and identical in both SDKs, so a Python and a TypeScript project can share one database. The DDL both SDKs embed comes from a single file in the repository, [`sqlite/schema.sql`](https://github.com/confident-ai/deepeval/blob/main/sqlite/schema.sql), and each SDK's test suite fails if its copy drifts from it. Each table below is shown as its `CREATE TABLE` statement — not for you to run, but because it is the most precise description of what you can query back. Print the live version from your own file with `sqlite3 deepeval.db .schema`. +`deepeval.db` holds five tables, created the first time the file is opened and identical in both SDKs, so a Python and a TypeScript project can share one database. The DDL both SDKs embed comes from a single file in the repository, [`sqlite/schema.sql`](https://github.com/confident-ai/deepeval/blob/main/sqlite/schema.sql) (the current full schema, built up from the versioned steps in [`sqlite/migrations/`](https://github.com/confident-ai/deepeval/tree/main/sqlite/migrations) — see [Schema versions](#schema-versions)), and each SDK's test suite fails if its copy drifts from it. Each table below is shown as its `CREATE TABLE` statement — not for you to run, but because it is the most precise description of what you can query back. Print the live version from your own file with `sqlite3 deepeval.db .schema`. Across all tables, the named columns are a **promoted subset** of each object, chosen so you can filter, join and aggregate without parsing JSON. The complete object — every field the JSON export would contain, in the same camelCase shape — is `payload_json`, and any field in it is reachable with SQLite's built-in [`json_extract`](https://sqlite.org/json1.html). -The `test_runs` row always has its `payload_json`: it is the whole run, and it is what and read back, so nothing is ever lost. The `test_cases`, `traces` and `spans` rows have the same column but leave it `NULL` unless you opt in: +The `test_runs` row always has its `payload_json`: it is the whole run — every test case, trace and span included — and it is the only thing and read back. They never reconstruct a run from the other tables, so nothing is ever lost and the option below has no effect on them. The `test_cases`, `traces` and `spans` rows have the same column but leave it `NULL` unless you opt in: ```bash export DEEPEVAL_SQLITE_INCLUDE_ROW_JSON=1 ``` -With it on, every child row carries its own full object, so fields that were not promoted — a test case's `turns`, a trace's `toolsCalled`, a span's `prompt` — can be queried right on that row instead of dug out of the run-level JSON. The examples marked *row JSON on* below need it. +With it on, every child row carries its own full object, so fields that were not promoted — a test case's `turns`, a trace's `toolsCalled`, a span's `prompt` — can be queried right on that row instead of dug out of the run-level JSON. The examples marked _row JSON on_ below need it. :::warning[Larger database] Row JSON stores everything twice: the run-level `payload_json` already contains every test case, trace and span, and each child row then repeats its slice of it. Expect `deepeval.db` to grow to roughly two to three times the size of the equivalent JSON export, and more for trace-heavy runs where spans carry large inputs and outputs. SQLite does not compress, so leave this off unless you actually query those fields. @@ -381,6 +381,14 @@ ORDER BY m.score ASC LIMIT 10; Writes are single transactions with a 30-second busy timeout, so parallel `test run` workers can share one database. Any storage failure — read-only filesystem, locked file — is reported as a warning after the results are printed rather than failing the evaluation. +### Schema versions + +`deepeval.db` records its schema version in SQLite's built-in `PRAGMA user_version`. When a newer `deepeval` opens an older file it upgrades it in place — every pending migration plus the version stamp run in one transaction, the first time the file is opened — so there is nothing to run by hand, and a failed upgrade leaves the file exactly as it was. + +Upgrades are forward-only: an older `deepeval` refuses to open a file written by a newer one and asks you to upgrade. Copying a database between machines therefore only requires the reader to be at least as new as the writer. + +The migrations live in [`sqlite/migrations/`](https://github.com/confident-ai/deepeval/tree/main/sqlite/migrations) (`0001_initial.sql`, then one file per later version). The run itself in `test_runs.payload_json` is versioned by the `TestRun` model, not by the table schema — that is what keeps re-reading or uploading old runs independent of local schema changes. + ### Exporting to other platforms Because it is a plain SQLite file, `deepeval.db` opens in anything that speaks SQL, which is the main reason to pick this backend over JSON. A few common destinations: @@ -423,7 +431,9 @@ import { resolveDbPath } from "deepeval/sqlite-store"; const db = new DatabaseSync(resolveDbPath(), { readOnly: true }); const scores = db - .prepare("SELECT test_run_id, name, score, success FROM metric_data WHERE owner_type = 'test_case'") + .prepare( + "SELECT test_run_id, name, score, success FROM metric_data WHERE owner_type = 'test_case'" + ) .all(); db.close(); @@ -440,7 +450,7 @@ console.table(scores); ## Reading results back -Whatever the backend, the [](/docs/command-line-interface#inspect) TUI opens a stored run and lets you walk its traces and spans with per-span scores and reasons: +Whatever the backend, the [](/docs/command-line-interface#inspect) TUI opens a stored run and lets you walk its traces and spans with per-span scores and reasons. With no arguments it opens the latest run from `deepeval.db` when `DEEPEVAL_LOCAL_STORE=sqlite`, otherwise the latest JSON file, and a folder argument prefers whichever of the two the current mode would have written. Either way it reads one thing — the run's full JSON, which in SQLite is the `test_runs.payload_json` column — so a run opens identically from either backend and the other tables are never involved: @@ -477,10 +487,14 @@ npx deepeval inspect --run-id 7 # SQLite: open a specific run Or load a run in code: ```typescript -import { listTestRuns, loadTestRunPayload, resolveDbPath } from "deepeval/sqlite-store"; +import { + listTestRuns, + loadTestRunPayload, + resolveDbPath, +} from "deepeval/sqlite-store"; const db = resolveDbPath(); -const runs = listTestRuns(db, 10); // newest first, summary columns only +const runs = listTestRuns(db, 10); // newest first, summary columns only const { runId, payload } = loadTestRunPayload(db); // latest run ``` @@ -488,7 +502,9 @@ const { runId, payload } = loadTestRunPayload(db); // latest run - reports which `DEEPEVAL_LOCAL_STORE` and `DEEPEVAL_RESULTS_FOLDER` values won and where they came from, which is the first thing to check if runs are landing somewhere you did not expect. + reports which `DEEPEVAL_LOCAL_STORE` +and `DEEPEVAL_RESULTS_FOLDER` values won and where they came from, which is the first +thing to check if runs are landing somewhere you did not expect. ## What is not stored locally diff --git a/sqlite/migrations/0001_initial.sql b/sqlite/migrations/0001_initial.sql new file mode 100644 index 000000000..87c1df19e --- /dev/null +++ b/sqlite/migrations/0001_initial.sql @@ -0,0 +1,105 @@ +-- Migration 0001: initial schema (brings PRAGMA user_version to 1). +-- +-- Applied by both SDKs when they open a deepeval.db whose user_version is 0. +-- Never edit a migration after it has shipped; add the next NNNN_.sql +-- and update sqlite/schema.sql to match the result of applying them all. + +CREATE TABLE IF NOT EXISTS test_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL, + identifier TEXT, + test_file TEXT, + dataset_alias TEXT, + dataset_id TEXT, + test_passed INTEGER, + test_failed INTEGER, + run_duration REAL, + evaluation_cost REAL, + official INTEGER NOT NULL DEFAULT 0, + confident_test_run_id TEXT, -- set once the run is uploaded to Confident AI + hyperparameters_json TEXT, + prompts_json TEXT, + metrics_scores_json TEXT, + payload_json TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_test_runs_confident ON test_runs(confident_test_run_id); + +CREATE TABLE IF NOT EXISTS test_cases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + test_run_id INTEGER NOT NULL REFERENCES test_runs(id) ON DELETE CASCADE, + kind TEXT NOT NULL, -- 'single-turn' | 'multi-turn' + "order" INTEGER, + name TEXT, + input TEXT, + actual_output TEXT, + expected_output TEXT, + success INTEGER, + run_duration REAL, + evaluation_cost REAL, + tags_json TEXT, + payload_json TEXT -- full object; only with DEEPEVAL_SQLITE_INCLUDE_ROW_JSON=1 +); +CREATE INDEX IF NOT EXISTS idx_test_cases_run ON test_cases(test_run_id); + +CREATE TABLE IF NOT EXISTS traces ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + test_run_id INTEGER NOT NULL REFERENCES test_runs(id) ON DELETE CASCADE, + test_case_id INTEGER NOT NULL REFERENCES test_cases(id) ON DELETE CASCADE, + uuid TEXT NOT NULL, + name TEXT, + status TEXT, + start_time TEXT, + end_time TEXT, + thread_id TEXT, + user_id TEXT, + environment TEXT, + input_json TEXT, + output_json TEXT, + metadata_json TEXT, + tags_json TEXT, + payload_json TEXT -- full object; only with DEEPEVAL_SQLITE_INCLUDE_ROW_JSON=1 +); +CREATE INDEX IF NOT EXISTS idx_traces_run ON traces(test_run_id); +CREATE INDEX IF NOT EXISTS idx_traces_case ON traces(test_case_id); + +CREATE TABLE IF NOT EXISTS spans ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + trace_id INTEGER NOT NULL REFERENCES traces(id) ON DELETE CASCADE, + uuid TEXT NOT NULL, + parent_uuid TEXT, + type TEXT NOT NULL, + name TEXT, + status TEXT, + start_time TEXT, + end_time TEXT, + error TEXT, + model TEXT, + provider TEXT, + input_token_count REAL, + output_token_count REAL, + input_json TEXT, + output_json TEXT, + payload_json TEXT -- full object; only with DEEPEVAL_SQLITE_INCLUDE_ROW_JSON=1 +); +CREATE INDEX IF NOT EXISTS idx_spans_trace_parent ON spans(trace_id, parent_uuid); + +CREATE TABLE IF NOT EXISTS metric_data ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + test_run_id INTEGER NOT NULL REFERENCES test_runs(id) ON DELETE CASCADE, + owner_type TEXT NOT NULL, + owner_id INTEGER NOT NULL, + name TEXT NOT NULL, + score REAL, + threshold REAL, + success INTEGER, + flaky INTEGER NOT NULL DEFAULT 0, + strict_mode INTEGER NOT NULL DEFAULT 0, + reason TEXT, + error TEXT, + evaluation_model TEXT, + evaluation_cost REAL, + input_tokens INTEGER, + output_tokens INTEGER +); +CREATE INDEX IF NOT EXISTS idx_metric_data_run_name ON metric_data(test_run_id, name); +CREATE INDEX IF NOT EXISTS idx_metric_data_owner ON metric_data(owner_type, owner_id); diff --git a/sqlite/schema.sql b/sqlite/schema.sql index 090e55b83..cecd5ddb4 100644 --- a/sqlite/schema.sql +++ b/sqlite/schema.sql @@ -1,12 +1,18 @@ -- SQLite schema for deepeval's local backend store (`DEEPEVAL_LOCAL_STORE=sqlite`). -- --- This file is the single source of truth for the DDL that both SDKs embed: --- Python: deepeval/sqlite_store/store.py (_SCHEMA) --- TypeScript: typescript/src/sqlite-store/store.ts (SCHEMA) --- Parity tests in each SDK fail if the embedded copy drifts from this file, and --- the docs page (docs/content/docs/evaluation-local-backend-storage.mdx) is --- checked against it too. Edit here first, then mirror into both SDKs and bump --- SCHEMA_VERSION. +-- This file is the FULL CURRENT schema: what you get by applying every file in +-- sqlite/migrations/ in order to an empty database. Both SDKs embed the +-- migrations, and a parity test in each fails if they drift: +-- Python: deepeval/sqlite_store/store.py (_MIGRATIONS, SCHEMA_SQL) +-- TypeScript: typescript/src/sqlite-store/store.ts (MIGRATIONS, SCHEMA) +-- A further test checks that migrations-applied-in-order == this file, and the +-- docs page (docs/content/docs/evaluation-local-backend-storage.mdx) is checked +-- against this file too. +-- +-- To change the schema: add sqlite/migrations/NNNN_.sql (NNNN = the +-- user_version it brings the database to), mirror it into both SDKs' migration +-- registries, bump SCHEMA_VERSION, then update this file to the new full +-- schema. Never edit a migration that has shipped. -- -- The `test_runs` row always keeps the complete serialized run in `payload_json` -- (that is what `inspect` reloads). `test_cases`, `traces` and `spans` have the diff --git a/tests/test_core/test_sqlite_store.py b/tests/test_core/test_sqlite_store.py index 4c599a920..71ebe4e51 100644 --- a/tests/test_core/test_sqlite_store.py +++ b/tests/test_core/test_sqlite_store.py @@ -212,7 +212,9 @@ class TestResolveDbPath: monkeypatch.delenv("DEEPEVAL_RESULTS_FOLDER", raising=False) from deepeval.constants import HIDDEN_DIR - assert sqlite_store.resolve_db_path() == Path(HIDDEN_DIR) / "deepeval.db" + assert ( + sqlite_store.resolve_db_path() == Path(HIDDEN_DIR) / "deepeval.db" + ) def test_results_folder_and_subfolder(self, tmp_path: Path): got = sqlite_store.resolve_db_path(str(tmp_path), "sweep") @@ -252,8 +254,16 @@ class TestSchema: r[0] for r in _q(db, "SELECT name FROM sqlite_master WHERE type='table'") } - assert {"test_runs", "test_cases", "traces", "spans", "metric_data"} <= tables - assert _q(db, "PRAGMA user_version")[0][0] == sqlite_store.SCHEMA_VERSION + assert { + "test_runs", + "test_cases", + "traces", + "spans", + "metric_data", + } <= tables + assert ( + _q(db, "PRAGMA user_version")[0][0] == sqlite_store.SCHEMA_VERSION + ) def test_reopen_is_idempotent(self, tmp_path: Path): db = tmp_path / "deepeval.db" @@ -264,15 +274,104 @@ class TestSchema: def test_newer_schema_is_rejected(self, tmp_path: Path): db = tmp_path / "deepeval.db" with closing(sqlite3.connect(str(db))) as conn: - conn.execute(f"PRAGMA user_version = {sqlite_store.SCHEMA_VERSION + 1}") + conn.execute( + f"PRAGMA user_version = {sqlite_store.SCHEMA_VERSION + 1}" + ) with pytest.raises(sqlite3.OperationalError): sqlite_store.connect(db) +_PROBE_MIGRATION = "ALTER TABLE test_runs ADD COLUMN _probe TEXT;" + + +def _columns(db: Path, table: str) -> set: + with closing(sqlite3.connect(str(db))) as conn: + return {r[1] for r in conn.execute(f"PRAGMA table_info({table})")} + + +def _user_version(db: Path) -> int: + with closing(sqlite3.connect(str(db))) as conn: + return conn.execute("PRAGMA user_version").fetchone()[0] + + +class TestSchemaUpgrade: + """Simulate a future release by patching SCHEMA_VERSION=2 + one ALTER step.""" + + @pytest.fixture + def future_release(self, monkeypatch): + """Patch the module the way a release with one more migration would.""" + + def activate(): + store = sqlite_store.store + monkeypatch.setattr(store, "SCHEMA_VERSION", 2) + monkeypatch.setitem(store._MIGRATIONS, 2, _PROBE_MIGRATION) + return store + + return activate + + def test_existing_v1_file_gets_only_step_2( + self, tmp_path: Path, future_release + ): + db = tmp_path / "deepeval.db" + # A genuine v1 file with data, written by "today's" release. + sqlite_store.write_test_run(_make_test_run(with_cases=False), db) + assert _user_version(db) == 1 + assert "_probe" not in _columns(db, "test_runs") + + store = future_release() + # Step 1 must NOT re-run (harmless here thanks to IF NOT EXISTS, but + # the loop must start at version + 1). Prove it by making step 1 blow + # up if executed. + store._MIGRATIONS[1] = "SELECT RAISE(ABORT, 'step 1 re-ran');" + try: + sqlite_store.connect(db).close() + finally: + store._MIGRATIONS[1] = sqlite_store.SCHEMA_SQL + + assert _user_version(db) == 2 + assert "_probe" in _columns(db, "test_runs") + # Existing data survives the in-place upgrade. + assert len(sqlite_store.list_test_runs(db)) == 1 + + def test_second_connect_is_a_noop(self, tmp_path: Path, future_release): + db = tmp_path / "deepeval.db" + future_release() + sqlite_store.connect(db).close() + assert _user_version(db) == 2 + # A second ALTER of the same column would fail if the loop re-ran. + sqlite_store.connect(db).close() + assert _user_version(db) == 2 + + def test_fresh_file_applies_all_steps(self, tmp_path: Path, future_release): + db = tmp_path / "deepeval.db" + future_release() + sqlite_store.connect(db).close() + assert _user_version(db) == 2 + cols = _columns(db, "test_runs") + assert {"id", "payload_json", "_probe"} <= cols + + def test_failed_step_rolls_back_and_keeps_old_version( + self, tmp_path: Path, future_release + ): + db = tmp_path / "deepeval.db" + sqlite_store.connect(db).close() # v1 file + store = future_release() + store._MIGRATIONS[2] = ( + "ALTER TABLE test_runs ADD COLUMN _probe TEXT;" + "ALTER TABLE no_such_table ADD COLUMN x TEXT;" + ) + with pytest.raises(sqlite3.OperationalError): + sqlite_store.connect(db) + assert _user_version(db) == 1 + assert "_probe" not in _columns(db, "test_runs") + + class TestWriteTestRun: def test_round_trip_payload_equals_model_dump(self, tmp_path: Path): db = tmp_path / "deepeval.db" - run = _make_test_run(hyperparameters={"model": "x", "t": 0.2}, identifier="base") + run = _make_test_run( + hyperparameters={"model": "x", "t": 0.2}, identifier="base" + ) run_id = sqlite_store.write_test_run(run, db) assert run_id == 1 @@ -322,9 +421,9 @@ class TestWriteTestRun: assert cases == _q(db, "SELECT count(*) FROM test_cases")[0][0] > 0 assert traces == _q(db, "SELECT count(*) FROM traces")[0][0] > 0 assert spans == _q(db, "SELECT count(*) FROM spans")[0][0] > 0 - (uuid, payload) = _q( - db, "SELECT uuid, payload_json FROM traces LIMIT 1" - )[0] + uuid, payload = _q(db, "SELECT uuid, payload_json FROM traces LIMIT 1")[ + 0 + ] assert json.loads(payload)["uuid"] == uuid @pytest.mark.parametrize("raw", ["0", "false", "no", ""]) @@ -334,9 +433,7 @@ class TestWriteTestRun: sqlite_store.write_test_run(_make_test_run(), db) assert self._row_json_counts(db) == (0, 0, 0) - def test_row_json_argument_overrides_env( - self, tmp_path: Path, monkeypatch - ): + def test_row_json_argument_overrides_env(self, tmp_path: Path, monkeypatch): monkeypatch.setenv(sqlite_store.INCLUDE_ROW_JSON_ENV_VAR, "1") db = tmp_path / "deepeval.db" sqlite_store.write_test_run( @@ -351,8 +448,12 @@ class TestWriteTestRun: def test_confident_test_run_id_null_until_set(self, tmp_path: Path): db = tmp_path / "deepeval.db" run_id = sqlite_store.write_test_run(_make_test_run(), db) - assert _q(db, "SELECT confident_test_run_id FROM test_runs")[0] == (None,) - assert sqlite_store.list_test_runs(db)[0]["confident_test_run_id"] is None + assert _q(db, "SELECT confident_test_run_id FROM test_runs")[0] == ( + None, + ) + assert ( + sqlite_store.list_test_runs(db)[0]["confident_test_run_id"] is None + ) sqlite_store.set_confident_test_run_id(db, run_id, "cai_abc123") assert _q(db, "SELECT confident_test_run_id FROM test_runs")[0] == ( @@ -395,14 +496,25 @@ class TestWriteTestRun: rows = _q( db, 'SELECT kind, "order", name, input, actual_output, expected_output, ' - "success, tags_json FROM test_cases ORDER BY \"order\"", + 'success, tags_json FROM test_cases ORDER BY "order"', ) assert rows[0][:7] == ( - "single-turn", 0, "case-0", "What is 1+1?", "2", "2", 1, + "single-turn", + 0, + "case-0", + "What is 1+1?", + "2", + "2", + 1, ) assert json.loads(rows[0][7]) == ["math"] assert rows[1][:6] == ( - "multi-turn", 1, "conv-0", "Book a flight", None, "Flight booked", + "multi-turn", + 1, + "conv-0", + "Book a flight", + None, + "Flight booked", ) def test_traces_and_spans_flattened(self, tmp_path: Path): @@ -414,9 +526,17 @@ class TestWriteTestRun: "SELECT uuid, name, thread_id, user_id, environment, tags_json, " "test_case_id FROM traces", ) - assert trace[:5] == ("trace-1", "my-agent", "thread-1", "user-1", "testing") + assert trace[:5] == ( + "trace-1", + "my-agent", + "thread-1", + "user-1", + "testing", + ) assert json.loads(trace[5]) == ["a", "b"] - (case_id,) = _q(db, "SELECT id FROM test_cases WHERE kind='single-turn'")[0] + (case_id,) = _q( + db, "SELECT id FROM test_cases WHERE kind='single-turn'" + )[0] assert trace[6] == case_id spans = _q( @@ -427,12 +547,21 @@ class TestWriteTestRun: ) by_uuid = {s[0]: s for s in spans} assert set(by_uuid) == { - "span-root", "span-llm", "span-tool", "span-ret", "span-base", + "span-root", + "span-llm", + "span-tool", + "span-ret", + "span-base", } assert by_uuid["span-root"][1] is None assert by_uuid["span-root"][2] == "agent" assert by_uuid["span-llm"][1:7] == ( - "span-root", "llm", "gpt-4o-mini", "openai", 12.0, 34.0, + "span-root", + "llm", + "gpt-4o-mini", + "openai", + 12.0, + 34.0, ) assert by_uuid["span-tool"][7:] == ("boom", "ERRORED") assert by_uuid["span-base"][1] == "span-llm" @@ -468,7 +597,9 @@ class TestWriteTestRun: (span_owner_id,) = _q( db, "SELECT owner_id FROM metric_data WHERE owner_type='span'" )[0] - (span_uuid,) = _q(db, "SELECT uuid FROM spans WHERE id = ?", span_owner_id)[0] + (span_uuid,) = _q( + db, "SELECT uuid FROM spans WHERE id = ?", span_owner_id + )[0] assert span_uuid == "span-root" def test_cross_run_sql_query(self, tmp_path: Path): @@ -493,7 +624,8 @@ class TestWriteTestRun: def worker(i: int): try: sqlite_store.write_test_run( - _make_test_run(hyperparameters={"i": i}, with_cases=False), db + _make_test_run(hyperparameters={"i": i}, with_cases=False), + db, ) except Exception as e: # pragma: no cover errors.append(e) @@ -629,7 +761,9 @@ class TestTestRunManagerIntegration: for temp in [0.0, 0.4, 0.8]: mgr = _TestRunManager() mgr.set_test_run( - _make_test_run(hyperparameters={"temperature": temp}, with_cases=False) + _make_test_run( + hyperparameters={"temperature": temp}, with_cases=False + ) ) mgr.configure_local_store(results_folder=str(tmp_path)) mgr.save_test_run_locally() @@ -642,16 +776,21 @@ class TestTestRunManagerIntegration: assert temps == [0.0, 0.4, 0.8] assert mgr.last_saved_run_id == 3 - def test_confident_id_stamped_after_upload(self, tmp_path: Path, monkeypatch): + def test_confident_id_stamped_after_upload( + self, tmp_path: Path, monkeypatch + ): """Logged-in + sqlite: the row is written first, then stamped with the - id Confident AI returned, so upload failures never lose the local copy.""" + id Confident AI returned, so upload failures never lose the local copy. + """ monkeypatch.setenv("DEEPEVAL_LOCAL_STORE", "sqlite") mgr = _TestRunManager() mgr.set_test_run(_make_test_run(with_cases=False)) mgr.configure_local_store(results_folder=str(tmp_path)) mgr.save_test_run_locally() db = tmp_path / "deepeval.db" - assert _q(db, "SELECT confident_test_run_id FROM test_runs")[0] == (None,) + assert _q(db, "SELECT confident_test_run_id FROM test_runs")[0] == ( + None, + ) mgr._record_confident_test_run_id("cai_from_post") assert _q(db, "SELECT confident_test_run_id FROM test_runs")[0] == ( @@ -678,7 +817,10 @@ class TestTestRunManagerIntegration: mgr.save_test_run_locally() mgr.last_saved_run_id = 999 # row that does not exist mgr._record_confident_test_run_id("cai_x") # must not raise - assert "could not record Confident AI test run id" in capsys.readouterr().err + assert ( + "could not record Confident AI test run id" + in capsys.readouterr().err + ) def test_read_only_env_is_noop(self, tmp_path: Path, monkeypatch): monkeypatch.setenv("DEEPEVAL_LOCAL_STORE", "sqlite") @@ -717,6 +859,7 @@ class TestTestRunManagerIntegration: _REPO_ROOT = Path(__file__).resolve().parents[2] _SHARED_SCHEMA = _REPO_ROOT / "sqlite" / "schema.sql" +_MIGRATIONS_DIR = _REPO_ROOT / "sqlite" / "migrations" _DOCS_PAGE = ( _REPO_ROOT / "docs" @@ -746,6 +889,51 @@ class TestSchemaParity: "sqlite/schema.sql; update both (and SCHEMA_VERSION) together." ) + def test_each_migration_matches_its_shared_file(self): + """`_MIGRATIONS[n]` is the embedded copy of `sqlite/migrations/000n_*.sql`.""" + assert set(sqlite_store.MIGRATIONS) == set( + range(1, sqlite_store.SCHEMA_VERSION + 1) + ) + for version, embedded in sqlite_store.MIGRATIONS.items(): + matches = sorted(_MIGRATIONS_DIR.glob(f"{version:04d}_*.sql")) + assert len(matches) == 1, ( + f"expected exactly one sqlite/migrations/{version:04d}_*.sql, " + f"got {[m.name for m in matches]}" + ) + assert _normalize_sql(embedded) == _normalize_sql( + matches[0].read_text(encoding="utf-8") + ), f"_MIGRATIONS[{version}] has drifted from {matches[0].name}" + + def test_no_orphan_migration_files(self): + on_disk = sorted( + int(p.name[:4]) + for p in _MIGRATIONS_DIR.glob("[0-9][0-9][0-9][0-9]_*.sql") + ) + assert on_disk == sorted(sqlite_store.MIGRATIONS) + + def test_migrations_in_order_equal_full_schema(self): + """Applying every migration to an empty DB == applying schema.sql.""" + + def objects(sql_scripts): + with closing(sqlite3.connect(":memory:")) as conn: + for script in sql_scripts: + conn.executescript(script) + rows = conn.execute( + "SELECT type, name, sql FROM sqlite_master " + "WHERE sql IS NOT NULL ORDER BY type, name" + ).fetchall() + return [(t, n, _normalize_sql(s)) for t, n, s in rows] + + migrated = objects( + p.read_text(encoding="utf-8") + for p in sorted(_MIGRATIONS_DIR.glob("[0-9][0-9][0-9][0-9]_*.sql")) + ) + snapshot = objects([_SHARED_SCHEMA.read_text(encoding="utf-8")]) + assert migrated == snapshot, ( + "sqlite/schema.sql is stale: it must equal the result of applying " + "sqlite/migrations/*.sql in order." + ) + def test_shared_schema_is_valid_sqlite(self): with closing(sqlite3.connect(":memory:")) as conn: conn.executescript(_SHARED_SCHEMA.read_text(encoding="utf-8")) @@ -755,7 +943,13 @@ class TestSchemaParity: "SELECT name FROM sqlite_master WHERE type='table'" ) } - assert tables >= {"test_runs", "test_cases", "traces", "spans", "metric_data"} + assert tables >= { + "test_runs", + "test_cases", + "traces", + "spans", + "metric_data", + } @pytest.mark.skipif( not _DOCS_PAGE.exists(), reason="docs page not in this checkout" diff --git a/typescript/src/sqlite-store/index.ts b/typescript/src/sqlite-store/index.ts index b2a499659..0da719c27 100644 --- a/typescript/src/sqlite-store/index.ts +++ b/typescript/src/sqlite-store/index.ts @@ -16,6 +16,7 @@ export { } from "@/sqlite-store/mode"; export { DB_FILENAME, + MIGRATIONS, SCHEMA, SCHEMA_VERSION, TestRunNotFoundError, diff --git a/typescript/src/sqlite-store/mode.ts b/typescript/src/sqlite-store/mode.ts index ca4810c04..a014270b1 100644 --- a/typescript/src/sqlite-store/mode.ts +++ b/typescript/src/sqlite-store/mode.ts @@ -6,7 +6,9 @@ import { DEEPEVAL_LOCAL_STORE } from "@/constants"; export const LOCAL_STORE_JSON = "json"; export const LOCAL_STORE_SQLITE = "sqlite"; -export type LocalStoreMode = typeof LOCAL_STORE_JSON | typeof LOCAL_STORE_SQLITE; +export type LocalStoreMode = + | typeof LOCAL_STORE_JSON + | typeof LOCAL_STORE_SQLITE; /** First Node major whose `node:sqlite` is stable (no experimental warning). */ export const MIN_NODE_MAJOR_FOR_SQLITE = 24; diff --git a/typescript/src/sqlite-store/store.ts b/typescript/src/sqlite-store/store.ts index b8599e53f..b9e044b08 100644 --- a/typescript/src/sqlite-store/store.ts +++ b/typescript/src/sqlite-store/store.ts @@ -179,6 +179,25 @@ CREATE INDEX IF NOT EXISTS idx_metric_data_run_name ON metric_data(test_run_id, CREATE INDEX IF NOT EXISTS idx_metric_data_owner ON metric_data(owner_type, owner_id); `; +/** + * Ordered schema migrations keyed by the `PRAGMA user_version` each brings the + * database *to*. `ensureSchema` applies every entry above the file's current + * version, in order, inside one transaction. Each value is the embedded copy + * of `sqlite/migrations/_*.sql`; parity tests keep them in sync. Never + * edit a shipped entry: add the next one and bump `SCHEMA_VERSION`. + */ +export const MIGRATIONS: Readonly> = Object.freeze({ + 1: SCHEMA, // 0001_initial.sql +}); + +for (let v = 1; v <= SCHEMA_VERSION; v++) { + if (!(v in MIGRATIONS)) { + throw new Error( + `deepeval/sqlite-store: MIGRATIONS is missing step ${v} (SCHEMA_VERSION=${SCHEMA_VERSION}).`, + ); + } +} + /** * `DEEPEVAL_SQLITE_INCLUDE_ROW_JSON` -> boolean (default false). Controls whether * `test_cases`, `traces` and `spans` rows also carry their full serialized @@ -186,7 +205,9 @@ CREATE INDEX IF NOT EXISTS idx_metric_data_owner ON metric_data(owner_type, owne * what `loadTestRunPayload` / `deepeval inspect` read back. */ export function resolveIncludeRowJson(): boolean { - const raw = (process.env[DEEPEVAL_SQLITE_INCLUDE_ROW_JSON] ?? "").trim().toLowerCase(); + const raw = (process.env[DEEPEVAL_SQLITE_INCLUDE_ROW_JSON] ?? "") + .trim() + .toLowerCase(); return ["1", "true", "yes", "y", "on"].includes(raw); } @@ -292,22 +313,55 @@ export function connect(dbPath: string): NodeSqliteDatabase { } function ensureSchema(db: NodeSqliteDatabase): void { + applyMigrations(db, { + migrations: MIGRATIONS, + targetVersion: SCHEMA_VERSION, + }); +} + +export interface ApplyMigrationsOptions { + /** Registry keyed by the version each script brings the database to. */ + migrations: Readonly>; + /** Version to upgrade the file to (usually `SCHEMA_VERSION`). */ + targetVersion: number; +} + +/** + * Bring `db` from its current `PRAGMA user_version` to `targetVersion` by + * running each pending migration in order, then stamp the version, all in one + * transaction. No-op when already current; throws when the file is newer than + * `targetVersion`. Exported for tests; `connect()` calls it with the real + * `MIGRATIONS` / `SCHEMA_VERSION`. + */ +export function applyMigrations( + db: NodeSqliteDatabase, + { migrations, targetVersion }: ApplyMigrationsOptions, +): void { const row = db.prepare("PRAGMA user_version").get() as | { user_version: number | bigint } | undefined; const version = Number(row?.user_version ?? 0); - if (version === SCHEMA_VERSION) return; - if (version > SCHEMA_VERSION) { + if (version === targetVersion) return; + if (version > targetVersion) { throw new Error( `deepeval.db schema version ${version} is newer than this deepeval ` + - `supports (${SCHEMA_VERSION}); please upgrade deepeval.`, + `supports (${targetVersion}); please upgrade deepeval.`, ); } + // Every pending step plus the version stamp land atomically; a crash midway + // leaves the file at its old version for the next open to retry. db.exec("BEGIN"); try { - db.exec(SCHEMA); - // Future migrations: `if (version < 2) { ... }` blocks go here. - db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`); + for (let step = version + 1; step <= targetVersion; step++) { + const script = migrations[step]; + if (script === undefined) { + throw new Error( + `deepeval/sqlite-store: no migration for schema version ${step}.`, + ); + } + db.exec(script); + } + db.exec(`PRAGMA user_version = ${targetVersion}`); db.exec("COMMIT"); } catch (e) { db.exec("ROLLBACK"); @@ -384,7 +438,8 @@ export function writeTestRun( for (const persisted of cases) { if (!isRecord(persisted)) continue; const entry = isRecord(persisted.entry) ? persisted.entry : persisted; - const kind = persisted.conversational === true ? "multi-turn" : "single-turn"; + const kind = + persisted.conversational === true ? "multi-turn" : "single-turn"; insertTestCase(db, runId, kind, entry, persisted, includeRowJson); } db.exec("COMMIT"); @@ -658,10 +713,7 @@ export function setConfidentTestRunId( } } -export function listTestRuns( - dbPath: string, - limit = 20, -): TestRunSummaryRow[] { +export function listTestRuns(dbPath: string, limit = 20): TestRunSummaryRow[] { if (!fs.existsSync(dbPath)) return []; const db = connect(dbPath); try { diff --git a/typescript/test/test-core/sqlite-store.test.ts b/typescript/test/test-core/sqlite-store.test.ts index 753cc3b10..e00bba228 100644 --- a/typescript/test/test-core/sqlite-store.test.ts +++ b/typescript/test/test-core/sqlite-store.test.ts @@ -12,9 +12,11 @@ import { } from "@/sqlite-store/mode"; import { DB_FILENAME, + MIGRATIONS, SCHEMA, SCHEMA_VERSION, TestRunNotFoundError, + applyMigrations, connect, formatSource, latestRunId, @@ -26,10 +28,7 @@ import { setConfidentTestRunId, writeTestRun, } from "@/sqlite-store/store"; -import { - exportTestRun, - type LocalTestRun, -} from "@/evaluate/test-run/local"; +import { exportTestRun, type LocalTestRun } from "@/evaluate/test-run/local"; import { loadTestRun, resolveInspectTarget, @@ -284,13 +283,18 @@ describe("sqlite-store mode / Node gate", () => { }); test("settings schema rejects sqlite on unsupported Node, junk always", () => { - const junk = settingsSchema.safeParse({ [DEEPEVAL_LOCAL_STORE]: "postgres" }); + const junk = settingsSchema.safeParse({ + [DEEPEVAL_LOCAL_STORE]: "postgres", + }); expect(junk.success).toBe(false); - const sqlite = settingsSchema.safeParse({ [DEEPEVAL_LOCAL_STORE]: "sqlite" }); + const sqlite = settingsSchema.safeParse({ + [DEEPEVAL_LOCAL_STORE]: "sqlite", + }); if (isSqliteSupported()) { expect(sqlite.success).toBe(true); - if (sqlite.success) expect(sqlite.data.DEEPEVAL_LOCAL_STORE).toBe("sqlite"); + if (sqlite.success) + expect(sqlite.data.DEEPEVAL_LOCAL_STORE).toBe("sqlite"); } else { expect(sqlite.success).toBe(false); if (!sqlite.success) { @@ -306,7 +310,9 @@ describe("sqlite-store mode / Node gate", () => { describe("sqlite-store paths / source specs", () => { test("resolveDbPath honours the results folder, else the cache dir", () => { - expect(resolveDbPath("/tmp/evals")).toBe(path.join("/tmp/evals", DB_FILENAME)); + expect(resolveDbPath("/tmp/evals")).toBe( + path.join("/tmp/evals", DB_FILENAME), + ); expect(resolveDbPath(undefined)).toBe(path.join(".deepeval", DB_FILENAME)); expect(resolveDbPath(" ")).toBe(path.join(".deepeval", DB_FILENAME)); }); @@ -346,7 +352,13 @@ describeWithSqlite("sqlite-store write / read", () => { "SELECT name FROM sqlite_master WHERE type='table'", ) as Array<{ name: string }>; const names = new Set(tables.map((t) => t.name)); - for (const t of ["test_runs", "test_cases", "traces", "spans", "metric_data"]) { + for (const t of [ + "test_runs", + "test_cases", + "traces", + "spans", + "metric_data", + ]) { expect(names.has(t)).toBe(true); } const [{ user_version }] = query(nested, "PRAGMA user_version") as Array<{ @@ -374,7 +386,10 @@ describeWithSqlite("sqlite-store write / read", () => { const rowJsonCounts = () => ["test_cases", "traces", "spans"].map((t) => { - const [row] = query(db, `SELECT count(payload_json) AS n FROM ${t}`) as Array<{ + const [row] = query( + db, + `SELECT count(payload_json) AS n FROM ${t}`, + ) as Array<{ n: number | bigint; }>; return Number(row!.n); @@ -383,7 +398,8 @@ describeWithSqlite("sqlite-store write / read", () => { describe("DEEPEVAL_SQLITE_INCLUDE_ROW_JSON", () => { const saved = process.env[DEEPEVAL_SQLITE_INCLUDE_ROW_JSON]; afterEach(() => { - if (saved === undefined) delete process.env[DEEPEVAL_SQLITE_INCLUDE_ROW_JSON]; + if (saved === undefined) + delete process.env[DEEPEVAL_SQLITE_INCLUDE_ROW_JSON]; else process.env[DEEPEVAL_SQLITE_INCLUDE_ROW_JSON] = saved; }); @@ -396,7 +412,9 @@ describeWithSqlite("sqlite-store write / read", () => { expect(loadTestRunPayload(db, runId).payload).toEqual( JSON.parse(JSON.stringify(makeRun())), ); - const [{ n }] = query(db, "SELECT count(*) AS n FROM spans") as Array<{ n: number }>; + const [{ n }] = query(db, "SELECT count(*) AS n FROM spans") as Array<{ + n: number; + }>; expect(Number(n)).toBeGreaterThan(0); }); @@ -406,7 +424,10 @@ describeWithSqlite("sqlite-store write / read", () => { writeTestRun(makeRun(), db); const counts = rowJsonCounts(); expect(counts.every((n) => n > 0)).toBe(true); - const [row] = query(db, "SELECT uuid, payload_json FROM traces LIMIT 1") as Array<{ + const [row] = query( + db, + "SELECT uuid, payload_json FROM traces LIMIT 1", + ) as Array<{ uuid: string; payload_json: string; }>; @@ -432,7 +453,13 @@ describeWithSqlite("sqlite-store write / read", () => { test("confident_test_run_id: NULL when not logged in, filled from the payload when posted", () => { const local = writeTestRun(makeRun(), db); - const posted = writeTestRun(makeRun({ testRunId: "cai_abc123", link: "https://app.confident-ai.com/x" }), db); + const posted = writeTestRun( + makeRun({ + testRunId: "cai_abc123", + link: "https://app.confident-ai.com/x", + }), + db, + ); const rows = query( db, "SELECT id, confident_test_run_id FROM test_runs ORDER BY id", @@ -442,19 +469,29 @@ describeWithSqlite("sqlite-store write / read", () => { [posted, "cai_abc123"], ]); const summaries = listTestRuns(db); - expect(summaries.map((r) => r.confident_test_run_id)).toEqual(["cai_abc123", null]); + expect(summaries.map((r) => r.confident_test_run_id)).toEqual([ + "cai_abc123", + null, + ]); }); test("setConfidentTestRunId stamps a stored run", () => { const runId = writeTestRun(makeRun(), db); setConfidentTestRunId(db, runId, "cai_later"); - const [row] = query(db, "SELECT confident_test_run_id FROM test_runs") as Array<{ + const [row] = query( + db, + "SELECT confident_test_run_id FROM test_runs", + ) as Array<{ confident_test_run_id: string | null; }>; expect(row!.confident_test_run_id).toBe("cai_later"); // Payload untouched: the id lives in the column only. - expect(loadTestRunPayload(db, runId).payload).toEqual(JSON.parse(JSON.stringify(makeRun()))); - expect(() => setConfidentTestRunId(db, 999, "x")).toThrow(TestRunNotFoundError); + expect(loadTestRunPayload(db, runId).payload).toEqual( + JSON.parse(JSON.stringify(makeRun())), + ); + expect(() => setConfidentTestRunId(db, 999, "x")).toThrow( + TestRunNotFoundError, + ); }); test("normalized test_runs columns", () => { @@ -670,7 +707,8 @@ describeWithSqlite("sqlite-store write / read", () => { afterEach(() => { if (savedMode === undefined) delete process.env[DEEPEVAL_LOCAL_STORE]; else process.env[DEEPEVAL_LOCAL_STORE] = savedMode; - if (savedFolder === undefined) delete process.env[DEEPEVAL_RESULTS_FOLDER]; + if (savedFolder === undefined) + delete process.env[DEEPEVAL_RESULTS_FOLDER]; else process.env[DEEPEVAL_RESULTS_FOLDER] = savedFolder; }); @@ -683,7 +721,9 @@ describeWithSqlite("sqlite-store write / read", () => { // Folder with only a DB (no JSON) resolves to the DB in json mode too. delete process.env[DEEPEVAL_LOCAL_STORE]; expect(resolveInspectTarget(dir)).toBe(db); - expect(resolveInspectTarget(dir, undefined, { runId: 1 })).toBe(`${db}#1`); + expect(resolveInspectTarget(dir, undefined, { runId: 1 })).toBe( + `${db}#1`, + ); // Folder with both prefers JSON in json mode, DB in sqlite mode. const jsonFile = path.join(dir, "test_run_20260914_060000.json"); fs.writeFileSync(jsonFile, JSON.stringify(makeRun()), "utf-8"); @@ -708,7 +748,8 @@ describeWithSqlite("sqlite-store write / read", () => { afterEach(() => { if (savedMode === undefined) delete process.env[DEEPEVAL_LOCAL_STORE]; else process.env[DEEPEVAL_LOCAL_STORE] = savedMode; - if (savedFolder === undefined) delete process.env[DEEPEVAL_RESULTS_FOLDER]; + if (savedFolder === undefined) + delete process.env[DEEPEVAL_RESULTS_FOLDER]; else process.env[DEEPEVAL_RESULTS_FOLDER] = savedFolder; jest.restoreAllMocks(); }); @@ -743,7 +784,9 @@ describeWithSqlite("sqlite-store write / read", () => { const second = exportTestRun(makeRun({ identifier: "two" }, false)); expect(first).toEqual({ mode: "sqlite", path: db, runId: 1 }); expect(second).toEqual({ mode: "sqlite", path: db, runId: 2 }); - expect(fs.readdirSync(dir).filter((f) => f.endsWith(".json"))).toEqual([]); + expect(fs.readdirSync(dir).filter((f) => f.endsWith(".json"))).toEqual( + [], + ); expect(listTestRuns(db).map((r) => r.identifier)).toEqual(["two", "one"]); }); }); @@ -757,14 +800,192 @@ const SHARED_SCHEMA = path.resolve(__dirname, "../../../sqlite/schema.sql"); /** Strip `--` comments and collapse whitespace so formatting can't fail us. */ function normalizeSql(sql: string): string { - return sql.replace(/--[^\n]*/g, "").replace(/\s+/g, " ").trim(); + return sql + .replace(/--[^\n]*/g, "") + .replace(/\s+/g, " ") + .trim(); } const describeIfRepo = fs.existsSync(SHARED_SCHEMA) ? describe : describe.skip; +const MIGRATIONS_DIR = path.resolve(__dirname, "../../../sqlite/migrations"); + +function migrationFiles(): string[] { + return fs + .readdirSync(MIGRATIONS_DIR) + .filter((f) => /^\d{4}_.*\.sql$/.test(f)) + .sort(); +} + describeIfRepo("sqlite-store schema parity", () => { it("embedded SCHEMA matches sqlite/schema.sql", () => { const shared = normalizeSql(fs.readFileSync(SHARED_SCHEMA, "utf8")); expect(normalizeSql(SCHEMA)).toBe(shared); }); + + it("MIGRATIONS covers 1..SCHEMA_VERSION and matches sqlite/migrations/", () => { + const versions = Object.keys(MIGRATIONS) + .map(Number) + .sort((a, b) => a - b); + expect(versions).toEqual( + Array.from({ length: SCHEMA_VERSION }, (_, i) => i + 1), + ); + const files = migrationFiles(); + expect(files.map((f) => Number(f.slice(0, 4)))).toEqual(versions); + for (const file of files) { + const version = Number(file.slice(0, 4)); + const onDisk = fs.readFileSync(path.join(MIGRATIONS_DIR, file), "utf8"); + expect(normalizeSql(MIGRATIONS[version]!)).toBe(normalizeSql(onDisk)); + } + }); + + (hasNodeSqlite ? it : it.skip)( + "applying migrations in order equals sqlite/schema.sql", + () => { + const { DatabaseSync } = ( + process as unknown as { + getBuiltinModule: (id: string) => { + DatabaseSync: new (p: string) => { + exec(sql: string): void; + prepare(sql: string): { all(): unknown[] }; + close(): void; + }; + }; + } + ).getBuiltinModule("node:sqlite"); + + const objects = (scripts: string[]) => { + const db = new DatabaseSync(":memory:"); + try { + for (const script of scripts) db.exec(script); + const rows = db + .prepare( + "SELECT type, name, sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY type, name", + ) + .all() as { type: string; name: string; sql: string }[]; + return rows.map((r) => [r.type, r.name, normalizeSql(r.sql)]); + } finally { + db.close(); + } + }; + + const migrated = objects( + migrationFiles().map((f) => + fs.readFileSync(path.join(MIGRATIONS_DIR, f), "utf8"), + ), + ); + const snapshot = objects([fs.readFileSync(SHARED_SCHEMA, "utf8")]); + expect(migrated).toEqual(snapshot); + }, + ); +}); + +describeWithSqlite("sqlite-store schema upgrade", () => { + const PROBE = "ALTER TABLE test_runs ADD COLUMN _probe TEXT;"; + type RawDb = Parameters[0] & { close(): void }; + const { DatabaseSync } = ( + process as unknown as { + getBuiltinModule: (id: string) => { + DatabaseSync: new (p: string) => RawDb; + }; + } + ).getBuiltinModule("node:sqlite"); + + let dir: string; + let db: string; + + beforeEach(() => { + dir = tempDir(); + db = path.join(dir, DB_FILENAME); + }); + afterEach(() => fs.rmSync(dir, { recursive: true, force: true })); + + // Open without `connect()`: once a file is at v2 the real (v1) `connect()` + // correctly refuses it, which is itself asserted below. + const withRaw = (fn: (raw: RawDb) => T): T => { + const raw = new DatabaseSync(db); + try { + return fn(raw); + } finally { + raw.close(); + } + }; + const userVersion = () => + withRaw((raw) => + Number( + ( + raw.prepare("PRAGMA user_version").get() as { + user_version: number | bigint; + } + ).user_version, + ), + ); + const columns = (table: string) => + withRaw( + (raw) => + new Set( + ( + raw.prepare(`PRAGMA table_info(${table})`).all() as { + name: string; + }[] + ).map((r) => r.name), + ), + ); + const upgrade = ( + migrations: Readonly>, + targetVersion = 2, + ) => withRaw((raw) => applyMigrations(raw, { migrations, targetVersion })); + + test("existing v1 file gets only step 2 and keeps its data", () => { + writeTestRun(makeRun(), db); + expect(userVersion()).toBe(SCHEMA_VERSION); + expect(columns("test_runs").has("_probe")).toBe(false); + + // Step 1 must not re-run: make it explode if the loop starts at 1. + upgrade({ 1: "SELECT RAISE(ABORT, 'step 1 re-ran');", 2: PROBE }); + + expect(userVersion()).toBe(2); + expect(columns("test_runs").has("_probe")).toBe(true); + expect( + withRaw((raw) => + raw.prepare("SELECT count(*) AS n FROM test_runs").get(), + ), + ).toMatchObject({ n: 1 }); + // ...and today's release now refuses the newer file. + expect(() => connect(db)).toThrow(/newer than this deepeval/); + }); + + test("second upgrade pass is a no-op", () => { + const migrations = { ...MIGRATIONS, 2: PROBE }; + connect(db).close(); // v1 + upgrade(migrations); + // Re-running the ALTER would throw "duplicate column"; a no-op does not. + upgrade(migrations); + expect(userVersion()).toBe(2); + }); + + test("fresh file applies every step", () => { + upgrade({ ...MIGRATIONS, 2: PROBE }); + expect(userVersion()).toBe(2); + const cols = columns("test_runs"); + expect(cols.has("payload_json")).toBe(true); + expect(cols.has("_probe")).toBe(true); + }); + + test("a failing step rolls back and leaves the old version", () => { + connect(db).close(); // v1 + expect(() => + upgrade({ + ...MIGRATIONS, + 2: PROBE + "ALTER TABLE no_such_table ADD COLUMN x TEXT;", + }), + ).toThrow(); + expect(userVersion()).toBe(1); + expect(columns("test_runs").has("_probe")).toBe(false); + }); + + test("refuses a file newer than the target", () => { + connect(db).close(); // v1 + expect(() => upgrade(MIGRATIONS, 0)).toThrow(/newer than this deepeval/); + }); });