DBOS/Postgres server support (#367)

This commit is contained in:
Adrian Lyjak
2026-02-11 19:06:53 -05:00
committed by GitHub
parent 79159f0be0
commit c2e7f1784b
61 changed files with 4267 additions and 1322 deletions
+5
View File
@@ -0,0 +1,5 @@
---
"llama-agents-dbos": minor
---
Add postgres and DBOS support to the workflow server
+1 -4
View File
@@ -106,12 +106,9 @@ jobs:
python-version: "3.14"
enable-cache: true
- name: Run non-Docker tests with coverage
run: uv run --all-extras --directory packages/llama-agents-integration-tests -- pytest -m "not docker" --cov --cov-report=
- name: Run Docker tests with coverage (append)
# Use longer timeout for Docker tests since container startup can take 30-60s in CI
run: uv run --all-extras --directory packages/llama-agents-integration-tests -- pytest --timeout=120 -m docker --cov --cov-append --cov-report=xml
run: uv run --all-extras --all-packages dev --timeout=120 -m docker --cov --cov-append --cov-report=xml
- name: Report Coveralls
uses: coverallsapp/github-action@v2
+1
View File
@@ -0,0 +1 @@
.last_handler_id
@@ -0,0 +1,9 @@
services:
postgres:
image: postgres:16
environment:
POSTGRES_DB: workflows
POSTGRES_USER: workflows
POSTGRES_PASSWORD: workflows
ports:
- "5433:5432"
+240
View File
@@ -0,0 +1,240 @@
#!/usr/bin/env python3
"""
Multi-Replica Demo
==================
Demonstrates durable workflow execution across multiple server replicas
backed by a shared Postgres database with DBOS.
- Two WorkflowServer replicas share a Postgres-backed event store
- A counter workflow is triggered on Replica A (port 8001)
- Events are streamed in real-time from Replica B (port 8002)
- Ctrl+C interrupts the workflow mid-flight
- --resume picks up exactly where it left off via DBOS recovery
Usage:
python examples/multi_replica/run.py # Start new
python examples/multi_replica/run.py --resume # Resume after Ctrl+C
python examples/multi_replica/run.py --clean # Tear down everything
"""
from __future__ import annotations
import argparse
import asyncio
import os
import signal
import subprocess
import sys
import time
from datetime import datetime
from pathlib import Path
import httpx
from llama_agents.client import WorkflowClient
_DIR = Path(__file__).parent
_HANDLER_FILE = _DIR / ".last_handler_id"
_COMPOSE_FILE = _DIR / "docker-compose.yml"
# -- Pretty output -----------------------------------------------------------
BLUE = "\033[94m"
GREEN = "\033[92m"
YELLOW = "\033[93m"
CYAN = "\033[96m"
DIM = "\033[2m"
BOLD = "\033[1m"
RESET = "\033[0m"
def ts() -> str:
return datetime.now().strftime("%H:%M:%S")
def log(msg: str, color: str = DIM) -> None:
print(f" {color}{ts()}{RESET} {msg}")
# -- Infrastructure -----------------------------------------------------------
def run_cmd(*args: str, **kwargs: object) -> subprocess.CompletedProcess[str]:
return subprocess.run(args, check=True, text=True, capture_output=True, **kwargs) # type: ignore[arg-type]
def start_postgres() -> None:
log("Starting Postgres container...", BLUE)
run_cmd("docker", "compose", "-f", str(_COMPOSE_FILE), "up", "-d")
for _ in range(30):
try:
run_cmd(
"docker",
"compose",
"-f",
str(_COMPOSE_FILE),
"exec",
"-T",
"postgres",
"pg_isready",
"-U",
"workflows",
)
log("Postgres ready", GREEN)
return
except subprocess.CalledProcessError:
time.sleep(1)
raise RuntimeError("Postgres failed to start")
def start_replica(port: int) -> subprocess.Popen[str]:
return subprocess.Popen(
[sys.executable, str(_DIR / "serve.py"), "--port", str(port)],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
start_new_session=True,
)
def wait_for_server(port: int, timeout: float = 30.0) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
resp = httpx.get(f"http://localhost:{port}/workflows", timeout=2.0)
if resp.status_code == 200:
return
except httpx.ConnectError:
pass
time.sleep(0.5)
raise RuntimeError(f"Server on port {port} did not start in {timeout}s")
# -- Workflow operations ------------------------------------------------------
async def start_workflow(client: WorkflowClient) -> str:
handler = await client.run_workflow_nowait("counter")
return handler.handler_id
async def stream_events(
client: WorkflowClient, handler_id: str, resume: bool = False
) -> bool:
after: int | str = "now" if resume else -1
completed = False
async for event in client.get_workflow_events(handler_id, after_sequence=after):
event_type = event.type
event_data = event.value or {}
if event_type == "Tick":
count = event_data.get("count", "?")
bar = "#" * int(count) + "." * (20 - int(count))
log(f"Tick {count:>2}/20 [{bar}]", CYAN)
elif event_type == "CounterResult":
final = event_data.get("final_count", "?")
log(f"Done! final_count={final}", GREEN)
completed = True
else:
log(f"{event_type}: {event_data}", DIM)
return completed
# -- Main ---------------------------------------------------------------------
async def async_main() -> None:
parser = argparse.ArgumentParser(description="Multi-Replica Demo")
parser.add_argument("--resume", action="store_true", help="Resume last workflow")
parser.add_argument("--clean", action="store_true", help="Tear down everything")
args = parser.parse_args()
if args.clean:
if _HANDLER_FILE.exists():
_HANDLER_FILE.unlink()
subprocess.run(
["docker", "compose", "-f", str(_COMPOSE_FILE), "down", "-v"],
check=False,
)
print("Cleaned up.")
return
print()
print(f" {BOLD}Multi-Replica Workflow Demo{RESET}")
print(f" {DIM}Two servers, one Postgres, durable execution{RESET}")
print()
replicas: list[subprocess.Popen[str]] = []
def cleanup() -> None:
for proc in replicas:
proc.kill()
for proc in replicas:
proc.wait()
def handle_sigint() -> None:
print()
log("Interrupted. Workflow state is safe in Postgres.", YELLOW)
log(f"Run with {BOLD}--resume{RESET} to continue where you left off.", YELLOW)
print()
cleanup()
os._exit(130)
asyncio.get_running_loop().add_signal_handler(signal.SIGINT, handle_sigint)
replica_a = WorkflowClient(base_url="http://localhost:8001")
replica_b = WorkflowClient(base_url="http://localhost:8002")
try:
# --- Postgres ---
start_postgres()
print()
# --- Replicas ---
log(
f"Starting Replica A on :8001 {DIM}(executor_id=replica-8001){RESET}", BLUE
)
replicas.append(start_replica(8001))
log(
f"Starting Replica B on :8002 {DIM}(executor_id=replica-8002){RESET}", BLUE
)
replicas.append(start_replica(8002))
wait_for_server(8001)
log("Replica A ready", GREEN)
wait_for_server(8002)
log("Replica B ready", GREEN)
print()
# --- Workflow ---
if args.resume and _HANDLER_FILE.exists():
handler_id = _HANDLER_FILE.read_text().strip()
log(f"Resuming workflow handler_id={BOLD}{handler_id}{RESET}", YELLOW)
log(f"{DIM}DBOS recovers the workflow on the owning replica{RESET}", DIM)
else:
log("Triggering counter workflow on Replica A (:8001)...", BLUE)
handler_id = await start_workflow(replica_a)
_HANDLER_FILE.write_text(handler_id)
log(f"Workflow started handler_id={BOLD}{handler_id}{RESET}", GREEN)
print()
log("Streaming events from Replica B (:8002)...", BLUE)
log(f"{DIM}Events flow: Replica A -> Postgres -> Replica B -> here{RESET}", DIM)
print()
completed = await stream_events(replica_b, handler_id, resume=args.resume)
print()
if completed:
log(f"{GREEN}{BOLD}Workflow completed across replicas!{RESET}", GREEN)
if _HANDLER_FILE.exists():
_HANDLER_FILE.unlink()
print()
finally:
cleanup()
def main() -> None:
asyncio.run(async_main())
if __name__ == "__main__":
main()
+99
View File
@@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""
Multi-Replica Server
Single replica server that can be run standalone.
Usage:
python examples/multi_replica/serve.py --port 8001
"""
from __future__ import annotations
import argparse
import asyncio
from dbos import DBOS
from llama_agents.dbos import DBOSRuntime
from llama_agents.server import WorkflowServer
from pydantic import Field
from workflows import Context, Workflow, step
from workflows.events import Event, StartEvent, StopEvent
POSTGRES_DSN = "postgresql://workflows:workflows@localhost:5433/workflows"
class Tick(Event):
count: int = Field(description="Current count")
class WaitDone(Event):
count: int = Field(description="Current count after waiting")
class CounterResult(StopEvent):
final_count: int = Field(description="Final counter value")
class CounterWorkflow(Workflow):
"""Counts to 20 with 1s delays, emitting Tick stream events.
Split into a slow wait step and a fast tick step so that
the stream event is the last thing written before the next
wait. This minimizes duplicate ticks on DBOS replay.
"""
@step
async def start(self, ctx: Context, ev: StartEvent) -> WaitDone:
print("[Start] Initializing counter")
return WaitDone(count=0)
@step
async def tick(self, ctx: Context, ev: WaitDone) -> Tick | CounterResult:
count = ev.count + 1
await ctx.store.set("count", count)
ctx.write_event_to_stream(Tick(count=count))
print(f"[Tick {count:2d}] count = {count}")
if count >= 20:
return CounterResult(final_count=count)
return Tick(count=count)
@step
async def wait(self, ctx: Context, ev: Tick) -> WaitDone:
await asyncio.sleep(1.0)
return WaitDone(count=ev.count)
async def main() -> None:
parser = argparse.ArgumentParser(description="Multi-Replica Server")
parser.add_argument("--port", type=int, default=8001)
args = parser.parse_args()
DBOS(
config={
"name": "multi-replica",
"system_database_url": POSTGRES_DSN,
"run_admin_server": False,
"executor_id": f"replica-{args.port}",
}
)
runtime = DBOSRuntime()
server = WorkflowServer(
workflow_store=runtime.create_workflow_store(),
runtime=runtime.build_server_runtime(),
)
server.add_workflow("counter", CounterWorkflow(runtime=runtime))
print(f"Serving on port {args.port}")
await server.start()
try:
await server.serve(host="0.0.0.0", port=args.port)
finally:
await server.stop()
if __name__ == "__main__":
asyncio.run(main())
+9 -1
View File
@@ -5,6 +5,8 @@ build-backend = "uv_build"
[dependency-groups]
dev = [
"basedpyright>=1.31.1",
"llama-agents-integration-tests",
"testcontainers[postgres]>=4.0.0",
"pytest>=8.4.0",
"pytest-asyncio>=1.0.0",
"pytest-cov>=6.1.1",
@@ -22,6 +24,7 @@ license = "MIT"
requires-python = ">=3.9"
dependencies = [
"dbos>=2.11.0; python_full_version >= '3.10.0'",
"llama-agents-server[asyncpg]",
"llama-index-workflows>=2.12.0,<3.0.0"
]
@@ -34,10 +37,15 @@ asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "module"
asyncio_default_test_loop_scope = "module"
testpaths = ["tests"]
addopts = "-nauto --timeout=30"
addopts = "-nauto --timeout=60 -m 'not docker'"
markers = [
"docker: marks tests as requiring Docker (testcontainers/PostgreSQL)"
]
[tool.uv.build-backend]
module-name = "llama_agents.dbos"
[tool.uv.sources]
llama-index-workflows = {workspace = true}
llama-agents-server = {workspace = true}
llama-agents-integration-tests = {workspace = true}
@@ -1,85 +1,107 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
"""CRUD operations and table definitions for the workflow journal."""
"""CRUD operations for the workflow journal using native database drivers."""
from __future__ import annotations
from sqlalchemy import Column, Integer, MetaData, String, Table, text
from sqlalchemy.engine import Connection, Engine
import re
import sqlite3
from abc import ABC, abstractmethod
from contextlib import contextmanager
from typing import Iterator
import asyncpg
JOURNAL_TABLE_NAME = "workflow_journal"
_VALID_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
class JournalCrud:
"""Database operations for the workflow journal table.
Initialized with table configuration (name, schema), then provides
methods for inserting, loading, and migrating journal entries.
"""
def _quote_identifier(name: str) -> str:
"""Quote a SQL identifier, raising on invalid names."""
if not _VALID_IDENTIFIER.match(name):
msg = f"Invalid SQL identifier: {name!r}"
raise ValueError(msg)
return f'"{name}"'
def _qualified_table_ref(table_name: str, schema: str | None = None) -> str:
"""Build a safely-quoted qualified table reference."""
ref = _quote_identifier(table_name)
if schema:
ref = f"{_quote_identifier(schema)}.{ref}"
return ref
class JournalCrud(ABC):
"""Abstract base for journal CRUD operations."""
@abstractmethod
async def insert(self, run_id: str, seq_num: int, task_key: str) -> None: ...
@abstractmethod
async def load(self, run_id: str) -> list[str]: ...
class PostgresJournalCrud(JournalCrud):
"""Journal CRUD using asyncpg."""
def __init__(
self,
pool: asyncpg.Pool,
table_name: str = JOURNAL_TABLE_NAME,
schema: str | None = None,
) -> None:
self.table_name = table_name
self.schema = schema
self._pool = pool
self._table_ref = _qualified_table_ref(table_name, schema)
@property
def _table_ref(self) -> str:
if self.schema:
return f"{self.schema}.{self.table_name}"
return self.table_name
def _define_table(self, metadata: MetaData) -> Table:
return Table(
self.table_name,
metadata,
Column("id", Integer, primary_key=True, autoincrement=True),
Column("run_id", String(255), nullable=False, index=True),
Column("seq_num", Integer, nullable=False),
Column("task_key", String(512), nullable=False),
async def insert(self, run_id: str, seq_num: int, task_key: str) -> None:
await self._pool.execute(
f"INSERT INTO {self._table_ref} (run_id, seq_num, task_key) VALUES ($1, $2, $3)",
run_id,
seq_num,
task_key,
)
def insert(
async def load(self, run_id: str) -> list[str]:
rows = await self._pool.fetch(
f"SELECT task_key FROM {self._table_ref} WHERE run_id = $1 ORDER BY seq_num ASC",
run_id,
)
return [row["task_key"] for row in rows]
class SqliteJournalCrud(JournalCrud):
"""Journal CRUD using sqlite3."""
def __init__(
self,
conn: Connection,
run_id: str,
seq_num: int,
task_key: str,
db_path: str,
table_name: str = JOURNAL_TABLE_NAME,
) -> None:
"""Insert a new journal entry."""
conn.execute(
text(f"""
INSERT INTO {self._table_ref} (run_id, seq_num, task_key)
VALUES (:run_id, :seq_num, :task_key)
"""), # noqa: S608
{
"run_id": run_id,
"seq_num": seq_num,
"task_key": task_key,
},
)
self._db_path = db_path
self._table_ref = _quote_identifier(table_name)
def load(self, conn: Connection, run_id: str) -> list[str]:
"""Load journal entries for a run, ordered by sequence number."""
result = conn.execute(
text(f"""
SELECT task_key FROM {self._table_ref}
WHERE run_id = :run_id
ORDER BY seq_num ASC
"""), # noqa: S608
{"run_id": run_id},
)
return [row[0] for row in result.fetchall()]
@contextmanager
def _connect(self) -> Iterator[sqlite3.Connection]:
conn = sqlite3.connect(self._db_path)
try:
yield conn
finally:
conn.close()
def run_migrations(self, engine: Engine) -> None:
"""Create the journal table if it doesn't exist."""
metadata = MetaData(schema=self.schema)
table = self._define_table(metadata)
async def insert(self, run_id: str, seq_num: int, task_key: str) -> None:
with self._connect() as conn:
conn.execute(
f"INSERT INTO {self._table_ref} (run_id, seq_num, task_key) VALUES (?, ?, ?)",
(run_id, seq_num, task_key),
)
conn.commit()
with engine.begin() as conn:
is_postgres = engine.dialect.name == "postgresql"
if is_postgres and self.schema:
conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {self.schema}")) # noqa: S608
table.create(bind=conn, checkfirst=True)
async def load(self, run_id: str) -> list[str]:
with self._connect() as conn:
cursor = conn.execute(
f"SELECT task_key FROM {self._table_ref} WHERE run_id = ? ORDER BY seq_num ASC",
(run_id,),
)
return [row[0] for row in cursor.fetchall()]
@@ -4,11 +4,6 @@
from __future__ import annotations
import asyncio
from typing import Any
from sqlalchemy.engine import Engine
from .crud import JournalCrud
@@ -27,41 +22,29 @@ class TaskJournal:
def __init__(
self,
run_id: str,
engine: Engine | None = None,
crud: JournalCrud | None = None,
) -> None:
"""Initialize the task journal.
Args:
run_id: Workflow run ID for this journal.
engine: SQLAlchemy engine. If None, operates in-memory only.
crud: Journal CRUD operations. If None, uses default JournalCrud().
crud: Journal CRUD operations. If None, operates in-memory only.
"""
self._run_id = run_id
self._engine = engine
self._crud = crud or JournalCrud()
self._crud = crud
self._entries: list[str] | None = None # Lazy loaded
self._replay_index: int = 0
async def _run_sync(self, fn: Any, *args: Any) -> Any:
"""Run a synchronous function in the default executor."""
loop = asyncio.get_event_loop()
return await loop.run_in_executor(None, fn, *args)
async def load(self) -> None:
"""Load journal from database. Idempotent - only loads once."""
if self._entries is not None:
return
if self._engine is None:
if self._crud is None:
self._entries = []
return
def _load_sync() -> list[str]:
with self._engine.connect() as conn: # type: ignore[union-attr]
return self._crud.load(conn, self._run_id)
self._entries = await self._run_sync(_load_sync)
self._entries = await self._crud.load(self._run_id)
def is_replaying(self) -> bool:
"""True if there are more journal entries to replay."""
@@ -84,13 +67,8 @@ class TaskJournal:
self._entries.append(key)
self._replay_index += 1
if self._engine is not None:
def _insert_sync() -> None:
with self._engine.begin() as conn: # type: ignore[union-attr]
self._crud.insert(conn, self._run_id, seq_num, key)
await self._run_sync(_insert_sync)
if self._crud is not None:
await self._crud.insert(self._run_id, seq_num, key)
def advance(self) -> None:
"""Advance replay index after processing a replayed task."""
@@ -13,9 +13,11 @@ import asyncio
import logging
import sys
import time
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from typing import Any, AsyncGenerator, TypedDict
from typing import Any, AsyncGenerator, TypedDict, cast
import asyncpg
from llama_index_instrumentation.dispatcher import active_instrument_tags
from pydantic import BaseModel
from typing_extensions import Unpack
@@ -46,7 +48,7 @@ from workflows.runtime.types.ticks import WorkflowTick
from workflows.workflow import Workflow
try:
from dbos import DBOS, SetWorkflowID
from dbos import DBOS, SetWorkflowID, WorkflowHandleAsync
from dbos._dbos import _get_dbos_instance
except ImportError as e:
# if 3.9, give a detailed error that dbos is not supported on this version of python
@@ -57,15 +59,90 @@ except ImportError as e:
) from e
raise
from llama_agents.client.protocol.serializable_events import (
EventEnvelopeWithMetadata,
)
from llama_agents.server._runtime.event_interceptor import EventInterceptorDecorator
from llama_agents.server._store.abstract_workflow_store import (
AbstractWorkflowStore,
HandlerQuery,
PersistentHandler,
StoredEvent,
StoredTick,
)
from llama_agents.server._store.postgres_state_store import PostgresStateStore
from llama_agents.server._store.postgres_workflow_store import (
PostgresWorkflowStore,
)
from llama_agents.server._store.sqlite.sqlite_state_store import SqliteStateStore
from llama_agents.server._store.sqlite.sqlite_workflow_store import SqliteWorkflowStore
from sqlalchemy.engine import URL as SaURL
from sqlalchemy.engine import Engine
from .journal.crud import JOURNAL_TABLE_NAME, JournalCrud
from .journal.crud import JOURNAL_TABLE_NAME, PostgresJournalCrud, SqliteJournalCrud
from .journal.task_journal import TaskJournal
from .state_store import STATE_TABLE_NAME, SqlStateStore
STATE_TABLE_NAME = "workflow_state"
logger = logging.getLogger(__name__)
class DBOSWorkflowStore(AbstractWorkflowStore):
"""Lazy proxy that defers dialect resolution until first use.
Wraps a factory callable that produces the real store (Postgres or Sqlite).
The factory is called once on first access; all abstract methods delegate
to the resolved store.
"""
def __init__(self, factory: Callable[[], AbstractWorkflowStore]) -> None:
self._factory = factory
self._inner: AbstractWorkflowStore | None = None
def _resolve(self) -> AbstractWorkflowStore:
if self._inner is None:
self._inner = self._factory()
return self._inner
@property
def poll_interval(self) -> float: # type: ignore[override]
return self._resolve().poll_interval
def create_state_store(
self,
run_id: str,
state_type: type[Any] | None = None,
serialized_state: dict[str, Any] | None = None,
serializer: BaseSerializer | None = None,
) -> StateStore[Any]:
return self._resolve().create_state_store(
run_id, state_type, serialized_state, serializer
)
async def query(self, query: HandlerQuery) -> list[PersistentHandler]:
return await self._resolve().query(query)
async def update(self, handler: PersistentHandler) -> None:
await self._resolve().update(handler)
async def delete(self, query: HandlerQuery) -> int:
return await self._resolve().delete(query)
async def append_event(self, run_id: str, event: EventEnvelopeWithMetadata) -> None:
await self._resolve().append_event(run_id, event)
async def query_events(
self, run_id: str, after_sequence: int | None = None, limit: int | None = None
) -> list[StoredEvent]:
return await self._resolve().query_events(run_id, after_sequence, limit)
async def append_tick(self, run_id: str, tick_data: dict[str, Any]) -> None:
await self._resolve().append_tick(run_id, tick_data)
async def get_ticks(self, run_id: str) -> list[StoredTick]:
return await self._resolve().get_ticks(run_id)
class DBOSRuntimeConfig(TypedDict, total=False):
"""Configuration options for DBOSRuntime.
@@ -95,6 +172,18 @@ def _resolve_schema(config: DBOSRuntimeConfig, engine: Engine) -> str | None:
return "dbos" if is_postgres else None
def _sqlalchemy_url_to_asyncpg_dsn(url: SaURL) -> str:
"""Convert a SQLAlchemy URL to an asyncpg-compatible DSN.
Strips dialect driver suffixes (e.g. postgresql+psycopg2 -> postgresql)
and renders the URL as a plain connection string.
"""
# url is a sqlalchemy.engine.URL object
# Set the drivername to plain 'postgresql' for asyncpg
plain_url = url.set(drivername="postgresql")
return plain_url.render_as_string(hide_password=False)
# Very long timeout for unbounded waits - encourages workflow to sleep.
# DBOS's default 60s is too short and gets recorded to event logs.
_UNBOUNDED_WAIT_TIMEOUT_SECONDS = 60 * 60 * 24 # 1 day
@@ -138,6 +227,7 @@ class DBOSRuntime(Runtime):
state_table_name: State table name. Default "workflow_state".
journal_table_name: Journal table name. Default "workflow_journal".
"""
super().__init__()
self.config: DBOSRuntimeConfig = dict(kwargs) # type: ignore[assignment]
# Workflow tracking state
@@ -150,7 +240,15 @@ class DBOSRuntime(Runtime):
self._sql_engine: Engine | None = None
self._migrations_run = False
def _track_task(self, task: asyncio.Task[None]) -> None:
# Native driver resources (resolved at launch time)
self._pool: asyncpg.Pool | None = None
self._pool_lock: asyncio.Lock = asyncio.Lock()
self._dsn: str | None = None # asyncpg DSN for lazy pool creation
self._db_path: str | None = None # sqlite path
self._schema: str | None = None
self._workflow_store: AbstractWorkflowStore | None = None
def _track_task(self, task: asyncio.Task[Any]) -> None:
self._tasks.append(task)
task.add_done_callback(self._tasks.remove)
@@ -180,7 +278,13 @@ class DBOSRuntime(Runtime):
Called at launch() time for each tracked workflow.
Uses workflow.workflow_name for stable DBOS registration names.
Idempotent: returns existing registration if already registered.
"""
# Return existing registration if already registered
existing = self._registered.get(id(workflow))
if existing is not None:
return existing
# Use workflow's name directly
name = workflow.workflow_name
@@ -191,6 +295,10 @@ class DBOSRuntime(Runtime):
start_event: StartEvent | None = None,
tags: dict[str, Any] = {},
) -> StopEvent:
# Eagerly resolve the asyncpg pool so the adapter can use it
# synchronously in get_state_store / is_replaying.
if self._dsn is not None:
await self._ensure_pool()
workflow_run_fn = create_workflow_run_function(workflow)
return await workflow_run_fn(init_state, start_event, tags)
@@ -200,9 +308,11 @@ class DBOSRuntime(Runtime):
for step_name, step in as_step_worker_functions(workflow).items()
}
return RegisteredWorkflow(
registered = RegisteredWorkflow(
workflow=workflow, workflow_run_fn=_dbos_control_loop, steps=wrapped_steps
)
self._registered[id(workflow)] = registered
return registered
def _get_sql_engine(self) -> Engine:
"""Get the SQLAlchemy engine from DBOS for state storage.
@@ -231,11 +341,28 @@ class DBOSRuntime(Runtime):
self._sql_engine = sys_db.engine
return self._sql_engine
def run_migrations(self) -> None:
"""Run database migrations for workflow state and journal tables.
async def _ensure_pool(self) -> asyncpg.Pool:
"""Get or lazily create the asyncpg connection pool.
Creates the workflow_state and workflow_journal tables if they don't exist.
Idempotent - safe to call multiple times.
Only valid for postgres dialect. Raises RuntimeError for sqlite.
"""
if self._pool is not None:
return self._pool
async with self._pool_lock:
if self._pool is not None:
return self._pool
if self._dsn is None:
raise RuntimeError(
"No asyncpg DSN configured. Either not launched or using sqlite dialect."
)
self._pool = await asyncpg.create_pool(dsn=self._dsn)
return self._pool
def run_migrations(self) -> None:
"""Run database migrations for all workflow tables.
Uses the file-based migration system to create/update workflow store,
state, and journal tables. Idempotent - safe to call multiple times.
Can be called explicitly before launch() when run_migrations_on_launch=False,
allowing for custom migration timing (e.g., during application startup).
@@ -247,19 +374,16 @@ class DBOSRuntime(Runtime):
engine = self._get_sql_engine()
schema = _resolve_schema(self.config, engine)
state_table = self.config.get("state_table_name", DEFAULT_STATE_TABLE_NAME)
journal_table = self.config.get(
"journal_table_name", DEFAULT_JOURNAL_TABLE_NAME
)
SqlStateStore.run_migrations(engine, table_name=state_table, schema=schema)
# Create workflow_journal table
journal_crud = JournalCrud(table_name=journal_table, schema=schema)
journal_crud.run_migrations(engine)
if engine.dialect.name == "postgresql":
dsn = _sqlalchemy_url_to_asyncpg_dsn(engine.url)
PostgresWorkflowStore.run_migrations_sync(dsn, schema=schema)
else:
db_path = str(engine.url.database) if engine.url.database else ":memory:"
SqliteWorkflowStore.run_migrations(db_path)
self._migrations_run = True
logger.info("Database migrations completed (workflow_state, workflow_journal)")
logger.info("Database migrations completed")
def run_workflow(
self,
@@ -299,23 +423,30 @@ class DBOSRuntime(Runtime):
)
# Capture values needed in the async task closure
engine = self._get_sql_engine()
active_serializer = serializer or JsonSerializer()
async def _run_workflow() -> None:
async def _run_workflow() -> WorkflowHandleAsync[Any]:
with SetWorkflowID(run_id):
# Write initial state to DB before starting workflow (non-blocking to caller)
if serialized_state:
store = SqlStateStore(
run_id=run_id,
engine=engine,
state_type=infer_state_type(workflow),
serializer=active_serializer,
schema=_resolve_schema(self.config, engine),
table_name=self.config.get(
"state_table_name", DEFAULT_STATE_TABLE_NAME
),
)
if self._dsn is not None:
pool = await self._ensure_pool()
store: StateStore[Any] = PostgresStateStore(
pool=pool,
run_id=run_id,
state_type=infer_state_type(workflow),
serializer=active_serializer,
schema=self._schema,
)
elif self._db_path is not None:
store = SqliteStateStore(
db_path=self._db_path,
run_id=run_id,
state_type=infer_state_type(workflow),
serializer=active_serializer,
)
else:
raise RuntimeError("No pool or db_path configured.")
# Deserialize and save the initial state
state = deserialize_state_from_dict(
serialized_state,
@@ -325,7 +456,7 @@ class DBOSRuntime(Runtime):
await store.set_state(state)
try:
await DBOS.start_workflow_async(
return await DBOS.start_workflow_async(
registered.workflow_run_fn,
init_state,
start_event,
@@ -367,13 +498,16 @@ class DBOSRuntime(Runtime):
run_id,
engine,
state_type,
schema=_resolve_schema(self.config, engine),
schema=self._schema,
state_table_name=self.config.get(
"state_table_name", DEFAULT_STATE_TABLE_NAME
),
journal_table_name=self.config.get(
"journal_table_name", DEFAULT_JOURNAL_TABLE_NAME
),
ensure_pool=self._ensure_pool if self._dsn is not None else None,
pool=self._pool,
db_path=self._db_path,
)
def get_external_adapter(self, run_id: str) -> ExternalRunAdapter:
@@ -383,6 +517,51 @@ class DBOSRuntime(Runtime):
)
return ExternalDBOSAdapter(run_id, self.config.get("polling_interval_sec", 1.0))
def create_workflow_store(self) -> AbstractWorkflowStore:
"""Return the cached workflow store, creating it on first call.
Detects the engine dialect and creates the appropriate store:
- PostgreSQL: PostgresWorkflowStore using asyncpg with LISTEN/NOTIFY
- SQLite: SqliteWorkflowStore using raw sqlite3
Returns a lazy proxy so this can be called before launch(). The real
store is resolved on first use (which happens after launch()).
"""
if self._workflow_store is not None:
return self._workflow_store
def _factory() -> AbstractWorkflowStore:
engine = self._get_sql_engine()
schema = _resolve_schema(self.config, engine)
if engine.dialect.name == "postgresql":
dsn = _sqlalchemy_url_to_asyncpg_dsn(engine.url)
logger.info(
"Using PostgresWorkflowStore (asyncpg) for workflow storage"
)
return PostgresWorkflowStore(dsn=dsn, schema=schema)
db_path = str(engine.url.database) if engine.url.database else ":memory:"
logger.info("Using SqliteWorkflowStore for workflow storage")
return SqliteWorkflowStore(db_path=db_path, auto_migrate=False)
self._workflow_store = DBOSWorkflowStore(_factory)
return self._workflow_store
def build_server_runtime(self) -> Runtime:
"""Build the decorator chain for use with WorkflowServer.
Wraps the DBOS runtime with:
- EventInterceptorDecorator (blocks events from reaching DBOS streams)
DBOS handles persistence and resumption internally, and idle detection
is not supported (would require cancelling and resuming a new workflow).
The returned runtime should be passed as the ``runtime`` argument
to ``WorkflowServer``.
"""
return EventInterceptorDecorator(self)
def launch(self) -> None:
"""
Launch DBOS and register all tracked workflows.
@@ -403,6 +582,16 @@ class DBOSRuntime(Runtime):
DBOS.launch()
self._dbos_launched = True
# Resolve native driver config from SQLAlchemy engine
engine = self._get_sql_engine()
self._schema = _resolve_schema(self.config, engine)
if engine.dialect.name == "postgresql":
self._dsn = _sqlalchemy_url_to_asyncpg_dsn(engine.url)
else:
self._db_path = (
str(engine.url.database) if engine.url.database else ":memory:"
)
# Run migrations after DBOS is launched (if configured)
if self.config.get("run_migrations_on_launch", True):
self.run_migrations()
@@ -421,6 +610,36 @@ class DBOSRuntime(Runtime):
self._dbos_launched = False
self._sql_engine = None
self._migrations_run = False
self._dsn = None
self._db_path = None
self._schema = None
if self._pool is not None:
try:
self._pool.terminate()
except Exception:
logger.debug(
"Failed to terminate asyncpg pool during destroy", exc_info=True
)
self._pool = None
if self._workflow_store is not None:
inner = (
self._workflow_store._inner
if isinstance(self._workflow_store, DBOSWorkflowStore)
else self._workflow_store
)
if isinstance(inner, PostgresWorkflowStore):
pool = inner._pool
if pool is not None:
try:
pool.terminate()
except Exception:
logger.debug(
"Failed to terminate workflow store pool during destroy",
exc_info=True,
)
inner._pool = None
inner._listen_conn = None
self._workflow_store = None
for task in self._tasks:
if not task.done():
task.cancel()
@@ -428,6 +647,8 @@ class DBOSRuntime(Runtime):
DBOS.destroy()
EnsurePoolFn = Callable[[], Awaitable[asyncpg.Pool]]
_IO_STREAM_PUBLISHED_EVENTS_NAME = "published_events"
_IO_STREAM_TICK_TOPIC = "ticks"
@@ -452,6 +673,9 @@ class InternalDBOSAdapter(InternalRunAdapter):
schema: str | None = None,
state_table_name: str = DEFAULT_STATE_TABLE_NAME,
journal_table_name: str = DEFAULT_JOURNAL_TABLE_NAME,
ensure_pool: EnsurePoolFn | None = None,
pool: asyncpg.Pool | None = None,
db_path: str | None = None,
) -> None:
self._run_id = run_id
self._engine = engine
@@ -459,8 +683,11 @@ class InternalDBOSAdapter(InternalRunAdapter):
self._schema = schema
self._state_table_name = state_table_name
self._journal_table_name = journal_table_name
self._ensure_pool = ensure_pool
self._resolved_pool: asyncpg.Pool | None = pool
self._db_path = db_path
self._closed = False
self._state_store: SqlStateStore[Any] | None = None
self._state_store: StateStore[Any] | None = None
# Journal for deterministic task ordering - lazily initialized
self._journal: TaskJournal | None = None
@@ -524,26 +751,74 @@ class InternalDBOSAdapter(InternalRunAdapter):
),
)
def _get_or_create_state_store(self) -> SqlStateStore[Any]:
"""Get or lazily create the state store."""
if self._state_store is None:
self._state_store = SqlStateStore(
run_id=self._run_id,
engine=self._engine,
state_type=self._state_type,
schema=self._schema,
table_name=self._state_table_name,
async def _resolve_pool(self) -> asyncpg.Pool:
"""Resolve the asyncpg pool, lazily creating it via the runtime callback."""
if self._resolved_pool is not None:
return self._resolved_pool
if self._ensure_pool is None:
raise RuntimeError(
"No asyncpg pool configured. Either not launched or using sqlite dialect."
)
self._resolved_pool = await self._ensure_pool()
return self._resolved_pool
def _get_or_create_state_store(self) -> StateStore[Any]:
"""Get or lazily create the state store.
For PostgreSQL, the pool must be resolved first via _resolve_pool().
Call _ensure_resources() before accessing the state store.
"""
if self._state_store is None:
if self._resolved_pool is not None:
self._state_store = PostgresStateStore(
pool=self._resolved_pool,
run_id=self._run_id,
state_type=cast(type[Any], self._state_type),
schema=self._schema,
)
elif self._db_path is not None:
self._state_store = SqliteStateStore(
db_path=self._db_path,
run_id=self._run_id,
state_type=cast(type[Any], self._state_type),
)
else:
raise RuntimeError(
"No pool or db_path configured for state store. "
"Ensure the runtime pool is initialized before accessing state."
)
return self._state_store
def get_state_store(self) -> StateStore[Any] | None:
return self._get_or_create_state_store()
def is_replaying(self) -> bool:
if (
self._journal is None
and self._resolved_pool is None
and self._db_path is None
):
return False
journal = self._get_or_create_journal()
return journal.is_replaying()
def _get_or_create_journal(self) -> TaskJournal:
"""Get or lazily create the task journal."""
if self._journal is None:
crud = JournalCrud(table_name=self._journal_table_name, schema=self._schema)
self._journal = TaskJournal(self._run_id, self._engine, crud)
if self._resolved_pool is not None:
crud = PostgresJournalCrud(
pool=self._resolved_pool,
table_name=self._journal_table_name,
schema=self._schema,
)
elif self._db_path is not None:
crud = SqliteJournalCrud(
db_path=self._db_path,
table_name=self._journal_table_name,
)
else:
raise RuntimeError("No pool or db_path configured for journal.")
self._journal = TaskJournal(self._run_id, crud)
return self._journal
async def wait_for_next_task(
@@ -556,6 +831,23 @@ class InternalDBOSAdapter(InternalRunAdapter):
During replay, waits for the specific task that completed in the original run.
During fresh execution, waits for any task and records the completion order.
**Journal ordering caveat:** The journal records task completion *before*
the control loop publishes events produced by the completed step. If a
crash occurs between the journal write and the event write, replay skips
the step (already journaled) but the event was never persisted.
- In the standalone DBOS path, ``write_to_event_stream`` publishes to
DBOS streams which are separately journaled, so the gap is less of an
issue.
- In the server path, ``_ServerInternalRunAdapter`` makes event writes
idempotent by comparing against already-persisted events, working
around the ordering gap.
- Self-publishing events (``InputRequiredEvent`` / ``HumanInputRequired``
subtypes) are affected: if a crash occurs after the journal records the
step but before the event is published, the event is lost on replay.
- Proper fix: defer journal recording until after all commands from a
tick are processed.
Args:
task_set: List of NamedTasks with stable string keys for identification
timeout: Timeout in seconds, None for no timeout
@@ -567,6 +859,10 @@ class InternalDBOSAdapter(InternalRunAdapter):
if not tasks:
return None
# Ensure pool is resolved before journal creation (needed for postgres)
if self._ensure_pool is not None and self._resolved_pool is None:
await self._resolve_pool()
journal = self._get_or_create_journal()
await journal.load()
@@ -616,11 +912,12 @@ class ExternalDBOSAdapter(ExternalRunAdapter):
self,
run_id: str,
polling_interval_sec: float = 1.0,
startup_task: asyncio.Task[None] | None = None,
startup_task: asyncio.Task[WorkflowHandleAsync[Any]] | None = None,
) -> None:
self._run_id = run_id
self._polling_interval_sec = polling_interval_sec
self._startup_task = startup_task # None means workflow already started
self._handle: WorkflowHandleAsync[Any] | None = None
@property
def run_id(self) -> str:
@@ -637,12 +934,27 @@ class ExternalDBOSAdapter(ExternalRunAdapter):
yield event
async def get_result(self) -> StopEvent:
await self._ensure_workflow_started()
handle = await DBOS.retrieve_workflow_async(self.run_id)
handle = await self._ensure_workflow_started()
return await handle.get_result(polling_interval_sec=self._polling_interval_sec)
async def _ensure_workflow_started(self) -> None:
"""Wait for the workflow startup task to complete if one was provided."""
async def _ensure_workflow_started(self) -> WorkflowHandleAsync[Any]:
"""Wait for the workflow startup task to complete and return the handle."""
if self._startup_task is not None:
await self._startup_task
self._handle = await self._startup_task
self._startup_task = None # Clear after awaiting
if self._handle is None:
# Fallback: workflow was started elsewhere, retrieve with retry since
# there can be a race between start_workflow_async completing and the
# workflow becoming retrievable in DBOS.
from dbos._error import DBOSNonExistentWorkflowError
for attempt in range(20):
try:
self._handle = await DBOS.retrieve_workflow_async(self.run_id)
break
except DBOSNonExistentWorkflowError:
if attempt == 19:
raise
await asyncio.sleep(0.1 * (attempt + 1))
assert self._handle is not None
return self._handle
@@ -1,588 +0,0 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
"""
SQL-backed StateStore implementations for durable workflow state.
Provides PostgreSQL and SQLite state stores that persist workflow state
to a database, enabling durable and distributed workflow execution.
"""
from __future__ import annotations
import asyncio
import functools
import json
import logging
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from typing import (
Any,
AsyncGenerator,
Callable,
Generic,
Literal,
Type,
)
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from sqlalchemy import (
Column,
Connection,
DateTime,
MetaData,
String,
Table,
Text,
select,
text,
)
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.engine import Engine
from typing_extensions import TypeVar
from workflows.context.serializers import BaseSerializer, JsonSerializer
from workflows.context.state_store import (
MAX_DEPTH,
DictState,
InMemorySerializedState,
assign_path_step,
deserialize_state_from_dict,
traverse_path_step,
)
logger = logging.getLogger(__name__)
MODEL_T = TypeVar("MODEL_T", bound=BaseModel)
class SqlSerializedState(BaseModel):
"""Serialized state referencing a database row (from SqlStateStore)."""
model_config = ConfigDict(populate_by_name=True)
store_type: Literal["sql"] = "sql"
run_id: str
db_schema: str | None = Field(default=None, alias="schema")
# Note: No state_data - actual data lives in the database
def parse_serialized_state(
data: dict[str, Any],
) -> InMemorySerializedState | SqlSerializedState:
"""Parse raw dict into appropriate format type.
Args:
data: Serialized state payload from to_dict().
Returns:
InMemorySerializedState or SqlSerializedState based on store_type.
Raises:
ValueError: If store_type is unknown.
"""
store_type = data.get("store_type")
if store_type == "sql":
return SqlSerializedState.model_validate(data)
elif store_type == "in_memory" or store_type is None:
# Backwards compat: missing store_type = InMemory
return InMemorySerializedState.model_validate(data)
else:
raise ValueError(f"Unknown store_type: {store_type}")
def _utc_now() -> datetime:
"""Get current UTC timestamp."""
return datetime.now(timezone.utc)
STATE_TABLE_NAME = "workflow_state"
def _state_columns() -> list[Column]:
"""Return fresh Column instances for the workflow_state table.
Must return new instances each call because SQLAlchemy Column objects
can't be shared across Table instances.
"""
return [
Column("run_id", String(255), primary_key=True),
Column("state_json", Text, nullable=False),
Column("created_at", DateTime(timezone=True), nullable=False),
Column("updated_at", DateTime(timezone=True), nullable=False),
]
class SqlStateStore(Generic[MODEL_T]):
"""
SQL-backed StateStore implementation.
Persists workflow state to a database table. Supports PostgreSQL and SQLite
dialects with automatic detection based on the engine.
Thread-safety is achieved through database-level locking during
transactional edits via the `edit_state` context manager.
"""
known_unserializable_keys = ("memory",)
state_type: Type[MODEL_T]
def __init__(
self,
run_id: str,
state_type: Type[MODEL_T] | None = None,
engine: Engine | None = None,
serializer: BaseSerializer | None = None,
schema: str | None = None,
table_name: str = STATE_TABLE_NAME,
) -> None:
self._run_id = run_id
self.state_type = state_type or DictState # type: ignore[assignment]
self._engine = engine
self._serializer = serializer or JsonSerializer()
self._schema = schema
self._table_name = table_name
self._metadata = MetaData(schema=self._schema)
self._table = self._define_table()
self._initialized = False
self._pending_state: dict[str, Any] | None = None
@property
def run_id(self) -> str:
"""Get the workflow run ID."""
return self._run_id
@property
def engine(self) -> Engine:
"""Get the SQLAlchemy engine, raising if not set."""
if self._engine is None:
raise RuntimeError(
"Engine not set. Provide an engine at construction or set it before use."
)
return self._engine
@engine.setter
def engine(self, engine: Engine) -> None:
"""Set the SQLAlchemy engine."""
self._engine = engine
@property
def _is_postgres(self) -> bool:
"""Check if the engine is PostgreSQL."""
return self.engine.dialect.name == "postgresql"
@property
def _table_ref(self) -> str:
"""Get the fully qualified table reference."""
if self._schema:
return f"{self._schema}.{self._table_name}"
return self._table_name
def _define_table(self) -> Table:
"""Define the workflow_state table schema."""
return Table(
self._table_name,
self._metadata,
*_state_columns(),
)
@classmethod
def run_migrations(
cls,
engine: Engine,
table_name: str = STATE_TABLE_NAME,
schema: str | None = None,
) -> None:
"""Create schema and table if they don't exist."""
metadata = MetaData(schema=schema)
table = Table(
table_name,
metadata,
*_state_columns(),
)
is_postgres = engine.dialect.name == "postgresql"
with engine.begin() as conn:
if is_postgres and schema:
conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {schema}")) # noqa: S608
table.create(bind=conn, checkfirst=True)
@functools.cached_property
def _lock(self) -> asyncio.Lock:
"""Lazy lock for Python 3.14+ compatibility."""
return asyncio.Lock()
async def _run_sync(self, fn: Callable[..., Any], *args: Any) -> Any:
"""Run a synchronous function in the default executor."""
loop = asyncio.get_event_loop()
return await loop.run_in_executor(None, fn, *args)
def _ensure_initialized(self) -> None:
"""Ensure the table exists and apply any pending state."""
if self._initialized:
return
self._run_instance_migrations()
self._initialized = True
# Apply any pending state from InMemory format deserialization
if self._pending_state is not None:
self._apply_pending_state_sync()
def _apply_pending_state_sync(self) -> None:
"""Write pending state to database (called after migrations)."""
if self._pending_state is None:
return
serialized_state = self._pending_state
self._pending_state = None
state = deserialize_state_from_dict(
serialized_state, self._serializer, state_type=self.state_type
)
state_json = self._serialize_state(state) # type: ignore[arg-type]
with self.engine.begin() as conn:
self._upsert_state(conn, state_json, _utc_now())
def _run_instance_migrations(self) -> None:
"""Create schema and table if they don't exist (instance-level)."""
with self.engine.begin() as conn:
if self._is_postgres and self._schema:
conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {self._schema}")) # noqa: S608
self._table.create(bind=conn, checkfirst=True)
def _lock_row_for_update(self, conn: Connection) -> dict[str, Any] | None:
"""Lock and return row data for this run_id."""
for_update = "FOR UPDATE" if self._is_postgres else ""
result = conn.execute(
text(f"""
SELECT state_json
FROM {self._table_ref}
WHERE run_id = :run_id
{for_update}
"""), # noqa: S608
{"run_id": self._run_id},
)
row = result.fetchone()
if row is None:
return None
return {"state_json": row[0]}
def _upsert_state(
self,
conn: Connection,
state_json: str,
now: datetime,
) -> None:
"""Perform database-specific upsert operation."""
if self._is_postgres:
stmt = pg_insert(self._table).values(
run_id=self._run_id,
state_json=state_json,
created_at=now,
updated_at=now,
)
stmt = stmt.on_conflict_do_update(
index_elements=["run_id"],
set_={
"state_json": stmt.excluded.state_json,
"updated_at": stmt.excluded.updated_at,
},
)
conn.execute(stmt)
else:
# SQLite upsert
conn.execute(
text(f"""
INSERT INTO {self._table_ref}
(run_id, state_json, created_at, updated_at)
VALUES (:run_id, :state_json, :created_at, :updated_at)
ON CONFLICT (run_id) DO UPDATE SET
state_json = excluded.state_json,
updated_at = excluded.updated_at
"""), # noqa: S608
{
"run_id": self._run_id,
"state_json": state_json,
"created_at": now.isoformat(),
"updated_at": now.isoformat(),
},
)
def _serialize_state(self, state: MODEL_T) -> str:
"""Serialize state model to JSON string."""
if isinstance(state, DictState):
serialized_data: dict[str, Any] = {}
for key, value in state.items():
try:
serialized_data[key] = self._serializer.serialize(value)
except Exception:
if key in self.known_unserializable_keys:
logger.warning(f"Skipping unserializable key: {key}")
continue
raise
return json.dumps({"_data": serialized_data})
return self._serializer.serialize(state)
def _deserialize_state(self, state_json: str) -> MODEL_T:
"""Deserialize state from JSON string."""
if issubclass(self.state_type, DictState):
data = json.loads(state_json)
deserialized = {
k: self._serializer.deserialize(v)
for k, v in data.get("_data", {}).items()
}
return DictState(_data=deserialized) # type: ignore[return-value]
return self._serializer.deserialize(state_json)
def _create_default_state(self) -> MODEL_T:
"""Create a default instance of the state type."""
return self.state_type()
def _load_state_sync(self) -> MODEL_T:
"""Load state from database synchronously."""
self._ensure_initialized()
with self.engine.connect() as conn:
result = conn.execute(
select(self._table.c.state_json).where(
self._table.c.run_id == self._run_id
)
)
row = result.fetchone()
if row is None:
state = self._create_default_state()
self._save_state_sync(state, conn)
conn.commit()
return state
return self._deserialize_state(row[0])
def _save_state_sync(self, state: MODEL_T, conn: Connection) -> None:
"""Save state to database synchronously."""
now = _utc_now()
self._upsert_state(conn, self._serialize_state(state), now)
async def get_state(self) -> MODEL_T:
"""Return a copy of the current state model."""
state = await self._run_sync(self._load_state_sync)
return state.model_copy()
async def set_state(self, state: MODEL_T) -> None:
"""Replace or merge into the current state model."""
def _set_state_sync() -> None:
self._ensure_initialized()
with self.engine.begin() as conn:
result = conn.execute(
select(self._table.c.state_json).where(
self._table.c.run_id == self._run_id
)
)
row = result.fetchone()
if row is None:
self._save_state_sync(state, conn)
return
current_state = self._deserialize_state(row[0])
current_type = type(current_state)
new_type = type(state)
if isinstance(state, current_type):
self._save_state_sync(state, conn)
elif issubclass(current_type, new_type):
parent_data = state.model_dump()
merged = current_type.model_validate(
{**current_state.model_dump(), **parent_data}
)
self._save_state_sync(merged, conn)
else:
raise ValueError(
f"State must be of type {current_type.__name__} or parent, "
f"got {new_type.__name__}"
)
await self._run_sync(_set_state_sync)
async def get(self, path: str, default: Any = ...) -> Any:
"""Get a nested value using dot-separated paths."""
state = await self._run_sync(self._load_state_sync)
segments = path.split(".") if path else []
if len(segments) > MAX_DEPTH:
raise ValueError(f"Path length exceeds {MAX_DEPTH} segments")
try:
value: Any = state
for segment in segments:
value = traverse_path_step(value, segment)
except Exception:
if default is not ...:
return default
raise ValueError(f"Path '{path}' not found in state")
return value
async def set(self, path: str, value: Any) -> None:
"""Set a nested value using dot-separated paths."""
if not path:
raise ValueError("Path cannot be empty")
segments = path.split(".")
if len(segments) > MAX_DEPTH:
raise ValueError(f"Path length exceeds {MAX_DEPTH} segments")
async with self.edit_state() as state:
current: Any = state
for segment in segments[:-1]:
try:
current = traverse_path_step(current, segment)
except (KeyError, AttributeError, IndexError, TypeError):
intermediate: Any = {}
assign_path_step(current, segment, intermediate)
current = intermediate
assign_path_step(current, segments[-1], value)
async def clear(self) -> None:
"""Reset the state to its type defaults."""
try:
await self.set_state(self._create_default_state())
except ValidationError:
raise ValueError("State must have defaults for all fields")
@asynccontextmanager
async def edit_state(self) -> AsyncGenerator[MODEL_T, None]:
"""Edit state transactionally under a database lock."""
def _edit_with_lock() -> tuple[
MODEL_T, Callable[[MODEL_T], None], Callable[[], None]
]:
self._ensure_initialized()
conn = self.engine.connect()
trans = conn.begin()
finalized = False
try:
row_data = self._lock_row_for_update(conn)
if row_data is None:
state = self._create_default_state()
else:
state = self._deserialize_state(row_data["state_json"])
def commit_fn(updated_state: MODEL_T) -> None:
nonlocal finalized
if finalized:
return
try:
self._save_state_sync(updated_state, conn)
trans.commit()
finally:
finalized = True
conn.close()
def rollback_fn() -> None:
nonlocal finalized
if finalized:
return
try:
if trans.is_active:
trans.rollback()
finally:
finalized = True
conn.close()
return state, commit_fn, rollback_fn
except Exception:
trans.rollback()
conn.close()
raise
async with self._lock:
state, commit_fn, rollback_fn = await self._run_sync(_edit_with_lock)
try:
yield state
await self._run_sync(commit_fn, state)
except Exception:
try:
await self._run_sync(rollback_fn)
except Exception:
logger.exception("Failed to rollback edit_state transaction")
raise
def to_dict(self, serializer: BaseSerializer) -> dict[str, Any]:
"""Serialize state store metadata for persistence.
Returns a SqlSerializedState payload that can be restored by from_dict().
The actual state data lives in the database, so this only serializes
connection metadata (run_id, schema).
"""
payload = SqlSerializedState.model_validate(
{
"run_id": self._run_id,
"schema": self._schema,
}
)
return payload.model_dump(by_alias=True)
@classmethod
def from_dict(
cls,
serialized_state: dict[str, Any],
serializer: BaseSerializer,
state_type: type[BaseModel] = DictState,
run_id: str | None = None,
) -> SqlStateStore[Any]:
"""Restore a state store from serialized payload.
Handles both InMemorySerializedState and SqlSerializedState formats:
- InMemorySerializedState: Stores the serialized data internally and
writes it to the database when the engine is first used (via
_ensure_initialized). This enables restoring state from in-memory
format into a SQL-backed store.
- SqlSerializedState: Creates a store pointing at the existing database
row. If a different run_id is provided, the store will use that run_id
(data copying must be handled separately if needed).
Note: The engine must be set separately after restoration.
Args:
serialized_state: Payload from to_dict() of either store type.
serializer: Serializer for data handling.
state_type: The state model type for deserialization.
run_id: Optional override run_id. If not provided, uses the run_id
from SqlSerializedState or generates one for InMemory format.
Returns:
A new SqlStateStore instance configured from the payload.
Raises:
ValueError: If serialized_state is empty.
"""
import uuid
if not serialized_state:
raise ValueError("Cannot restore SqlStateStore from empty dict")
parsed = parse_serialized_state(serialized_state)
if isinstance(parsed, InMemorySerializedState):
# InMemory format: store data internally, apply when engine is set
effective_run_id = run_id or str(uuid.uuid4())
store = cls(
run_id=effective_run_id,
state_type=state_type, # type: ignore[arg-type]
serializer=serializer,
)
# Store the serialized state to apply when engine is available
store._pending_state = serialized_state
return store
else:
# SqlSerializedState format: create store pointing at existing row
effective_run_id = run_id or parsed.run_id
schema = parsed.db_schema
return cls(
run_id=effective_run_id,
state_type=state_type, # type: ignore[arg-type]
serializer=serializer,
schema=schema,
)
+39 -5
View File
@@ -2,20 +2,54 @@
# Copyright (c) 2026 LlamaIndex Inc.
from __future__ import annotations
import tempfile
from collections.abc import Generator
from pathlib import Path
import pytest
from llama_agents.dbos.journal.crud import JournalCrud
from llama_agents.server._store.sqlite.sqlite_workflow_store import (
SqliteWorkflowStore,
)
from llama_agents_integration_tests.postgres import (
get_asyncpg_dsn,
)
from llama_agents_integration_tests.postgres import (
postgres_container as _postgres_container,
)
from sqlalchemy import create_engine
from sqlalchemy.engine import Engine
from sqlalchemy.pool import StaticPool
from testcontainers.postgres import PostgresContainer
@pytest.fixture
def sqlite_engine() -> Engine:
"""Create an in-memory SQLite engine with journal table."""
def journal_db_path() -> Generator[str]:
"""Create a temporary SQLite database with migrations applied."""
with tempfile.TemporaryDirectory() as tmp:
db_path = str(Path(tmp) / "test.db")
SqliteWorkflowStore.run_migrations(db_path)
yield db_path
@pytest.fixture
def sqlite_engine(journal_db_path: str) -> Engine:
"""Create a SQLAlchemy engine pointing at the migrated test database."""
engine = create_engine(
"sqlite:///:memory:",
f"sqlite:///{journal_db_path}",
connect_args={"check_same_thread": False},
poolclass=StaticPool,
)
JournalCrud().run_migrations(engine)
return engine
@pytest.fixture(scope="module")
def postgres_container() -> Generator[PostgresContainer, None, None]:
"""Module-scoped PostgreSQL container for docker-marked tests."""
with _postgres_container() as pg:
yield pg
@pytest.fixture(scope="module")
def postgres_dsn(postgres_container: PostgresContainer) -> str:
"""Return a plain postgresql:// DSN suitable for asyncpg."""
return get_asyncpg_dsn(postgres_container)
+28 -140
View File
@@ -8,7 +8,7 @@ response simulation.
Usage:
python /path/to/packages/llama-agents-dbos/tests/fixtures/runner.py \
--workflow "tests.fixtures.workflows.hitl:TestWorkflow" \
--workflow "tests.fixtures.sample_workflows.hitl:TestWorkflow" \
--db-url "sqlite+pysqlite:///path/to/db" \
--run-id "test-001" \
--config '{"interrupt_on": "AskInputEvent"}'
@@ -26,118 +26,24 @@ from __future__ import annotations
import argparse
import asyncio
import importlib
import json
import os
import sys
from pathlib import Path
from types import ModuleType
from typing import Any
# Add package source directories to sys.path for imports
# Runner is at: packages/llama-agents-dbos/tests/fixtures/runner.py
# We need to add:
# - packages/llama-agents-dbos/src for llama_agents.dbos
# - packages/llama-index-workflows/src for workflows.*
# - packages/llama-agents-dbos (parent of tests/) so tests.fixtures.workflows.* can be imported
TESTS_DIR = Path(__file__).parent.parent
DBOS_PACKAGE_DIR = TESTS_DIR.parent
DBOS_PACKAGE_SRC_PATH = str(DBOS_PACKAGE_DIR / "src")
WORKFLOWS_PACKAGE_SRC_PATH = str(
DBOS_PACKAGE_DIR.parent / "llama-index-workflows" / "src"
# Add fixtures dir to find runner_common; safe now that fixtures/workflows
# was renamed to fixtures/sample_workflows to avoid shadowing the real package
sys.path.insert(0, str(Path(__file__).parent))
from dbos import DBOS # noqa: E402
from runner_common import ( # noqa: E402 # ty: ignore[unresolved-import]
get_event_class_by_name,
import_workflow,
setup_dbos,
)
# Insert at front of path so these packages take precedence
# Add the parent of tests/ so "import tests.fixtures.workflows..." works
sys.path.insert(0, str(DBOS_PACKAGE_DIR))
sys.path.insert(0, DBOS_PACKAGE_SRC_PATH)
sys.path.insert(0, WORKFLOWS_PACKAGE_SRC_PATH)
from dbos import DBOS, DBOSConfig # noqa: E402
from llama_agents.dbos import DBOSRuntime # noqa: E402
from workflows.context import Context # noqa: E402
from workflows.events import Event, InputRequiredEvent, StartEvent # noqa: E402
from workflows.workflow import Workflow # noqa: E402
def import_workflow(path: str) -> tuple[type[Workflow], ModuleType]:
"""Import a workflow class from a module path.
Args:
path: Module path with class name, e.g., "tests.fixtures.workflows.hitl:TestWorkflow"
Returns:
Tuple of (workflow_class, module) for accessing classes defined in the module.
Raises:
ValueError: If path format is invalid.
ImportError: If module cannot be imported.
AttributeError: If class not found in module.
"""
if ":" not in path:
raise ValueError(
f"Invalid workflow path format: {path}. Expected 'module.path:ClassName'"
)
module_path, class_name = path.rsplit(":", 1)
module = importlib.import_module(module_path)
workflow_class = getattr(module, class_name)
if not (isinstance(workflow_class, type) and issubclass(workflow_class, Workflow)):
raise TypeError(f"{class_name} is not a Workflow subclass")
return workflow_class, module
def get_event_class_by_name(module: ModuleType, name: str) -> type[Event] | None:
"""Find an event class in a module by its name.
Searches through all attributes of the module to find an Event subclass
with a matching class name.
Args:
module: The module to search in.
name: The class name to find.
Returns:
The event class if found, None otherwise.
"""
for attr_name in dir(module):
attr = getattr(module, attr_name)
if isinstance(attr, type) and issubclass(attr, Event) and attr.__name__ == name:
return attr
return None
def parse_config(config_json: str | None) -> dict[str, Any]:
"""Parse the JSON config string.
Args:
config_json: JSON string with configuration, or None.
Returns:
Parsed config dict, or empty dict if None.
"""
if not config_json:
return {}
return json.loads(config_json)
def setup_dbos(db_url: str, app_name: str = "test-workflow") -> DBOSRuntime:
"""Set up DBOS with the given database URL.
Args:
db_url: SQLite database URL.
app_name: Application name for DBOS config.
Returns:
Configured DBOSRuntime instance.
"""
config: DBOSConfig = {
"name": app_name,
"system_database_url": db_url,
"run_admin_server": False,
"notification_listener_polling_interval_sec": 0.01,
}
DBOS(config=config)
return DBOSRuntime(polling_interval_sec=0.01)
from workflows.handler import WorkflowHandler # noqa: E402
async def run_workflow(
@@ -146,14 +52,7 @@ async def run_workflow(
run_id: str,
config: dict[str, Any],
) -> None:
"""Run the workflow with the specified configuration.
Args:
workflow_path: Module path with class name.
db_url: SQLite database URL.
run_id: Unique run ID for the workflow.
config: Configuration dict with interrupt_on and/or respond settings.
"""
"""Run the workflow with the specified configuration."""
# Import workflow and get module for event class lookup
workflow_class, module = import_workflow(workflow_path)
@@ -194,7 +93,6 @@ async def run_workflow(
f"ERROR:ValueError:Response event class '{response_event_name}' not found in module"
)
sys.exit(1)
# Both trigger_class and response_class are narrowed after sys.exit(1) guards
assert trigger_class is not None
assert response_class is not None
response_map[trigger_class] = (response_class, response_fields)
@@ -207,8 +105,17 @@ async def run_workflow(
runtime.launch()
try:
ctx = Context(wf)
handler = ctx._workflow_run(wf, StartEvent(), run_id=run_id)
# Check if the workflow already exists (i.e., we're resuming after interrupt).
# DBOS auto-recovers pending workflows on launch(), so we just need to
# attach to the existing run instead of starting a new one.
existing = await DBOS.get_workflow_status_async(run_id)
if existing is not None:
# Attach to the auto-recovered workflow
external_adapter = runtime.get_external_adapter(run_id)
handler = WorkflowHandler(wf, external_adapter)
else:
# Fresh run - start the workflow
handler = wf.run(start_event=StartEvent(), run_id=run_id)
async for event in handler.stream_events():
event_name = type(event).__name__
@@ -218,7 +125,6 @@ async def run_workflow(
if interrupt_event_class is not None and isinstance(
event, interrupt_event_class
):
# Check condition fields if present
should_interrupt = True
if interrupt_condition:
for field, expected_value in interrupt_condition.items():
@@ -256,37 +162,19 @@ def main() -> None:
parser = argparse.ArgumentParser(
description="Run workflows in isolated subprocesses for testing"
)
parser.add_argument(
"--workflow",
required=True,
help="Module path with class name (e.g., 'tests.fixtures.workflows.hitl:TestWorkflow')",
)
parser.add_argument(
"--db-url",
required=True,
help="SQLite database URL",
)
parser.add_argument(
"--run-id",
required=True,
help="Unique run ID for the workflow",
)
parser.add_argument(
"--config",
default=None,
help="JSON string with configuration",
)
parser.add_argument("--workflow", required=True)
parser.add_argument("--db-url", required=True)
parser.add_argument("--run-id", required=True)
parser.add_argument("--config", default=None)
args = parser.parse_args()
config = parse_config(args.config)
asyncio.run(
run_workflow(
workflow_path=args.workflow,
db_url=args.db_url,
run_id=args.run_id,
config=config,
config=json.loads(args.config) if args.config else {},
)
)
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
"""Shared utilities for subprocess test runners.
Importing this module adds all necessary package source directories to sys.path
as a side effect, so that runner scripts can import workflows, llama_agents, etc.
"""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
from types import ModuleType
# Compute package source directories relative to this file
TESTS_DIR = Path(__file__).parent.parent
DBOS_PACKAGE_DIR = TESTS_DIR.parent
_SYS_PATHS = [
str(DBOS_PACKAGE_DIR),
str(DBOS_PACKAGE_DIR / "src"),
str(DBOS_PACKAGE_DIR.parent / "llama-index-workflows" / "src"),
str(DBOS_PACKAGE_DIR.parent / "llama-agents-server" / "src"),
str(DBOS_PACKAGE_DIR.parent / "llama-agents-client" / "src"),
str(DBOS_PACKAGE_DIR.parent / "llama-index-instrumentation" / "src"),
]
for _p in _SYS_PATHS:
if _p not in sys.path:
sys.path.insert(0, _p)
from dbos import DBOS, DBOSConfig # noqa: E402
from llama_agents.dbos import DBOSRuntime # noqa: E402
from workflows.events import Event # noqa: E402
from workflows.workflow import Workflow # noqa: E402
def import_workflow(path: str) -> tuple[type[Workflow], ModuleType]:
"""Import a workflow class from a module path like 'module.path:ClassName'."""
if ":" not in path:
raise ValueError(f"Invalid workflow path format: {path}")
module_path, class_name = path.rsplit(":", 1)
module = importlib.import_module(module_path)
workflow_class = getattr(module, class_name)
if not (isinstance(workflow_class, type) and issubclass(workflow_class, Workflow)):
raise TypeError(f"{class_name} is not a Workflow subclass")
return workflow_class, module
def get_event_class_by_name(module: ModuleType, name: str) -> type[Event] | None:
"""Find an event class in a module by its name."""
for attr_name in dir(module):
attr = getattr(module, attr_name)
if isinstance(attr, type) and issubclass(attr, Event) and attr.__name__ == name:
return attr
return None
def setup_dbos(db_url: str, app_name: str = "test-workflow") -> DBOSRuntime:
"""Set up DBOS with the given database URL and return a DBOSRuntime."""
config: DBOSConfig = {
"name": app_name,
"system_database_url": db_url,
"run_admin_server": False,
"notification_listener_polling_interval_sec": 0.01,
}
DBOS(config=config)
return DBOSRuntime(polling_interval_sec=0.01)
@@ -7,7 +7,12 @@ from __future__ import annotations
from pydantic import Field
from workflows.context import Context
from workflows.decorators import step
from workflows.events import Event, InputRequiredEvent, StartEvent, StopEvent
from workflows.events import (
HumanResponseEvent,
InputRequiredEvent,
StartEvent,
StopEvent,
)
from workflows.workflow import Workflow
@@ -15,7 +20,7 @@ class AskInputEvent(InputRequiredEvent):
prefix: str = Field(default="Enter: ")
class UserInput(Event):
class UserInput(HumanResponseEvent):
response: str = Field(default="")
@@ -10,7 +10,13 @@ import random
from pydantic import Field
from workflows.context import Context
from workflows.decorators import step
from workflows.events import Event, InputRequiredEvent, StartEvent, StopEvent
from workflows.events import (
Event,
HumanResponseEvent,
InputRequiredEvent,
StartEvent,
StopEvent,
)
from workflows.workflow import Workflow
@@ -22,7 +28,7 @@ class WaitForInputEvent(InputRequiredEvent):
prompt: str = Field(default="")
class UserContinueEvent(Event):
class UserContinueEvent(HumanResponseEvent):
continue_value: str = Field(default="")
@@ -36,7 +36,7 @@ class FanOutComplete(Event):
class StreamingInterruptWorkflow(Workflow):
@step
async def fan_out(self, ctx: Context, ev: StartEvent) -> FanOutComplete:
async def fan_out(self, ctx: Context, ev: StartEvent) -> WorkItem | FanOutComplete:
for i in range(15):
ctx.write_event_to_stream(ProgressEvent(progress=i))
ctx.send_event(WorkItem(item_id=i))
@@ -32,7 +32,7 @@ class FanOutComplete(Event):
class StreamingStressWorkflow(Workflow):
@step
async def fan_out(self, ctx: Context, ev: StartEvent) -> FanOutComplete:
async def fan_out(self, ctx: Context, ev: StartEvent) -> WorkItem | FanOutComplete:
# Fire many stream writes and internal events concurrently
# This creates many background tasks that call DBOS operations
for i in range(15):
@@ -7,7 +7,12 @@ from __future__ import annotations
from pydantic import Field
from workflows.context import Context
from workflows.decorators import step
from workflows.events import Event, InputRequiredEvent, StartEvent, StopEvent
from workflows.events import (
HumanResponseEvent,
InputRequiredEvent,
StartEvent,
StopEvent,
)
from workflows.workflow import Workflow
@@ -15,7 +20,7 @@ class NameInputEvent(InputRequiredEvent):
prefix: str = Field(default="Name: ")
class NameResponseEvent(Event):
class NameResponseEvent(HumanResponseEvent):
response: str = Field(default="")
@@ -23,7 +28,7 @@ class QuestInputEvent(InputRequiredEvent):
prefix: str = Field(default="Quest: ")
class QuestResponseEvent(Event):
class QuestResponseEvent(HumanResponseEvent):
response: str = Field(default="")
@@ -0,0 +1,196 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
"""Subprocess runner for DBOS + WorkflowServer integration tests.
Runs a workflow through the full WorkflowServer decorator chain with the store
created by DBOSRuntime, enabling end-to-end event flow testing including
interrupt/resume scenarios.
Usage:
python server_runner.py \
--workflow "tests.fixtures.sample_workflows.chained:ChainedWorkflow" \
--db-url "postgresql://user:pass@localhost/db" \
--run-id "test-001" \
--check-streams --check-events
python server_runner.py \
--workflow "tests.fixtures.sample_workflows.chained:ChainedWorkflow" \
--db-url "postgresql://user:pass@localhost/db" \
--run-id "test-001" \
--interrupt-after StepTwoEvent
Flags:
--check-streams: After workflow completes, query dbos.streams table
and print STREAMS_COUNT:<N>
--check-events: After workflow completes, query wf_events table
and print EVENTS_COUNT:<N> and EVENT_JSON:<json>
--interrupt-after EVENT_NAME: Kill the process (os._exit) after seeing
a StepStateChanged with output_event_name matching
EVENT_NAME. Simulates a crash for resume testing.
"""
from __future__ import annotations
import argparse
import asyncio
import os
import sys
from pathlib import Path
import asyncpg
# Add fixtures dir to find runner_common; safe now that fixtures/workflows
# was renamed to fixtures/sample_workflows to avoid shadowing the real package
sys.path.insert(0, str(Path(__file__).parent))
from llama_agents.server import WorkflowServer # noqa: E402
from runner_common import ( # noqa: E402 # ty: ignore[unresolved-import]
import_workflow,
setup_dbos,
)
async def check_streams_count(db_url: str, run_id: str) -> None:
"""Query DBOS streams table and print count of published_events rows."""
try:
dsn = db_url
if "+psycopg2" in dsn or "+psycopg" in dsn:
dsn = "postgresql://" + dsn.split("://", 1)[1]
conn = await asyncpg.connect(dsn)
try:
count = await conn.fetchval(
"SELECT COUNT(*) FROM dbos.streams "
"WHERE workflow_uuid = $1 AND key = 'published_events'",
run_id,
)
print(f"STREAMS_COUNT:{count}", flush=True)
finally:
await conn.close()
except Exception as e:
print(f"ERROR:{type(e).__name__}:Failed to check streams: {e}", flush=True)
async def check_events(db_url: str, run_id: str, schema: str | None = None) -> None:
"""Query wf_events table and print all events."""
try:
dsn = db_url
if "+psycopg2" in dsn or "+psycopg" in dsn:
dsn = "postgresql://" + dsn.split("://", 1)[1]
events_table = "wf_events" if schema is None else f"{schema}.wf_events"
conn = await asyncpg.connect(dsn)
try:
rows = await conn.fetch(
f"SELECT event_json FROM {events_table} "
f"WHERE run_id = $1 ORDER BY sequence",
run_id,
)
print(f"EVENTS_COUNT:{len(rows)}", flush=True)
for row in rows:
print(f"EVENT_JSON:{row['event_json']}", flush=True)
finally:
await conn.close()
except Exception as e:
print(f"ERROR:{type(e).__name__}:Failed to check events: {e}", flush=True)
async def run_workflow_with_server(
workflow_path: str,
db_url: str,
run_id: str,
do_check_streams: bool,
do_check_events: bool,
interrupt_after: str | None = None,
) -> None:
"""Run workflow through the full WorkflowServer + DBOS decorator chain."""
workflow_class, _module = import_workflow(workflow_path)
dbos_runtime = setup_dbos(db_url, app_name="test-server-workflow")
wf = workflow_class(runtime=dbos_runtime)
dbos_runtime.launch()
store = dbos_runtime.create_workflow_store()
server_runtime = dbos_runtime.build_server_runtime()
server = WorkflowServer(
runtime=server_runtime,
workflow_store=store,
idle_timeout=60.0,
)
server.add_workflow("test", wf)
schema = dbos_runtime._schema
try:
async with server.contextmanager():
wf_ref = server._service._runtime.get_workflow("test")
assert wf_ref is not None
handler_data = await server._service.start_workflow(wf_ref, run_id)
actual_run_id = handler_data.run_id
assert actual_run_id is not None
async for stored_event in store.subscribe_events(actual_run_id):
event_type = stored_event.event.type
print(f"EVENT:{event_type}", flush=True)
# Check for interrupt: StepStateChanged events carry
# output_event_name as a class repr string.
if interrupt_after is not None:
data = stored_event.event.value or {}
output_name = data.get("output_event_name") or ""
if interrupt_after in output_name:
print("INTERRUPTING", flush=True)
os._exit(0)
print("SUCCESS", flush=True)
if do_check_streams:
await check_streams_count(db_url, actual_run_id)
if do_check_events:
await check_events(db_url, actual_run_id, schema)
except Exception as e:
print(f"ERROR:{type(e).__name__}:{e}", flush=True)
raise
finally:
dbos_runtime.destroy()
def main() -> None:
"""Entry point for the subprocess runner."""
parser = argparse.ArgumentParser(
description="Run workflows through WorkflowServer + DBOS for testing"
)
parser.add_argument("--workflow", required=True)
parser.add_argument("--db-url", required=True)
parser.add_argument("--run-id", required=True)
parser.add_argument("--check-streams", action="store_true")
parser.add_argument("--check-events", action="store_true")
parser.add_argument(
"--interrupt-after",
default=None,
help="Event name to interrupt after (simulates crash)",
)
args = parser.parse_args()
asyncio.run(
run_workflow_with_server(
workflow_path=args.workflow,
db_url=args.db_url,
run_id=args.run_id,
do_check_streams=args.check_streams,
do_check_events=args.check_events,
interrupt_after=args.interrupt_after,
)
)
if __name__ == "__main__":
main()
@@ -37,16 +37,17 @@ def run_scenario(
db_url: str,
run_id: str,
config: dict[str, Any] | None = None,
timeout: float = 30.0,
timeout: float = 45.0,
) -> subprocess.CompletedProcess[str]:
"""Run a workflow scenario in a subprocess.
Args:
workflow: Module path with class name (e.g., "tests.fixtures.workflows.hitl:TestWorkflow")
workflow: Module path with class name (e.g., "tests.fixtures.sample_workflows.hitl:TestWorkflow")
db_url: SQLite database URL
run_id: Unique run ID for the workflow
config: Optional config dict with interrupt_on and/or respond settings
timeout: Subprocess timeout in seconds
timeout: Subprocess timeout in seconds. Keep below pytest-timeout (60s)
so we can capture output on timeout instead of losing it.
Returns:
CompletedProcess with stdout and stderr captured.
@@ -63,7 +64,17 @@ def run_scenario(
]
if config:
cmd.extend(["--config", json.dumps(config)])
return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
try:
return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
except subprocess.TimeoutExpired as e:
stdout = e.stdout.decode() if isinstance(e.stdout, bytes) else (e.stdout or "")
stderr = e.stderr.decode() if isinstance(e.stderr, bytes) else (e.stderr or "")
pytest.fail(
f"Subprocess timed out after {timeout}s\n"
f"stdout:\n{stdout}\n"
f"stderr:\n{stderr}"
)
raise AssertionError("unreachable") # pytest.fail always raises # noqa: B904
def assert_no_determinism_errors(result: subprocess.CompletedProcess[str]) -> None:
@@ -78,8 +89,11 @@ def assert_no_determinism_errors(result: subprocess.CompletedProcess[str]) -> No
f"stderr: {result.stderr}"
)
# Catch any unhandled Python exception
if "Traceback (most recent call last)" in combined:
# Catch unhandled Python exceptions in stdout (main process output).
# We only check stdout because stderr may contain logged tracebacks from
# DBOS background tasks (e.g., SQLite locking retries) that don't affect
# the workflow result.
if "Traceback (most recent call last)" in result.stdout:
pytest.fail(
f"Subprocess exception!\nstdout: {result.stdout}\nstderr: {result.stderr}"
)
@@ -104,7 +118,7 @@ def test_determinism_on_resume_after_interrupt(test_db_path: Path) -> None:
db_url = f"sqlite+pysqlite:///{test_db_path}?check_same_thread=false"
result1 = run_scenario(
workflow="tests.fixtures.workflows.hitl:TestWorkflow",
workflow="tests.fixtures.sample_workflows.hitl:TestWorkflow",
db_url=db_url,
run_id=run_id,
config={"interrupt_on": "AskInputEvent"},
@@ -115,7 +129,7 @@ def test_determinism_on_resume_after_interrupt(test_db_path: Path) -> None:
assert "INTERRUPTING" in result1.stdout, "Should have interrupted"
result2 = run_scenario(
workflow="tests.fixtures.workflows.hitl:TestWorkflow",
workflow="tests.fixtures.sample_workflows.hitl:TestWorkflow",
db_url=db_url,
run_id=run_id,
config={
@@ -146,7 +160,7 @@ def test_chained_steps_determinism_on_resume(test_db_path: Path) -> None:
db_url = f"sqlite+pysqlite:///{test_db_path}?check_same_thread=false"
result1 = run_scenario(
workflow="tests.fixtures.workflows.chained:ChainedWorkflow",
workflow="tests.fixtures.sample_workflows.chained:ChainedWorkflow",
db_url=db_url,
run_id=run_id,
config={"interrupt_on": "StepTwoEvent"},
@@ -156,7 +170,7 @@ def test_chained_steps_determinism_on_resume(test_db_path: Path) -> None:
assert "STEP:one:complete" in result1.stdout, "Step one should complete"
result2 = run_scenario(
workflow="tests.fixtures.workflows.chained:ChainedWorkflow",
workflow="tests.fixtures.sample_workflows.chained:ChainedWorkflow",
db_url=db_url,
run_id=run_id,
)
@@ -176,7 +190,7 @@ def test_hitl_three_step_determinism(test_db_path: Path) -> None:
db_url = f"sqlite+pysqlite:///{test_db_path}?check_same_thread=false"
result1 = run_scenario(
workflow="tests.fixtures.workflows.three_step_hitl:HITLWorkflow",
workflow="tests.fixtures.sample_workflows.three_step_hitl:HITLWorkflow",
db_url=db_url,
run_id=run_id,
config={
@@ -196,7 +210,7 @@ def test_hitl_three_step_determinism(test_db_path: Path) -> None:
assert "INTERRUPTING" in result1.stdout, "Should interrupt at quest"
result2 = run_scenario(
workflow="tests.fixtures.workflows.three_step_hitl:HITLWorkflow",
workflow="tests.fixtures.sample_workflows.three_step_hitl:HITLWorkflow",
db_url=db_url,
run_id=run_id,
config={
@@ -231,7 +245,7 @@ def test_parallel_steps_determinism(test_db_path: Path) -> None:
db_url = f"sqlite+pysqlite:///{test_db_path}?check_same_thread=false"
result1 = run_scenario(
workflow="tests.fixtures.workflows.parallel:ParallelWorkflow",
workflow="tests.fixtures.sample_workflows.parallel:ParallelWorkflow",
db_url=db_url,
run_id=run_id,
)
@@ -254,7 +268,7 @@ def test_concurrent_workers_determinism(test_db_path: Path) -> None:
db_url = f"sqlite+pysqlite:///{test_db_path}?check_same_thread=false"
result1 = run_scenario(
workflow="tests.fixtures.workflows.concurrent_workers:ConcurrentWorkersWorkflow",
workflow="tests.fixtures.sample_workflows.concurrent_workers:ConcurrentWorkersWorkflow",
db_url=db_url,
run_id=run_id,
)
@@ -277,7 +291,7 @@ def test_sequential_hitl_interrupt_resume(test_db_path: Path) -> None:
db_url = f"sqlite+pysqlite:///{test_db_path}?check_same_thread=false"
result1 = run_scenario(
workflow="tests.fixtures.workflows.sequential_hitl:SequentialHITLWorkflow",
workflow="tests.fixtures.sample_workflows.sequential_hitl:SequentialHITLWorkflow",
db_url=db_url,
run_id=run_id,
config={"interrupt_on": "WaitForInputEvent"},
@@ -288,7 +302,7 @@ def test_sequential_hitl_interrupt_resume(test_db_path: Path) -> None:
assert "INTERRUPTING" in result1.stdout
result2 = run_scenario(
workflow="tests.fixtures.workflows.sequential_hitl:SequentialHITLWorkflow",
workflow="tests.fixtures.sample_workflows.sequential_hitl:SequentialHITLWorkflow",
db_url=db_url,
run_id=run_id,
config={
@@ -313,6 +327,7 @@ def test_sequential_hitl_interrupt_resume(test_db_path: Path) -> None:
# =============================================================================
@pytest.mark.timeout(60)
@pytest.mark.parametrize("iteration", range(5))
def test_parallel_steps_stress(test_db_path: Path, iteration: int) -> None:
"""Stress test parallel steps - run 5 times to catch timing issues."""
@@ -320,7 +335,7 @@ def test_parallel_steps_stress(test_db_path: Path, iteration: int) -> None:
db_url = f"sqlite+pysqlite:///{test_db_path}?check_same_thread=false"
result = run_scenario(
workflow="tests.fixtures.workflows.parallel:ParallelWorkflow",
workflow="tests.fixtures.sample_workflows.parallel:ParallelWorkflow",
db_url=db_url,
run_id=run_id,
)
@@ -331,6 +346,7 @@ def test_parallel_steps_stress(test_db_path: Path, iteration: int) -> None:
assert_no_determinism_errors(result)
@pytest.mark.timeout(60)
@pytest.mark.parametrize("iteration", range(5))
def test_concurrent_workers_stress(test_db_path: Path, iteration: int) -> None:
"""Stress test concurrent workers - run 5 times to catch timing issues."""
@@ -338,7 +354,7 @@ def test_concurrent_workers_stress(test_db_path: Path, iteration: int) -> None:
db_url = f"sqlite+pysqlite:///{test_db_path}?check_same_thread=false"
result = run_scenario(
workflow="tests.fixtures.workflows.concurrent_workers:ConcurrentWorkersWorkflow",
workflow="tests.fixtures.sample_workflows.concurrent_workers:ConcurrentWorkersWorkflow",
db_url=db_url,
run_id=run_id,
)
@@ -360,7 +376,7 @@ def test_streaming_stress_determinism(test_db_path: Path) -> None:
db_url = f"sqlite+pysqlite:///{test_db_path}?check_same_thread=false"
result = run_scenario(
workflow="tests.fixtures.workflows.streaming_stress:StreamingStressWorkflow",
workflow="tests.fixtures.sample_workflows.streaming_stress:StreamingStressWorkflow",
db_url=db_url,
run_id=run_id,
)
@@ -378,7 +394,7 @@ def test_streaming_interrupt_resume(test_db_path: Path) -> None:
db_url = f"sqlite+pysqlite:///{test_db_path}?check_same_thread=false"
result1 = run_scenario(
workflow="tests.fixtures.workflows.streaming_interrupt:StreamingInterruptWorkflow",
workflow="tests.fixtures.sample_workflows.streaming_interrupt:StreamingInterruptWorkflow",
db_url=db_url,
run_id=run_id,
config={
@@ -393,7 +409,7 @@ def test_streaming_interrupt_resume(test_db_path: Path) -> None:
assert "INTERRUPTING" in result1.stdout, "Should have interrupted"
result2 = run_scenario(
workflow="tests.fixtures.workflows.streaming_interrupt:StreamingInterruptWorkflow",
workflow="tests.fixtures.sample_workflows.streaming_interrupt:StreamingInterruptWorkflow",
db_url=db_url,
run_id=run_id,
)
@@ -405,6 +421,7 @@ def test_streaming_interrupt_resume(test_db_path: Path) -> None:
)
@pytest.mark.timeout(60)
@pytest.mark.parametrize("iteration", range(5))
def test_streaming_stress_repeated(test_db_path: Path, iteration: int) -> None:
"""Stress test streaming - run 5 times to catch timing issues."""
@@ -412,7 +429,7 @@ def test_streaming_stress_repeated(test_db_path: Path, iteration: int) -> None:
db_url = f"sqlite+pysqlite:///{test_db_path}?check_same_thread=false"
result = run_scenario(
workflow="tests.fixtures.workflows.streaming_stress:StreamingStressWorkflow",
workflow="tests.fixtures.sample_workflows.streaming_stress:StreamingStressWorkflow",
db_url=db_url,
run_id=run_id,
)
@@ -10,26 +10,18 @@ from __future__ import annotations
import asyncio
from contextlib import suppress
from pathlib import Path
from typing import Any, Generator, cast
from typing import Any, Generator
from unittest.mock import patch
import pytest
from dbos import DBOS, DBOSConfig
from llama_agents.dbos import DBOSRuntime
from llama_agents.dbos.journal.crud import SqliteJournalCrud
from llama_agents.dbos.journal.task_journal import TaskJournal
from llama_agents.dbos.runtime import InternalDBOSAdapter
from llama_agents.dbos.state_store import (
SqlSerializedState,
SqlStateStore,
parse_serialized_state,
)
from pydantic import Field
from sqlalchemy import create_engine
from sqlalchemy.engine import Engine
from sqlalchemy.pool import QueuePool
from workflows.context import Context
from workflows.context.state_store import InMemorySerializedState
from workflows.decorators import step
from workflows.events import Event, StartEvent, StopEvent
from workflows.runtime.types.named_task import NamedTask
@@ -261,16 +253,20 @@ async def test_run_workflow_does_not_create_store(dbos_runtime: DBOSRuntime) ->
@pytest.mark.asyncio
async def test_replay_wait_for_next_task_timeout_returns_none(
journal_db_path: str,
sqlite_engine: Engine,
) -> None:
"""Replay wait timeout should return None and not raise."""
run_id = "replay-timeout-run"
journal = TaskJournal(run_id, sqlite_engine)
crud = SqliteJournalCrud(db_path=journal_db_path)
journal = TaskJournal(run_id, crud)
await journal.load()
await journal.record("step_a:0")
adapter = InternalDBOSAdapter(run_id=run_id, engine=sqlite_engine)
adapter = InternalDBOSAdapter(
run_id=run_id, engine=sqlite_engine, db_path=journal_db_path
)
task = asyncio.create_task(asyncio.sleep(5.0))
try:
@@ -283,113 +279,3 @@ async def test_replay_wait_for_next_task_timeout_returns_none(
task.cancel()
with suppress(asyncio.CancelledError):
await task
# ============================================================================
# SqlSerializedState and parse_serialized_state Tests
# ============================================================================
def test_parse_serialized_state_sql_store_type() -> None:
"""Test that store_type='sql' parses as SqlSerializedState."""
serialized = {
"store_type": "sql",
"run_id": "run-12345",
"schema": "public",
}
result = parse_serialized_state(serialized)
assert isinstance(result, SqlSerializedState)
assert result.store_type == "sql"
assert result.run_id == "run-12345"
assert result.db_schema == "public"
def test_parse_serialized_state_sql_with_null_schema() -> None:
"""Test that SqlSerializedState accepts null schema."""
serialized = {
"store_type": "sql",
"run_id": "run-67890",
"schema": None,
}
result = parse_serialized_state(serialized)
assert isinstance(result, SqlSerializedState)
assert result.db_schema is None
def test_parse_serialized_state_in_memory_format() -> None:
"""Test that in_memory format is still handled."""
serialized = {
"store_type": "in_memory",
"state_type": "DictState",
"state_module": "workflows.context.state_store",
"state_data": {"_data": {"counter": 42}},
}
result = parse_serialized_state(serialized)
assert isinstance(result, InMemorySerializedState)
assert result.store_type == "in_memory"
def test_parse_serialized_state_unknown_store_type_raises() -> None:
"""Test that unknown store_type raises ValueError."""
serialized = {
"store_type": "redis", # Unknown store type
"state_type": "SomeState",
"state_module": "some.module",
}
with pytest.raises(ValueError, match="Unknown store_type"):
parse_serialized_state(serialized)
@pytest.mark.asyncio
async def test_edit_state_rolls_back_and_closes_on_error(sqlite_engine: Engine) -> None:
"""Errors inside edit_state should rollback/close and leave store usable."""
store = SqlStateStore(run_id="state-run", engine=sqlite_engine)
with pytest.raises(RuntimeError, match="boom"):
async with store.edit_state() as state:
state["transient"] = "value"
raise RuntimeError("boom")
assert await store.get("transient", default=None) is None
await store.set("after", "ok")
assert await store.get("after") == "ok"
with sqlite_engine.connect() as conn:
raw = conn.connection
assert not bool(getattr(raw, "in_transaction", False))
@pytest.mark.asyncio
async def test_edit_state_failure_releases_checked_out_connection(
tmp_path: Path,
) -> None:
"""Failed edit_state should not leak checked-out pooled connections."""
db_file = tmp_path / "state.sqlite3"
engine = create_engine(
f"sqlite:///{db_file}",
connect_args={"check_same_thread": False},
poolclass=QueuePool,
)
store = SqlStateStore(run_id="pool-run", engine=engine)
pool = cast(QueuePool, engine.pool)
assert pool.checkedout() == 0
with pytest.raises(ValueError, match="force failure"):
async with store.edit_state() as state:
state["x"] = 1
raise ValueError("force failure")
assert pool.checkedout() == 0
await store.set("y", 2)
assert await store.get("y") == 2
assert pool.checkedout() == 0
@@ -0,0 +1,184 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
"""End-to-end DBOS + WorkflowServer + PostgresWorkflowStore integration tests.
These tests verify:
1. Event interceptor prevents published events from reaching dbos.streams
2. Events are stored as clean JSON in our wf_events table
3. subscribe_events works across the full server chain
4. Interrupt/resume produces no duplicate events (replay safety)
All tests require Docker (testcontainers) and use subprocess isolation for
DBOS global state safety.
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
import pytest
SERVER_RUNNER_PATH = str(Path(__file__).parent / "fixtures" / "server_runner.py")
pytestmark = [pytest.mark.docker]
def run_server_scenario(
workflow: str,
db_url: str,
run_id: str,
check_streams: bool = False,
check_events: bool = False,
interrupt_after: str | None = None,
timeout: float = 60.0,
) -> subprocess.CompletedProcess[str]:
"""Run a workflow scenario through the WorkflowServer + DBOS chain."""
cmd = [
sys.executable,
SERVER_RUNNER_PATH,
"--workflow",
workflow,
"--db-url",
db_url,
"--run-id",
run_id,
]
if check_streams:
cmd.append("--check-streams")
if check_events:
cmd.append("--check-events")
if interrupt_after:
cmd.extend(["--interrupt-after", interrupt_after])
return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
def assert_no_errors(result: subprocess.CompletedProcess[str]) -> None:
"""Check subprocess result for crashes and errors."""
if result.returncode != 0:
pytest.fail(
f"Subprocess exited with code {result.returncode}\n"
f"stdout: {result.stdout}\nstderr: {result.stderr}"
)
# Only fail on tracebacks in stdout — stderr tracebacks during DBOS
# shutdown are noisy but harmless when exit code is 0.
if "Traceback (most recent call last)" in result.stdout:
pytest.fail(f"Exception!\nstdout: {result.stdout}\nstderr: {result.stderr}")
def extract_line(output: str, prefix: str) -> str | None:
"""Extract a line starting with prefix from output."""
for line in output.splitlines():
if line.startswith(prefix):
return line[len(prefix) :]
return None
def extract_all_lines(output: str, prefix: str) -> list[str]:
"""Extract all lines starting with prefix from output."""
return [
line[len(prefix) :] for line in output.splitlines() if line.startswith(prefix)
]
def test_event_interceptor_no_dbos_streams(postgres_dsn: str) -> None:
"""Run a workflow via the server chain and verify no events in dbos.streams."""
result = run_server_scenario(
workflow="tests.fixtures.sample_workflows.chained:ChainedWorkflow",
db_url=postgres_dsn,
run_id="test-interceptor-001",
check_streams=True,
check_events=True,
)
assert_no_errors(result)
assert "SUCCESS" in result.stdout
streams_count = extract_line(result.stdout, "STREAMS_COUNT:")
assert streams_count is not None, (
f"No STREAMS_COUNT found.\nstdout: {result.stdout}"
)
assert int(streams_count) == 0, (
f"Expected 0 events in dbos.streams, got {streams_count}"
)
def test_events_stored_as_json(postgres_dsn: str) -> None:
"""Verify events are stored in wf_events with valid JSON."""
result = run_server_scenario(
workflow="tests.fixtures.sample_workflows.chained:ChainedWorkflow",
db_url=postgres_dsn,
run_id="test-events-json-001",
check_events=True,
)
assert_no_errors(result)
assert "SUCCESS" in result.stdout
events_count = extract_line(result.stdout, "EVENTS_COUNT:")
assert events_count is not None, f"No EVENTS_COUNT.\nstdout: {result.stdout}"
count = int(events_count)
assert count >= 3, f"Expected at least 3 events, got {count}"
event_jsons = extract_all_lines(result.stdout, "EVENT_JSON:")
for i, event_json in enumerate(event_jsons):
try:
parsed = json.loads(event_json)
assert "type" in parsed, f"Event {i} missing 'type' field"
except json.JSONDecodeError:
pytest.fail(f"Event {i} is not valid JSON: {event_json}")
def test_subscribe_events_receives_all_events(postgres_dsn: str) -> None:
"""Verify subscribe_events receives all events in order during workflow execution."""
result = run_server_scenario(
workflow="tests.fixtures.sample_workflows.chained:ChainedWorkflow",
db_url=postgres_dsn,
run_id="test-subscribe-001",
)
assert_no_errors(result)
assert "SUCCESS" in result.stdout
event_names = extract_all_lines(result.stdout, "EVENT:")
assert len(event_names) >= 3, f"Expected at least 3 events, got {event_names}"
# StopEvent should be the last event
assert event_names[-1] == "StopEvent", (
f"Last event should be StopEvent, got {event_names[-1]}"
)
def test_no_duplicate_events_after_replay(postgres_dsn: str) -> None:
"""Interrupt a workflow, resume it, and verify no duplicate events."""
run_id = "test-replay-dedup-001"
# Run 1: interrupt after step_two produces StepTwoEvent
result1 = run_server_scenario(
workflow="tests.fixtures.sample_workflows.chained:ChainedWorkflow",
db_url=postgres_dsn,
run_id=run_id,
interrupt_after="StepTwoEvent",
)
assert "INTERRUPTING" in result1.stdout, (
f"Should have interrupted.\nstdout: {result1.stdout}\nstderr: {result1.stderr}"
)
# Run 2: resume to completion with same run_id, check events
result2 = run_server_scenario(
workflow="tests.fixtures.sample_workflows.chained:ChainedWorkflow",
db_url=postgres_dsn,
run_id=run_id,
check_events=True,
check_streams=True,
)
assert_no_errors(result2)
assert "SUCCESS" in result2.stdout, (
f"Resume should succeed.\nstdout: {result2.stdout}\nstderr: {result2.stderr}"
)
# Verify no events in dbos.streams
streams_count = extract_line(result2.stdout, "STREAMS_COUNT:")
if streams_count is not None:
assert int(streams_count) == 0, (
f"Expected 0 events in dbos.streams after replay, got {streams_count}"
)
@@ -5,14 +5,19 @@
from __future__ import annotations
import pytest
from llama_agents.dbos.journal.crud import SqliteJournalCrud
from llama_agents.dbos.journal.task_journal import TaskJournal
from sqlalchemy.engine import Engine
def _make_journal(run_id: str, db_path: str) -> TaskJournal:
crud = SqliteJournalCrud(db_path=db_path)
return TaskJournal(run_id, crud)
@pytest.mark.asyncio
async def test_fresh_journal_has_no_entries(sqlite_engine: Engine) -> None:
async def test_fresh_journal_has_no_entries(journal_db_path: str) -> None:
"""Fresh journal returns None for next_expected_key."""
journal = TaskJournal("test-run", sqlite_engine)
journal = _make_journal("test-run", journal_db_path)
await journal.load()
assert journal.next_expected_key() is None
@@ -20,23 +25,23 @@ async def test_fresh_journal_has_no_entries(sqlite_engine: Engine) -> None:
@pytest.mark.asyncio
async def test_record_adds_entry(sqlite_engine: Engine) -> None:
async def test_record_adds_entry(journal_db_path: str) -> None:
"""Recording a key adds it to the journal."""
journal = TaskJournal("test-run", sqlite_engine)
journal = _make_journal("test-run", journal_db_path)
await journal.load()
await journal.record("step_a:0")
# Verify by loading a new journal for the same run
journal2 = TaskJournal("test-run", sqlite_engine)
journal2 = _make_journal("test-run", journal_db_path)
await journal2.load()
assert journal2.next_expected_key() == "step_a:0"
@pytest.mark.asyncio
async def test_record_multiple_entries(sqlite_engine: Engine) -> None:
async def test_record_multiple_entries(journal_db_path: str) -> None:
"""Multiple records append to journal in order."""
journal = TaskJournal("test-run", sqlite_engine)
journal = _make_journal("test-run", journal_db_path)
await journal.load()
await journal.record("step_a:0")
@@ -44,7 +49,7 @@ async def test_record_multiple_entries(sqlite_engine: Engine) -> None:
await journal.record("step_b:1")
# Verify order by loading a new journal
journal2 = TaskJournal("test-run", sqlite_engine)
journal2 = _make_journal("test-run", journal_db_path)
await journal2.load()
assert journal2.next_expected_key() == "step_a:0"
journal2.advance()
@@ -56,17 +61,17 @@ async def test_record_multiple_entries(sqlite_engine: Engine) -> None:
@pytest.mark.asyncio
async def test_replay_returns_entries_in_order(sqlite_engine: Engine) -> None:
async def test_replay_returns_entries_in_order(journal_db_path: str) -> None:
"""Replaying journal returns entries in recorded order."""
# Set up initial data
journal1 = TaskJournal("replay-run", sqlite_engine)
journal1 = _make_journal("replay-run", journal_db_path)
await journal1.load()
await journal1.record("step_a:0")
await journal1.record("step_b:1")
await journal1.record("__pull__:2")
# Load fresh journal and replay
journal = TaskJournal("replay-run", sqlite_engine)
journal = _make_journal("replay-run", journal_db_path)
await journal.load()
assert journal.is_replaying()
@@ -84,15 +89,15 @@ async def test_replay_returns_entries_in_order(sqlite_engine: Engine) -> None:
@pytest.mark.asyncio
async def test_load_is_idempotent(sqlite_engine: Engine) -> None:
async def test_load_is_idempotent(journal_db_path: str) -> None:
"""Calling load() multiple times doesn't reset state."""
# Set up initial data
journal1 = TaskJournal("idempotent-run", sqlite_engine)
journal1 = _make_journal("idempotent-run", journal_db_path)
await journal1.load()
await journal1.record("step_a:0")
# Load and advance
journal = TaskJournal("idempotent-run", sqlite_engine)
journal = _make_journal("idempotent-run", journal_db_path)
await journal.load()
journal.advance()
assert journal.next_expected_key() is None
@@ -103,9 +108,9 @@ async def test_load_is_idempotent(sqlite_engine: Engine) -> None:
@pytest.mark.asyncio
async def test_none_engine_works_in_memory() -> None:
"""Journal works without engine (in-memory only)."""
journal = TaskJournal("memory-run", engine=None)
async def test_none_crud_works_in_memory() -> None:
"""Journal works without crud (in-memory only)."""
journal = TaskJournal("memory-run", crud=None)
await journal.load()
assert journal.next_expected_key() is None
@@ -113,16 +118,16 @@ async def test_none_engine_works_in_memory() -> None:
await journal.record("step_a:0")
await journal.record("step_b:1")
# New journal with same run_id but no engine won't see the entries
journal2 = TaskJournal("memory-run", engine=None)
# New journal with same run_id but no crud won't see the entries
journal2 = TaskJournal("memory-run", crud=None)
await journal2.load()
assert journal2.next_expected_key() is None
@pytest.mark.asyncio
async def test_record_advances_index(sqlite_engine: Engine) -> None:
async def test_record_advances_index(journal_db_path: str) -> None:
"""Recording advances the replay index to stay in sync."""
journal = TaskJournal("index-run", sqlite_engine)
journal = _make_journal("index-run", journal_db_path)
await journal.load()
# After recording, index should advance
@@ -136,15 +141,15 @@ async def test_record_advances_index(sqlite_engine: Engine) -> None:
@pytest.mark.asyncio
async def test_mixed_replay_and_fresh_execution(sqlite_engine: Engine) -> None:
async def test_mixed_replay_and_fresh_execution(journal_db_path: str) -> None:
"""Journal transitions from replay to fresh execution correctly."""
# Set up initial data
journal1 = TaskJournal("mixed-run", sqlite_engine)
journal1 = _make_journal("mixed-run", journal_db_path)
await journal1.load()
await journal1.record("step_a:0")
# Load fresh journal
journal = TaskJournal("mixed-run", sqlite_engine)
journal = _make_journal("mixed-run", journal_db_path)
await journal.load()
# Replay the existing entry
@@ -156,7 +161,7 @@ async def test_mixed_replay_and_fresh_execution(sqlite_engine: Engine) -> None:
await journal.record("step_b:1")
# Verify both entries persisted
journal2 = TaskJournal("mixed-run", sqlite_engine)
journal2 = _make_journal("mixed-run", journal_db_path)
await journal2.load()
assert journal2.next_expected_key() == "step_a:0"
journal2.advance()
@@ -164,9 +169,9 @@ async def test_mixed_replay_and_fresh_execution(sqlite_engine: Engine) -> None:
@pytest.mark.asyncio
async def test_empty_journal_is_valid(sqlite_engine: Engine) -> None:
async def test_empty_journal_is_valid(journal_db_path: str) -> None:
"""Empty journal (no entries) is a valid state."""
journal = TaskJournal("empty-run", sqlite_engine)
journal = _make_journal("empty-run", journal_db_path)
await journal.load()
assert journal.next_expected_key() is None
@@ -174,21 +179,21 @@ async def test_empty_journal_is_valid(sqlite_engine: Engine) -> None:
@pytest.mark.asyncio
async def test_run_id_isolation(sqlite_engine: Engine) -> None:
async def test_run_id_isolation(journal_db_path: str) -> None:
"""Journals with different run_ids are isolated."""
journal1 = TaskJournal("run-1", sqlite_engine)
journal1 = _make_journal("run-1", journal_db_path)
await journal1.load()
await journal1.record("step_a:0")
journal2 = TaskJournal("run-2", sqlite_engine)
journal2 = _make_journal("run-2", journal_db_path)
await journal2.load()
await journal2.record("step_b:0")
# Each journal sees only its own entries
check1 = TaskJournal("run-1", sqlite_engine)
check1 = _make_journal("run-1", journal_db_path)
await check1.load()
assert check1.next_expected_key() == "step_a:0"
check2 = TaskJournal("run-2", sqlite_engine)
check2 = _make_journal("run-2", journal_db_path)
await check2.load()
assert check2.next_expected_key() == "step_b:0"
@@ -45,7 +45,7 @@ asyncio_default_fixture_loop_scope = "module"
asyncio_default_test_loop_scope = "module"
testpaths = ["tests"]
# Skip docker tests by default - run with `pytest -m docker` to include them
addopts = "-nauto --timeout=60 -m 'not docker'"
addopts = "-nauto --timeout=120 -m 'not docker'"
markers = [
"docker: marks tests as requiring Docker (testcontainers/PostgreSQL)"
]
@@ -0,0 +1,31 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
"""Reusable PostgreSQL testcontainers utilities for integration tests."""
from __future__ import annotations
from collections.abc import Generator
from contextlib import contextmanager
from testcontainers.postgres import PostgresContainer
@contextmanager
def postgres_container(
image: str = "postgres:16",
) -> Generator[PostgresContainer, None, None]:
"""Start a disposable Postgres container. Yields the container object.
Use ``get_connection_url()`` on the result to get a connection string.
The ``driver=None`` argument ensures the raw ``postgresql://`` scheme
is used (no psycopg2/psycopg suffix).
"""
with PostgresContainer(image, driver=None) as pg:
yield pg
def get_asyncpg_dsn(container: PostgresContainer) -> str:
"""Return a plain ``postgresql://`` DSN suitable for asyncpg."""
url = container.get_connection_url()
# testcontainers may return psycopg2-style URLs
return url.replace("postgresql+psycopg2://", "postgresql://")
@@ -9,6 +9,9 @@ from llama_agents_integration_tests.helpers import (
make_text_response,
response_generator_from_list,
)
from llama_agents_integration_tests.postgres import (
postgres_container as _postgres_container,
)
from llama_index.core.agent.workflow import (
AgentWorkflow,
FunctionAgent,
@@ -145,8 +148,8 @@ def postgres_container() -> Generator[PostgresContainer, None, None]:
Requires Docker to be running. Used by tests marked with @pytest.mark.docker.
"""
with PostgresContainer("postgres:16", driver=None) as postgres:
yield postgres
with _postgres_container() as pg:
yield pg
@pytest.fixture(scope="module")
@@ -1,18 +0,0 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
"""Basic Docker/PostgreSQL connectivity test."""
from __future__ import annotations
import pytest
from sqlalchemy import text
from sqlalchemy.engine import Engine
@pytest.mark.docker
def test_postgres_connection(postgres_engine: Engine) -> None:
"""Test basic PostgreSQL connectivity with SELECT 1."""
with postgres_engine.connect() as conn:
result = conn.execute(text("SELECT 1"))
assert result.scalar() == 1
@@ -0,0 +1,447 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
"""Parameterized live HTTP server integration tests across storage backends."""
from __future__ import annotations
import asyncio
import socket
import time
from pathlib import Path
from typing import Any, AsyncGenerator
import httpx
import pytest
import uvicorn
from dbos import DBOS, DBOSConfig
from llama_agents.client.client import WorkflowClient
from llama_agents.dbos import DBOSRuntime
from llama_agents.server import MemoryWorkflowStore, SqliteWorkflowStore, WorkflowServer
from testcontainers.postgres import PostgresContainer
from workflows import Context, Workflow, step
from workflows.events import (
Event,
HumanResponseEvent,
InputRequiredEvent,
StartEvent,
StopEvent,
)
# Workflow definitions
class StreamEvent(Event):
message: str
sequence: int
class StreamingWorkflow(Workflow):
@step
async def stream_data(self, ctx: Context, ev: StartEvent) -> StopEvent:
count = getattr(ev, "count", 3)
for i in range(count):
ctx.write_event_to_stream(StreamEvent(message=f"event_{i}", sequence=i))
await asyncio.sleep(0.01)
return StopEvent(result=f"completed_{count}_events")
class RequestedExternalEvent(InputRequiredEvent):
message: str
class ExternalEvent(HumanResponseEvent):
response: str
class InteractiveWorkflow(Workflow):
@step
async def start(self, ctx: Context, ev: StartEvent) -> RequestedExternalEvent:
return RequestedExternalEvent(message="ping")
@step
async def end(self, ctx: Context, ev: ExternalEvent) -> StopEvent:
return StopEvent(result=f"received: {ev.response}")
class CumulativeWorkflow(Workflow):
@step
async def accumulate(self, ctx: Context, ev: StartEvent) -> StopEvent:
current_count = await ctx.store.get("count", 0)
increment = getattr(ev, "increment", 1)
new_count = current_count + increment
await ctx.store.set("count", new_count)
run_history = await ctx.store.get("run_history", [])
run_history.append(f"run_{len(run_history) + 1}")
await ctx.store.set("run_history", run_history)
return StopEvent(result=f"count: {new_count}, runs: {len(run_history)}")
class SimpleTestWorkflow(Workflow):
@step
async def process(self, ctx: Context, ev: StartEvent) -> StopEvent:
message = await ctx.store.get("test_param", None)
if message is None:
message = getattr(ev, "message", "default")
return StopEvent(result=f"processed: {message}")
class WaitableExternalEvent(Event):
response: str
class WaitingWorkflow(Workflow):
@step
async def start_and_wait(self, ctx: Context, ev: StartEvent) -> StopEvent:
external = await ctx.wait_for_event(WaitableExternalEvent)
return StopEvent(result=f"received: {external.response}")
# Helper functions
async def wait_for_passing(
func: Any, max_duration: float = 5.0, interval: float = 0.05
) -> Any:
start_time = time.monotonic()
last_exception = None
while time.monotonic() - start_time < max_duration:
remaining = max_duration - (time.monotonic() - start_time)
try:
return await asyncio.wait_for(func(), timeout=remaining)
except Exception as e:
last_exception = e
await asyncio.sleep(interval)
raise last_exception or TimeoutError(
f"wait_for_passing timed out after {max_duration}s"
)
class live_server:
def __init__(self, server_factory: Any) -> None:
self.server_factory = server_factory
self.sock: socket.socket | None = None
self.task: Any = None
self.uv_server: Any = None
async def __aenter__(self) -> tuple[str, WorkflowServer]:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(("127.0.0.1", 0))
self.sock.listen(128)
port = self.sock.getsockname()[1]
self.server = self.server_factory()
await self.server.start()
config = uvicorn.Config(
self.server.app,
host="127.0.0.1",
port=port,
log_level="error",
loop="asyncio",
)
self.uv_server = uvicorn.Server(config)
self.task = asyncio.create_task(self.uv_server.serve(sockets=[self.sock]))
base_url = f"http://127.0.0.1:{port}"
async with httpx.AsyncClient(base_url=base_url, timeout=1.0) as client:
for _ in range(50):
try:
resp = await client.get("/health")
if resp.status_code == 200:
break
except Exception:
pass
await asyncio.sleep(0.01)
else:
self.uv_server.should_exit = True
await self.task
raise RuntimeError("Live server did not start in time")
return base_url, self.server
async def __aexit__(self, *args: Any) -> None:
if self.uv_server:
self.uv_server.should_exit = True
if self.task:
try:
await self.task
except Exception:
pass
if hasattr(self.server, "stop"):
await self.server.stop()
if self.sock:
try:
self.sock.close()
except Exception:
pass
def _get_backend_params() -> list[Any]:
return [
pytest.param("memory", id="memory"),
pytest.param("sqlite", id="sqlite"),
pytest.param("postgres", marks=pytest.mark.docker, id="postgres"),
]
@pytest.fixture(scope="module")
def postgres_container(request: Any) -> Any:
# Only start the container when docker-marked tests will actually run
if not any(
item.get_closest_marker("docker")
for item in request.session.items
if item.module is request.module
):
yield None
return
with PostgresContainer("postgres:17", driver=None) as container:
yield container
def _add_all_workflows(server: WorkflowServer) -> None:
server.add_workflow("SimpleTestWorkflow", SimpleTestWorkflow())
server.add_workflow("StreamingWorkflow", StreamingWorkflow())
server.add_workflow("InteractiveWorkflow", InteractiveWorkflow())
server.add_workflow("CumulativeWorkflow", CumulativeWorkflow())
server.add_workflow("WaitingWorkflow", WaitingWorkflow())
_pg_server_state: dict[str, Any] = {}
async def _start_postgres_server(
postgres_container: Any,
) -> tuple[str, WorkflowServer]:
"""Start a persistent postgres-backed server (called once per module)."""
connection_url = postgres_container.get_connection_url()
dbos_config: DBOSConfig = {
"name": "wf-server-http-pg",
"system_database_url": connection_url,
"run_admin_server": False,
"notification_listener_polling_interval_sec": 0.01,
}
DBOS(config=dbos_config)
runtime = DBOSRuntime(polling_interval_sec=0.01)
runtime.launch()
store = runtime.create_workflow_store()
server_runtime = runtime.build_server_runtime()
server = WorkflowServer(workflow_store=store, runtime=server_runtime)
_add_all_workflows(server)
await server.start()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(("127.0.0.1", 0))
sock.listen(128)
port = sock.getsockname()[1]
config = uvicorn.Config(
server.app, host="127.0.0.1", port=port, log_level="error", loop="asyncio"
)
uv_server = uvicorn.Server(config)
serve_task = asyncio.create_task(uv_server.serve(sockets=[sock]))
base_url = f"http://127.0.0.1:{port}"
async with httpx.AsyncClient(base_url=base_url, timeout=1.0) as client:
for _ in range(50):
try:
resp = await client.get("/health")
if resp.status_code == 200:
break
except Exception:
pass
await asyncio.sleep(0.01)
else:
uv_server.should_exit = True
await serve_task
raise RuntimeError("Postgres live server did not start in time")
_pg_server_state["runtime"] = runtime
_pg_server_state["uv_server"] = uv_server
_pg_server_state["serve_task"] = serve_task
_pg_server_state["sock"] = sock
_pg_server_state["server"] = server
_pg_server_state["base_url"] = base_url
return base_url, server
async def _stop_postgres_server() -> None:
if "uv_server" in _pg_server_state:
_pg_server_state["uv_server"].should_exit = True
try:
await _pg_server_state["serve_task"]
except Exception:
pass
_pg_server_state["sock"].close()
await _pg_server_state["server"].stop()
_pg_server_state.clear()
@pytest.fixture(scope="module")
async def postgres_server(
postgres_container: Any,
) -> AsyncGenerator[tuple[str, WorkflowServer] | None, None]:
"""Module-scoped postgres server — DBOS is created and destroyed once."""
if postgres_container is None:
yield None
return
result = await _start_postgres_server(postgres_container)
yield result
await _stop_postgres_server()
@pytest.fixture
async def backend_server(
request: Any,
tmp_path: Path,
postgres_server: tuple[str, WorkflowServer] | None,
) -> AsyncGenerator[tuple[str, WorkflowServer], None]:
backend = request.param
if backend == "memory":
def factory() -> WorkflowServer:
store = MemoryWorkflowStore()
server = WorkflowServer(workflow_store=store)
_add_all_workflows(server)
return server
async with live_server(factory) as (base_url, server):
yield base_url, server
elif backend == "sqlite":
db_path = tmp_path / "test.db"
def factory() -> WorkflowServer:
store = SqliteWorkflowStore(db_path=str(db_path))
server = WorkflowServer(workflow_store=store)
_add_all_workflows(server)
return server
async with live_server(factory) as (base_url, server):
yield base_url, server
elif backend == "postgres":
assert postgres_server is not None
yield postgres_server
else:
raise ValueError(f"Unknown backend: {backend}")
# Tests
@pytest.mark.asyncio
@pytest.mark.parametrize("backend_server", _get_backend_params(), indirect=True)
async def test_basic_run_and_result(
backend_server: tuple[str, WorkflowServer],
) -> None:
base_url, server = backend_server
client = WorkflowClient(base_url=base_url)
start_event = StartEvent(message="test_message") # type: ignore[call-arg]
result = await client.run_workflow("SimpleTestWorkflow", start_event=start_event)
assert result.result is not None
assert result.result.value["result"] == "processed: test_message"
@pytest.mark.asyncio
@pytest.mark.parametrize("backend_server", _get_backend_params(), indirect=True)
async def test_streaming_and_interactive(
backend_server: tuple[str, WorkflowServer],
) -> None:
base_url, server = backend_server
client = WorkflowClient(base_url=base_url)
started = await client.run_workflow_nowait("InteractiveWorkflow")
handler_id = started.handler_id
saw_prompt = False
async for ev in client.get_workflow_events(handler_id):
event = ev.load_event()
if isinstance(event, RequestedExternalEvent):
saw_prompt = True
sent = await client.send_event(handler_id, ExternalEvent(response="pong"))
assert sent.status == "sent"
break
assert saw_prompt
async def check_completed() -> str:
handler = await client.get_handler(handler_id)
assert handler.status == "completed"
assert handler.result is not None
return handler.result.value["result"]
result = await wait_for_passing(check_completed, max_duration=5.0)
assert result == "received: pong"
@pytest.mark.asyncio
@pytest.mark.parametrize("backend_server", _get_backend_params(), indirect=True)
async def test_reconnect_stream(
backend_server: tuple[str, WorkflowServer],
) -> None:
base_url, server = backend_server
client = WorkflowClient(base_url=base_url)
started = await client.run_workflow_nowait("InteractiveWorkflow")
handler_id = started.handler_id
saw_prompt = False
events_before = 0
async for ev in client.get_workflow_events(handler_id):
events_before += 1
event = ev.load_event()
if isinstance(event, RequestedExternalEvent):
saw_prompt = True
break
assert saw_prompt
stop_seen = asyncio.Event()
async def consume_again() -> None:
async for ev in client.get_workflow_events(handler_id):
event = ev.load_event()
if isinstance(event, StopEvent):
stop_seen.set()
break
consume_task = asyncio.create_task(consume_again())
await asyncio.sleep(0.05)
sent = await client.send_event(handler_id, ExternalEvent(response="reconnect_test"))
assert sent.status == "sent"
await asyncio.wait_for(stop_seen.wait(), timeout=5.0)
consume_task.cancel()
async def check_completed() -> str:
handler = await client.get_handler(handler_id)
assert handler.status == "completed"
assert handler.result is not None
return handler.result.value["result"]
result = await wait_for_passing(check_completed, max_duration=5.0)
assert result == "received: reconnect_test"
@pytest.mark.asyncio
@pytest.mark.parametrize("backend_server", _get_backend_params(), indirect=True)
async def test_cumulative_rerun(
backend_server: tuple[str, WorkflowServer],
) -> None:
base_url, server = backend_server
client = WorkflowClient(base_url=base_url)
start_event1 = StartEvent(increment=5) # type: ignore[call-arg]
result1 = await client.run_workflow("CumulativeWorkflow", start_event=start_event1)
assert result1.result is not None
result_str1 = result1.result.value["result"]
assert "count: 5" in result_str1
assert "runs: 1" in result_str1
handler_id = result1.handler_id
start_event2 = StartEvent(increment=3) # type: ignore[call-arg]
result2 = await client.run_workflow(
"CumulativeWorkflow", handler_id=handler_id, start_event=start_event2
)
assert result2.result is not None
result_str2 = result2.result.value["result"]
assert "count: 8" in result_str2
assert "runs: 2" in result_str2
@@ -10,11 +10,15 @@ Tests the StateStore protocol across InMemoryStateStore and SqlStateStore
from __future__ import annotations
import asyncio
import sqlite3
from pathlib import Path
from typing import Any, AsyncGenerator, Generator
import asyncpg
import pytest
from llama_agents.dbos.state_store import SqlStateStore
from llama_agents.server._store.postgres_state_store import PostgresStateStore
from llama_agents.server._store.sqlite.migrate import run_migrations
from llama_agents.server._store.sqlite.sqlite_state_store import SqliteStateStore
from pydantic import (
BaseModel,
ConfigDict,
@@ -22,8 +26,6 @@ from pydantic import (
field_serializer,
field_validator,
)
from sqlalchemy import create_engine
from sqlalchemy.engine import Engine
from testcontainers.postgres import PostgresContainer
from workflows.context.serializers import JsonSerializer
from workflows.context.state_store import DictState, InMemoryStateStore, StateStore
@@ -86,35 +88,58 @@ def postgres_container() -> Generator[PostgresContainer, None, None]:
@pytest.fixture(scope="module")
def postgres_engine(
def postgres_dsn(
postgres_container: PostgresContainer,
) -> Generator[Engine, None, None]:
"""Module-scoped PostgreSQL engine for state store tests."""
# Get connection URL and convert to use psycopg (psycopg3) driver
) -> str:
"""Module-scoped PostgreSQL DSN for state store tests."""
connection_url = postgres_container.get_connection_url()
# Replace postgresql:// or postgresql+psycopg2:// with postgresql+psycopg://
if "postgresql+psycopg2://" in connection_url:
connection_url = connection_url.replace(
"postgresql+psycopg2://", "postgresql+psycopg://"
"postgresql+psycopg2://", "postgresql://"
)
elif connection_url.startswith("postgresql://"):
elif "postgresql+psycopg://" in connection_url:
connection_url = connection_url.replace(
"postgresql://", "postgresql+psycopg://", 1
"postgresql+psycopg://", "postgresql://"
)
engine = create_engine(connection_url)
yield engine
engine.dispose()
return connection_url
async def _create_postgres_pool(dsn: str) -> asyncpg.Pool:
"""Create a pool and ensure schema/table exist."""
pool = await asyncpg.create_pool(dsn=dsn)
assert pool is not None
async with pool.acquire() as conn:
await conn.execute("CREATE SCHEMA IF NOT EXISTS dbos")
await conn.execute("""
CREATE TABLE IF NOT EXISTS dbos.workflow_state (
run_id VARCHAR(255) PRIMARY KEY,
state_json TEXT NOT NULL,
state_type VARCHAR(255),
state_module VARCHAR(255),
created_at TIMESTAMPTZ,
updated_at TIMESTAMPTZ
)
""")
return pool
@pytest.fixture(scope="module")
def sqlite_engine(
def sqlite_db_path(
tmp_path_factory: pytest.TempPathFactory,
) -> Generator[Engine, None, None]:
"""Module-scoped SQLite engine for state store tests."""
) -> str:
"""Module-scoped SQLite database path for state store tests."""
db_file: Path = tmp_path_factory.mktemp("state_store") / "test.sqlite3"
engine = create_engine(f"sqlite:///{db_file}?check_same_thread=false")
yield engine
engine.dispose()
db_path = str(db_file)
# Run migrations to create tables
conn = sqlite3.connect(db_path)
try:
run_migrations(conn)
conn.commit()
finally:
conn.close()
return db_path
def _get_store_params() -> list[Any]:
@@ -137,7 +162,7 @@ def _get_sql_params() -> list[Any]:
@pytest.fixture(params=_get_store_params())
async def state_store(
request: pytest.FixtureRequest,
sqlite_engine: Engine,
sqlite_db_path: str,
) -> AsyncGenerator[StateStore[DictState], None]:
"""Parametrized fixture yielding a fresh StateStore for each test."""
# Use unique run_id per test to avoid state bleeding
@@ -146,31 +171,35 @@ async def state_store(
if request.param == "in_memory":
yield InMemoryStateStore(DictState())
elif request.param == "sqlite":
store = SqlStateStore(run_id=run_id, engine=sqlite_engine)
store = SqliteStateStore(db_path=sqlite_db_path, run_id=run_id)
yield store
elif request.param == "postgres":
pg_engine: Engine = request.getfixturevalue("postgres_engine")
store = SqlStateStore(run_id=run_id, engine=pg_engine, schema="dbos")
dsn: str = request.getfixturevalue("postgres_dsn")
pool = await _create_postgres_pool(dsn)
store = PostgresStateStore(pool=pool, run_id=run_id, schema="dbos")
yield store
await pool.close()
@pytest.fixture(params=_get_sql_params())
async def sql_engine_and_schema(
async def sql_store_factory(
request: pytest.FixtureRequest,
sqlite_engine: Engine,
) -> AsyncGenerator[tuple[Engine, str | None], None]:
"""Parametrized fixture yielding (engine, schema) for SQL backend tests."""
sqlite_db_path: str,
) -> AsyncGenerator[tuple[str, str | None, asyncpg.Pool | None], None]:
"""Parametrized fixture yielding (backend, db_path_or_schema, pool) for SQL backend tests."""
if request.param == "sqlite":
yield sqlite_engine, None
yield "sqlite", sqlite_db_path, None
elif request.param == "postgres":
pg_engine: Engine = request.getfixturevalue("postgres_engine")
yield pg_engine, "dbos"
dsn: str = request.getfixturevalue("postgres_dsn")
pool = await _create_postgres_pool(dsn)
yield "postgres", "dbos", pool
await pool.close()
@pytest.fixture(params=_get_store_params())
async def custom_state_store(
request: pytest.FixtureRequest,
sqlite_engine: Engine,
sqlite_db_path: str,
) -> AsyncGenerator[StateStore[MyState], None]:
"""Parametrized fixture yielding a StateStore with custom typed state."""
run_id = f"test-custom-{id(request)}"
@@ -184,23 +213,25 @@ async def custom_state_store(
if request.param == "in_memory":
yield InMemoryStateStore(initial_state)
elif request.param == "sqlite":
store = SqlStateStore(
store = SqliteStateStore(
db_path=sqlite_db_path,
run_id=run_id,
state_type=MyState,
engine=sqlite_engine,
)
await store.set_state(initial_state)
yield store
elif request.param == "postgres":
pg_engine: Engine = request.getfixturevalue("postgres_engine")
store = SqlStateStore(
dsn: str = request.getfixturevalue("postgres_dsn")
pool = await _create_postgres_pool(dsn)
store = PostgresStateStore(
pool=pool,
run_id=run_id,
state_type=MyState,
engine=pg_engine,
schema="dbos",
)
await store.set_state(initial_state)
yield store
await pool.close()
# -- Basic Operations Tests --
@@ -387,29 +418,40 @@ async def test_to_dict_from_dict_roundtrip(state_store: StateStore[DictState]) -
@pytest.mark.asyncio
async def test_sql_persistence(
sql_engine_and_schema: tuple[Engine, str | None],
sql_store_factory: tuple[str, str, asyncpg.Pool | None],
) -> None:
"""Test that state persists across store instances."""
engine, schema = sql_engine_and_schema
backend, db_path_or_schema, pool = sql_store_factory
run_id = "persistence-test"
store1 = SqlStateStore(run_id=run_id, engine=engine, schema=schema)
await store1.set("persistent_key", "persistent_value")
if backend == "sqlite":
store1 = SqliteStateStore(db_path=db_path_or_schema, run_id=run_id)
await store1.set("persistent_key", "persistent_value")
store2 = SqliteStateStore(db_path=db_path_or_schema, run_id=run_id)
else:
assert pool is not None
store1 = PostgresStateStore(pool=pool, run_id=run_id, schema=db_path_or_schema)
await store1.set("persistent_key", "persistent_value")
store2 = PostgresStateStore(pool=pool, run_id=run_id, schema=db_path_or_schema)
store2 = SqlStateStore(run_id=run_id, engine=engine, schema=schema)
result = await store2.get("persistent_key")
assert result == "persistent_value"
@pytest.mark.asyncio
async def test_sql_isolation(
sql_engine_and_schema: tuple[Engine, str | None],
sql_store_factory: tuple[str, str, asyncpg.Pool | None],
) -> None:
"""Test that different run_ids have isolated state."""
engine, schema = sql_engine_and_schema
store1 = SqlStateStore(run_id="run-1", engine=engine, schema=schema)
store2 = SqlStateStore(run_id="run-2", engine=engine, schema=schema)
backend, db_path_or_schema, pool = sql_store_factory
if backend == "sqlite":
store1 = SqliteStateStore(db_path=db_path_or_schema, run_id="run-1")
store2 = SqliteStateStore(db_path=db_path_or_schema, run_id="run-2")
else:
assert pool is not None
store1 = PostgresStateStore(pool=pool, run_id="run-1", schema=db_path_or_schema)
store2 = PostgresStateStore(pool=pool, run_id="run-2", schema=db_path_or_schema)
await store1.set("key", "value1")
await store2.set("key", "value2")
@@ -420,12 +462,18 @@ async def test_sql_isolation(
@pytest.mark.asyncio
async def test_sql_concurrent_edits(
sql_engine_and_schema: tuple[Engine, str | None],
sql_store_factory: tuple[str, str, asyncpg.Pool | None],
) -> None:
"""Test concurrent edit_state calls are serialized correctly."""
engine, schema = sql_engine_and_schema
backend, db_path_or_schema, pool = sql_store_factory
run_id = "concurrent-test"
store = SqlStateStore(run_id=run_id, engine=engine, schema=schema)
if backend == "sqlite":
store = SqliteStateStore(db_path=db_path_or_schema, run_id=run_id)
else:
assert pool is not None
store = PostgresStateStore(pool=pool, run_id=run_id, schema=db_path_or_schema)
await store.set("counter", 0)
async def increment() -> None:
@@ -442,10 +490,10 @@ async def test_sql_concurrent_edits(
@pytest.mark.asyncio
async def test_sql_custom_state_persistence(
sql_engine_and_schema: tuple[Engine, str | None],
sql_store_factory: tuple[str, str, asyncpg.Pool | None],
) -> None:
"""Test that custom typed state persists correctly."""
engine, schema = sql_engine_and_schema
backend, db_path_or_schema, pool = sql_store_factory
run_id = "custom-persistence-test"
initial_state = MyState(
@@ -455,21 +503,36 @@ async def test_sql_custom_state_persistence(
age=100,
)
store1 = SqlStateStore(
run_id=run_id,
state_type=MyState,
engine=engine,
schema=schema,
)
await store1.set_state(initial_state)
await store1.set("name", "Modified")
if backend == "sqlite":
store1 = SqliteStateStore(
db_path=db_path_or_schema,
run_id=run_id,
state_type=MyState,
)
await store1.set_state(initial_state)
await store1.set("name", "Modified")
store2 = SqliteStateStore(
db_path=db_path_or_schema,
run_id=run_id,
state_type=MyState,
)
else:
assert pool is not None
store1 = PostgresStateStore(
pool=pool,
run_id=run_id,
state_type=MyState,
schema=db_path_or_schema,
)
await store1.set_state(initial_state)
await store1.set("name", "Modified")
store2 = PostgresStateStore(
pool=pool,
run_id=run_id,
state_type=MyState,
schema=db_path_or_schema,
)
store2 = SqlStateStore(
run_id=run_id,
state_type=MyState,
engine=engine,
schema=schema,
)
state = await store2.get_state()
assert state.name == "Modified"
@@ -481,24 +544,25 @@ async def test_sql_custom_state_persistence(
@pytest.mark.docker
@pytest.mark.asyncio
async def test_postgres_uses_dbos_schema(postgres_engine: Engine) -> None:
"""Test that SqlStateStore with schema='dbos' creates the table in the dbos schema."""
run_id = "pg-schema-test"
store = SqlStateStore(run_id=run_id, engine=postgres_engine, schema="dbos")
async def test_postgres_uses_dbos_schema(postgres_dsn: str) -> None:
"""Test that PostgresStateStore with schema='dbos' uses the table in the dbos schema."""
pool = await _create_postgres_pool(postgres_dsn)
try:
run_id = "pg-schema-test"
store = PostgresStateStore(pool=pool, run_id=run_id, schema="dbos")
# Trigger table creation by accessing state
await store.set("test", "value")
await store.set("test", "value")
# Verify the table was created in the dbos schema
with postgres_engine.connect() as conn:
result = conn.exec_driver_sql(
"""
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'dbos'
AND table_name = 'workflow_state'
async with pool.acquire() as conn:
exists = await conn.fetchval(
"""
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'dbos'
AND table_name = 'workflow_state'
)
"""
)
"""
)
exists = result.scalar()
assert exists is True
assert exists is True
finally:
await pool.close()
+13 -2
View File
@@ -12,7 +12,9 @@ dev = [
"pytest-timeout>=2.4.0",
"pytest-xdist>=3.8.0",
"time-machine>=2.19.0,<3.0.0",
"llama-agents-integration-tests"
"llama-agents-integration-tests",
"asyncpg>=0.29.0",
"testcontainers[postgres]>=4.0.0"
]
[project]
@@ -29,6 +31,9 @@ dependencies = [
"httpx>=0.27.0"
]
[project.optional-dependencies]
asyncpg = ["asyncpg>=0.29.0"]
[project.scripts]
llama-agents-server = "llama_agents.server.__main__:run_server"
@@ -43,7 +48,13 @@ asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "module"
asyncio_default_test_loop_scope = "module"
testpaths = ["tests"]
addopts = "-nauto --timeout=10"
addopts = "-nauto --timeout=60 -m 'not docker'"
markers = [
"docker: marks tests as requiring Docker (testcontainers/PostgreSQL)"
]
filterwarnings = [
"ignore:The @wait_container_is_ready decorator is deprecated:DeprecationWarning"
]
[tool.uv.build-backend]
module-name = "llama_agents.server"
@@ -398,7 +398,18 @@ class _WorkflowAPI:
try:
handler_data = await self._service.await_workflow(started)
status = 200 if handler_data.status == "completed" else 500
if handler_data.status == "completed":
status = 200
else:
logger.error(
"Workflow %s finished with status=%s error=%s (handler=%s, run=%s)",
handler_data.workflow_name,
handler_data.status,
handler_data.error,
handler_data.handler_id,
handler_data.run_id,
)
status = 500
except Exception as e:
logger.error(f"Error running workflow: {e}", exc_info=True)
handler_data = await self._service.load_handler(handler_id)
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
"""
EventInterceptorDecorator: blocks write_to_event_stream from reaching the
inner runtime while allowing all other operations (ticks, send/recv, close)
to pass through normally.
Used when the ServerRuntimeDecorator already writes events to the workflow
store and forwarding them to the inner runtime (e.g. DBOS) would cause
duplicate writes.
"""
from __future__ import annotations
import logging
from typing_extensions import override
from workflows.events import Event
from workflows.runtime.types.plugin import InternalRunAdapter
from workflows.workflow import Workflow
from .runtime_decorators import (
BaseInternalRunAdapterDecorator,
BaseRuntimeDecorator,
)
logger = logging.getLogger(__name__)
class _InterceptorInternalAdapter(BaseInternalRunAdapterDecorator):
"""Internal adapter that swallows write_to_event_stream calls."""
@override
async def write_to_event_stream(self, event: Event) -> None:
# No-op: do NOT forward to inner adapter.
pass
class EventInterceptorDecorator(BaseRuntimeDecorator):
"""Runtime decorator that prevents published events from reaching the
inner runtime's event stream.
All other methods (on_tick, send_event, wait_receive, close, etc.)
pass through to the inner runtime normally.
"""
@override
def get_internal_adapter(self, workflow: Workflow) -> InternalRunAdapter:
inner = self._decorated.get_internal_adapter(workflow)
return _InterceptorInternalAdapter(inner)
@@ -257,7 +257,7 @@ class PersistenceDecorator(BaseRuntimeDecorator):
conn = sqlite3.connect(state_store._db_path)
try:
row = conn.execute(
"SELECT 1 FROM state WHERE run_id = ?", (run_id,)
"SELECT 1 FROM workflow_state WHERE run_id = ?", (run_id,)
).fetchone()
if row is not None:
return
@@ -137,6 +137,9 @@ class BaseInternalRunAdapterDecorator(InternalRunAdapter):
async def finalize_step(self) -> None:
await self._decorated.finalize_step()
def is_replaying(self) -> bool:
return self._decorated.is_replaying()
async def on_tick(self, tick: WorkflowTick) -> None:
await self._decorated.on_tick(tick)
@@ -22,7 +22,6 @@ from llama_agents.server._runtime.runtime_decorators import (
from typing_extensions import override
from workflows.context.serializers import BaseSerializer
from workflows.context.state_store import (
InMemoryStateStore,
StateStore,
infer_state_type,
)
@@ -34,7 +33,6 @@ from workflows.events import (
WorkflowFailedEvent,
WorkflowTimedOutEvent,
)
from workflows.handler import WorkflowHandler
from workflows.runtime.types.internal_state import BrokerState
from workflows.runtime.types.plugin import (
ExternalRunAdapter,
@@ -81,46 +79,50 @@ class _ServerInternalRunAdapter(BaseInternalRunAdapterDecorator):
def get_state_store(self) -> StateStore[Any]:
if self._state_store is not None:
return self._state_store
store = self._store.create_state_store(self.run_id, self._state_type)
# Seed with initial context state if provided at run start
initial = self._runtime._initial_state.pop(self.run_id, None)
if initial is not None and isinstance(store, InMemoryStateStore):
store._state = initial
if initial is not None:
serialized_state, serializer = initial
store = self._store.create_state_store(
self.run_id, self._state_type, serialized_state, serializer
)
else:
store = self._store.create_state_store(self.run_id, self._state_type)
self._state_store = store
return store
@override
async def write_to_event_stream(self, event: Event) -> None:
"""
Monitors for writes to the event stream that indicate a workflow has terminated.
"""
if isinstance(event, WorkflowFailedEvent):
await self._runtime._handle_status_update(
run_id=self.run_id,
status="failed",
error=event.exception_message,
)
elif isinstance(event, WorkflowTimedOutEvent):
await self._runtime._handle_status_update(
run_id=self.run_id,
status="failed",
error=f"Workflow timed out after {event.timeout}s",
)
elif isinstance(event, WorkflowCancelledEvent):
await self._runtime._handle_status_update(
run_id=self.run_id, status="cancelled"
)
elif isinstance(event, StopEvent):
await self._runtime._handle_status_update(
run_id=self.run_id,
status="completed",
result=event,
)
"""Record events to the workflow store, skipping duplicates on replay."""
replaying = self.is_replaying()
envelope = EventEnvelopeWithMetadata.from_event(event)
await self._store.append_event(self.run_id, envelope)
if not replaying:
if isinstance(event, WorkflowFailedEvent):
await self._runtime._handle_status_update(
run_id=self.run_id,
status="failed",
error=event.exception_message,
)
elif isinstance(event, WorkflowTimedOutEvent):
await self._runtime._handle_status_update(
run_id=self.run_id,
status="failed",
error=f"Workflow timed out after {event.timeout}s",
)
elif isinstance(event, WorkflowCancelledEvent):
await self._runtime._handle_status_update(
run_id=self.run_id, status="cancelled"
)
elif isinstance(event, StopEvent):
await self._runtime._handle_status_update(
run_id=self.run_id,
status="completed",
result=event,
)
# Forward to inner adapter (e.g. _DurableInternalRunAdapter for idle detection)
envelope = EventEnvelopeWithMetadata.from_event(event)
await self._store.append_event(self.run_id, envelope)
# Always forward to inner adapter (e.g. idle detection, DBOS stream)
await super().write_to_event_stream(event)
@@ -220,18 +222,20 @@ class ServerRuntimeDecorator(BaseRuntimeDecorator):
serialized_state: dict[str, Any] | None = None,
serializer: BaseSerializer | None = None,
) -> ExternalRunAdapter:
# Intercept serialized state: we handle seeding ourselves in get_state_store
# so non-InMemory formats don't leak to the base runtime.
passthrough_state = serialized_state
if serialized_state and serializer:
try:
seed_store = InMemoryStateStore.from_dict(serialized_state, serializer)
self._initial_state[run_id] = seed_store._state
except Exception:
pass
self._initial_state[run_id] = (serialized_state, serializer)
store_type = serialized_state.get("store_type")
if store_type is not None and store_type != "in_memory":
passthrough_state = None
return super().run_workflow(
run_id,
workflow,
init_state,
start_event=start_event,
serialized_state=serialized_state,
serialized_state=passthrough_state,
serializer=serializer,
)
@@ -249,9 +253,14 @@ class ServerRuntimeDecorator(BaseRuntimeDecorator):
self,
handler_id: str,
workflow_name: str,
handler: WorkflowHandler,
) -> WorkflowHandler:
"""Persist initial handler record to store, then notify decorator chain."""
run_id: str,
) -> None:
"""Persist initial handler record to store.
Must be called before the workflow is started so that
``update_handler_status`` can find the handler row when
the workflow completes.
"""
started_at = datetime.now(timezone.utc)
await self._retry_store_write(
@@ -260,11 +269,9 @@ class ServerRuntimeDecorator(BaseRuntimeDecorator):
handler_id=handler_id,
workflow_name=workflow_name,
status="running",
run_id=handler.run_id,
run_id=run_id,
started_at=started_at,
updated_at=started_at,
)
)
)
return handler
@@ -22,8 +22,10 @@ from llama_agents.client.protocol.serializable_events import (
from llama_agents.server._runtime.server_runtime import ServerRuntimeDecorator
from llama_index_instrumentation.dispatcher import instrument_tags
from workflows import Context
from workflows.context.serializers import JsonSerializer
from workflows.events import Event, StartEvent
from workflows.handler import WorkflowHandler
from workflows.utils import _nanoid as nanoid
from workflows.workflow import Workflow
from ._store.abstract_workflow_store import (
@@ -199,12 +201,20 @@ class _WorkflowService:
context: Context | None = None,
) -> HandlerData:
with instrument_tags({"handler_id": handler_id}):
handler = workflow.run(
if context is None:
context = await self._context_from_handler_id(workflow, handler_id)
# Pre-generate run_id and persist the handler record BEFORE starting
# the workflow. This prevents a race where a fast workflow completes
# and tries to update_handler_status before the handler row exists,
# causing the status update to be silently skipped.
run_id = nanoid()
await self._runtime.run_workflow_handler(
handler_id, workflow.workflow_name, run_id
)
_ = workflow.run(
ctx=context,
start_event=start_event,
)
await self._runtime.run_workflow_handler(
handler_id, workflow.workflow_name, handler
run_id=run_id,
)
handler_data = await self.load_handler(handler_id)
if handler_data is None:
@@ -219,7 +229,13 @@ class _WorkflowService:
try:
await run
except Exception:
pass
logger.error(
"Workflow %s (handler=%s, run=%s) raised an exception",
handler.workflow_name,
handler.handler_id,
handler.run_id,
exc_info=True,
)
handler_data = await self.load_handler(handler.handler_id)
if handler_data is None:
raise HandlerNotFoundError()
@@ -241,6 +257,42 @@ class _WorkflowService:
# Private helpers
# ------------------------------------------------------------------
async def _context_from_handler_id(
self, workflow: Workflow, handler_id: str
) -> Context | None:
"""Look up a completed handler's final state and build a Context from it.
Returns the lightweight serialized state reference so that SQL-backed
stores can do an optimized copy rather than round-tripping through memory.
Returns None if the handler doesn't exist, isn't completed, or has no state.
"""
found = await self._store.query(HandlerQuery(handler_id_in=[handler_id]))
if not found:
return None
handler = found[0]
if not is_terminal_status(handler.status) or handler.run_id is None:
return None
try:
serializer = JsonSerializer()
old_state_store = self._store.create_state_store(handler.run_id)
state_dict = old_state_store.to_dict(serializer)
if not state_dict:
return None
return Context.from_dict(
workflow=workflow,
data={"version": 1, "state": state_dict},
serializer=serializer,
)
except Exception:
logger.warning(
"Failed to read state from previous handler %s",
handler_id,
exc_info=True,
)
return None
def _workflow_run_handler(self, workflow_name: str, run_id: str) -> WorkflowHandler:
workflow = self._runtime.get_workflow(workflow_name)
if workflow is None:
@@ -20,6 +20,7 @@ from pydantic import (
field_validator,
)
from workflows.context import JsonSerializer
from workflows.context.serializers import BaseSerializer
from workflows.context.state_store import StateStore
from workflows.events import StopEvent
@@ -110,9 +111,17 @@ class AbstractWorkflowStore(ABC):
@abstractmethod
def create_state_store(
self, run_id: str, state_type: type[Any] | None = None
self,
run_id: str,
state_type: type[Any] | None = None,
serialized_state: dict[str, Any] | None = None,
serializer: BaseSerializer | None = None,
) -> StateStore[Any]:
"""Create a persistent state store for the given run. see e.g. InMemoryStateStore for a reference implementation."""
"""Create a persistent state store for the given run.
If *serialized_state* and *serializer* are provided, the store is
seeded from the serialized data during construction.
"""
@abstractmethod
async def query(self, query: HandlerQuery) -> List[PersistentHandler]: ...
@@ -1,12 +1,14 @@
from __future__ import annotations
import asyncio
import logging
import weakref
from collections.abc import AsyncIterator
from datetime import datetime, timezone
from typing import Any, Dict, List
from llama_agents.client.protocol.serializable_events import EventEnvelopeWithMetadata
from workflows.context.serializers import BaseSerializer
from workflows.context.state_store import DictState, InMemoryStateStore
from .abstract_workflow_store import (
@@ -17,6 +19,8 @@ from .abstract_workflow_store import (
StoredTick,
)
logger = logging.getLogger(__name__)
def _matches_query(handler: PersistentHandler, query: HandlerQuery) -> bool:
# Empty lists should match nothing (short-circuit)
@@ -63,12 +67,27 @@ class MemoryWorkflowStore(AbstractWorkflowStore):
)
def create_state_store(
self, run_id: str, state_type: type[Any] | None = None
self,
run_id: str,
state_type: type[Any] | None = None,
serialized_state: dict[str, Any] | None = None,
serializer: BaseSerializer | None = None,
) -> InMemoryStateStore[Any]:
if run_id not in self.state_stores:
self.state_stores[run_id] = InMemoryStateStore(
state_type() if state_type else DictState()
)
if serialized_state is not None and serializer is not None:
try:
self.state_stores[run_id] = InMemoryStateStore.from_dict(
serialized_state, serializer
)
except Exception:
logger.warning("Failed to seed InMemoryStateStore", exc_info=True)
self.state_stores[run_id] = InMemoryStateStore(
state_type() if state_type else DictState()
)
else:
self.state_stores[run_id] = InMemoryStateStore(
state_type() if state_type else DictState()
)
return self.state_stores[run_id]
async def query(self, query: HandlerQuery) -> List[PersistentHandler]:
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
@@ -0,0 +1,104 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
from __future__ import annotations
try:
from importlib.resources.abc import Traversable # type: ignore
except ImportError: # pre 3.11
from importlib.abc import Traversable # type: ignore
import logging
import re
from importlib import import_module, resources
import asyncpg
logger = logging.getLogger(__name__)
_MIGRATIONS_PKG = "llama_agents.server._store.postgres.migrations"
_VERSION_PATTERN = re.compile(r"--\s*migration:\s*(\d+)")
# Arbitrary but fixed int64 used as a pg_advisory_xact_lock key so that
# concurrent replicas serialize their migration runs.
_LOCK_ID = 7_201_407_233_458_173
_VALID_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
def _quote_identifier(name: str) -> str:
"""Quote a SQL identifier, raising on invalid names."""
if not _VALID_IDENTIFIER.match(name):
msg = f"Invalid SQL identifier: {name!r}"
raise ValueError(msg)
return f'"{name}"'
def _iter_migration_files() -> list[Traversable]:
"""Yield packaged SQL migration files in lexicographic order."""
pkg = import_module(_MIGRATIONS_PKG)
root = resources.files(pkg)
files = (p for p in root.iterdir() if p.name.endswith(".sql"))
return sorted(files, key=lambda p: p.name) # type: ignore
def _parse_target_version(sql_text: str) -> int | None:
"""Return target schema version declared in the first line comment."""
first_line = sql_text.splitlines()[0] if sql_text else ""
match = _VERSION_PATTERN.search(first_line)
return int(match.group(1)) if match else None
async def run_migrations(conn: asyncpg.Connection, schema: str | None = None) -> None:
"""Apply pending migrations found under the migrations package.
Each migration file should start with a `-- migration: N` line.
Files are applied in lexicographic order and only when N > current_version.
A session-level advisory lock ensures that concurrent replicas serialize
their migration runs so DDL and version bookkeeping never race.
"""
# Acquire a session-level advisory lock *before* any DDL so that
# concurrent callers (e.g. multiple replicas starting up) don't race
# on CREATE SCHEMA / CREATE TABLE.
await conn.execute("SELECT pg_advisory_lock($1)", _LOCK_ID)
try:
await _run_migrations_locked(conn, schema)
finally:
await conn.execute("SELECT pg_advisory_unlock($1)", _LOCK_ID)
async def _run_migrations_locked(conn: asyncpg.Connection, schema: str | None) -> None:
if schema:
quoted = _quote_identifier(schema)
await conn.execute(f"CREATE SCHEMA IF NOT EXISTS {quoted}")
await conn.execute(f"SET search_path TO {quoted}")
await conn.execute("""
CREATE TABLE IF NOT EXISTS schema_migrations (
version INTEGER PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
""")
row = await conn.fetchval("SELECT MAX(version) FROM schema_migrations")
current_version = row if row is not None else 0
for path in _iter_migration_files():
sql_text = path.read_text()
target_version = _parse_target_version(sql_text) or 0
if target_version <= current_version:
continue
try:
logger.debug(
"Applying migration %s -> target version %s", path.name, target_version
)
async with conn.transaction():
await conn.execute(sql_text)
await conn.execute(
"INSERT INTO schema_migrations (version) VALUES ($1)",
target_version,
)
except Exception:
logger.error("Failed migration %s", path.name)
raise
current_version = target_version
@@ -0,0 +1,54 @@
-- migration: 1
CREATE TABLE IF NOT EXISTS wf_handlers (
handler_id VARCHAR(255) PRIMARY KEY,
workflow_name VARCHAR(255) NOT NULL,
status VARCHAR(50) NOT NULL,
run_id VARCHAR(255),
error TEXT,
result TEXT,
started_at TIMESTAMPTZ,
updated_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
idle_since TIMESTAMPTZ
);
CREATE TABLE IF NOT EXISTS wf_events (
id SERIAL PRIMARY KEY,
run_id VARCHAR(255) NOT NULL,
sequence INTEGER NOT NULL,
timestamp TIMESTAMPTZ NOT NULL,
event_json TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_wf_events_run_id ON wf_events (run_id);
CREATE INDEX IF NOT EXISTS idx_wf_handlers_run_id ON wf_handlers (run_id);
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_constraint WHERE conname = 'uq_wf_events_run_id_sequence'
) THEN
ALTER TABLE wf_events
ADD CONSTRAINT uq_wf_events_run_id_sequence UNIQUE (run_id, sequence);
END IF;
END
$$;
CREATE TABLE IF NOT EXISTS workflow_state (
run_id VARCHAR(255) PRIMARY KEY,
state_json TEXT NOT NULL,
state_type VARCHAR(255),
state_module VARCHAR(255),
created_at TIMESTAMPTZ,
updated_at TIMESTAMPTZ
);
CREATE TABLE IF NOT EXISTS workflow_journal (
id SERIAL PRIMARY KEY,
run_id VARCHAR(255) NOT NULL,
seq_num INTEGER NOT NULL,
task_key VARCHAR(512) NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_workflow_journal_run_id ON workflow_journal (run_id);
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
@@ -0,0 +1,294 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
from __future__ import annotations
import asyncio
import functools
import json
import logging
import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from typing import Any, AsyncGenerator, Generic, Literal, Type
import asyncpg
from pydantic import BaseModel
from typing_extensions import TypeVar
from workflows.context.serializers import BaseSerializer, JsonSerializer
from workflows.context.state_store import (
DictState,
create_cleared_state,
deserialize_dict_state_data,
deserialize_state_from_dict,
get_by_path,
merge_state,
parse_in_memory_state,
serialize_dict_state_data,
set_by_path,
)
logger = logging.getLogger(__name__)
MODEL_T = TypeVar("MODEL_T", bound=BaseModel, default=DictState) # type: ignore[reportGeneralTypeIssues]
class PostgresSerializedState(BaseModel):
"""Serialized state referencing a postgres database row."""
store_type: Literal["postgres"] = "postgres"
run_id: str
def _utc_now() -> datetime:
return datetime.now(timezone.utc)
class PostgresStateStore(Generic[MODEL_T]):
"""Asyncpg-backed StateStore implementation.
Every get() reads from the database, every set() writes through.
No in-memory cache — the database is the source of truth.
"""
state_type: Type[MODEL_T]
def __init__(
self,
pool: asyncpg.Pool,
run_id: str,
state_type: Type[MODEL_T] | None = None,
serializer: BaseSerializer | None = None,
schema: str | None = None,
) -> None:
self._pool = pool
self._run_id = run_id
self.state_type = state_type or DictState # type: ignore[assignment]
self._serializer = serializer or JsonSerializer()
self._schema = schema
self._pending_seed: tuple[dict[str, Any], BaseSerializer] | None = None
@property
def run_id(self) -> str:
return self._run_id
@property
def _table_ref(self) -> str:
if self._schema:
return f"{self._schema}.workflow_state"
return "workflow_state"
@functools.cached_property
def _lock(self) -> asyncio.Lock:
"""Lazy lock initialization for Python 3.14+ compatibility."""
return asyncio.Lock()
def _serialize_state(self, state: MODEL_T) -> str:
"""Serialize state model to JSON string."""
if isinstance(state, DictState):
return json.dumps(serialize_dict_state_data(state, self._serializer))
return self._serializer.serialize(state)
def _deserialize_state(self, state_json: str) -> MODEL_T:
"""Deserialize state from JSON string."""
if issubclass(self.state_type, DictState):
data = json.loads(state_json)
return deserialize_dict_state_data(data, self._serializer) # type: ignore[return-value]
return self._serializer.deserialize(state_json)
def _create_default_state(self) -> MODEL_T:
return self.state_type()
async def _write_in_memory_state(self, serialized_state: dict[str, Any]) -> None:
"""Migrate InMemory-format state into the database."""
state = deserialize_state_from_dict(serialized_state, self._serializer)
await self._save_state(state) # type: ignore[arg-type]
async def _flush_pending_seed(self) -> None:
"""Flush pending seed data to the database if present."""
if self._pending_seed is None:
return
serialized_state, serializer = self._pending_seed
self._pending_seed = None
store_type = serialized_state.get("store_type")
if store_type == "postgres":
source_run_id = serialized_state.get("run_id")
if source_run_id and source_run_id != self._run_id:
await self._copy_state_from_run(source_run_id)
else:
await self._write_in_memory_state(serialized_state)
async def _copy_state_from_run(self, source_run_id: str) -> None:
"""Copy state from another run_id using SQL INSERT...SELECT."""
async with self._pool.acquire() as conn:
now = _utc_now()
await conn.execute(
f"""
INSERT INTO {self._table_ref} (run_id, state_json, state_type, state_module, created_at, updated_at)
SELECT $1, state_json, state_type, state_module, $2, $3
FROM {self._table_ref} WHERE run_id = $4
ON CONFLICT(run_id) DO UPDATE SET
state_json = EXCLUDED.state_json,
state_type = EXCLUDED.state_type,
state_module = EXCLUDED.state_module,
updated_at = EXCLUDED.updated_at
""",
self._run_id,
now,
now,
source_run_id,
)
async def _load_state(
self,
conn: asyncpg.Connection | None = None,
) -> MODEL_T:
"""Load state from database. Creates default if row doesn't exist."""
await self._flush_pending_seed()
should_release = conn is None
if conn is None:
conn = await self._pool.acquire() # type: ignore[assignment]
try:
row = await conn.fetchrow( # type: ignore[union-attr]
f"SELECT state_json FROM {self._table_ref} WHERE run_id = $1",
self._run_id,
)
if row is None:
state = self._create_default_state()
await self._save_state(state, conn)
return state
return self._deserialize_state(row["state_json"])
finally:
if should_release:
await self._pool.release(conn) # type: ignore[arg-type]
async def _save_state(
self,
state: MODEL_T,
conn: asyncpg.Connection | asyncpg.pool.PoolConnectionProxy | None = None,
) -> None:
"""Save state to database via upsert."""
should_release = conn is None
if conn is None:
conn = await self._pool.acquire() # type: ignore[assignment]
try:
now = _utc_now()
state_json = self._serialize_state(state)
await conn.execute( # type: ignore[union-attr]
f"""
INSERT INTO {self._table_ref} (run_id, state_json, state_type, state_module, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT(run_id) DO UPDATE SET
state_json = EXCLUDED.state_json,
state_type = EXCLUDED.state_type,
state_module = EXCLUDED.state_module,
updated_at = EXCLUDED.updated_at
""",
self._run_id,
state_json,
type(state).__name__,
type(state).__module__,
now,
now,
)
finally:
if should_release:
await self._pool.release(conn) # type: ignore[arg-type]
async def get_state(self) -> MODEL_T:
"""Return a copy of the current state model."""
state = await self._load_state()
return state.model_copy()
async def set_state(self, state: MODEL_T) -> None:
"""Replace or merge into the current state model."""
async with self._pool.acquire() as conn:
row = await conn.fetchrow(
f"SELECT state_json FROM {self._table_ref} WHERE run_id = $1",
self._run_id,
)
if row is None:
await self._save_state(state, conn)
return
current_state = self._deserialize_state(row["state_json"])
merged = merge_state(current_state, state)
await self._save_state(merged, conn) # type: ignore[arg-type]
async def get(self, path: str, default: Any = ...) -> Any:
"""Get a nested value using dot-separated paths."""
state = await self._load_state()
return get_by_path(state, path, default)
async def set(self, path: str, value: Any) -> None:
"""Set a nested value using dot-separated paths."""
async with self.edit_state() as state:
set_by_path(state, path, value)
async def clear(self) -> None:
"""Reset the state to its type defaults."""
await self.set_state(create_cleared_state(self.state_type))
@asynccontextmanager
async def edit_state(self) -> AsyncGenerator[MODEL_T, None]:
"""Edit state transactionally under a lock."""
async with self._lock:
state = await self._load_state()
yield state
await self._save_state(state)
def to_dict(self, serializer: BaseSerializer) -> dict[str, Any]:
"""Serialize state store metadata for persistence.
Returns metadata only — actual state lives in the database.
"""
payload = PostgresSerializedState(run_id=self._run_id)
return payload.model_dump()
@classmethod
def from_dict(
cls,
serialized_state: dict[str, Any],
serializer: BaseSerializer,
pool: asyncpg.Pool | None = None,
state_type: type[BaseModel] | None = None,
run_id: str | None = None,
schema: str | None = None,
) -> PostgresStateStore[Any]:
"""Restore a state store from serialized payload.
Handles both InMemorySerializedState (migrates data to DB on first use)
and PostgresSerializedState (reconnects to existing row).
"""
if not serialized_state:
raise ValueError("Cannot restore PostgresStateStore from empty dict")
if pool is None:
raise ValueError("pool is required for PostgresStateStore.from_dict()")
store_type = serialized_state.get("store_type")
if store_type == "postgres":
parsed = PostgresSerializedState.model_validate(serialized_state)
effective_run_id = run_id or parsed.run_id
return cls(
pool=pool,
run_id=effective_run_id,
state_type=state_type, # type: ignore[arg-type]
serializer=serializer,
schema=schema,
)
# InMemory format — will need async migration
parse_in_memory_state(serialized_state)
effective_run_id = run_id or str(uuid.uuid4())
store = cls(
pool=pool,
run_id=effective_run_id,
state_type=state_type, # type: ignore[arg-type]
serializer=serializer,
schema=schema,
)
# Note: caller must await store._write_in_memory_state(serialized_state)
# since from_dict is synchronous but migration requires async DB access
return store
@@ -0,0 +1,433 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
from __future__ import annotations
import asyncio
import concurrent.futures
import json
import logging
import weakref
from collections.abc import AsyncIterator
from datetime import datetime, timezone
from typing import Any, List, Sequence, cast
import asyncpg
from llama_agents.client.protocol.serializable_events import EventEnvelopeWithMetadata
from workflows.context import JsonSerializer
from workflows.context.serializers import BaseSerializer
from .abstract_workflow_store import (
AbstractWorkflowStore,
HandlerQuery,
PersistentHandler,
StoredEvent,
StoredTick,
)
from .postgres.migrate import run_migrations as _run_migrations
from .postgres_state_store import PostgresStateStore
logger = logging.getLogger(__name__)
def _utc_now() -> datetime:
return datetime.now(timezone.utc)
class PostgresWorkflowStore(AbstractWorkflowStore):
"""Async Postgres workflow store using asyncpg with LISTEN/NOTIFY."""
def __init__(
self,
dsn: str,
schema: str | None = None,
poll_interval: float = 1.0,
handlers_table_name: str = "wf_handlers",
events_table_name: str = "wf_events",
pool_min_size: int = 2,
pool_max_size: int = 10,
) -> None:
self._dsn = dsn
self._schema = schema
self.poll_interval = poll_interval
self._handlers_table_name = handlers_table_name
self._events_table_name = events_table_name
self._pool_min_size = pool_min_size
self._pool_max_size = pool_max_size
self._pool: asyncpg.Pool | None = None
self._listen_conn: asyncpg.Connection | None = None
self._conditions: weakref.WeakValueDictionary[str, asyncio.Condition] = (
weakref.WeakValueDictionary()
)
@property
def _handlers_ref(self) -> str:
if self._schema:
return f"{self._schema}.{self._handlers_table_name}"
return self._handlers_table_name
@property
def _events_ref(self) -> str:
if self._schema:
return f"{self._schema}.{self._events_table_name}"
return self._events_table_name
@property
def _notify_channel(self) -> str:
return self._events_table_name
async def start(self) -> None:
"""Create the connection pool and set up the LISTEN connection."""
if self._pool is not None:
return
self._pool = await asyncpg.create_pool(
self._dsn,
min_size=self._pool_min_size,
max_size=self._pool_max_size,
)
await self._setup_listener()
async def _setup_listener(self) -> None:
"""Set up a dedicated connection for LISTEN/NOTIFY."""
assert self._pool is not None
conn = cast(asyncpg.Connection, await self._pool.acquire())
await conn.add_listener(self._notify_channel, self._on_notify)
self._listen_conn = conn
def _on_notify(
self,
connection: asyncpg.Connection,
pid: int,
channel: str,
payload: str,
) -> None:
"""Handle NOTIFY callback — wake up subscribers for the given run_id."""
run_id = payload
cond = self._conditions.get(run_id)
if cond is not None:
# Schedule the notify on the event loop since this callback
# may fire from a non-async context
loop = asyncio.get_event_loop()
loop.create_task(self._notify_condition(cond))
@staticmethod
async def _notify_condition(cond: asyncio.Condition) -> None:
async with cond:
cond.notify_all()
async def close(self) -> None:
"""Tear down the LISTEN connection and close the pool."""
if self._listen_conn is not None:
try:
await self._listen_conn.remove_listener(
self._notify_channel, self._on_notify
)
except Exception:
logger.debug("Failed to remove listener during close", exc_info=True)
try:
await self._pool.release(self._listen_conn) # type: ignore[union-attr]
except Exception:
logger.debug(
"Failed to release listen connection during close", exc_info=True
)
self._listen_conn = None
if self._pool is not None:
await self._pool.close()
self._pool = None
async def _ensure_pool(self) -> asyncpg.Pool:
if self._pool is None:
await self.start()
assert self._pool is not None
return self._pool
def _get_or_create_condition(self, run_id: str) -> asyncio.Condition:
cond = self._conditions.get(run_id)
if cond is None:
cond = asyncio.Condition()
self._conditions[run_id] = cond
return cond
def create_state_store(
self,
run_id: str,
state_type: type[Any] | None = None,
serialized_state: dict[str, Any] | None = None,
serializer: BaseSerializer | None = None,
) -> PostgresStateStore[Any]:
if self._pool is None:
raise RuntimeError(
"PostgresWorkflowStore pool not initialized. Call start() first."
)
store = PostgresStateStore(
pool=self._pool,
run_id=run_id,
state_type=state_type,
schema=self._schema,
)
if serialized_state is not None and serializer is not None:
store._pending_seed = (serialized_state, serializer)
return store
# ── Migrations ──────────────────────────────────────────────────────
async def run_migrations(self) -> None:
"""Apply file-based migrations to create/update schema."""
pool = await self._ensure_pool()
async with pool.acquire() as conn:
await _run_migrations(cast(asyncpg.Connection, conn), schema=self._schema)
@staticmethod
def run_migrations_sync(dsn: str, schema: str | None = None) -> None:
"""Run migrations synchronously, handling event loop detection.
Safe to call from both sync and async contexts. When called from
within a running event loop, runs migrations in a background thread.
"""
async def _migrate() -> None:
store = PostgresWorkflowStore(dsn=dsn, schema=schema)
await store.start()
try:
await store.run_migrations()
finally:
await store.close()
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop is not None and loop.is_running():
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
executor.submit(lambda: asyncio.run(_migrate())).result()
else:
asyncio.run(_migrate())
# ── Handlers ────────────────────────────────────────────────────────
async def query(self, query: HandlerQuery) -> List[PersistentHandler]:
filter_spec = self._build_filters(query)
if filter_spec is None:
return []
clauses, params = filter_spec
sql = f"""
SELECT handler_id, workflow_name, status, run_id, error, result,
started_at, updated_at, completed_at, idle_since
FROM {self._handlers_ref}
"""
if clauses:
sql = f"{sql} WHERE {' AND '.join(clauses)}"
pool = await self._ensure_pool()
async with pool.acquire() as conn:
rows = await conn.fetch(sql, *params)
return [self._row_to_handler(row) for row in rows]
async def update(self, handler: PersistentHandler) -> None:
result_json = None
if handler.result is not None:
result_json = JsonSerializer().serialize(handler.result)
pool = await self._ensure_pool()
async with pool.acquire() as conn:
await conn.execute(
f"""
INSERT INTO {self._handlers_ref}
(handler_id, workflow_name, status, run_id, error, result,
started_at, updated_at, completed_at, idle_since)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (handler_id) DO UPDATE SET
workflow_name = EXCLUDED.workflow_name,
status = EXCLUDED.status,
run_id = EXCLUDED.run_id,
error = EXCLUDED.error,
result = EXCLUDED.result,
started_at = EXCLUDED.started_at,
updated_at = EXCLUDED.updated_at,
completed_at = EXCLUDED.completed_at,
idle_since = EXCLUDED.idle_since
""",
handler.handler_id,
handler.workflow_name,
handler.status,
handler.run_id,
handler.error,
result_json,
handler.started_at,
handler.updated_at,
handler.completed_at,
handler.idle_since,
)
async def delete(self, query: HandlerQuery) -> int:
filter_spec = self._build_filters(query)
if filter_spec is None:
return 0
clauses, params = filter_spec
if not clauses:
return 0
sql = f"DELETE FROM {self._handlers_ref} WHERE {' AND '.join(clauses)}"
pool = await self._ensure_pool()
async with pool.acquire() as conn:
result = await conn.execute(sql, *params)
# asyncpg returns "DELETE N"
return int(result.split()[-1])
# ── Events ──────────────────────────────────────────────────────────
_MAX_SEQUENCE_RETRIES = 5
async def append_event(self, run_id: str, event: EventEnvelopeWithMetadata) -> None:
now = _utc_now()
event_json = event.model_dump_json()
pool = await self._ensure_pool()
insert_sql = f"""
INSERT INTO {self._events_ref} (run_id, sequence, timestamp, event_json)
VALUES (
$1,
COALESCE((SELECT MAX(sequence) FROM {self._events_ref} WHERE run_id = $1::varchar), -1) + 1,
$2,
$3
)
"""
# Retry on unique constraint violation from concurrent sequence assignment
for attempt in range(self._MAX_SEQUENCE_RETRIES):
try:
async with pool.acquire() as conn:
await conn.execute(insert_sql, run_id, now, event_json)
await conn.execute(
"SELECT pg_notify($1, $2)",
self._notify_channel,
run_id,
)
return
except asyncpg.UniqueViolationError:
if attempt == self._MAX_SEQUENCE_RETRIES - 1:
raise
logger.debug(
"Sequence conflict for run_id=%s, retrying (attempt %d)",
run_id,
attempt + 1,
)
async def query_events(
self,
run_id: str,
after_sequence: int | None = None,
limit: int | None = None,
) -> list[StoredEvent]:
sql = f"""
SELECT run_id, sequence, timestamp, event_json
FROM {self._events_ref}
WHERE run_id = $1
"""
params: list[Any] = [run_id]
param_idx = 2
if after_sequence is not None:
sql += f" AND sequence > ${param_idx}"
params.append(after_sequence)
param_idx += 1
sql += " ORDER BY sequence"
if limit is not None:
sql += f" LIMIT ${param_idx}"
params.append(limit)
pool = await self._ensure_pool()
async with pool.acquire() as conn:
rows = await conn.fetch(sql, *params)
return [
StoredEvent(
run_id=row["run_id"],
sequence=row["sequence"],
timestamp=row["timestamp"],
event=EventEnvelopeWithMetadata.model_validate_json(row["event_json"]),
)
for row in rows
]
async def subscribe_events(
self, run_id: str, after_sequence: int = -1
) -> AsyncIterator[StoredEvent]:
condition = self._get_or_create_condition(run_id)
cursor = after_sequence
while True:
events = await self.query_events(run_id, after_sequence=max(cursor - 1, -1))
for event in events:
if event.sequence > cursor:
yield event
cursor = event.sequence
if self._is_terminal_event(event):
return
if not events:
async with condition:
try:
await asyncio.wait_for(
condition.wait(), timeout=self.poll_interval
)
except TimeoutError:
pass
# ── Ticks (not supported) ───────────────────────────────────────────
async def append_tick(self, run_id: str, tick_data: dict[str, Any]) -> None:
raise NotImplementedError("PostgresWorkflowStore does not support ticks")
async def get_ticks(self, run_id: str) -> list[StoredTick]:
raise NotImplementedError("PostgresWorkflowStore does not support ticks")
# ── Helpers ─────────────────────────────────────────────────────────
def _build_filters(self, query: HandlerQuery) -> tuple[list[str], list[Any]] | None:
clauses: list[str] = []
params: list[Any] = []
param_idx = 1
def add_in_clause(column: str, values: Sequence[str]) -> None:
nonlocal param_idx
placeholders = ", ".join([f"${param_idx + i}" for i in range(len(values))])
clauses.append(f"{column} IN ({placeholders})")
params.extend(values)
param_idx += len(values)
for field, column in [
(query.workflow_name_in, "workflow_name"),
(query.handler_id_in, "handler_id"),
(query.run_id_in, "run_id"),
(query.status_in, "status"),
]:
if field is not None:
if len(field) == 0:
return None
add_in_clause(column, field)
if query.is_idle is not None:
if query.is_idle:
clauses.append("idle_since IS NOT NULL")
else:
clauses.append("idle_since IS NULL")
return clauses, params
@staticmethod
def _row_to_handler(row: asyncpg.Record) -> PersistentHandler:
return PersistentHandler(
handler_id=row["handler_id"],
workflow_name=row["workflow_name"],
status=row["status"],
run_id=row["run_id"],
error=row["error"],
result=json.loads(row["result"]) if row["result"] else None,
started_at=row["started_at"],
updated_at=row["updated_at"],
completed_at=row["completed_at"],
idle_since=row["idle_since"],
)
@@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS ticks (
CREATE INDEX IF NOT EXISTS idx_ticks_run_id ON ticks (run_id);
CREATE INDEX IF NOT EXISTS idx_ticks_run_id_sequence ON ticks (run_id, sequence);
CREATE TABLE IF NOT EXISTS state (
CREATE TABLE IF NOT EXISTS workflow_state (
run_id TEXT PRIMARY KEY,
state_json TEXT NOT NULL DEFAULT '{}',
state_type TEXT NOT NULL DEFAULT 'DictState',
@@ -31,3 +31,12 @@ CREATE TABLE IF NOT EXISTS events (
CREATE INDEX IF NOT EXISTS idx_events_run_id_sequence ON events (run_id, sequence);
CREATE INDEX IF NOT EXISTS idx_handlers_run_id ON handlers (run_id);
CREATE TABLE IF NOT EXISTS workflow_journal (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id TEXT NOT NULL,
seq_num INTEGER NOT NULL,
task_key TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_workflow_journal_run_id ON workflow_journal (run_id);
@@ -82,6 +82,38 @@ class SqliteStateStore(Generic[MODEL_T]):
state = deserialize_state_from_dict(serialized_state, self._serializer)
self._save_state(state) # type: ignore[arg-type]
def _seed_from_serialized(
self, serialized_state: dict[str, Any], serializer: BaseSerializer
) -> None:
"""Seed this store from serialized state data.
Handles both sqlite references (SQL-level copy) and InMemory format.
"""
store_type = serialized_state.get("store_type")
if store_type == "sqlite":
source_run_id = serialized_state.get("run_id")
if source_run_id and source_run_id != self._run_id:
self._copy_state_from_run(source_run_id)
else:
self._write_in_memory_state(serialized_state)
def _copy_state_from_run(self, source_run_id: str) -> None:
"""Copy state from another run_id using SQL INSERT...SELECT."""
conn = self._connect()
try:
now = _utc_now().isoformat()
conn.execute(
"""
INSERT OR REPLACE INTO workflow_state (run_id, state_json, state_type, state_module, created_at, updated_at)
SELECT ?, state_json, state_type, state_module, ?, ?
FROM workflow_state WHERE run_id = ?
""",
(self._run_id, now, now, source_run_id),
)
conn.commit()
finally:
conn.close()
def _serialize_state(self, state: MODEL_T) -> str:
"""Serialize state model to JSON string."""
if isinstance(state, DictState):
@@ -104,7 +136,7 @@ class SqliteStateStore(Generic[MODEL_T]):
try:
cursor = conn.cursor()
cursor.execute(
"SELECT state_json FROM state WHERE run_id = ?",
"SELECT state_json FROM workflow_state WHERE run_id = ?",
(self._run_id,),
)
row = cursor.fetchone()
@@ -129,7 +161,7 @@ class SqliteStateStore(Generic[MODEL_T]):
state_json = self._serialize_state(state)
conn.execute(
"""
INSERT INTO state (run_id, state_json, state_type, state_module, created_at, updated_at)
INSERT INTO workflow_state (run_id, state_json, state_type, state_module, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(run_id) DO UPDATE SET
state_json = excluded.state_json,
@@ -163,7 +195,7 @@ class SqliteStateStore(Generic[MODEL_T]):
try:
cursor = conn.cursor()
cursor.execute(
"SELECT state_json FROM state WHERE run_id = ?",
"SELECT state_json FROM workflow_state WHERE run_id = ?",
(self._run_id,),
)
row = cursor.fetchone()
@@ -7,11 +7,13 @@ import json
import sqlite3
import weakref
from collections.abc import AsyncIterator
from contextlib import contextmanager
from datetime import datetime
from typing import Any, List, Sequence
from typing import Any, Iterator, List, Sequence
from llama_agents.client.protocol.serializable_events import EventEnvelopeWithMetadata
from workflows.context import JsonSerializer
from workflows.context.serializers import BaseSerializer
from ..abstract_workflow_store import (
AbstractWorkflowStore,
@@ -20,25 +22,46 @@ from ..abstract_workflow_store import (
StoredEvent,
StoredTick,
)
from .migrate import run_migrations
from .migrate import run_migrations as _run_migrations
from .sqlite_state_store import SqliteStateStore
class SqliteWorkflowStore(AbstractWorkflowStore):
def __init__(self, db_path: str, poll_interval: float = 1.0) -> None:
def __init__(
self,
db_path: str,
poll_interval: float = 1.0,
auto_migrate: bool = True,
) -> None:
self.db_path = db_path
self.poll_interval = poll_interval
self._conditions: weakref.WeakValueDictionary[str, asyncio.Condition] = (
weakref.WeakValueDictionary()
)
self._init_db()
if auto_migrate:
self._run_migrations()
@contextmanager
def _connect(self) -> Iterator[sqlite3.Connection]:
conn = sqlite3.connect(self.db_path)
try:
yield conn
finally:
conn.close()
def create_state_store(
self, run_id: str, state_type: type[Any] | None = None
self,
run_id: str,
state_type: type[Any] | None = None,
serialized_state: dict[str, Any] | None = None,
serializer: BaseSerializer | None = None,
) -> SqliteStateStore[Any]:
return SqliteStateStore(
store = SqliteStateStore(
db_path=self.db_path, run_id=run_id, state_type=state_type
)
if serialized_state is not None and serializer is not None:
store._seed_from_serialized(serialized_state, serializer)
return store
def _get_or_create_condition(self, run_id: str) -> asyncio.Condition:
"""Get or create a condition for a run_id.
@@ -52,10 +75,18 @@ class SqliteWorkflowStore(AbstractWorkflowStore):
self._conditions[run_id] = cond
return cond
def _init_db(self) -> None:
conn = sqlite3.connect(self.db_path)
def _run_migrations(self) -> None:
self.run_migrations(self.db_path)
@staticmethod
def run_migrations(db_path: str) -> None:
"""Run all pending SQLite schema migrations.
Safe to call multiple times — only applies migrations not yet applied.
"""
conn = sqlite3.connect(db_path)
try:
run_migrations(conn)
_run_migrations(conn)
conn.commit()
finally:
conn.close()
@@ -70,53 +101,47 @@ class SqliteWorkflowStore(AbstractWorkflowStore):
started_at, updated_at, completed_at, idle_since FROM handlers"""
if clauses:
sql = f"{sql} WHERE {' AND '.join(clauses)}"
conn = sqlite3.connect(self.db_path)
try:
with self._connect() as conn:
cursor = conn.cursor()
cursor.execute(sql, tuple(params))
rows = cursor.fetchall()
finally:
conn.close()
return [_row_to_persistent_handler(row) for row in rows]
async def update(self, handler: PersistentHandler) -> None:
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute(
"""
INSERT INTO handlers (handler_id, workflow_name, status, run_id, error, result,
started_at, updated_at, completed_at, idle_since)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(handler_id) DO UPDATE SET
workflow_name = excluded.workflow_name,
status = excluded.status,
run_id = excluded.run_id,
error = excluded.error,
result = excluded.result,
started_at = excluded.started_at,
updated_at = excluded.updated_at,
completed_at = excluded.completed_at,
idle_since = excluded.idle_since
""",
(
handler.handler_id,
handler.workflow_name,
handler.status,
handler.run_id,
handler.error,
JsonSerializer().serialize(handler.result)
if handler.result is not None
else None,
handler.started_at.isoformat() if handler.started_at else None,
handler.updated_at.isoformat() if handler.updated_at else None,
handler.completed_at.isoformat() if handler.completed_at else None,
handler.idle_since.isoformat() if handler.idle_since else None,
),
)
conn.commit()
conn.close()
with self._connect() as conn:
conn.execute(
"""
INSERT INTO handlers (handler_id, workflow_name, status, run_id, error, result,
started_at, updated_at, completed_at, idle_since)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(handler_id) DO UPDATE SET
workflow_name = excluded.workflow_name,
status = excluded.status,
run_id = excluded.run_id,
error = excluded.error,
result = excluded.result,
started_at = excluded.started_at,
updated_at = excluded.updated_at,
completed_at = excluded.completed_at,
idle_since = excluded.idle_since
""",
(
handler.handler_id,
handler.workflow_name,
handler.status,
handler.run_id,
handler.error,
JsonSerializer().serialize(handler.result)
if handler.result is not None
else None,
handler.started_at.isoformat() if handler.started_at else None,
handler.updated_at.isoformat() if handler.updated_at else None,
handler.completed_at.isoformat() if handler.completed_at else None,
handler.idle_since.isoformat() if handler.idle_since else None,
),
)
conn.commit()
async def delete(self, query: HandlerQuery) -> int:
filter_spec = self._build_filters(query)
@@ -128,20 +153,16 @@ class SqliteWorkflowStore(AbstractWorkflowStore):
return 0
sql = f"DELETE FROM handlers WHERE {' AND '.join(clauses)}"
conn = sqlite3.connect(self.db_path)
try:
with self._connect() as conn:
cursor = conn.cursor()
cursor.execute(sql, tuple(params))
deleted = cursor.rowcount
conn.commit()
finally:
conn.close()
return int(deleted)
async def append_event(self, run_id: str, event: EventEnvelopeWithMetadata) -> None:
conn = sqlite3.connect(self.db_path)
try:
with self._connect() as conn:
conn.execute(
"""INSERT INTO events (run_id, sequence, timestamp, event_json)
VALUES (?, COALESCE((SELECT MAX(sequence) FROM events WHERE run_id = ?), -1) + 1, CURRENT_TIMESTAMP, ?)""",
@@ -152,8 +173,6 @@ class SqliteWorkflowStore(AbstractWorkflowStore):
),
)
conn.commit()
finally:
conn.close()
condition = self._conditions.get(run_id)
if condition is not None:
async with condition:
@@ -165,22 +184,19 @@ class SqliteWorkflowStore(AbstractWorkflowStore):
after_sequence: int | None = None,
limit: int | None = None,
) -> list[StoredEvent]:
conn = sqlite3.connect(self.db_path)
try:
sql = "SELECT run_id, sequence, timestamp, event_json FROM events WHERE run_id = ?"
params: list[Any] = [run_id]
if after_sequence is not None:
sql += " AND sequence > ?"
params.append(after_sequence)
sql += " ORDER BY sequence"
if limit is not None:
sql += " LIMIT ?"
params.append(limit)
sql = "SELECT run_id, sequence, timestamp, event_json FROM events WHERE run_id = ?"
params: list[Any] = [run_id]
if after_sequence is not None:
sql += " AND sequence > ?"
params.append(after_sequence)
sql += " ORDER BY sequence"
if limit is not None:
sql += " LIMIT ?"
params.append(limit)
with self._connect() as conn:
cursor = conn.cursor()
cursor.execute(sql, params)
rows = cursor.fetchall()
finally:
conn.close()
return [
StoredEvent(
run_id=row[0],
@@ -214,8 +230,7 @@ class SqliteWorkflowStore(AbstractWorkflowStore):
pass
async def append_tick(self, run_id: str, tick_data: dict[str, Any]) -> None:
conn = sqlite3.connect(self.db_path)
try:
with self._connect() as conn:
conn.execute(
"""INSERT INTO ticks (run_id, sequence, timestamp, tick_data)
VALUES (?, COALESCE((SELECT MAX(sequence) FROM ticks WHERE run_id = ?), -1) + 1, CURRENT_TIMESTAMP, ?)""",
@@ -226,20 +241,15 @@ class SqliteWorkflowStore(AbstractWorkflowStore):
),
)
conn.commit()
finally:
conn.close()
async def get_ticks(self, run_id: str) -> List[StoredTick]:
conn = sqlite3.connect(self.db_path)
try:
with self._connect() as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT run_id, sequence, timestamp, tick_data FROM ticks WHERE run_id = ? ORDER BY sequence",
(run_id,),
)
rows = cursor.fetchall()
finally:
conn.close()
return [
StoredTick(
run_id=row[0],
@@ -252,8 +262,7 @@ class SqliteWorkflowStore(AbstractWorkflowStore):
def get_legacy_ctx(self, run_id: str) -> dict[str, Any] | None:
"""Read the old ctx column for a run_id, if present."""
conn = sqlite3.connect(self.db_path)
try:
with self._connect() as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT ctx FROM handlers WHERE run_id = ?",
@@ -269,8 +278,6 @@ class SqliteWorkflowStore(AbstractWorkflowStore):
return data
except (json.JSONDecodeError, TypeError):
return None
finally:
conn.close()
def _build_filters(self, query: HandlerQuery) -> tuple[list[str], list[str]] | None:
clauses: list[str] = []
@@ -1,2 +1,18 @@
# Re-export fixtures from server_test_fixtures for pytest discovery
from __future__ import annotations
from collections.abc import Generator
import pytest
from llama_agents_integration_tests.postgres import (
get_asyncpg_dsn,
postgres_container,
)
from server_test_fixtures import * # noqa: F401, F403
@pytest.fixture(scope="module")
def postgres_dsn() -> Generator[str, None, None]:
"""Module-scoped disposable Postgres via testcontainers; yields an asyncpg DSN."""
with postgres_container() as pg:
yield get_asyncpg_dsn(pg)
@@ -8,6 +8,7 @@ import asyncio
import json
import sqlite3
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import pytest
@@ -1027,3 +1028,69 @@ async def test_failed_workflow_after_reload(
handler = await wait_handler_status(memory_store, "fail-reload-1", "failed")
assert handler.error is not None
assert "Error response received" in handler.error
# --- State persistence across handler runs (counter pattern) ---
class IncrementEvent(HumanResponseEvent):
pass
class CounterWorkflow(Workflow):
"""Workflow that increments a persistent counter each time it receives an event."""
@step
async def start(self, ctx: Context, ev: StartEvent) -> None:
count = await ctx.store.get("count", 0)
await ctx.store.set("count", count + 1)
@step
async def wait_and_increment(self, ctx: Context, ev: IncrementEvent) -> StopEvent:
count = await ctx.store.get("count", 0)
new_count = count + 1
await ctx.store.set("count", new_count)
return StopEvent(result=f"count={new_count}")
@pytest.mark.asyncio
@pytest.mark.parametrize(
"store_factory",
[
pytest.param(lambda tmp: MemoryWorkflowStore(), id="memory"),
pytest.param(
lambda tmp: SqliteWorkflowStore(str(tmp / "test.db")), id="sqlite"
),
],
)
async def test_counter_state_persists_across_idle_reload(
tmp_path: Path,
store_factory: Any,
) -> None:
"""A counter workflow increments on start, goes idle, reloads on event,
and the count reflects both increments."""
store = store_factory(tmp_path)
handler_id = "counter-1"
wf = CounterWorkflow()
server = WorkflowServer(workflow_store=store, idle_timeout=0.01)
server.add_workflow("counter", wf, additional_events=[IncrementEvent])
async with server.contextmanager():
handler_data = await server._service.start_workflow(wf, handler_id)
run_id = handler_data.run_id
assert run_id is not None
idle_release = _get_idle_release(server)
await wait_run_released(idle_release, run_id)
# Verify count=1 after the start step
state_store = store.create_state_store(run_id)
assert await state_store.get("count") == 1
# Send event to reload and increment again
await server._service.send_event(handler_id, IncrementEvent())
handler = await wait_handler_status(store, handler_id, "completed")
assert handler.result is not None
assert handler.result.result == "count=2"
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
"""Tests for EventInterceptorDecorator."""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock
from llama_agents.server._runtime.event_interceptor import (
EventInterceptorDecorator,
)
from workflows.context.state_store import StateStore
from workflows.events import Event, StopEvent
from workflows.runtime.types.plugin import (
ExternalRunAdapter,
InternalRunAdapter,
RegisteredWorkflow,
Runtime,
WaitResult,
WaitResultTimeout,
)
from workflows.runtime.types.ticks import WorkflowTick
from workflows.workflow import Workflow
class _RecordingInternalAdapter(InternalRunAdapter):
"""Adapter that records calls for assertion."""
def __init__(self) -> None:
self.events_written: list[Event] = []
self.ticks_sent: list[WorkflowTick] = []
self.closed = False
@property
def run_id(self) -> str:
return "test-run"
async def write_to_event_stream(self, event: Event) -> None:
self.events_written.append(event)
async def get_now(self) -> float:
return 1.0
async def send_event(self, tick: WorkflowTick) -> None:
self.ticks_sent.append(tick)
async def wait_receive(self, timeout_seconds: float | None = None) -> WaitResult:
return WaitResultTimeout()
async def close(self) -> None:
self.closed = True
def get_state_store(self) -> StateStore[Any] | None:
return None
class _StubRuntime(Runtime):
def __init__(self, adapter: InternalRunAdapter) -> None:
super().__init__()
self._adapter = adapter
def register(self, workflow: Any) -> RegisteredWorkflow:
return RegisteredWorkflow(
workflow=workflow, workflow_run_fn=MagicMock(), steps={}
)
def run_workflow(
self,
run_id: str,
workflow: Any,
init_state: Any,
start_event: Any = None,
serialized_state: dict[str, Any] | None = None,
serializer: Any = None,
) -> ExternalRunAdapter:
raise NotImplementedError
def get_internal_adapter(self, workflow: Any) -> InternalRunAdapter:
return self._adapter
def get_external_adapter(self, run_id: str) -> ExternalRunAdapter:
raise NotImplementedError
def launch(self) -> None:
pass
def destroy(self) -> None:
pass
async def test_write_to_event_stream_is_blocked() -> None:
inner_adapter = _RecordingInternalAdapter()
runtime = _StubRuntime(inner_adapter)
decorator = EventInterceptorDecorator(runtime)
wf = MagicMock(spec=Workflow)
adapter = decorator.get_internal_adapter(wf)
await adapter.write_to_event_stream(StopEvent(result="hello"))
assert inner_adapter.events_written == [], "Events should not reach inner adapter"
async def test_other_methods_pass_through() -> None:
inner_adapter = _RecordingInternalAdapter()
runtime = _StubRuntime(inner_adapter)
decorator = EventInterceptorDecorator(runtime)
wf = MagicMock(spec=Workflow)
adapter = decorator.get_internal_adapter(wf)
assert adapter.run_id == "test-run"
assert await adapter.get_now() == 1.0
await adapter.close()
assert inner_adapter.closed
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
from __future__ import annotations
import asyncio
import asyncpg
import pytest
from llama_agents.server._store.postgres.migrate import (
_iter_migration_files,
_parse_target_version,
run_migrations,
)
# ── Unit tests (no DB) ──────────────────────────────────────────────
def test_parse_target_version_valid() -> None:
assert _parse_target_version("-- migration: 1\nCREATE TABLE ...") == 1
assert _parse_target_version("-- migration: 42\n") == 42
def test_parse_target_version_missing() -> None:
assert _parse_target_version("CREATE TABLE ...") is None
assert _parse_target_version("") is None
def test_iter_migration_files_returns_sorted_sql() -> None:
files = _iter_migration_files()
assert len(files) >= 1
assert all(f.name.endswith(".sql") for f in files)
names = [f.name for f in files]
assert names == sorted(names)
def test_first_migration_has_version_1() -> None:
files = _iter_migration_files()
sql = files[0].read_text()
assert _parse_target_version(sql) == 1
# ── Integration tests (require Docker) ──────────────────────────────
@pytest.mark.docker
async def test_run_migrations_fresh_db(postgres_dsn: str) -> None:
conn = await asyncpg.connect(postgres_dsn)
schema = "test_migrate_fresh"
try:
await conn.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE")
await run_migrations(conn, schema=schema)
version = await conn.fetchval(
f"SELECT MAX(version) FROM {schema}.schema_migrations"
)
assert version == 1
tables = await conn.fetch(
"SELECT table_name FROM information_schema.tables WHERE table_schema = $1",
schema,
)
table_names = {r["table_name"] for r in tables}
assert "wf_handlers" in table_names
assert "wf_events" in table_names
assert "workflow_state" in table_names
assert "schema_migrations" in table_names
finally:
await conn.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE")
await conn.close()
@pytest.mark.docker
async def test_run_migrations_idempotent(postgres_dsn: str) -> None:
conn = await asyncpg.connect(postgres_dsn)
schema = "test_migrate_idempotent"
try:
await conn.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE")
await run_migrations(conn, schema=schema)
await run_migrations(conn, schema=schema)
version = await conn.fetchval(
f"SELECT MAX(version) FROM {schema}.schema_migrations"
)
assert version == 1
count = await conn.fetchval(f"SELECT COUNT(*) FROM {schema}.schema_migrations")
assert count == 1
finally:
await conn.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE")
await conn.close()
@pytest.mark.docker
async def test_run_migrations_no_schema(postgres_dsn: str) -> None:
conn = await asyncpg.connect(postgres_dsn)
try:
await conn.execute("DROP TABLE IF EXISTS schema_migrations CASCADE")
await conn.execute("DROP TABLE IF EXISTS wf_handlers CASCADE")
await conn.execute("DROP TABLE IF EXISTS wf_events CASCADE")
await conn.execute("DROP TABLE IF EXISTS workflow_state CASCADE")
await conn.execute("DROP TABLE IF EXISTS workflow_journal CASCADE")
await run_migrations(conn, schema=None)
version = await conn.fetchval("SELECT MAX(version) FROM schema_migrations")
assert version == 1
finally:
await conn.execute("DROP TABLE IF EXISTS schema_migrations CASCADE")
await conn.execute("DROP TABLE IF EXISTS wf_handlers CASCADE")
await conn.execute("DROP TABLE IF EXISTS wf_events CASCADE")
await conn.execute("DROP TABLE IF EXISTS workflow_state CASCADE")
await conn.execute("DROP TABLE IF EXISTS workflow_journal CASCADE")
await conn.close()
@pytest.mark.docker
@pytest.mark.timeout(30)
async def test_concurrent_migrations_with_advisory_lock(
postgres_dsn: str,
) -> None:
"""Run migrations from N concurrent connections — only one should win the
race; the rest should observe the lock and skip gracefully."""
schema = "test_concurrent_mig"
concurrency = 8
conn = await asyncpg.connect(postgres_dsn)
try:
await conn.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE")
finally:
await conn.close()
async def migrate_once() -> None:
c = await asyncpg.connect(postgres_dsn)
try:
await run_migrations(c, schema=schema)
finally:
await c.close()
results = await asyncio.gather(
*[migrate_once() for _ in range(concurrency)],
return_exceptions=True,
)
for i, result in enumerate(results):
assert not isinstance(result, Exception), f"Migration task {i} failed: {result}"
conn = await asyncpg.connect(postgres_dsn)
try:
count = await conn.fetchval(
f"SELECT COUNT(*) FROM {schema}.schema_migrations WHERE version = 1"
)
assert count == 1, f"Expected 1 migration row, got {count}"
tables = await conn.fetch(
"SELECT table_name FROM information_schema.tables WHERE table_schema = $1",
schema,
)
table_names = {r["table_name"] for r in tables}
assert "wf_handlers" in table_names
assert "wf_events" in table_names
finally:
await conn.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE")
await conn.close()
@@ -0,0 +1,264 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
from __future__ import annotations
from typing import AsyncGenerator
import asyncpg
import pytest
from llama_agents.server._store.postgres_state_store import (
PostgresStateStore,
)
from pydantic import BaseModel
from workflows.context.serializers import JsonSerializer
from workflows.context.state_store import DictState, InMemoryStateStore
SCHEMA = "test_pg_state"
class CounterState(BaseModel):
count: int = 0
label: str = "default"
class ExtendedCounterState(CounterState):
extra: str = "extra_default"
@pytest.fixture
async def pool(postgres_dsn: str) -> AsyncGenerator[asyncpg.Pool, None]:
p = await asyncpg.create_pool(postgres_dsn, min_size=1, max_size=5)
async with p.acquire() as conn:
await conn.execute(f"CREATE SCHEMA IF NOT EXISTS {SCHEMA}")
await conn.execute(f"""
CREATE TABLE IF NOT EXISTS {SCHEMA}.workflow_state (
run_id VARCHAR(255) PRIMARY KEY,
state_json TEXT NOT NULL,
state_type VARCHAR(255),
state_module VARCHAR(255),
created_at TIMESTAMPTZ,
updated_at TIMESTAMPTZ
)
""")
await conn.execute(f"DELETE FROM {SCHEMA}.workflow_state")
yield p
await p.close()
@pytest.mark.docker
async def test_get_returns_default_dict_state(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-1", schema=SCHEMA
)
state = await store.get_state()
assert isinstance(state, DictState)
assert dict(state) == {}
@pytest.mark.docker
async def test_set_and_get_path(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-path", schema=SCHEMA
)
await store.set("foo", 42)
value = await store.get("foo")
assert value == 42
@pytest.mark.docker
async def test_set_nested_path(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-nested", schema=SCHEMA
)
await store.set("a.b.c", "deep")
value = await store.get("a.b.c")
assert value == "deep"
@pytest.mark.docker
async def test_get_missing_path_raises(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-missing", schema=SCHEMA
)
with pytest.raises(ValueError, match="not found"):
await store.get("nonexistent")
@pytest.mark.docker
async def test_get_missing_path_returns_default(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-default", schema=SCHEMA
)
value = await store.get("nonexistent", default="fallback")
assert value == "fallback"
@pytest.mark.docker
async def test_set_state_replaces_dict_state(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-replace", schema=SCHEMA
)
await store.set("x", 1)
new_state = DictState(y=2)
await store.set_state(new_state)
state = await store.get_state()
assert "y" in state
assert "x" not in state
@pytest.mark.docker
async def test_typed_state_get_returns_default(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[CounterState] = PostgresStateStore(
pool=pool, run_id="run-typed", state_type=CounterState, schema=SCHEMA
)
state = await store.get_state()
assert isinstance(state, CounterState)
assert state.count == 0
assert state.label == "default"
@pytest.mark.docker
async def test_typed_state_set_and_get(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[CounterState] = PostgresStateStore(
pool=pool, run_id="run-typed-set", state_type=CounterState, schema=SCHEMA
)
await store.set_state(CounterState(count=5, label="updated"))
state = await store.get_state()
assert state.count == 5
assert state.label == "updated"
@pytest.mark.docker
async def test_set_state_parent_type_merge(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[ExtendedCounterState] = PostgresStateStore(
pool=pool, run_id="run-merge", state_type=ExtendedCounterState, schema=SCHEMA
)
await store.set_state(ExtendedCounterState(count=1, label="init", extra="mine"))
parent = CounterState(count=10, label="merged")
await store.set_state(parent) # type: ignore[arg-type]
state = await store.get_state()
assert state.count == 10
assert state.label == "merged"
assert state.extra == "mine"
@pytest.mark.docker
async def test_edit_state_dict(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-edit", schema=SCHEMA
)
await store.set("counter", 0)
async with store.edit_state() as state:
state["counter"] = state["counter"] + 1
value = await store.get("counter")
assert value == 1
@pytest.mark.docker
async def test_edit_state_typed(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[CounterState] = PostgresStateStore(
pool=pool, run_id="run-edit-typed", state_type=CounterState, schema=SCHEMA
)
async with store.edit_state() as state:
state.count += 10
result = await store.get_state()
assert result.count == 10
@pytest.mark.docker
async def test_clear_resets_state(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-clear", schema=SCHEMA
)
await store.set("x", 99)
await store.clear()
state = await store.get_state()
assert dict(state) == {}
@pytest.mark.docker
async def test_clear_resets_typed_state(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[CounterState] = PostgresStateStore(
pool=pool, run_id="run-clear-typed", state_type=CounterState, schema=SCHEMA
)
await store.set_state(CounterState(count=100, label="dirty"))
await store.clear()
state = await store.get_state()
assert state.count == 0
assert state.label == "default"
@pytest.mark.docker
async def test_different_run_ids_are_isolated(pool: asyncpg.Pool) -> None:
store_a: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-a", schema=SCHEMA
)
store_b: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-b", schema=SCHEMA
)
await store_a.set("x", "from-a")
await store_b.set("x", "from-b")
assert await store_a.get("x") == "from-a"
assert await store_b.get("x") == "from-b"
@pytest.mark.docker
async def test_to_dict_returns_metadata_only(pool: asyncpg.Pool) -> None:
store: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-todict", schema=SCHEMA
)
await store.set("key", "value")
serializer = JsonSerializer()
d = store.to_dict(serializer)
assert d["store_type"] == "postgres"
assert d["run_id"] == "run-todict"
assert "state_data" not in d
@pytest.mark.docker
async def test_from_dict_postgres_format(pool: asyncpg.Pool) -> None:
store1: PostgresStateStore[DictState] = PostgresStateStore(
pool=pool, run_id="run-fromdict", schema=SCHEMA
)
await store1.set("saved", True)
serializer = JsonSerializer()
payload = store1.to_dict(serializer)
store2 = PostgresStateStore.from_dict(
payload, serializer, pool=pool, state_type=DictState, schema=SCHEMA
)
value = await store2.get("saved")
assert value is True
@pytest.mark.docker
async def test_from_dict_in_memory_format_migrates(pool: asyncpg.Pool) -> None:
serializer = JsonSerializer()
in_memory_store = InMemoryStateStore(DictState(migrated_key="migrated_value"))
payload = in_memory_store.to_dict(serializer)
store = PostgresStateStore.from_dict(
payload,
serializer,
pool=pool,
state_type=DictState,
run_id="run-migrate",
schema=SCHEMA,
)
await store._write_in_memory_state(payload)
value = await store.get("migrated_key")
assert value == "migrated_value"
async def test_from_dict_empty_raises() -> None:
with pytest.raises(ValueError, match="Cannot restore"):
PostgresStateStore.from_dict({}, JsonSerializer())
async def test_from_dict_no_pool_raises() -> None:
with pytest.raises(ValueError, match="pool is required"):
PostgresStateStore.from_dict(
{"store_type": "postgres", "run_id": "x"}, JsonSerializer()
)
@@ -0,0 +1,223 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LlamaIndex Inc.
from __future__ import annotations
import asyncio
from datetime import datetime, timezone
from unittest.mock import MagicMock
import pytest
from llama_agents.client.protocol.serializable_events import EventEnvelopeWithMetadata
from llama_agents.server._store.abstract_workflow_store import (
HandlerQuery,
PersistentHandler,
Status,
)
from llama_agents.server._store.postgres_workflow_store import PostgresWorkflowStore
from workflows.events import Event, StopEvent
def _make_event() -> EventEnvelopeWithMetadata:
class TestEvent(Event):
key: str = "value"
return EventEnvelopeWithMetadata.from_event(TestEvent())
def _make_stop_event() -> EventEnvelopeWithMetadata:
return EventEnvelopeWithMetadata.from_event(StopEvent(result="done"))
def _make_handler(
handler_id: str = "h1",
workflow_name: str = "test_workflow",
status: Status = "running",
run_id: str = "run-1",
started_at: datetime | None = None,
updated_at: datetime | None = None,
completed_at: datetime | None = None,
idle_since: datetime | None = None,
error: str | None = None,
) -> PersistentHandler:
now = datetime.now(timezone.utc)
return PersistentHandler(
handler_id=handler_id,
workflow_name=workflow_name,
status=status,
run_id=run_id,
started_at=started_at or now,
updated_at=updated_at or now,
completed_at=completed_at,
idle_since=idle_since,
error=error,
)
# ── Unit tests (no Postgres needed, test logic with mocks) ──────────
async def test_ticks_raise_not_implemented() -> None:
store = PostgresWorkflowStore(dsn="postgresql://localhost/test")
with pytest.raises(NotImplementedError):
await store.append_tick("run-1", {"step": "init"})
with pytest.raises(NotImplementedError):
await store.get_ticks("run-1")
async def test_create_state_store_without_pool_raises() -> None:
store = PostgresWorkflowStore(dsn="postgresql://localhost/test")
with pytest.raises(RuntimeError, match="pool not initialized"):
store.create_state_store("run-1")
async def test_build_filters_empty_in_returns_none() -> None:
store = PostgresWorkflowStore(dsn="postgresql://localhost/test")
assert store._build_filters(HandlerQuery(handler_id_in=[])) is None
assert store._build_filters(HandlerQuery(run_id_in=[])) is None
assert store._build_filters(HandlerQuery(status_in=[])) is None
assert store._build_filters(HandlerQuery(workflow_name_in=[])) is None
async def test_build_filters_produces_correct_clauses() -> None:
store = PostgresWorkflowStore(dsn="postgresql://localhost/test")
result = store._build_filters(HandlerQuery(handler_id_in=["h1", "h2"]))
assert result is not None
clauses, params = result
assert len(clauses) == 1
assert "handler_id IN" in clauses[0]
assert params == ["h1", "h2"]
result = store._build_filters(HandlerQuery(is_idle=True))
assert result is not None
clauses, params = result
assert "idle_since IS NOT NULL" in clauses[0]
assert params == []
result = store._build_filters(HandlerQuery(is_idle=False))
assert result is not None
clauses, params = result
assert "idle_since IS NULL" in clauses[0]
async def test_on_notify_wakes_condition() -> None:
store = PostgresWorkflowStore(dsn="postgresql://localhost/test")
cond = store._get_or_create_condition("run-1")
notified = asyncio.Event()
async def waiter() -> None:
async with cond:
await cond.wait()
notified.set()
task = asyncio.create_task(waiter())
await asyncio.sleep(0.01)
# Simulate the NOTIFY callback
store._on_notify(MagicMock(), 0, "wf_events", "run-1")
await asyncio.wait_for(notified.wait(), timeout=1.0)
await task
async def test_close_without_start_is_safe() -> None:
store = PostgresWorkflowStore(dsn="postgresql://localhost/test")
await store.close() # Should not raise
# ── Integration tests (require Docker) ──────────────────────────────
@pytest.mark.docker
async def test_integration_migrations_idempotent(postgres_dsn: str) -> None:
store = PostgresWorkflowStore(dsn=postgres_dsn, schema="test_pg_store")
try:
await store.start()
await store.run_migrations()
await store.run_migrations() # Should be idempotent
finally:
await store.close()
@pytest.mark.docker
async def test_integration_handler_crud(postgres_dsn: str) -> None:
store = PostgresWorkflowStore(dsn=postgres_dsn, schema="test_pg_store")
try:
await store.start()
await store.run_migrations()
handler = _make_handler(handler_id="pg-h1", run_id="pg-run-1")
await store.update(handler)
results = await store.query(HandlerQuery(handler_id_in=["pg-h1"]))
assert len(results) == 1
assert results[0].handler_id == "pg-h1"
count = await store.delete(HandlerQuery(handler_id_in=["pg-h1"]))
assert count == 1
results = await store.query(HandlerQuery(handler_id_in=["pg-h1"]))
assert len(results) == 0
finally:
await store.close()
@pytest.mark.docker
async def test_integration_event_append_and_query(postgres_dsn: str) -> None:
store = PostgresWorkflowStore(dsn=postgres_dsn, schema="test_pg_store")
try:
await store.start()
await store.run_migrations()
await store.append_event("pg-run-ev", _make_event())
await store.append_event("pg-run-ev", _make_event())
await store.append_event("pg-run-ev", _make_event())
events = await store.query_events("pg-run-ev")
assert len(events) == 3
assert events[0].sequence == 0
assert events[1].sequence == 1
assert events[2].sequence == 2
events = await store.query_events("pg-run-ev", after_sequence=0, limit=1)
assert len(events) == 1
assert events[0].sequence == 1
finally:
await store.close()
@pytest.mark.docker
async def test_integration_subscribe_events(postgres_dsn: str) -> None:
store = PostgresWorkflowStore(
dsn=postgres_dsn, schema="test_pg_store", poll_interval=0.05
)
try:
await store.start()
await store.run_migrations()
run_id = "pg-run-sub"
async def append_events() -> None:
await asyncio.sleep(0.05)
await store.append_event(run_id, _make_event())
await asyncio.sleep(0.05)
await store.append_event(run_id, _make_event())
await asyncio.sleep(0.05)
await store.append_event(run_id, _make_stop_event())
async def subscribe() -> list[object]:
collected = []
async for event in store.subscribe_events(run_id):
collected.append(event)
return collected
append_task = asyncio.create_task(append_events())
subscribe_task = asyncio.create_task(subscribe())
collected = await asyncio.wait_for(subscribe_task, timeout=5.0)
await append_task
assert len(collected) == 3
finally:
await store.close()
@@ -348,9 +348,7 @@ async def test_run_workflow_handler_persists_initial_record() -> None:
decorator = ServerRuntimeDecorator(StubRuntime(), store=store)
decorator._persistence_backoff = [0, 0]
mock_handler = MagicMock(run_id="test-run")
await decorator.run_workflow_handler("h-init", "my_workflow", mock_handler)
await decorator.run_workflow_handler("h-init", "my_workflow", "test-run")
found = await store.query(HandlerQuery(handler_id_in=["h-init"]))
assert len(found) == 1
@@ -24,6 +24,7 @@ from typing import (
)
from workflows.context.state_store import StateStore
from workflows.events import Event, StartEvent, StopEvent
from workflows.runtime.types.named_task import NamedTask
if TYPE_CHECKING:
@@ -32,8 +33,6 @@ if TYPE_CHECKING:
from workflows.runtime.types.internal_state import BrokerState
from workflows.runtime.types.step_function import StepWorkerFunction
from workflows.workflow import Workflow
from workflows.events import Event, StartEvent, StopEvent
from workflows.runtime.types.ticks import TickCancelRun, WorkflowTick
# Context variable for implicit runtime scoping
@@ -181,6 +180,14 @@ class InternalRunAdapter(ABC):
"""
pass
def is_replaying(self) -> bool:
"""Whether the adapter is currently replaying recorded operations.
During replay, side effects like persisting events to external stores
should be skipped to avoid duplicates. Default is False (live execution).
"""
return False
async def on_tick(self, tick: WorkflowTick) -> None:
"""
Called whenever a tick event is processed by the control loop.
Generated
+88
View File
@@ -243,6 +243,73 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" },
]
[[package]]
name = "asyncpg"
version = "0.31.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "async-timeout", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/fe/cc/d18065ce2380d80b1bcce927c24a2642efd38918e33fd724bc4bca904877/asyncpg-0.31.0.tar.gz", hash = "sha256:c989386c83940bfbd787180f2b1519415e2d3d6277a70d9d0f0145ac73500735", size = 993667, upload-time = "2025-11-24T23:27:00.812Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/c3/d9/507c80bdac2e95e5a525644af94b03fa7f9a44596a84bd48a6e80f854f92/asyncpg-0.31.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:831712dd3cf117eec68575a9b50da711893fd63ebe277fc155ecae1c6c9f0f61", size = 644865, upload-time = "2025-11-24T23:25:23.527Z" },
{ url = "https://files.pythonhosted.org/packages/ea/03/f93b5e543f65c5f504e91405e8d21bb9e600548be95032951a754781a41d/asyncpg-0.31.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0b17c89312c2f4ccea222a3a6571f7df65d4ba2c0e803339bfc7bed46a96d3be", size = 639297, upload-time = "2025-11-24T23:25:25.192Z" },
{ url = "https://files.pythonhosted.org/packages/e5/1e/de2177e57e03a06e697f6c1ddf2a9a7fcfdc236ce69966f54ffc830fd481/asyncpg-0.31.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3faa62f997db0c9add34504a68ac2c342cfee4d57a0c3062fcf0d86c7f9cb1e8", size = 2816679, upload-time = "2025-11-24T23:25:26.718Z" },
{ url = "https://files.pythonhosted.org/packages/d0/98/1a853f6870ac7ad48383a948c8ff3c85dc278066a4d69fc9af7d3d4b1106/asyncpg-0.31.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ea599d45c361dfbf398cb67da7fd052affa556a401482d3ff1ee99bd68808a1", size = 2867087, upload-time = "2025-11-24T23:25:28.399Z" },
{ url = "https://files.pythonhosted.org/packages/11/29/7e76f2a51f2360a7c90d2cf6d0d9b210c8bb0ae342edebd16173611a55c2/asyncpg-0.31.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:795416369c3d284e1837461909f58418ad22b305f955e625a4b3a2521d80a5f3", size = 2747631, upload-time = "2025-11-24T23:25:30.154Z" },
{ url = "https://files.pythonhosted.org/packages/5d/3f/716e10cb57c4f388248db46555e9226901688fbfabd0afb85b5e1d65d5a7/asyncpg-0.31.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a8d758dac9d2e723e173d286ef5e574f0b350ec00e9186fce84d0fc5f6a8e6b8", size = 2855107, upload-time = "2025-11-24T23:25:31.888Z" },
{ url = "https://files.pythonhosted.org/packages/7e/ec/3ebae9dfb23a1bd3f68acfd4f795983b65b413291c0e2b0d982d6ae6c920/asyncpg-0.31.0-cp310-cp310-win32.whl", hash = "sha256:2d076d42eb583601179efa246c5d7ae44614b4144bc1c7a683ad1222814ed095", size = 521990, upload-time = "2025-11-24T23:25:33.402Z" },
{ url = "https://files.pythonhosted.org/packages/20/b4/9fbb4b0af4e36d96a61d026dd37acab3cf521a70290a09640b215da5ab7c/asyncpg-0.31.0-cp310-cp310-win_amd64.whl", hash = "sha256:9ea33213ac044171f4cac23740bed9a3805abae10e7025314cfbd725ec670540", size = 581629, upload-time = "2025-11-24T23:25:34.846Z" },
{ url = "https://files.pythonhosted.org/packages/08/17/cc02bc49bc350623d050fa139e34ea512cd6e020562f2a7312a7bcae4bc9/asyncpg-0.31.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:eee690960e8ab85063ba93af2ce128c0f52fd655fdff9fdb1a28df01329f031d", size = 643159, upload-time = "2025-11-24T23:25:36.443Z" },
{ url = "https://files.pythonhosted.org/packages/a4/62/4ded7d400a7b651adf06f49ea8f73100cca07c6df012119594d1e3447aa6/asyncpg-0.31.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2657204552b75f8288de08ca60faf4a99a65deef3a71d1467454123205a88fab", size = 638157, upload-time = "2025-11-24T23:25:37.89Z" },
{ url = "https://files.pythonhosted.org/packages/d6/5b/4179538a9a72166a0bf60ad783b1ef16efb7960e4d7b9afe9f77a5551680/asyncpg-0.31.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a429e842a3a4b4ea240ea52d7fe3f82d5149853249306f7ff166cb9948faa46c", size = 2918051, upload-time = "2025-11-24T23:25:39.461Z" },
{ url = "https://files.pythonhosted.org/packages/e6/35/c27719ae0536c5b6e61e4701391ffe435ef59539e9360959240d6e47c8c8/asyncpg-0.31.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0807be46c32c963ae40d329b3a686356e417f674c976c07fa49f1b30303f109", size = 2972640, upload-time = "2025-11-24T23:25:41.512Z" },
{ url = "https://files.pythonhosted.org/packages/43/f4/01ebb9207f29e645a64699b9ce0eefeff8e7a33494e1d29bb53736f7766b/asyncpg-0.31.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e5d5098f63beeae93512ee513d4c0c53dc12e9aa2b7a1af5a81cddf93fe4e4da", size = 2851050, upload-time = "2025-11-24T23:25:43.153Z" },
{ url = "https://files.pythonhosted.org/packages/3e/f4/03ff1426acc87be0f4e8d40fa2bff5c3952bef0080062af9efc2212e3be8/asyncpg-0.31.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37fc6c00a814e18eef51833545d1891cac9aa69140598bb076b4cd29b3e010b9", size = 2962574, upload-time = "2025-11-24T23:25:44.942Z" },
{ url = "https://files.pythonhosted.org/packages/c7/39/cc788dfca3d4060f9d93e67be396ceec458dfc429e26139059e58c2c244d/asyncpg-0.31.0-cp311-cp311-win32.whl", hash = "sha256:5a4af56edf82a701aece93190cc4e094d2df7d33f6e915c222fb09efbb5afc24", size = 521076, upload-time = "2025-11-24T23:25:46.486Z" },
{ url = "https://files.pythonhosted.org/packages/28/fc/735af5384c029eb7f1ca60ccb8fa95521dbdaeef788edf4cecfc604c3cab/asyncpg-0.31.0-cp311-cp311-win_amd64.whl", hash = "sha256:480c4befbdf079c14c9ca43c8c5e1fe8b6296c96f1f927158d4f1e750aacc047", size = 584980, upload-time = "2025-11-24T23:25:47.938Z" },
{ url = "https://files.pythonhosted.org/packages/2a/a6/59d0a146e61d20e18db7396583242e32e0f120693b67a8de43f1557033e2/asyncpg-0.31.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b44c31e1efc1c15188ef183f287c728e2046abb1d26af4d20858215d50d91fad", size = 662042, upload-time = "2025-11-24T23:25:49.578Z" },
{ url = "https://files.pythonhosted.org/packages/36/01/ffaa189dcb63a2471720615e60185c3f6327716fdc0fc04334436fbb7c65/asyncpg-0.31.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c89ccf741c067614c9b5fc7f1fc6f3b61ab05ae4aaa966e6fd6b93097c7d20d", size = 638504, upload-time = "2025-11-24T23:25:51.501Z" },
{ url = "https://files.pythonhosted.org/packages/9f/62/3f699ba45d8bd24c5d65392190d19656d74ff0185f42e19d0bbd973bb371/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:12b3b2e39dc5470abd5e98c8d3373e4b1d1234d9fbdedf538798b2c13c64460a", size = 3426241, upload-time = "2025-11-24T23:25:53.278Z" },
{ url = "https://files.pythonhosted.org/packages/8c/d1/a867c2150f9c6e7af6462637f613ba67f78a314b00db220cd26ff559d532/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:aad7a33913fb8bcb5454313377cc330fbb19a0cd5faa7272407d8a0c4257b671", size = 3520321, upload-time = "2025-11-24T23:25:54.982Z" },
{ url = "https://files.pythonhosted.org/packages/7a/1a/cce4c3f246805ecd285a3591222a2611141f1669d002163abef999b60f98/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3df118d94f46d85b2e434fd62c84cb66d5834d5a890725fe625f498e72e4d5ec", size = 3316685, upload-time = "2025-11-24T23:25:57.43Z" },
{ url = "https://files.pythonhosted.org/packages/40/ae/0fc961179e78cc579e138fad6eb580448ecae64908f95b8cb8ee2f241f67/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd5b6efff3c17c3202d4b37189969acf8927438a238c6257f66be3c426beba20", size = 3471858, upload-time = "2025-11-24T23:25:59.636Z" },
{ url = "https://files.pythonhosted.org/packages/52/b2/b20e09670be031afa4cbfabd645caece7f85ec62d69c312239de568e058e/asyncpg-0.31.0-cp312-cp312-win32.whl", hash = "sha256:027eaa61361ec735926566f995d959ade4796f6a49d3bde17e5134b9964f9ba8", size = 527852, upload-time = "2025-11-24T23:26:01.084Z" },
{ url = "https://files.pythonhosted.org/packages/b5/f0/f2ed1de154e15b107dc692262395b3c17fc34eafe2a78fc2115931561730/asyncpg-0.31.0-cp312-cp312-win_amd64.whl", hash = "sha256:72d6bdcbc93d608a1158f17932de2321f68b1a967a13e014998db87a72ed3186", size = 597175, upload-time = "2025-11-24T23:26:02.564Z" },
{ url = "https://files.pythonhosted.org/packages/95/11/97b5c2af72a5d0b9bc3fa30cd4b9ce22284a9a943a150fdc768763caf035/asyncpg-0.31.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c204fab1b91e08b0f47e90a75d1b3c62174dab21f670ad6c5d0f243a228f015b", size = 661111, upload-time = "2025-11-24T23:26:04.467Z" },
{ url = "https://files.pythonhosted.org/packages/1b/71/157d611c791a5e2d0423f09f027bd499935f0906e0c2a416ce712ba51ef3/asyncpg-0.31.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:54a64f91839ba59008eccf7aad2e93d6e3de688d796f35803235ea1c4898ae1e", size = 636928, upload-time = "2025-11-24T23:26:05.944Z" },
{ url = "https://files.pythonhosted.org/packages/2e/fc/9e3486fb2bbe69d4a867c0b76d68542650a7ff1574ca40e84c3111bb0c6e/asyncpg-0.31.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0e0822b1038dc7253b337b0f3f676cadc4ac31b126c5d42691c39691962e403", size = 3424067, upload-time = "2025-11-24T23:26:07.957Z" },
{ url = "https://files.pythonhosted.org/packages/12/c6/8c9d076f73f07f995013c791e018a1cd5f31823c2a3187fc8581706aa00f/asyncpg-0.31.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bef056aa502ee34204c161c72ca1f3c274917596877f825968368b2c33f585f4", size = 3518156, upload-time = "2025-11-24T23:26:09.591Z" },
{ url = "https://files.pythonhosted.org/packages/ae/3b/60683a0baf50fbc546499cfb53132cb6835b92b529a05f6a81471ab60d0c/asyncpg-0.31.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0bfbcc5b7ffcd9b75ab1558f00db2ae07db9c80637ad1b2469c43df79d7a5ae2", size = 3319636, upload-time = "2025-11-24T23:26:11.168Z" },
{ url = "https://files.pythonhosted.org/packages/50/dc/8487df0f69bd398a61e1792b3cba0e47477f214eff085ba0efa7eac9ce87/asyncpg-0.31.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:22bc525ebbdc24d1261ecbf6f504998244d4e3be1721784b5f64664d61fbe602", size = 3472079, upload-time = "2025-11-24T23:26:13.164Z" },
{ url = "https://files.pythonhosted.org/packages/13/a1/c5bbeeb8531c05c89135cb8b28575ac2fac618bcb60119ee9696c3faf71c/asyncpg-0.31.0-cp313-cp313-win32.whl", hash = "sha256:f890de5e1e4f7e14023619399a471ce4b71f5418cd67a51853b9910fdfa73696", size = 527606, upload-time = "2025-11-24T23:26:14.78Z" },
{ url = "https://files.pythonhosted.org/packages/91/66/b25ccb84a246b470eb943b0107c07edcae51804912b824054b3413995a10/asyncpg-0.31.0-cp313-cp313-win_amd64.whl", hash = "sha256:dc5f2fa9916f292e5c5c8b2ac2813763bcd7f58e130055b4ad8a0531314201ab", size = 596569, upload-time = "2025-11-24T23:26:16.189Z" },
{ url = "https://files.pythonhosted.org/packages/3c/36/e9450d62e84a13aea6580c83a47a437f26c7ca6fa0f0fd40b6670793ea30/asyncpg-0.31.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f6b56b91bb0ffc328c4e3ed113136cddd9deefdf5f79ab448598b9772831df44", size = 660867, upload-time = "2025-11-24T23:26:17.631Z" },
{ url = "https://files.pythonhosted.org/packages/82/4b/1d0a2b33b3102d210439338e1beea616a6122267c0df459ff0265cd5807a/asyncpg-0.31.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:334dec28cf20d7f5bb9e45b39546ddf247f8042a690bff9b9573d00086e69cb5", size = 638349, upload-time = "2025-11-24T23:26:19.689Z" },
{ url = "https://files.pythonhosted.org/packages/41/aa/e7f7ac9a7974f08eff9183e392b2d62516f90412686532d27e196c0f0eeb/asyncpg-0.31.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98cc158c53f46de7bb677fd20c417e264fc02b36d901cc2a43bd6cb0dc6dbfd2", size = 3410428, upload-time = "2025-11-24T23:26:21.275Z" },
{ url = "https://files.pythonhosted.org/packages/6f/de/bf1b60de3dede5c2731e6788617a512bc0ebd9693eac297ee74086f101d7/asyncpg-0.31.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9322b563e2661a52e3cdbc93eed3be7748b289f792e0011cb2720d278b366ce2", size = 3471678, upload-time = "2025-11-24T23:26:23.627Z" },
{ url = "https://files.pythonhosted.org/packages/46/78/fc3ade003e22d8bd53aaf8f75f4be48f0b460fa73738f0391b9c856a9147/asyncpg-0.31.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19857a358fc811d82227449b7ca40afb46e75b33eb8897240c3839dd8b744218", size = 3313505, upload-time = "2025-11-24T23:26:25.235Z" },
{ url = "https://files.pythonhosted.org/packages/bf/e9/73eb8a6789e927816f4705291be21f2225687bfa97321e40cd23055e903a/asyncpg-0.31.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ba5f8886e850882ff2c2ace5732300e99193823e8107e2c53ef01c1ebfa1e85d", size = 3434744, upload-time = "2025-11-24T23:26:26.944Z" },
{ url = "https://files.pythonhosted.org/packages/08/4b/f10b880534413c65c5b5862f79b8e81553a8f364e5238832ad4c0af71b7f/asyncpg-0.31.0-cp314-cp314-win32.whl", hash = "sha256:cea3a0b2a14f95834cee29432e4ddc399b95700eb1d51bbc5bfee8f31fa07b2b", size = 532251, upload-time = "2025-11-24T23:26:28.404Z" },
{ url = "https://files.pythonhosted.org/packages/d3/2d/7aa40750b7a19efa5d66e67fc06008ca0f27ba1bd082e457ad82f59aba49/asyncpg-0.31.0-cp314-cp314-win_amd64.whl", hash = "sha256:04d19392716af6b029411a0264d92093b6e5e8285ae97a39957b9a9c14ea72be", size = 604901, upload-time = "2025-11-24T23:26:30.34Z" },
{ url = "https://files.pythonhosted.org/packages/ce/fe/b9dfe349b83b9dee28cc42360d2c86b2cdce4cb551a2c2d27e156bcac84d/asyncpg-0.31.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:bdb957706da132e982cc6856bb2f7b740603472b54c3ebc77fe60ea3e57e1bd2", size = 702280, upload-time = "2025-11-24T23:26:32Z" },
{ url = "https://files.pythonhosted.org/packages/6a/81/e6be6e37e560bd91e6c23ea8a6138a04fd057b08cf63d3c5055c98e81c1d/asyncpg-0.31.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6d11b198111a72f47154fa03b85799f9be63701e068b43f84ac25da0bda9cb31", size = 682931, upload-time = "2025-11-24T23:26:33.572Z" },
{ url = "https://files.pythonhosted.org/packages/a6/45/6009040da85a1648dd5bc75b3b0a062081c483e75a1a29041ae63a0bf0dc/asyncpg-0.31.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18c83b03bc0d1b23e6230f5bf8d4f217dc9bc08644ce0502a9d91dc9e634a9c7", size = 3581608, upload-time = "2025-11-24T23:26:35.638Z" },
{ url = "https://files.pythonhosted.org/packages/7e/06/2e3d4d7608b0b2b3adbee0d0bd6a2d29ca0fc4d8a78f8277df04e2d1fd7b/asyncpg-0.31.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e009abc333464ff18b8f6fd146addffd9aaf63e79aa3bb40ab7a4c332d0c5e9e", size = 3498738, upload-time = "2025-11-24T23:26:37.275Z" },
{ url = "https://files.pythonhosted.org/packages/7d/aa/7d75ede780033141c51d83577ea23236ba7d3a23593929b32b49db8ed36e/asyncpg-0.31.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3b1fbcb0e396a5ca435a8826a87e5c2c2cc0c8c68eb6fadf82168056b0e53a8c", size = 3401026, upload-time = "2025-11-24T23:26:39.423Z" },
{ url = "https://files.pythonhosted.org/packages/ba/7a/15e37d45e7f7c94facc1e9148c0e455e8f33c08f0b8a0b1deb2c5171771b/asyncpg-0.31.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8df714dba348efcc162d2adf02d213e5fab1bd9f557e1305633e851a61814a7a", size = 3429426, upload-time = "2025-11-24T23:26:41.032Z" },
{ url = "https://files.pythonhosted.org/packages/13/d5/71437c5f6ae5f307828710efbe62163974e71237d5d46ebd2869ea052d10/asyncpg-0.31.0-cp314-cp314t-win32.whl", hash = "sha256:1b41f1afb1033f2b44f3234993b15096ddc9cd71b21a42dbd87fc6a57b43d65d", size = 614495, upload-time = "2025-11-24T23:26:42.659Z" },
{ url = "https://files.pythonhosted.org/packages/3c/d7/8fb3044eaef08a310acfe23dae9a8e2e07d305edc29a53497e52bc76eca7/asyncpg-0.31.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bd4107bb7cdd0e9e65fae66a62afd3a249663b844fa34d479f6d5b3bef9c04c3", size = 706062, upload-time = "2025-11-24T23:26:44.086Z" },
{ url = "https://files.pythonhosted.org/packages/3d/f9/104361bb10203039569eb56fdd4eddb185d7480cec71d5f93d4c5454142d/asyncpg-0.31.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb3cde58321a1f89ce41812be3f2a98dddedc1e76d0838aba1d724f1e4e1a95", size = 645552, upload-time = "2025-11-24T23:26:45.659Z" },
{ url = "https://files.pythonhosted.org/packages/13/38/bbb09ea041a935dc7720e283b6876c3487ace4b180b8d58c07db6cd8c941/asyncpg-0.31.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e6974f36eb9a224d8fb428bcf66bd411aa12cf57c2967463178149e73d4de366", size = 639850, upload-time = "2025-11-24T23:26:47.236Z" },
{ url = "https://files.pythonhosted.org/packages/60/9f/1f9491f6b73096e1d5eb0da2207ddbde3f9b1fc0ea926183f0f5eadfdd05/asyncpg-0.31.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc2b685f400ceae428f79f78b58110470d7b4466929a7f78d455964b17ad1008", size = 2805292, upload-time = "2025-11-24T23:26:48.821Z" },
{ url = "https://files.pythonhosted.org/packages/96/9c/7426e5f4483acc5b2622091b2ae6dd44e1a490847aff434c0bc7c99d1242/asyncpg-0.31.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb223567dea5f47c45d347f2bde5486be8d9f40339f27217adb3fb1c3be51298", size = 2859461, upload-time = "2025-11-24T23:26:50.596Z" },
{ url = "https://files.pythonhosted.org/packages/13/68/cb5d6a43d34e5735928fb745b4087cee2d7e6b8b0cc902ad520520cfd16f/asyncpg-0.31.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:22be6e02381bab3101cd502d9297ac71e2f966c86e20e78caead9934c98a8af6", size = 2734644, upload-time = "2025-11-24T23:26:52.795Z" },
{ url = "https://files.pythonhosted.org/packages/af/3b/e9a33ab89fe9bd4c6fb7a9e0707f0e7656e07dd2af5fff8a5375c13b03b5/asyncpg-0.31.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:37a58919cfef2448a920df00d1b2f821762d17194d0dbf355d6dde8d952c04f9", size = 2844195, upload-time = "2025-11-24T23:26:55.224Z" },
{ url = "https://files.pythonhosted.org/packages/88/7a/04dcdc53e4e255e4f60e68fc1934e5523fa5654ec1b51e2c75d0657004a6/asyncpg-0.31.0-cp39-cp39-win32.whl", hash = "sha256:c1a9c5b71d2371a2290bc93336cd05ba4ec781683cab292adbddc084f89443c6", size = 522403, upload-time = "2025-11-24T23:26:56.806Z" },
{ url = "https://files.pythonhosted.org/packages/c5/03/ea5fd3fa18a26ba1aa663fce57620238d7e413d262dda284b71631ca8d2a/asyncpg-0.31.0-cp39-cp39-win_amd64.whl", hash = "sha256:c1e1ab5bc65373d92dd749d7308c5b26fb2dc0fbe5d3bf68a32b676aa3bcd24a", size = 582103, upload-time = "2025-11-24T23:26:58.716Z" },
]
[[package]]
name = "attrs"
version = "25.4.0"
@@ -1671,34 +1738,42 @@ version = "0.1.0"
source = { editable = "packages/llama-agents-dbos" }
dependencies = [
{ name = "dbos", marker = "python_full_version >= '3.10'" },
{ name = "llama-agents-server", extra = ["asyncpg"] },
{ name = "llama-index-workflows" },
]
[package.dev-dependencies]
dev = [
{ name = "basedpyright" },
{ name = "llama-agents-integration-tests" },
{ name = "pytest" },
{ name = "pytest-asyncio" },
{ name = "pytest-cov" },
{ name = "pytest-timeout" },
{ name = "pytest-xdist" },
{ name = "testcontainers", version = "4.13.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9.2'" },
{ name = "testcontainers", version = "4.13.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9.2' and python_full_version < '3.10'" },
{ name = "testcontainers", version = "4.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
{ name = "ty" },
]
[package.metadata]
requires-dist = [
{ name = "dbos", marker = "python_full_version >= '3.10'", specifier = ">=2.11.0" },
{ name = "llama-agents-server", extras = ["asyncpg"], editable = "packages/llama-agents-server" },
{ name = "llama-index-workflows", editable = "packages/llama-index-workflows" },
]
[package.metadata.requires-dev]
dev = [
{ name = "basedpyright", specifier = ">=1.31.1" },
{ name = "llama-agents-integration-tests", editable = "packages/llama-agents-integration-tests" },
{ name = "pytest", specifier = ">=8.4.0" },
{ name = "pytest-asyncio", specifier = ">=1.0.0" },
{ name = "pytest-cov", specifier = ">=6.1.1" },
{ name = "pytest-timeout", specifier = ">=2.4.0" },
{ name = "pytest-xdist", specifier = ">=3.0.0" },
{ name = "testcontainers", extras = ["postgres"], specifier = ">=4.0.0" },
{ name = "ty", specifier = ">=0.0.1,<0.0.9" },
]
@@ -1810,8 +1885,14 @@ dependencies = [
{ name = "uvicorn" },
]
[package.optional-dependencies]
asyncpg = [
{ name = "asyncpg" },
]
[package.dev-dependencies]
dev = [
{ name = "asyncpg" },
{ name = "hatch" },
{ name = "llama-agents-integration-tests" },
{ name = "pytest" },
@@ -1820,20 +1901,26 @@ dev = [
{ name = "pytest-timeout" },
{ name = "pytest-xdist" },
{ name = "pyyaml" },
{ name = "testcontainers", version = "4.13.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9.2'" },
{ name = "testcontainers", version = "4.13.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9.2' and python_full_version < '3.10'" },
{ name = "testcontainers", version = "4.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
{ name = "time-machine" },
]
[package.metadata]
requires-dist = [
{ name = "asyncpg", marker = "extra == 'asyncpg'", specifier = ">=0.29.0" },
{ name = "httpx", specifier = ">=0.27.0" },
{ name = "llama-agents-client", editable = "packages/llama-agents-client" },
{ name = "llama-index-workflows", editable = "packages/llama-index-workflows" },
{ name = "starlette", specifier = ">=0.39.0" },
{ name = "uvicorn", specifier = ">=0.32.0" },
]
provides-extras = ["asyncpg"]
[package.metadata.requires-dev]
dev = [
{ name = "asyncpg", specifier = ">=0.29.0" },
{ name = "hatch", specifier = ">=1.14.1" },
{ name = "llama-agents-integration-tests", editable = "packages/llama-agents-integration-tests" },
{ name = "pytest", specifier = ">=8.4.2" },
@@ -1842,6 +1929,7 @@ dev = [
{ name = "pytest-timeout", specifier = ">=2.4.0" },
{ name = "pytest-xdist", specifier = ">=3.8.0" },
{ name = "pyyaml", specifier = ">=6.0.2" },
{ name = "testcontainers", extras = ["postgres"], specifier = ">=4.0.0" },
{ name = "time-machine", specifier = ">=2.19.0,<3.0.0" },
]