feat: Add DAGs for managing maintenance tasks (#27489)

2026-02-04 03:01:23 +01:00 · 2025-01-14 14:03:53 -08:00
parent cb7547d021
commit bbd7e1f4c5
12 changed files with 422 additions and 127 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -76,3 +76,6 @@ pyrightconfig.json
 temp_test_run_data.json
 .temp-deepeval-cache.json
 .eslintcache
+
+# ignore all dagster tmp directories that may be created
+tmp*/
--- a/dags/init.py
+++ b/dags/init.py
--- a/dags/assets.py
+++ b/dags/assets.py
@@ -0,0 +1,34 @@
+import os
+import django
+
+from dagster import (
+    Config,
+    MaterializeResult,
+    asset,
+)
+
+# setup PostHog Django Project
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "posthog.settings")
+django.setup()
+
+from posthog.clickhouse.client import sync_execute  # noqa
+
+
+class ClickHouseConfig(Config):
+    result_path: str = "/tmp/clickhouse_version.txt"
+
+
+@asset
+def get_clickhouse_version(config: ClickHouseConfig) -> MaterializeResult:
+    version = sync_execute("SELECT version()")[0][0]
+    with open(config.result_path, "w") as f:
+        f.write(version)
+    return MaterializeResult(metadata={"version": version})
+
+
+@asset(deps=[get_clickhouse_version])
+def print_clickhouse_version(config: ClickHouseConfig):
+    with open(config.result_path) as f:
+        print(f.read())  # noqa
+    return MaterializeResult(metadata={"version": config.result_path})
--- a/dags/definitions.py
+++ b/dags/definitions.py
@@ -0,0 +1,9 @@
+from dagster import Definitions, load_assets_from_modules
+
+from . import assets
+
+all_assets = load_assets_from_modules([assets])
+
+defs = Definitions(
+    assets=all_assets,
+)
--- a/dags_test/init.py
+++ b/dags_test/init.py
--- a/dags_test/test_assets.py
+++ b/dags_test/test_assets.py
@@ -0,0 +1,79 @@
+import pytest
+from unittest.mock import patch, mock_open
+
+from dagster import build_op_context
+
+from dags.assets import (
+    get_clickhouse_version,
+    print_clickhouse_version,
+    ClickHouseConfig,
+)
+
+
+@pytest.fixture
+def mock_sync_execute():
+    with patch("dags.assets.sync_execute") as mock:
+        mock.return_value = [["23.8.1.2992"]]
+        yield mock
+
+
+@pytest.fixture
+def config():
+    return ClickHouseConfig(result_path="/tmp/test_clickhouse_version.txt")
+
+
+def test_get_clickhouse_version(mock_sync_execute, config):
+    # Create a test context with our config
+    context = build_op_context(resources={}, config=config)
+
+    # Mock the file write operation
+    mock_file = mock_open()
+    with patch("builtins.open", mock_file):
+        result = get_clickhouse_version(context)
+
+    # Verify the SQL query was executed
+    mock_sync_execute.assert_called_once_with("SELECT version()")
+
+    # Verify the version was written to file
+    mock_file().write.assert_called_once_with("23.8.1.2992")
+
+    # Verify the result metadata
+    assert result.metadata == {"version": "23.8.1.2992"}
+
+
+def test_print_clickhouse_version(config):
+    # Create a test context with our config
+    context = build_op_context(resources={}, config=config)
+
+    # Mock the file read operation
+    mock_file = mock_open(read_data="23.8.1.2992")
+    with patch("builtins.open", mock_file) as mock_open_:
+        with patch("builtins.print") as mock_print:
+            result = print_clickhouse_version(context)
+
+            # Verify the file was read
+            mock_open_.assert_called_once_with(config.result_path)
+
+            # Verify the version was printed
+            mock_print.assert_called_once_with("23.8.1.2992")
+
+            # Verify the result metadata
+            assert result.metadata == {"version": config.result_path}
+
+
+def test_assets_integration(mock_sync_execute, config):
+    """Test that both assets work together in sequence"""
+    context = build_op_context(resources={}, config=config)
+
+    # First run get_clickhouse_version
+    mock_write = mock_open()
+    with patch("builtins.open", mock_write):
+        get_clickhouse_version(context)
+        mock_write().write.assert_called_once_with("23.8.1.2992")
+
+    # Then run print_clickhouse_version
+    mock_read = mock_open(read_data="23.8.1.2992")
+    with patch("builtins.open", mock_read):
+        with patch("builtins.print") as mock_print:
+            print_clickhouse_version(context)
+            mock_print.assert_called_once_with("23.8.1.2992")
--- a/posthog/temporal/common/clickhouse.py
+++ b/posthog/temporal/common/clickhouse.py
@@ -464,7 +464,7 @@ async def get_client(
    Usage:

        async with get_client() as client:
-            await client.execute("SELECT 1")
+            await client.apost_query("SELECT 1")

    Note that this is not a connection pool, so you should not use this for
    queries that are run frequently.
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -100,3 +100,9 @@ combine-as-imports = true
 "./posthog/management/commands/test_migrations_are_safe.py" = ["T201"]
 "./posthog/management/commands/api_keys.py" = ["T201"]
 "./posthog/demo/matrix/manager.py" = ["T201"]
+
+[tool.dagster]
+python_version = "3.11"
+module_name = "dags.definitions"
+code_location_name = "dags"
+home = "/tmp/posthog-dagster/"
--- a/requirements-dev.in
+++ b/requirements-dev.in
@@ -52,7 +52,7 @@ pytest-xdist==3.6.1
 python-dateutil>=2.8.2
 responses==0.23.1
 syrupy~=4.6.0
-watchdog==2.1.8
+watchdog==5.0.3
 flaky==3.7.0
 aioresponses==0.7.6
 prance==23.06.21.0
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,12 +1,16 @@
-# This file was autogenerated by uv via the following command:
-#    uv pip compile requirements-dev.in -o requirements-dev.txt
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile requirements-dev.in
+#
 aiohappyeyeballs==2.4.4
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   aiohttp
 aiohttp==3.11.10
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   aioresponses
    #   datasets
    #   fsspec
@@ -16,15 +20,15 @@ aioresponses==0.7.6
    # via -r requirements-dev.in
 aiosignal==1.2.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   aiohttp
 annotated-types==0.7.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   pydantic
 anyio==4.6.2.post1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   httpx
    #   openai
 appdirs==1.4.4
@@ -33,18 +37,18 @@ argcomplete==2.0.0
    # via datamodel-code-generator
 asgiref==3.7.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   django
    #   django-stubs
 asttokens==2.4.1
    # via inline-snapshot
 async-timeout==4.0.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   redis
 attrs==23.2.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   aiohttp
    #   jsonschema
    #   referencing
@@ -53,44 +57,46 @@ black==23.9.1
    #   -r requirements-dev.in
    #   datamodel-code-generator
    #   inline-snapshot
-boto3-stubs==1.34.84
+boto3-stubs[s3]==1.34.84
    # via -r requirements-dev.in
 botocore-stubs==1.34.84
    # via boto3-stubs
 certifi==2019.11.28
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   httpcore
    #   httpx
    #   requests
    #   sentry-sdk
 cffi==1.16.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   cryptography
 chardet==5.2.0
    # via prance
 charset-normalizer==2.1.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   requests
 click==8.1.7
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   black
    #   inline-snapshot
    #   typer
 colorama==0.4.4
    # via pytest-watch
-coverage==5.5
-    # via pytest-cov
+coverage[toml]==5.5
+    # via
+    #   coverage
+    #   pytest-cov
 cryptography==39.0.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   types-paramiko
 dataclasses-json==0.6.7
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   langchain-community
 datamodel-code-generator==0.26.1
    # via -r requirements-dev.in
@@ -108,11 +114,11 @@ dill==0.3.8
    #   multiprocess
 distro==1.9.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   openai
 django==4.2.15
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   django-linear-migrations
    #   django-stubs
    #   django-stubs-ext
@@ -128,7 +134,7 @@ djangorestframework-stubs==3.14.5
    # via -r requirements-dev.in
 dnspython==2.2.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   email-validator
 docopt==0.6.2
    # via pytest-watch
@@ -142,11 +148,13 @@ executing==2.0.1
    # via inline-snapshot
 faker==17.5.0
    # via -r requirements-dev.in
-fakeredis==2.23.3
-    # via -r requirements-dev.in
+fakeredis[lua]==2.23.3
+    # via
+    #   -r requirements-dev.in
+    #   fakeredis
 filelock==3.12.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   datasets
    #   huggingface-hub
 flaky==3.7.0
@@ -155,36 +163,37 @@ freezegun==1.2.2
    # via -r requirements-dev.in
 frozenlist==1.4.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   aiohttp
    #   aiosignal
-fsspec==2023.10.0
+fsspec[http]==2023.10.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   datasets
+    #   fsspec
    #   huggingface-hub
 genson==1.2.2
    # via datamodel-code-generator
 googleapis-common-protos==1.60.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   opentelemetry-exporter-otlp-proto-grpc
 grpcio==1.63.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   deepeval
    #   opentelemetry-exporter-otlp-proto-grpc
 h11==0.13.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   httpcore
 httpcore==1.0.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   httpx
 httpx==0.26.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   langsmith
    #   openai
 huggingface-hub==0.26.2
@@ -193,7 +202,7 @@ icdiff==2.0.5
    # via pytest-icdiff
 idna==3.10
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   anyio
    #   email-validator
    #   httpx
@@ -211,45 +220,47 @@ inline-snapshot==0.12.1
    # via -r requirements-dev.in
 isort==5.2.2
    # via datamodel-code-generator
-jinja2==3.1.4
-    # via datamodel-code-generator
+jinja2==3.1.5
+    # via
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
+    #   datamodel-code-generator
 jiter==0.5.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   openai
 jsonpatch==1.33
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   langchain-core
 jsonpointer==3.0.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   jsonpatch
 jsonschema==4.20.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   openapi-schema-validator
    #   openapi-spec-validator
 jsonschema-path==0.3.2
    # via openapi-spec-validator
 jsonschema-specifications==2023.12.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   jsonschema
    #   openapi-schema-validator
 langchain==0.3.9
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   deepeval
    #   langchain-community
    #   ragas
 langchain-community==0.3.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   ragas
 langchain-core==0.3.21
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   deepeval
    #   langchain
    #   langchain-community
@@ -258,16 +269,16 @@ langchain-core==0.3.21
    #   ragas
 langchain-openai==0.2.11
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   deepeval
    #   ragas
 langchain-text-splitters==0.3.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   langchain
 langsmith==0.1.132
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   langchain
    #   langchain-community
    #   langchain-core
@@ -277,21 +288,23 @@ lupa==2.2
    # via fakeredis
 markdown-it-py==3.0.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   rich
-markupsafe==2.1.5
-    # via jinja2
+markupsafe==3.0.2
+    # via
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
+    #   jinja2
 marshmallow==3.23.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   dataclasses-json
 mdurl==0.1.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   markdown-it-py
 multidict==6.0.5
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   -r requirements-dev.in
    #   aiohttp
    #   yarl
@@ -305,7 +318,7 @@ mypy-boto3-s3==1.34.65
    # via boto3-stubs
 mypy-extensions==1.0.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   -r requirements-dev.in
    #   black
    #   mypy
@@ -314,7 +327,7 @@ nest-asyncio==1.6.0
    # via ragas
 numpy==1.23.3
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   datasets
    #   langchain
    #   langchain-community
@@ -323,7 +336,7 @@ numpy==1.23.3
    #   ragas
 openai==1.56.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   langchain-openai
    #   ragas
 openapi-schema-validator==0.6.2
@@ -351,11 +364,11 @@ opentelemetry-semantic-conventions==0.45b0
    # via opentelemetry-sdk
 orjson==3.10.7
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   langsmith
 packaging==24.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   -r requirements-dev.in
    #   black
    #   datamodel-code-generator
@@ -367,7 +380,7 @@ packaging==24.1
    #   pytest
 pandas==2.2.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   datasets
 parameterized==0.9.0
    # via -r requirements-dev.in
@@ -377,11 +390,11 @@ pathspec==0.12.1
    # via black
 platformdirs==3.11.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   black
 pluggy==1.5.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   pytest
 portalocker==2.10.1
    # via deepeval
@@ -391,46 +404,49 @@ prance==23.6.21.0
    # via -r requirements-dev.in
 propcache==0.2.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   aiohttp
    #   yarl
 protobuf==4.22.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   deepeval
    #   googleapis-common-protos
    #   opentelemetry-proto
 pyarrow==17.0.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   datasets
 pyarrow-hotfix==0.6
    # via datasets
 pycparser==2.20
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   cffi
-pydantic==2.9.2
+pydantic[email]==2.9.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   datamodel-code-generator
    #   deepeval
    #   langchain
    #   langchain-core
    #   langsmith
    #   openai
+    #   pydantic
    #   pydantic-settings
    #   ragas
 pydantic-core==2.23.4
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   pydantic
 pydantic-settings==2.7.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   langchain-community
-pygments==2.18.0
-    # via rich
+pygments==2.19.1
+    # via
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
+    #   rich
 pysbd==0.3.4
    # via ragas
 pytest==8.0.2
@@ -472,22 +488,22 @@ pytest-xdist==3.6.1
    #   deepeval
 python-dateutil==2.8.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   -r requirements-dev.in
    #   faker
    #   freezegun
    #   pandas
 python-dotenv==0.21.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   pydantic-settings
 pytz==2023.3
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   pandas
 pyyaml==6.0.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   datamodel-code-generator
    #   datasets
    #   huggingface-hub
@@ -500,21 +516,21 @@ ragas==0.2.5
    # via deepeval
 redis==4.5.4
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   fakeredis
 referencing==0.31.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   jsonschema
    #   jsonschema-path
    #   jsonschema-specifications
 regex==2023.12.25
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   tiktoken
 requests==2.32.3
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   datasets
    #   deepeval
    #   djangorestframework-stubs
@@ -530,20 +546,21 @@ requests==2.32.3
    #   tiktoken
 requests-toolbelt==1.0.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   langsmith
 responses==0.23.1
    # via -r requirements-dev.in
 rfc3339-validator==0.1.4
    # via openapi-schema-validator
-rich==13.7.1
+rich==13.9.4
    # via
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   deepeval
    #   inline-snapshot
    #   typer
 rpds-py==0.16.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   jsonschema
    #   referencing
 ruamel-yaml==0.18.6
@@ -554,50 +571,52 @@ ruff==0.8.1
    # via -r requirements-dev.in
 sentry-sdk==1.44.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   deepeval
 shellingham==1.5.4
    # via typer
 six==1.16.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   asttokens
    #   prance
    #   python-dateutil
    #   rfc3339-validator
 sniffio==1.3.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   anyio
    #   httpx
    #   openai
 sortedcontainers==2.4.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   fakeredis
 sqlalchemy==2.0.31
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   langchain
    #   langchain-community
 sqlparse==0.4.4
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   django
 syrupy==4.6.4
    # via -r requirements-dev.in
 tabulate==0.9.0
-    # via deepeval
+    # via
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
+    #   deepeval
 tenacity==8.4.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   deepeval
    #   langchain
    #   langchain-community
    #   langchain-core
 tiktoken==0.8.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   -r requirements-dev.in
    #   langchain-openai
    #   ragas
@@ -607,7 +626,7 @@ toml==0.10.2
    #   inline-snapshot
 tqdm==4.64.1
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   datasets
    #   deepeval
    #   huggingface-hub
@@ -652,7 +671,7 @@ types-tzlocal==5.1.0.1
    # via -r requirements-dev.in
 typing-extensions==4.12.2
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   boto3-stubs
    #   django-stubs
    #   django-stubs-ext
@@ -671,31 +690,32 @@ typing-extensions==4.12.2
    #   typing-inspect
 typing-inspect==0.9.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   dataclasses-json
 tzdata==2023.3
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   pandas
 urllib3==1.26.18
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   requests
    #   responses
    #   sentry-sdk
-watchdog==2.1.8
+watchdog==5.0.3
    # via
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   -r requirements-dev.in
    #   pytest-watch
 wrapt==1.15.0
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   deprecated
 xxhash==3.5.0
    # via datasets
 yarl==1.18.3
    # via
-    #   -c requirements.txt
+    #   -c /Users/james/opt/ph/posthog/requirements.txt
    #   aiohttp
 zipp==3.21.0
    # via importlib-metadata
--- a/requirements.in
+++ b/requirements.in
@@ -17,6 +17,11 @@ clickhouse-driver==0.2.7
 clickhouse-pool==0.5.3
 conditional-cache==1.2
 cryptography==39.0.2
+dagster==1.9.8
+dagster-webserver==1.9.8
+dagster-postgres==0.25.8
+dagster-celery==0.25.8
+# dagster-cloud==1.9.8
 deltalake==0.23.2
 dj-database-url==0.5.0
 Django~=4.2.15
@@ -115,3 +120,4 @@ lxml==4.9.4 # Do not change this version - it will break SAML
 grpcio~=1.63.2 # Version constrained so that `deepeval` can be installed in in dev
 tenacity~=8.4.2  # Version constrained so that `deepeval` can be installed in in dev
 markdown-it-py~=3.0.0
+tzlocal~=5.1
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,15 @@
-# This file was autogenerated by uv via the following command:
-#    uv pip compile requirements.in -o requirements.txt
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile requirements.in
+#
 aioboto3==12.0.0
    # via -r requirements.in
-aiobotocore==2.7.0
+aiobotocore[boto3]==2.7.0
    # via
    #   aioboto3
+    #   aiobotocore
    #   s3fs
 aiohappyeyeballs==2.4.4
    # via aiohttp
@@ -22,16 +27,23 @@ aiokafka==0.8.1
    # via -r requirements.in
 aiosignal==1.2.0
    # via aiohttp
+alembic==1.14.0
+    # via dagster
 amqp==5.1.1
    # via kombu
 annotated-types==0.7.0
    # via pydantic
 antlr4-python3-runtime==4.13.1
-    # via -r requirements.in
+    # via
+    #   -r requirements.in
+    #   dagster
 anyio==4.6.2.post1
    # via
+    #   gql
    #   httpx
    #   openai
+    #   starlette
+    #   watchfiles
 asgiref==3.7.2
    # via django
 asn1crypto==1.5.1
@@ -51,7 +63,10 @@ attrs==23.2.0
    #   trio
    #   zeep
 backoff==2.2.1
-    # via posthoganalytics
+    # via
+    #   gql
+    #   langfuse
+    #   posthoganalytics
 bcrypt==4.1.3
    # via paramiko
 billiard==4.1.0
@@ -73,6 +88,7 @@ celery==5.3.4
    # via
    #   -r requirements.in
    #   celery-redbeat
+    #   dagster-celery
    #   sentry-sdk
 celery-redbeat==2.1.1
    # via -r requirements.in
@@ -102,7 +118,11 @@ click==8.1.7
    #   click-didyoumean
    #   click-plugins
    #   click-repl
+    #   dagster
+    #   dagster-celery
+    #   dagster-webserver
    #   dlt
+    #   uvicorn
 click-didyoumean==0.3.0
    # via celery
 click-plugins==1.1.1
@@ -118,8 +138,12 @@ clickhouse-driver==0.2.7
    #   sentry-sdk
 clickhouse-pool==0.5.3
    # via -r requirements.in
+coloredlogs==14.0
+    # via dagster
 conditional-cache==1.2
    # via -r requirements.in
+croniter==5.0.1
+    # via dagster
 cryptography==39.0.2
    # via
    #   -r requirements.in
@@ -133,6 +157,23 @@ cssselect==1.1.0
    # via toronado
 cssutils==1.0.2
    # via toronado
+dagster==1.9.8
+    # via
+    #   -r requirements.in
+    #   dagster-celery
+    #   dagster-graphql
+    #   dagster-postgres
+    #   dagster-webserver
+dagster-celery==0.25.8
+    # via -r requirements.in
+dagster-graphql==1.9.8
+    # via dagster-webserver
+dagster-pipes==1.9.8
+    # via dagster
+dagster-postgres==0.25.8
+    # via -r requirements.in
+dagster-webserver==1.9.8
+    # via -r requirements.in
 dataclasses-json==0.6.7
    # via langchain-community
 decorator==5.1.1
@@ -216,6 +257,8 @@ dlt==1.3.0
    # via -r requirements.in
 dnspython==2.2.1
    # via -r requirements.in
+docstring-parser==0.16
+    # via dagster
 drf-exceptions-hog==0.4.0
    # via -r requirements.in
 drf-extensions==0.7.0
@@ -225,7 +268,9 @@ drf-spectacular==0.27.2
 et-xmlfile==1.1.0
    # via openpyxl
 filelock==3.12.0
-    # via snowflake-connector-python
+    # via
+    #   dagster
+    #   snowflake-connector-python
 frozenlist==1.4.1
    # via
    #   aiohttp
@@ -234,6 +279,7 @@ fsspec==2023.10.0
    # via
    #   dlt
    #   s3fs
+    #   universal-pathlib
 future==0.18.3
    # via lzstring
 geoip2==4.6.0
@@ -244,8 +290,9 @@ gitpython==3.1.40
    # via dlt
 giturlparse==0.12.0
    # via dlt
-google-api-core==2.11.1
+google-api-core[grpc]==2.11.1
    # via
+    #   google-api-core
    #   google-cloud-bigquery
    #   google-cloud-bigquery-storage
    #   google-cloud-core
@@ -273,12 +320,29 @@ googleapis-common-protos==1.60.0
    # via
    #   google-api-core
    #   grpcio-status
+gql[requests]==3.5.0
+    # via
+    #   dagster-graphql
+    #   gql
+graphene==3.4.3
+    # via dagster-graphql
+graphql-core==3.2.5
+    # via
+    #   gql
+    #   graphene
+    #   graphql-relay
+graphql-relay==3.2.0
+    # via graphene
 grpcio==1.63.2
    # via
    #   -r requirements.in
+    #   dagster
    #   google-api-core
+    #   grpcio-health-checking
    #   grpcio-status
    #   sqlalchemy-bigquery
+grpcio-health-checking==1.62.3
+    # via dagster
 grpcio-status==1.57.0
    # via google-api-core
 gunicorn==20.1.0
@@ -286,6 +350,7 @@ gunicorn==20.1.0
 h11==0.13.0
    # via
    #   httpcore
+    #   uvicorn
    #   wsproto
 hexbytes==1.0.0
    # via dlt
@@ -293,11 +358,15 @@ hogql-parser==1.0.47
    # via -r requirements.in
 httpcore==1.0.2
    # via httpx
+httptools==0.6.4
+    # via uvicorn
 httpx==0.26.0
    # via
    #   langgraph-sdk
    #   langsmith
    #   openai
+humanfriendly==10.0
+    # via coloredlogs
 humanize==4.9.0
    # via dlt
 idna==3.10
@@ -319,6 +388,8 @@ isodate==0.6.1
    # via
    #   python3-saml
    #   zeep
+jinja2==3.1.5
+    # via dagster
 jiter==0.5.0
    # via openai
 jmespath==1.0.0
@@ -389,8 +460,16 @@ lzstring==1.0.4
    # via -r requirements.in
 makefun==1.15.2
    # via dlt
+mako==1.3.8
+    # via alembic
 markdown-it-py==3.0.0
-    # via -r requirements.in
+    # via
+    #   -r requirements.in
+    #   rich
+markupsafe==3.0.2
+    # via
+    #   jinja2
+    #   mako
 marshmallow==3.23.1
    # via dataclasses-json
 maxminddb==2.2.0
@@ -451,6 +530,7 @@ outcome==1.3.0.post0
 packaging==24.1
    # via
    #   aiokafka
+    #   dagster
    #   dlt
    #   google-cloud-bigquery
    #   langchain-core
@@ -496,20 +576,26 @@ proto-plus==1.22.3
    # via google-cloud-bigquery-storage
 protobuf==4.22.1
    # via
+    #   dagster
    #   google-api-core
    #   google-cloud-bigquery-storage
    #   googleapis-common-protos
+    #   grpcio-health-checking
    #   grpcio-status
    #   proto-plus
    #   temporalio
 psutil==6.0.0
    # via -r requirements.in
-psycopg==3.1.20
-    # via -r requirements.in
+psycopg[binary]==3.1.20
+    # via
+    #   -r requirements.in
+    #   psycopg
 psycopg-binary==3.1.20
    # via psycopg
 psycopg2-binary==2.9.7
-    # via -r requirements.in
+    # via
+    #   -r requirements.in
+    #   dagster-postgres
 py==1.11.0
    # via retry
 pyarrow==17.0.0
@@ -528,6 +614,7 @@ pycparser==2.20
 pydantic==2.9.2
    # via
    #   -r requirements.in
+    #   dagster
    #   langchain
    #   langchain-core
    #   langsmith
@@ -537,7 +624,9 @@ pydantic-core==2.23.4
    # via pydantic
 pydantic-settings==2.7.0
    # via langchain-community
-pyjwt==2.4.0
+pygments==2.19.1
+    # via rich
+pyjwt[crypto]==2.4.0
    # via
    #   -r requirements.in
    #   simple-salesforce
@@ -565,13 +654,17 @@ python-dateutil==2.8.2
    #   botocore
    #   celery
    #   celery-redbeat
+    #   croniter
    #   google-cloud-bigquery
+    #   graphene
    #   pandas
    #   pendulum
    #   posthoganalytics
 python-dotenv==0.21.0
    # via
+    #   dagster
    #   pydantic-settings
+    #   uvicorn
    #   webdriver-manager
 python-statsd==2.1.0
    # via django-statsd
@@ -584,21 +677,24 @@ pytz==2023.3
    #   -r requirements.in
    #   clickhouse-connect
    #   clickhouse-driver
+    #   croniter
+    #   dagster
    #   dlt
    #   infi-clickhouse-orm
    #   pandas
    #   snowflake-connector-python
-    #   tzlocal
    #   zeep
 pytzdata==2020.1
    # via pendulum
 pyyaml==6.0.1
    # via
+    #   dagster
    #   dlt
    #   drf-spectacular
    #   langchain
    #   langchain-community
    #   langchain-core
+    #   uvicorn
 qrcode==7.4.2
    # via django-two-factor-auth
 redis==4.5.4
@@ -615,10 +711,13 @@ regex==2023.12.25
 requests==2.32.3
    # via
    #   -r requirements.in
+    #   dagster
+    #   dagster-graphql
    #   dlt
    #   geoip2
    #   google-api-core
    #   google-cloud-bigquery
+    #   gql
    #   infi-clickhouse-orm
    #   langchain
    #   langchain-community
@@ -641,12 +740,15 @@ requests-oauthlib==1.3.0
    # via social-auth-core
 requests-toolbelt==1.0.0
    # via
+    #   gql
    #   langsmith
    #   zeep
 requirements-parser==0.5.0
    # via dlt
 retry==0.9.2
    # via -r requirements.in
+rich==13.9.4
+    # via dagster
 rpds-py==0.16.2
    # via
    #   jsonschema
@@ -667,13 +769,10 @@ semantic-version==2.8.5
    # via -r requirements.in
 semver==3.0.2
    # via dlt
-sentry-sdk==1.44.1
-    # via -r requirements.in
-setuptools==75.0.0
+sentry-sdk[celery,clickhouse-driver,django,openai]==1.44.1
    # via
-    #   dlt
-    #   gunicorn
-    #   infi-clickhouse-orm
+    #   -r requirements.in
+    #   sentry-sdk
 simple-salesforce==1.12.6
    # via -r requirements.in
 simplejson==3.19.2
@@ -715,18 +814,26 @@ sortedcontainers==2.4.0
 sqlalchemy==2.0.31
    # via
    #   -r requirements.in
+    #   alembic
+    #   dagster
    #   langchain
    #   langchain-community
    #   snowflake-sqlalchemy
    #   sqlalchemy-bigquery
-sqlalchemy-bigquery==1.11.0
-    # via -r requirements.in
+sqlalchemy-bigquery[bqstorage]==1.11.0
+    # via
+    #   -r requirements.in
+    #   sqlalchemy-bigquery
 sqlparse==0.4.4
    # via
    #   -r requirements.in
    #   django
 sshtunnel==0.4.0
    # via -r requirements.in
+starlette==0.45.2
+    # via
+    #   dagster-graphql
+    #   dagster-webserver
 statshog==1.0.6
    # via -r requirements.in
 stripe==10.3.0
@@ -734,7 +841,10 @@ stripe==10.3.0
 structlog==23.2.0
    # via
    #   -r requirements.in
+    #   dagster
    #   django-structlog
+tabulate==0.9.0
+    # via dagster
 temporalio==1.7.1
    # via -r requirements.in
 tenacity==8.4.2
@@ -754,14 +864,20 @@ tiktoken==0.8.0
    #   sentry-sdk
 token-bucket==0.3.0
    # via -r requirements.in
+tomli==2.2.1
+    # via dagster
 tomlkit==0.12.3
    # via
    #   dlt
    #   snowflake-connector-python
+toposort==1.10
+    # via dagster
 toronado==0.1.0
    # via -r requirements.in
 tqdm==4.64.1
-    # via openai
+    # via
+    #   dagster
+    #   openai
 trio==0.26.0
    # via
    #   selenium
@@ -774,7 +890,10 @@ types-setuptools==69.0.0.0
    # via requirements-parser
 typing-extensions==4.12.2
    # via
+    #   alembic
+    #   dagster
    #   dlt
+    #   graphene
    #   langchain-core
    #   openai
    #   psycopg
@@ -794,13 +913,17 @@ tzdata==2023.3
    #   celery
    #   dlt
    #   pandas
-tzlocal==2.1
-    # via clickhouse-driver
+tzlocal==5.2
+    # via
+    #   -r requirements.in
+    #   clickhouse-driver
 unicodecsv==0.14.1
    # via djangorestframework-csv
+universal-pathlib==0.2.6
+    # via dagster
 uritemplate==4.1.1
    # via drf-spectacular
-urllib3==1.26.18
+urllib3[secure,socks]==1.26.18
    # via
    #   botocore
    #   clickhouse-connect
@@ -812,15 +935,25 @@ urllib3==1.26.18
    #   sentry-sdk
 urllib3-secure-extra==0.1.0
    # via urllib3
+uvicorn[standard]==0.34.0
+    # via dagster-webserver
+uvloop==0.21.0
+    # via uvicorn
 vine==5.0.0
    # via
    #   amqp
    #   celery
    #   kombu
+watchdog==5.0.3
+    # via dagster
+watchfiles==1.0.4
+    # via uvicorn
 wcwidth==0.2.6
    # via prompt-toolkit
 webdriver-manager==4.0.2
    # via -r requirements.in
+websockets==14.1
+    # via uvicorn
 wheel==0.42.0
    # via astunparse
 whitenoise==6.5.0
@@ -834,7 +967,9 @@ xmlsec==1.3.13
    #   -r requirements.in
    #   python3-saml
 yarl==1.18.3
-    # via aiohttp
+    # via
+    #   aiohttp
+    #   gql
 zeep==4.2.1
    # via simple-salesforce
 zstandard==0.23.0
@@ -843,3 +978,6 @@ zstd==1.5.5.1
    # via -r requirements.in
 zxcvbn==4.4.28
    # via -r requirements.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools