mirror of
https://github.com/langchain-ai/langsmith-self-hosted-workshops.git
synced 2026-07-01 20:44:14 -04:00
Add tests for Module 3 and Module 4 notebooks
This commit adds comprehensive test coverage for Module 3 (Production
Operations & Scaling) and Module 4 (Troubleshooting & Incident Response)
notebooks, following the same pattern established for Modules 1 and 2.
Test Infrastructure:
- tests/test_notebook_execution.py:
- Added TestModule3Notebooks class with syntax and execution tests
for 01_ops_sanity_checks.ipynb
- Added TestModule4Notebooks class with:
- Syntax tests for all 6 Module 4 notebooks (00, 01, 10, 20, 30, 40)
- Execution tests for setup/baseline notebooks (00, 01) - read-only
- Execution tests for failure labs (10, 20, 30, 40) - with warnings
about failure injection and secret modification
CI/CD Integration:
- .github/workflows/test-notebooks.yml:
- Added Module 3 and Module 4 syntax tests to CI pipeline
- All syntax tests now run automatically on PRs and pushes
Test Features:
- Respects CI_SKIP_EXECUTION environment variable (same as Module 1)
- Uses environment variables for configuration (cloud provider, region, etc.)
- Appropriate timeouts: 600s for ops checks, 900s for failure labs
- Safety warnings for failure lab execution tests
- Syntax tests always run (including in CI)
- Execution tests skip in CI when CI_SKIP_EXECUTION=true
Module 4 Test Structure:
- Syntax tests: All 6 notebooks validated for structure
- Setup/Baseline execution: Read-only validation notebooks (00, 01)
- Failure lab execution: Separate test method with warnings about
secret modification and failure injection (10, 20, 30, 40)
This ensures all workshop notebooks are validated for syntax correctness
and can be execution-tested when infrastructure is available, maintaining
consistency with existing test patterns.
This commit is contained in:
@@ -53,6 +53,8 @@ jobs:
|
||||
run: |
|
||||
pytest tests/test_notebook_execution.py::TestModule1Notebooks::test_module1_notebook_syntax -v
|
||||
pytest tests/test_notebook_execution.py::TestModule2Notebooks::test_module2_notebook_syntax -v
|
||||
pytest tests/test_notebook_execution.py::TestModule3Notebooks::test_module3_notebook_syntax -v
|
||||
pytest tests/test_notebook_execution.py::TestModule4Notebooks::test_module4_notebook_syntax -v
|
||||
|
||||
- name: Upload test artifacts
|
||||
if: always()
|
||||
|
||||
@@ -190,3 +190,94 @@ class TestModule2Notebooks(TestNotebookExecution):
|
||||
success, output = execute_notebook(notebook_path, timeout=300)
|
||||
assert success, f"Notebook execution failed:\n{output}"
|
||||
|
||||
|
||||
# Module 3 tests
|
||||
class TestModule3Notebooks(TestNotebookExecution):
|
||||
"""Test Module 3 notebooks."""
|
||||
|
||||
@pytest.mark.parametrize("notebook", [
|
||||
"01_ops_sanity_checks.ipynb",
|
||||
])
|
||||
def test_module3_notebook_syntax(self, notebook):
|
||||
"""Test Module 3 notebook syntax."""
|
||||
notebook_path = NOTEBOOKS_DIR / "module-3" / notebook
|
||||
self._validate_notebook_syntax(notebook_path)
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get("CI_SKIP_EXECUTION") == "true",
|
||||
reason="Skipping execution in CI (requires infrastructure)"
|
||||
)
|
||||
@pytest.mark.parametrize("notebook", [
|
||||
"01_ops_sanity_checks.ipynb",
|
||||
])
|
||||
def test_module3_notebook_execution(self, notebook):
|
||||
"""Test Module 3 notebook execution (only if infrastructure available)."""
|
||||
notebook_path = NOTEBOOKS_DIR / "module-3" / notebook
|
||||
# Ops sanity checks may take longer due to resource usage checks
|
||||
success, output = execute_notebook(notebook_path, timeout=600)
|
||||
assert success, f"Notebook execution failed:\n{output}"
|
||||
|
||||
|
||||
# Module 4 tests
|
||||
class TestModule4Notebooks(TestNotebookExecution):
|
||||
"""Test Module 4 notebooks."""
|
||||
|
||||
@pytest.mark.parametrize("notebook", [
|
||||
"00_setup_or_resume_environment.ipynb",
|
||||
"01_diagnostics_baseline.ipynb",
|
||||
"10_failure_lab_postgres.ipynb",
|
||||
"20_failure_lab_redis.ipynb",
|
||||
"30_failure_lab_clickhouse.ipynb",
|
||||
"40_failure_lab_blob_storage.ipynb",
|
||||
])
|
||||
def test_module4_notebook_syntax(self, notebook):
|
||||
"""Test Module 4 notebook syntax."""
|
||||
notebook_path = NOTEBOOKS_DIR / "module-4" / notebook
|
||||
self._validate_notebook_syntax(notebook_path)
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get("CI_SKIP_EXECUTION") == "true",
|
||||
reason="Skipping execution in CI (requires infrastructure)"
|
||||
)
|
||||
@pytest.mark.parametrize("notebook", [
|
||||
"00_setup_or_resume_environment.ipynb",
|
||||
"01_diagnostics_baseline.ipynb",
|
||||
])
|
||||
def test_module4_notebook_execution(self, notebook):
|
||||
"""
|
||||
Test Module 4 notebook execution (only if infrastructure available).
|
||||
|
||||
Tests setup and baseline notebooks which are read-only validation.
|
||||
Failure labs are syntax-tested only to avoid modifying production environments.
|
||||
"""
|
||||
notebook_path = NOTEBOOKS_DIR / "module-4" / notebook
|
||||
# Setup and baseline checks may take longer due to diagnostics collection
|
||||
success, output = execute_notebook(notebook_path, timeout=600)
|
||||
assert success, f"Notebook execution failed:\n{output}"
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get("CI_SKIP_EXECUTION") == "true",
|
||||
reason="Skipping execution in CI (requires infrastructure and failure injection)"
|
||||
)
|
||||
@pytest.mark.parametrize("notebook", [
|
||||
"10_failure_lab_postgres.ipynb",
|
||||
"20_failure_lab_redis.ipynb",
|
||||
"30_failure_lab_clickhouse.ipynb",
|
||||
"40_failure_lab_blob_storage.ipynb",
|
||||
])
|
||||
def test_module4_failure_lab_execution(self, notebook):
|
||||
"""
|
||||
Test Module 4 failure lab notebook execution (only if infrastructure available).
|
||||
|
||||
WARNING: These notebooks inject failures by modifying secrets and configurations.
|
||||
They should only be run in test environments, not production.
|
||||
|
||||
These tests validate that failure injection and remediation workflows function
|
||||
correctly. The notebooks include safety mechanisms (commented-out injection code)
|
||||
but should still be used with caution.
|
||||
"""
|
||||
notebook_path = NOTEBOOKS_DIR / "module-4" / notebook
|
||||
# Failure labs may take longer due to failure injection, observation, and remediation
|
||||
success, output = execute_notebook(notebook_path, timeout=900) # 15 minutes
|
||||
assert success, f"Notebook execution failed:\n{output}"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user