Add tests for Module 3 and Module 4 notebooks

This commit adds comprehensive test coverage for Module 3 (Production
Operations & Scaling) and Module 4 (Troubleshooting & Incident Response)
notebooks, following the same pattern established for Modules 1 and 2.

Test Infrastructure:
- tests/test_notebook_execution.py:
  - Added TestModule3Notebooks class with syntax and execution tests
    for 01_ops_sanity_checks.ipynb
  - Added TestModule4Notebooks class with:
    - Syntax tests for all 6 Module 4 notebooks (00, 01, 10, 20, 30, 40)
    - Execution tests for setup/baseline notebooks (00, 01) - read-only
    - Execution tests for failure labs (10, 20, 30, 40) - with warnings
      about failure injection and secret modification

CI/CD Integration:
- .github/workflows/test-notebooks.yml:
  - Added Module 3 and Module 4 syntax tests to CI pipeline
  - All syntax tests now run automatically on PRs and pushes

Test Features:
- Respects CI_SKIP_EXECUTION environment variable (same as Module 1)
- Uses environment variables for configuration (cloud provider, region, etc.)
- Appropriate timeouts: 600s for ops checks, 900s for failure labs
- Safety warnings for failure lab execution tests
- Syntax tests always run (including in CI)
- Execution tests skip in CI when CI_SKIP_EXECUTION=true

Module 4 Test Structure:
- Syntax tests: All 6 notebooks validated for structure
- Setup/Baseline execution: Read-only validation notebooks (00, 01)
- Failure lab execution: Separate test method with warnings about
  secret modification and failure injection (10, 20, 30, 40)

This ensures all workshop notebooks are validated for syntax correctness
and can be execution-tested when infrastructure is available, maintaining
consistency with existing test patterns.
This commit is contained in:
Cory Waddingham
2026-01-02 10:55:02 -08:00
parent f9c22ad3ea
commit b0b424eabd
2 changed files with 93 additions and 0 deletions
+2
View File
@@ -53,6 +53,8 @@ jobs:
run: |
pytest tests/test_notebook_execution.py::TestModule1Notebooks::test_module1_notebook_syntax -v
pytest tests/test_notebook_execution.py::TestModule2Notebooks::test_module2_notebook_syntax -v
pytest tests/test_notebook_execution.py::TestModule3Notebooks::test_module3_notebook_syntax -v
pytest tests/test_notebook_execution.py::TestModule4Notebooks::test_module4_notebook_syntax -v
- name: Upload test artifacts
if: always()
+91
View File
@@ -190,3 +190,94 @@ class TestModule2Notebooks(TestNotebookExecution):
success, output = execute_notebook(notebook_path, timeout=300)
assert success, f"Notebook execution failed:\n{output}"
# Module 3 tests
class TestModule3Notebooks(TestNotebookExecution):
"""Test Module 3 notebooks."""
@pytest.mark.parametrize("notebook", [
"01_ops_sanity_checks.ipynb",
])
def test_module3_notebook_syntax(self, notebook):
"""Test Module 3 notebook syntax."""
notebook_path = NOTEBOOKS_DIR / "module-3" / notebook
self._validate_notebook_syntax(notebook_path)
@pytest.mark.skipif(
os.environ.get("CI_SKIP_EXECUTION") == "true",
reason="Skipping execution in CI (requires infrastructure)"
)
@pytest.mark.parametrize("notebook", [
"01_ops_sanity_checks.ipynb",
])
def test_module3_notebook_execution(self, notebook):
"""Test Module 3 notebook execution (only if infrastructure available)."""
notebook_path = NOTEBOOKS_DIR / "module-3" / notebook
# Ops sanity checks may take longer due to resource usage checks
success, output = execute_notebook(notebook_path, timeout=600)
assert success, f"Notebook execution failed:\n{output}"
# Module 4 tests
class TestModule4Notebooks(TestNotebookExecution):
"""Test Module 4 notebooks."""
@pytest.mark.parametrize("notebook", [
"00_setup_or_resume_environment.ipynb",
"01_diagnostics_baseline.ipynb",
"10_failure_lab_postgres.ipynb",
"20_failure_lab_redis.ipynb",
"30_failure_lab_clickhouse.ipynb",
"40_failure_lab_blob_storage.ipynb",
])
def test_module4_notebook_syntax(self, notebook):
"""Test Module 4 notebook syntax."""
notebook_path = NOTEBOOKS_DIR / "module-4" / notebook
self._validate_notebook_syntax(notebook_path)
@pytest.mark.skipif(
os.environ.get("CI_SKIP_EXECUTION") == "true",
reason="Skipping execution in CI (requires infrastructure)"
)
@pytest.mark.parametrize("notebook", [
"00_setup_or_resume_environment.ipynb",
"01_diagnostics_baseline.ipynb",
])
def test_module4_notebook_execution(self, notebook):
"""
Test Module 4 notebook execution (only if infrastructure available).
Tests setup and baseline notebooks which are read-only validation.
Failure labs are syntax-tested only to avoid modifying production environments.
"""
notebook_path = NOTEBOOKS_DIR / "module-4" / notebook
# Setup and baseline checks may take longer due to diagnostics collection
success, output = execute_notebook(notebook_path, timeout=600)
assert success, f"Notebook execution failed:\n{output}"
@pytest.mark.skipif(
os.environ.get("CI_SKIP_EXECUTION") == "true",
reason="Skipping execution in CI (requires infrastructure and failure injection)"
)
@pytest.mark.parametrize("notebook", [
"10_failure_lab_postgres.ipynb",
"20_failure_lab_redis.ipynb",
"30_failure_lab_clickhouse.ipynb",
"40_failure_lab_blob_storage.ipynb",
])
def test_module4_failure_lab_execution(self, notebook):
"""
Test Module 4 failure lab notebook execution (only if infrastructure available).
WARNING: These notebooks inject failures by modifying secrets and configurations.
They should only be run in test environments, not production.
These tests validate that failure injection and remediation workflows function
correctly. The notebooks include safety mechanisms (commented-out injection code)
but should still be used with caution.
"""
notebook_path = NOTEBOOKS_DIR / "module-4" / notebook
# Failure labs may take longer due to failure injection, observation, and remediation
success, output = execute_notebook(notebook_path, timeout=900) # 15 minutes
assert success, f"Notebook execution failed:\n{output}"