Invalidate cache when running extract tests

* Run hourly extract tests to catch SDK schema drifts

* fix url

* fix prod/staging env

.github/workflows/hourly_extract_e2e.yml

@@ -0,0 +1,159 @@
+name: Hourly Extract E2E Tests
+
+on:
+  schedule:
+    - cron: "0 * * * *" # Runs every hour at the top of the hour
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Environment to run the tests in"
+        required: false
+        default: staging
+        type: choice
+        options:
+          - staging
+          - production
+      notify_slack:
+        description: "Notify Slack"
+        required: false
+        default: false
+        type: boolean
+  workflow_call:
+
+env:
+  UV_VERSION: "0.7.20"
+  PYTHON_VERSION: "3.12"
+  SLACK_CHANNEL_ID: C078PHNTF44 # Extract channel ID
+  API_E2E_LOG_PATH: ${{ github.workspace }}/extract-e2e.log
+
+jobs:
+  extract-e2e:
+    name: "Hourly Extract E2E Tests (${{ matrix.environment }})"
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.environment }}
+      cancel-in-progress: true
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: ${{ github.event_name == 'schedule' && fromJson('["staging", "production"]') || fromJson(format('["{0}"]', github.event.inputs.environment || 'staging')) }}
+    steps:
+      - name: Set runtime inputs
+        id: runtime
+        run: |
+          environment=${{ matrix.environment }}
+          notify_slack=${{ github.event.inputs.notify_slack || github.event_name == 'schedule' }}
+          echo "environment=${environment}" >> $GITHUB_OUTPUT
+          echo "notify_slack=${notify_slack}" >> $GITHUB_OUTPUT
+
+          if [ "${environment}" = "production" ]; then
+            echo "LLAMA_CLOUD_BASE_URL=https://api.cloud.llamaindex.ai" >> $GITHUB_ENV
+            api_key_secret="${{ secrets.LLAMA_CLOUD_API_KEY }}"
+            project_id_secret="${{ secrets.LLAMA_CLOUD_PROJECT_ID }}"
+          else
+            echo "LLAMA_CLOUD_BASE_URL=https://api.staging.llamaindex.ai" >> $GITHUB_ENV
+            api_key_secret="${{ secrets.LLAMA_CLOUD_API_KEY_STAGING }}"
+            project_id_secret="${{ secrets.LLAMA_CLOUD_PROJECT_ID_STAGING }}"
+          fi
+
+          if [ -n "$api_key_secret" ]; then
+            echo "LLAMA_CLOUD_API_KEY=$api_key_secret" >> $GITHUB_ENV
+          fi
+
+          if [ -n "$project_id_secret" ]; then
+            echo "LLAMA_CLOUD_PROJECT_ID=$project_id_secret" >> $GITHUB_ENV
+          fi
+
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: ${{ env.UV_VERSION }}
+
+      - name: Set up Python
+        run: uv python install ${{ env.PYTHON_VERSION }} && uv python pin ${{ env.PYTHON_VERSION }}
+
+      - name: Run Extract E2E tests
+        id: extract-tests
+        continue-on-error: true
+        working-directory: py
+        run: |
+          set -o pipefail
+          rm -f "$API_E2E_LOG_PATH"
+          uv run pytest -v -n 8 --timeout=300 --session-timeout=1740 tests/extract/ 2>&1 | tee "$API_E2E_LOG_PATH"
+
+      - name: Extract pytest failure summary
+        id: failed-tests
+        if: steps.extract-tests.outcome == 'failure' || cancelled()
+        run: |
+          summary="$(python3 - <<'PY'
+          import os
+          from pathlib import Path
+
+          log_path = Path(os.environ["API_E2E_LOG_PATH"])
+          if not log_path.exists():
+              print("Test log not found.")
+              raise SystemExit(0)
+
+          lines = log_path.read_text(errors="ignore").splitlines()
+
+          def find_section(keyword):
+              for i, line in enumerate(lines):
+                  if line.startswith("=") and keyword in line:
+                      return i
+              return None
+
+          start = (
+              find_section("FAILURES")
+              or find_section("ERRORS")
+              or find_section("short test summary info")
+          )
+
+          if start is not None:
+              snippet = lines[start : start + 200]
+          else:
+              snippet = lines[-200:]
+
+          print("\n".join(snippet).strip())
+          PY
+          )"
+          if [ -z "$summary" ]; then
+            summary="Failed test summary not available. Review the full run logs."
+          fi
+          {
+            printf 'summary<<EOF\n%s\nEOF\n' "$summary"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Check test results
+        if: always()
+        run: |
+          if [ "${{ steps.extract-tests.outcome }}" == "failure" ]; then
+            echo "Extract E2E tests failed"
+            exit 1
+          fi
+
+      - name: Post to Extract Slack channel
+        id: slack
+        if: (failure() || cancelled()) && steps.runtime.outputs.notify_slack == 'true'
+        uses: slackapi/slack-github-action@v1.27.0
+        with:
+          channel-id: ${{ env.SLACK_CHANNEL_ID }}
+          slack-message: |
+            ALERT: *Hourly Extract E2E Tests Failure*
+
+            *Environment*: ${{ steps.runtime.outputs.environment }}
+            *Repository*: ${{ github.repository }}
+            *Branch*: ${{ github.ref_name }}
+
+            <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Details>
+
+            Failed tests:
+            ```
+            ${{ steps.failed-tests.outputs.summary }}
+            ```
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

py/tests/extract/test_extract_api.py

@@ -87,7 +87,11 @@ def test_agent(llama_extract, test_agent_name, test_schema_dict, request):
     except Exception as e:
         print(f"Warning: Failed to cleanup existing agent: {e}")
 
-    agent = create_agent_with_retry(llama_extract, name=name, data_schema=schema)
+    # Use config with cache invalidation to ensure fresh results in tests
+    config = ExtractConfig(invalidate_cache=True)
+    agent = create_agent_with_retry(
+        llama_extract, name=name, data_schema=schema, config=config
+    )
 
     # Add agent to cleanup list via conftest helper
     register_agent_for_cleanup(agent.id)
@@ -237,7 +241,7 @@ class TestStatelessExtraction:
 
     @pytest.fixture
     def test_config(self):
-        return ExtractConfig(extraction_mode=ExtractMode.FAST)
+        return ExtractConfig(extraction_mode=ExtractMode.FAST, invalidate_cache=True)
 
     @pytest.fixture
     def test_schema_dict(self):

py/tests/extract/test_extract_e2e.py

@@ -56,10 +56,12 @@ def get_test_cases():
             input_files.append(file_path)
 
         settings = [
-            ExtractConfig(extraction_mode=ExtractMode.FAST),
-            ExtractConfig(extraction_mode=ExtractMode.BALANCED),
-            ExtractConfig(extraction_mode=ExtractMode.MULTIMODAL),
-            ExtractConfig(extraction_mode=ExtractMode.PREMIUM),
+            ExtractConfig(extraction_mode=ExtractMode.FAST, invalidate_cache=True),
+            ExtractConfig(extraction_mode=ExtractMode.BALANCED, invalidate_cache=True),
+            ExtractConfig(
+                extraction_mode=ExtractMode.MULTIMODAL, invalidate_cache=True
+            ),
+            ExtractConfig(extraction_mode=ExtractMode.PREMIUM, invalidate_cache=True),
         ]
 
         for input_file in sorted(input_files):