wire tests up to depot test results analysis (#8262)

* emit junit files

* wire up the actions

* note

* [autofix.ci] apply automated fixes

* fix overly broad error

* scope to master runs

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

.github/workflows/check.yml

@@ -86,6 +86,9 @@ jobs:
   test:
     needs: setup
     runs-on: depot-ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write # Authenticate to Depot via GitHub OIDC for test reporting
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
@@ -96,6 +99,13 @@ jobs:
           download-translations: 'false'
       - name: Test
         run: yarn test
+      - name: Report test results to Depot
+        uses: depot/test-report-action@2ad5779840a9c63e884fa26f11d019768a27a941 # v1.0.1
+        if: ${{ !cancelled() && github.event_name == 'push' }}
+        with:
+          path: |
+            bin/test-results/junit*.xml
+            packages/*/test-results/junit*.xml
   check-gh-actions:
     runs-on: depot-ubuntu-latest
     permissions:

.github/workflows/e2e-test.yml

@@ -92,10 +92,41 @@ jobs:
           path: packages/desktop-client/test-results/
           retention-days: 30
           overwrite: true
+      - name: Upload E2E JUnit report
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: ${{ !cancelled() }}
+        with:
+          name: e2e-junit-${{ matrix.shard }}
+          path: packages/desktop-client/test-results/junit.xml
+          retention-days: 1
+          overwrite: true
+
+  report-e2e:
+    name: Report E2E results to Depot
+    needs: functional
+    runs-on: depot-ubuntu-latest
+    if: ${{ !cancelled() && github.event_name == 'merge_group' }}
+    permissions:
+      contents: read
+      id-token: write # Authenticate to Depot via GitHub OIDC for test reporting
+    steps:
+      - name: Download E2E JUnit reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: e2e-junit
+          pattern: e2e-junit-*
+      - name: Report test results to Depot
+        uses: depot/test-report-action@2ad5779840a9c63e884fa26f11d019768a27a941 # v1.0.1
+        if: ${{ !cancelled() }}
+        with:
+          path: e2e-junit
 
   functional-desktop-app:
     name: Functional Desktop App
     runs-on: depot-ubuntu-latest-4
+    permissions:
+      contents: read
+      id-token: write # Authenticate to Depot via GitHub OIDC for test reporting
     container:
       image: mcr.microsoft.com/playwright:v1.59.1-jammy
     steps:
@@ -124,6 +155,11 @@ jobs:
           path: packages/desktop-electron/e2e/test-results/
           retention-days: 30
           overwrite: true
+      - name: Report test results to Depot
+        uses: depot/test-report-action@2ad5779840a9c63e884fa26f11d019768a27a941 # v1.0.1
+        if: ${{ !cancelled() && github.event_name == 'merge_group' }}
+        with:
+          path: packages/desktop-electron/e2e/test-results/junit.xml
 
   vrt:
     name: Visual regression (shard ${{ matrix.shard }}/3)
@@ -161,12 +197,23 @@ jobs:
           path: packages/desktop-client/blob-report/
           retention-days: 1
           overwrite: true
+      - name: Upload VRT JUnit report
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: ${{ !cancelled() }}
+        with:
+          name: vrt-junit-${{ matrix.shard }}
+          path: packages/desktop-client/test-results/junit.xml
+          retention-days: 1
+          overwrite: true
 
   merge-vrt:
     name: Merge VRT Reports
     needs: vrt
     runs-on: depot-ubuntu-latest
     if: ${{ !cancelled() }}
+    permissions:
+      contents: read
+      id-token: write # Authenticate to Depot via GitHub OIDC for test reporting
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
@@ -179,6 +226,16 @@ jobs:
           path: packages/desktop-client/all-blob-reports
           pattern: vrt-blob-report-*
           merge-multiple: true
+      - name: Download VRT JUnit reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: vrt-junit
+          pattern: vrt-junit-*
+      - name: Report test results to Depot
+        uses: depot/test-report-action@2ad5779840a9c63e884fa26f11d019768a27a941 # v1.0.1
+        if: ${{ !cancelled() && github.event_name == 'merge_group' }}
+        with:
+          path: vrt-junit
       - name: Merge reports
         id: merge-reports
         run: yarn workspace @actual-app/web run playwright merge-reports --reporter html ./all-blob-reports

.gitignore

@@ -88,6 +88,9 @@ build/
 # Lage cache
 .lage/
 
+# Test results (JUnit XML reports, Playwright artifacts)
+**/test-results/
+
 *storybook.log
 storybook-static
 

bin/vitest.config.mts

@@ -5,5 +5,14 @@ export default defineConfig({
     globals: true,
     include: ['__tests__/**/*.test.ts'],
     environment: 'node',
+    reporters: process.env.CI
+      ? [
+          'default',
+          [
+            'junit',
+            { outputFile: './test-results/junit.xml', suiteName: 'bin' },
+          ],
+        ]
+      : ['default'],
   },
 });

packages/api/vite.config.mts

@@ -126,5 +126,14 @@ export default defineConfig({
       return type === 'stderr';
     },
     maxWorkers: 2,
+    reporters: process.env.CI
+      ? [
+          'default',
+          [
+            'junit',
+            { outputFile: './test-results/junit.xml', suiteName: 'api' },
+          ],
+        ]
+      : ['default'],
   },
 });

packages/ci-actions/vitest.config.mts

@@ -7,5 +7,14 @@ export default defineConfig({
     environment: 'node',
     maxWorkers: 1,
     isolate: false,
+    reporters: process.env.CI
+      ? [
+          'default',
+          [
+            'junit',
+            { outputFile: './test-results/junit.xml', suiteName: 'ci-actions' },
+          ],
+        ]
+      : ['default'],
   },
 });

packages/component-library/vitest.web.config.ts

@@ -7,6 +7,18 @@ export default defineConfig({
     globals: true,
     include: ['src/**/*.web.test.(js|jsx|ts|tsx)'],
     maxWorkers: 2,
+    reporters: process.env.CI
+      ? [
+          'default',
+          [
+            'junit',
+            {
+              outputFile: './test-results/junit-web.xml',
+              suiteName: 'component-library (web)',
+            },
+          ],
+        ]
+      : ['default'],
   },
   plugins: [react()],
 });

packages/crdt/vitest.config.mts

@@ -7,5 +7,14 @@ export default defineConfig({
     environment: 'node',
     maxWorkers: 1,
     isolate: false,
+    reporters: process.env.CI
+      ? [
+          'default',
+          [
+            'junit',
+            { outputFile: './test-results/junit.xml', suiteName: 'crdt' },
+          ],
+        ]
+      : ['default'],
   },
 });

packages/desktop-client/playwright.config.ts

@@ -9,7 +9,7 @@ export default defineConfig({
   workers: process.env.CI ? 4 : undefined,
   testDir: 'e2e/',
   reporter: process.env.CI
-    ? [['blob'], ['list']]
+    ? [['blob'], ['list'], ['junit', { outputFile: 'test-results/junit.xml' }]]
     : [['html', { open: 'never' }]],
   use: {
     userAgent: 'playwright',

packages/desktop-client/vite.config.mts

@@ -408,6 +408,18 @@ export default defineConfig(async ({ mode, command }) => {
         return type === 'stderr';
       },
       maxWorkers: 2,
+      reporters: process.env.CI
+        ? [
+            'default',
+            [
+              'junit',
+              {
+                outputFile: './test-results/junit.xml',
+                suiteName: 'desktop-client',
+              },
+            ],
+          ]
+        : ['default'],
     },
   };
 });

packages/desktop-electron/playwright.config.ts

@@ -4,7 +4,9 @@ export default defineConfig({
   timeout: 60000, // 60 seconds
   retries: 1,
   testDir: 'e2e/',
-  reporter: undefined,
+  reporter: process.env.CI
+    ? [['list'], ['junit', { outputFile: 'e2e/test-results/junit.xml' }]]
+    : undefined,
   outputDir: 'e2e/test-results/',
   snapshotPathTemplate:
     '{testDir}/__screenshots__/{testFilePath}/{arg}-{platform}{ext}',

packages/eslint-plugin-actual/vitest.config.mts

@@ -7,5 +7,17 @@ export default defineConfig({
     environment: 'node',
     maxWorkers: 1,
     isolate: false,
+    reporters: process.env.CI
+      ? [
+          'default',
+          [
+            'junit',
+            {
+              outputFile: './test-results/junit.xml',
+              suiteName: 'eslint-plugin-actual',
+            },
+          ],
+        ]
+      : ['default'],
   },
 });

packages/loot-core/vitest.config.ts

@@ -15,6 +15,18 @@ export default defineConfig({
       return type === 'stderr';
     },
     maxWorkers: 2,
+    reporters: process.env.CI
+      ? [
+          'default',
+          [
+            'junit',
+            {
+              outputFile: './test-results/junit-node.xml',
+              suiteName: 'loot-core (node)',
+            },
+          ],
+        ]
+      : ['default'],
   },
   ssr: {
     resolve: { conditions: ['electron', 'module', 'node', 'development'] },

packages/loot-core/vitest.web.config.ts

@@ -10,6 +10,18 @@ export default defineConfig({
       'src/platform/server/fs/index.test.ts',
     ],
     maxWorkers: 2,
+    reporters: process.env.CI
+      ? [
+          'default',
+          [
+            'junit',
+            {
+              outputFile: './test-results/junit-web.xml',
+              suiteName: 'loot-core (web)',
+            },
+          ],
+        ]
+      : ['default'],
   },
   plugins: [peggyLoader()],
 });

packages/sync-server/vitest.config.ts

@@ -10,5 +10,17 @@ export default {
     // All test files share account.sqlite. Running files in parallel races on
     // the auth table's PRIMARY KEY (e.g. UNIQUE constraint failed: auth.method).
     fileParallelism: false,
+    reporters: process.env.CI
+      ? [
+          'default',
+          [
+            'junit',
+            {
+              outputFile: './test-results/junit.xml',
+              suiteName: 'sync-server',
+            },
+          ],
+        ]
+      : ['default'],
   },
 };

upcoming-release-notes/depot-test-results.md

@@ -0,0 +1,6 @@
+---
+category: Maintenance
+authors: [matt-fidd]
+---
+
+Report Vitest and Playwright test results to Depot for CI test observability.