fix: e2e testing posthog capture (#17940)

* fix: e2e testing posthog capture

* wat

* wat

* wat:

.github/workflows/ci-e2e.yml

@@ -222,12 +222,16 @@ jobs:
             - name: Cypress run
               # these are required checks so, we can't skip entire sections
               if: needs.changes.outputs.shouldTriggerCypress == 'true'
-              uses: cypress-io/github-action@v5
+              uses: cypress-io/github-action@v6
               with:
                   config-file: cypress.e2e.config.ts
                   config: retries=2
                   spec: ${{ matrix.chunk }}
                   install: false
+              env:
+                  E2E_TESTING: 1
+                  OPT_OUT_CAPTURE: 0
+                  GITHUB_ACTION_RUN_URL: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'
 
             - name: Archive test screenshots
               uses: actions/upload-artifact@v3

cypress.e2e.config.ts

@@ -37,6 +37,8 @@ export default defineConfig({
         // We've imported your old cypress plugins here.
         // You may want to clean this up later by importing these.
         setupNodeEvents(on, config) {
+            config.env.E2E_TESTING = !!process.env.E2E_TESTING
+
             const options = {
                 webpackOptions: createEntry('cypress'),
                 watchOptions: {},

cypress/support/e2e.ts

@@ -9,6 +9,8 @@ try {
     // eslint-disable-next-line no-empty
 } catch {}
 
+const E2E_TESTING = Cypress.env('E2E_TESTING')
+
 // Add console errors into cypress logs. This helps with failures in Github Actions which otherwise swallows them.
 // From: https://github.com/cypress-io/cypress/issues/300#issuecomment-688915086
 Cypress.on('window:before:load', (win) => {
@@ -62,21 +64,17 @@ beforeEach(() => {
     }
 })
 
-afterEach(() => {
-    if (process.env.E2E_TESTING) {
-        cy.window().then((win) => {
-            ;(win as any).posthog?.capture('e2e_testing_test_passed')
-        })
-    }
-})
+afterEach(function () {
+    const { state, duration } = this.currentTest
+    const event = state === 'passed' ? 'e2e_testing_test_passed' : 'e2e_testing_test_failed'
 
-Cypress.on('fail', (error: Cypress.CypressError) => {
-    if (process.env.E2E_TESTING) {
+    if (E2E_TESTING) {
+        cy.log(`E2E_TESTING: ${event} ${Cypress.spec.name}`, { state, duration })
         cy.window().then((win) => {
-            ;(win as any).posthog?.capture('e2e_testing_test_failed', { e2e_testing_error_message: error.message })
+            cy.log('has posthog on window', { window, posthog: (win as any).posthog })
+            ;(win as any).posthog?.capture(event, { state, duration })
         })
     }
-    throw error // so the test still fails
 })
 
 const resizeObserverLoopErrRe = /^[^(ResizeObserver loop limit exceeded)]/