From d66de8a47bc5f64c07635f6763c65579ea77d83c Mon Sep 17 00:00:00 2001 From: yyh <92089059+lyzno1@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:32:11 +0800 Subject: [PATCH] test(e2e): harden behavior coverage and CI gates (#39043) Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .github/workflows/main-ci.yml | 1 + .github/workflows/web-e2e.yml | 75 ++++- e2e/AGENTS.md | 19 ++ .../accessibility/keyboard-navigation.feature | 30 ++ e2e/features/agent-v2/AGENTS.md | 5 +- e2e/features/agent-v2/access-point.feature | 19 -- e2e/features/agent-v2/agent-edit.feature | 9 - e2e/features/agent-v2/build-draft.feature | 30 +- e2e/features/agent-v2/files.feature | 16 - .../agent-v2/output-variables.feature | 11 - e2e/features/apps/share-app.feature | 3 +- ...n-publish.feature => workflow-run.feature} | 7 +- e2e/features/auth/sign-out.feature | 7 - .../keyboard-navigation.steps.ts | 83 ++++++ .../access-point-service-api.steps.ts | 7 - .../agent-v2/access-point-web-app.steps.ts | 82 +---- .../agent-v2/access-point.steps.ts | 6 +- .../agent-v2/advanced-settings.steps.ts | 17 +- .../agent-v2/agent-edit.steps.ts | 36 +-- .../agent-v2/build-draft.steps.ts | 91 ------ .../agent-v2/configure-helpers.ts | 97 +++--- .../agent-v2/env-editor.steps.ts | 107 +++---- .../step-definitions/agent-v2/files.steps.ts | 45 +-- .../agent-v2/output-variables.steps.ts | 36 +-- .../agent-v2/preflight.steps.ts | 11 - .../agent-v2/workflow-node.steps.ts | 3 +- .../step-definitions/apps/create-app.steps.ts | 11 +- .../apps/duplicate-app.steps.ts | 9 +- .../apps/publish-app.steps.ts | 2 +- .../step-definitions/apps/share-app.steps.ts | 11 +- .../apps/workflow-run.steps.ts | 4 +- e2e/features/support/hooks.ts | 18 +- e2e/package.json | 2 +- e2e/scripts/env.ts | 2 + e2e/scripts/run-cucumber.ts | 49 ++- e2e/scripts/setup.ts | 4 + e2e/support/cleanup.ts | 3 + e2e/support/cucumber-report.ts | 279 ++++++++++++++++++ e2e/test-env.ts | 11 + e2e/tests/cleanup.test.ts | 15 +- e2e/tests/cucumber-report.test.ts | 214 ++++++++++++++ e2e/tests/test-env.test.ts | 16 + .../apps/app-card-operations-flow.test.tsx | 50 ++-- .../app/overview/__tests__/app-card.spec.tsx | 16 +- web/app/components/app/overview/app-card.tsx | 4 + .../apps/__tests__/app-card.spec.tsx | 8 +- web/app/components/apps/app-card.tsx | 10 +- .../account-dropdown/__tests__/index.spec.tsx | 8 + .../header/account-dropdown/index.tsx | 15 +- .../main-nav/components/account-section.tsx | 2 +- .../workflow/run/__tests__/status.spec.tsx | 2 +- .../workflow/run/status-container.tsx | 1 + .../components/orchestrate/advanced/env.tsx | 41 +-- .../__tests__/agent-roster-list.spec.tsx | 6 + .../roster/components/agent-roster-list.tsx | 5 +- web/i18n/en-US/common.json | 1 + 56 files changed, 1078 insertions(+), 594 deletions(-) create mode 100644 e2e/features/accessibility/keyboard-navigation.feature rename e2e/features/apps/{workflow-run-publish.feature => workflow-run.feature} (61%) create mode 100644 e2e/features/step-definitions/accessibility/keyboard-navigation.steps.ts create mode 100644 e2e/support/cucumber-report.ts create mode 100644 e2e/tests/cucumber-report.test.ts create mode 100644 e2e/tests/test-env.test.ts diff --git a/.github/workflows/main-ci.yml b/.github/workflows/main-ci.yml index ceb98abd305..9906827433a 100644 --- a/.github/workflows/main-ci.yml +++ b/.github/workflows/main-ci.yml @@ -106,6 +106,7 @@ jobs: - 'docker/docker-compose.middleware.yaml' - 'docker/envs/middleware.env.example' - '.github/workflows/web-e2e.yml' + - '.github/workflows/main-ci.yml' - '.github/actions/setup-web/**' vdb: - 'api/core/rag/datasource/**' diff --git a/.github/workflows/web-e2e.yml b/.github/workflows/web-e2e.yml index 43108462a09..22d20584e61 100644 --- a/.github/workflows/web-e2e.yml +++ b/.github/workflows/web-e2e.yml @@ -19,6 +19,7 @@ jobs: test: name: Web Full-Stack E2E runs-on: depot-ubuntu-24.04-4 + timeout-minutes: 120 defaults: run: shell: bash @@ -58,10 +59,57 @@ jobs: E2E_ADMIN_EMAIL: e2e-admin@example.com E2E_ADMIN_NAME: E2E Admin E2E_ADMIN_PASSWORD: E2eAdmin12345 + E2E_CUCUMBER_REPORT_PROFILE: core E2E_FORCE_WEB_BUILD: "1" E2E_INIT_PASSWORD: E2eInit12345 run: vp run e2e:full + - name: Preserve Chromium E2E report and logs + if: ${{ !cancelled() }} + run: | + if [[ -d e2e/cucumber-report ]]; then + mv e2e/cucumber-report e2e/cucumber-report-non-external + fi + if [[ -d e2e/.logs ]]; then + mv e2e/.logs e2e/.logs-non-external + fi + + - name: Run WebKit keyboard and browser smoke tests + working-directory: ./e2e + env: + E2E_ADMIN_EMAIL: e2e-admin@example.com + E2E_ADMIN_NAME: E2E Admin + E2E_ADMIN_PASSWORD: E2eAdmin12345 + E2E_BROWSER: webkit + E2E_CUCUMBER_REPORT_PROFILE: webkit-browser-smoke + E2E_INIT_PASSWORD: E2eInit12345 + run: | + teardown_webkit_smoke() { + local run_status=$? + trap - EXIT + if ! vp run e2e:middleware:down; then + echo "::error title=E2E teardown failed::WebKit smoke middleware did not shut down cleanly." + if [[ "$run_status" -eq 0 ]]; then + run_status=1 + fi + fi + exit "$run_status" + } + + trap teardown_webkit_smoke EXIT + vp run e2e:middleware:up + vp run e2e -- --tags '@browser-smoke' + + - name: Preserve WebKit E2E report and logs + if: ${{ !cancelled() }} + run: | + if [[ -d e2e/cucumber-report ]]; then + mv e2e/cucumber-report e2e/cucumber-report-webkit + fi + if [[ -d e2e/.logs ]]; then + mv e2e/.logs e2e/.logs-webkit + fi + - name: Run external runtime E2E tests if: ${{ inputs.run-external-runtime }} working-directory: ./e2e @@ -72,8 +120,8 @@ jobs: E2E_AGENT_DECISION_MODEL_NAME: ${{ vars.E2E_AGENT_DECISION_MODEL_NAME || 'gpt-5.5' }} E2E_AGENT_DECISION_MODEL_PROVIDER: ${{ vars.E2E_AGENT_DECISION_MODEL_PROVIDER || 'openai' }} E2E_AGENT_DECISION_MODEL_TYPE: ${{ vars.E2E_AGENT_DECISION_MODEL_TYPE || 'llm' }} + E2E_CUCUMBER_REPORT_PROFILE: external E2E_EXTERNAL_RUNTIME_SEED_SPECS: ${{ vars.E2E_EXTERNAL_RUNTIME_SEED_SPECS }} - E2E_EXTERNAL_RUNTIME_TAGS: ${{ vars.E2E_EXTERNAL_RUNTIME_TAGS }} E2E_FORCE_WEB_BUILD: "1" E2E_INIT_PASSWORD: E2eInit12345 E2E_MARKETPLACE_API_URL: ${{ vars.E2E_MARKETPLACE_API_URL }} @@ -102,7 +150,19 @@ jobs: mv .logs .logs-non-external fi - trap 'vp run e2e:middleware:down' EXIT + teardown_external_runtime() { + local run_status=$? + trap - EXIT + if ! vp run e2e:middleware:down; then + echo "::error title=E2E teardown failed::External runtime middleware did not shut down cleanly." + if [[ "$run_status" -eq 0 ]]; then + run_status=1 + fi + fi + exit "$run_status" + } + + trap teardown_external_runtime EXIT vp run e2e:middleware:up vp run e2e:external:prepare vp run e2e:external @@ -115,6 +175,7 @@ jobs: path: | e2e/cucumber-report e2e/cucumber-report-non-external + e2e/cucumber-report-webkit retention-days: 7 - name: Upload E2E logs @@ -125,5 +186,15 @@ jobs: path: | e2e/.logs/*.log e2e/.logs-non-external/*.log + e2e/.logs-webkit/*.log include-hidden-files: true retention-days: 7 + + - name: Upload E2E seed report + if: ${{ !cancelled() && inputs.run-external-runtime }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-seed-report + path: e2e/seed-report + if-no-files-found: ignore + retention-days: 7 diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 10eacdbe44e..a5eed78c83a 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -64,6 +64,9 @@ pnpm -C e2e e2e:headed -- --tags @smoke # slow down browser actions for local debugging E2E_SLOW_MO=500 pnpm -C e2e e2e:headed -- --tags @smoke + +# focused keyboard and cross-browser smoke coverage +E2E_BROWSER=webkit pnpm -C e2e e2e -- --tags @browser-smoke ``` Frontend artifact behavior: @@ -128,7 +131,11 @@ This removes: - `e2e/.auth` - `e2e/.logs` - `e2e/.logs-non-external` +- `e2e/.logs-webkit` - `e2e/cucumber-report` +- `e2e/cucumber-report-non-external` +- `e2e/cucumber-report-webkit` +- `e2e/seed-report` Start the full middleware stack: @@ -170,9 +177,20 @@ Artifacts and diagnostics: - `cucumber-report/report.html`: HTML report - `cucumber-report/report.json`: JSON report - `cucumber-report/artifacts/`: failure screenshots and HTML captures +- `cucumber-report-non-external/`: Chromium core report preserved before later CI lanes +- `cucumber-report-webkit/`: focused WebKit keyboard/browser smoke report - `.logs/cucumber-api.log`: backend startup log - `.logs/cucumber-web.log`: frontend startup log - `.logs-non-external/`: non-external logs preserved before an external CI run +- `.logs-webkit/`: focused WebKit lane logs +- `seed-report/`: JSON readiness reports emitted by external runtime seed packs + +CI enables a JSON report gate after Cucumber exits. The gate asserts minimum selected and passed +scenario counts, maximum skipped counts, and zero unexplained skips. A skipped scenario counts as +an explained blocked precondition only when its skipped step attaches a `Blocked precondition:` +reason. This keeps feature-gated and preflight readiness reporting visible without allowing a +zero-coverage or silently skipped CI run to pass. Set `E2E_CUCUMBER_REPORT_PROFILE` to select the +checked-in `core`, `webkit-browser-smoke`, or `external` thresholds and allowed blocked tags. Open the HTML report locally with: @@ -212,6 +230,7 @@ Feature: Create dataset - `@external-model` — scenario execution can call a real model provider. Use this only for runtime requests, not for scenarios that only require an active model fixture. - `@external-tool` — scenario execution can call a real third-party tool provider. Use this only for runtime tool execution, not for plugin installation, discovery, or local deterministic tools. - `@microphone` — runs the scenario in an isolated Chromium instance backed by the checked-in fake audio fixture and grants microphone permission only to that scenario context. +- `@browser-smoke` — focused keyboard and navigation coverage that runs in Chromium with the core suite and again in WebKit on CI. - `@skip` — excluded from all runs External runtime commands are opt-in. `pnpm -C e2e e2e:external:prepare` reads `E2E_EXTERNAL_RUNTIME_SEED_SPECS`, defaulting to `agent-v2:external-runtime`, and runs the matching seed packs before the external suite. `pnpm -C e2e e2e:external` reads `E2E_EXTERNAL_RUNTIME_TAGS`, defaulting to `(@external-model or @external-tool) and not @feature-gated and not @skip and not @preview`. diff --git a/e2e/features/accessibility/keyboard-navigation.feature b/e2e/features/accessibility/keyboard-navigation.feature new file mode 100644 index 00000000000..229552affa8 --- /dev/null +++ b/e2e/features/accessibility/keyboard-navigation.feature @@ -0,0 +1,30 @@ +@accessibility @keyboard @browser-smoke +Feature: Keyboard navigation + + @authenticated + Scenario: Skip repeated navigation and move focus to the main content + Given I am signed in as the default E2E admin + When I open the default console entry + And I focus and activate the skip navigation link with the keyboard + Then the console main content should have keyboard focus + + @unauthenticated + Scenario: Sign in by following the form tab order + Given I am not signed in + When I open the sign-in page + And I complete the sign-in form using only the keyboard + Then I should be on the console home + + @authenticated + Scenario: Closing the account menu restores focus to its trigger + Given I am signed in as the default E2E admin + When I open the apps console + And I open and close the account menu using the keyboard + Then the account menu trigger should regain keyboard focus + + @authenticated + Scenario: Closing the create app menu restores focus to its trigger + Given I am signed in as the default E2E admin + When I open the apps console + And I open and close the create app menu using the keyboard + Then the create app menu trigger should regain keyboard focus diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 26ab4bd1ae8..b02d662c5a1 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -26,9 +26,8 @@ Use tags in three layers: - `@core` — stable non-runtime scenario expected to run in the regular Agent v2 suite when its explicit preconditions are met. Do not apply `@core` to Preview/Test Run, Web app chat runtime, or Backend service API chat runtime scenarios. - `@infra` — infrastructure or readiness checks. - `@build` — Build mode and Build draft behavior. -- `@build-unavailable-resources` — feature-gated Build chat recovery when the user requests unavailable Skills or Tools. - `@files` — Files section upload, display, and fixture behavior. -- `@files-limits` — file limit behavior. Multiple-file drop is stable core coverage; format and size rejection remain feature-gated until their product contracts are stable. +- `@files-limits` — stable file limit behavior, currently covering multiple-file drop rejection. - `@knowledge` — Knowledge Retrieval configuration display, persistence, and reference cleanup. - `@advanced-settings` — Env Editor, Content Moderation, and related Advanced Settings behavior. - `@agent-create` — Agent Roster creation and initial Configure navigation. @@ -232,6 +231,6 @@ Order blocked steps by the real owner of the first unresolved condition. If a sc Use partial coverage only when current product behavior is intentionally narrower than the written requirement and the test still asserts a real user-visible behavior. Example: Files are currently flat in Agent config files, so the flat Files list can be asserted while tree display remains blocked until product support exists. -Multiple-file drop is already covered as stable `@core @files-limits` behavior. File format and size rejection remain feature-gated until the product exposes stable Agent config file restrictions and user-visible error states. Do not convert those gated `@files-limits` scenarios to passing tests by relying on default environment behavior; first align the product contract or seed configuration. +Multiple-file drop is covered as stable `@core @files-limits` behavior. Add file format or size rejection coverage only after the product exposes stable Agent config file restrictions and user-visible error states; do not encode undefined behavior as permanently skipped scenarios. Do not mark a scenario as complete if it only proves setup state and does not assert the user-visible behavior or persisted product contract required by the case. diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 431133e50ac..4bc480b00d3 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -16,7 +16,6 @@ Feature: Agent v2 Access Point When I open the Agent v2 configure page from the Agent Roster And I switch to the Agent v2 Access Point section Then I should see the Agent v2 Web app access URL - And I record the current Agent v2 orchestration draft When I copy the Agent v2 Web app access URL Then the Agent v2 Web app access URL should show it was copied And the current Agent v2 orchestration draft should be unchanged @@ -30,7 +29,6 @@ Feature: Agent v2 Access Point When I open the Agent v2 configure page from the Agent Roster And I switch to the Agent v2 Access Point section Then I should see the Agent v2 Web app access URL - And I record the current Agent v2 orchestration draft When I launch the Agent v2 Web app Then the Agent v2 Web app should open in a new tab And the current Agent v2 orchestration draft should be unchanged @@ -42,7 +40,6 @@ Feature: Agent v2 Access Point And Agent v2 Web app access has been enabled via API When I open the Agent v2 configure page from the Agent Roster And I switch to the Agent v2 Access Point section - And I record the current Agent v2 orchestration draft And I open Agent v2 Embedded configuration Then I should see the Agent v2 Embedded configuration dialog And the current Agent v2 orchestration draft should be unchanged @@ -54,7 +51,6 @@ Feature: Agent v2 Access Point And Agent v2 Web app access has been enabled via API When I open the Agent v2 configure page from the Agent Roster And I switch to the Agent v2 Access Point section - And I record the current Agent v2 orchestration draft And I open Agent v2 Web app customization Then I should see the Agent v2 Web app customization dialog And the current Agent v2 orchestration draft should be unchanged @@ -66,7 +62,6 @@ Feature: Agent v2 Access Point And Agent v2 Web app access has been enabled via API When I open the Agent v2 configure page from the Agent Roster And I switch to the Agent v2 Access Point section - And I record the current Agent v2 orchestration draft And I open Agent v2 Web app settings Then I should see the Agent v2 Web app settings dialog And the current Agent v2 orchestration draft should be unchanged @@ -86,20 +81,6 @@ Feature: Agent v2 Access Point When I refresh the current page Then Agent v2 Web app access should be in service - @web-app-access @published-web-app @feature-gated - Scenario: Disabled Web app public URL shows an unavailable state - Given I am signed in as the default E2E admin - And Agent v2 disabled Web app public unavailable state is available - And a basic configured Agent v2 test agent has been created via API - And the Agent v2 draft has been published via API - And Agent v2 Web app access has been enabled via API - When I open the Agent v2 configure page from the Agent Roster - And I switch to the Agent v2 Access Point section - And I disable Agent v2 Web app access - Then Agent v2 Web app access should be out of service - When I open the disabled Agent v2 Web app URL - Then the disabled Agent v2 Web app should show an unavailable state - @core @workflow-reference Scenario: Workflow access shows the referencing workflow Given I am signed in as the default E2E admin diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index f9fadd314dd..aa6e7800e8f 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -31,15 +31,6 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster Then I should see the Agent v2 tool state fixture tools - @tool-error-state @tool-states-agent @feature-gated - Scenario: Tool credential error states are visible on the Agent Edit page - Given I am signed in as the default E2E admin - And Agent v2 Tool credential error state is available - And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available - And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" includes the tool state fixture configuration - When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster - Then Agent v2 Tool credential error state should be available - @core @dual-retrieval-fixture Scenario: Dual Knowledge Retrieval settings are visible on the Agent Edit page Given I am signed in as the default E2E admin diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 0968a32ccc8..12b045486a3 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -50,23 +50,15 @@ Feature: Agent v2 build draft When I open the Agent v2 configure page Then I should see the Agent v2 Build draft pending changes And I should see the small Agent v2 file in the Files section - And I should see the e2e-summary-skill Skill in the Skills section - And I should see the supported E2E environment variable in Advanced Settings And the normal Agent v2 draft should still use the normal E2E prompt When I discard the Agent v2 Build draft - Then I should see the normal E2E prompt in the Agent v2 prompt editor - And I should not see the small Agent v2 file in the Files section - And I should not see the e2e-summary-skill Skill in the Skills section - And I should not see the supported E2E environment variable in Advanced Settings - And the Agent v2 draft should not include the supported Build draft config + Then the Agent v2 draft should not include the supported Build draft config And the Agent v2 Build draft should no longer be active When I refresh the current page Then I should see the normal E2E prompt in the Agent v2 prompt editor And I should not see the small Agent v2 file in the Files section And I should not see the e2e-summary-skill Skill in the Skills section And I should not see the supported E2E environment variable in Advanced Settings - And the Agent v2 draft should not include the supported Build draft config - And the Agent v2 Build draft should no longer be active @external-model @agent-backend-runtime @stable-model Scenario: Applying a pending Build draft updates the normal Agent configuration @@ -95,9 +87,6 @@ Feature: Agent v2 build draft When I open the Agent v2 configure page Then I should see the Agent v2 Build draft pending changes And I should see the updated E2E prompt in the Agent v2 prompt editor - And I should see the small Agent v2 file in the Files section - And I should see the e2e-summary-skill Skill in the Skills section - And I should see the supported E2E environment variable in Advanced Settings And the normal Agent v2 draft should still use the normal E2E prompt When I apply the Agent v2 Build draft via API Then the Agent v2 draft should include the supported Build draft config @@ -138,20 +127,3 @@ Feature: Agent v2 build draft Then I should see the Agent v2 Build draft pending changes And I should see the updated E2E prompt in the Agent v2 prompt editor And the normal Agent v2 draft should still use the normal E2E prompt - - @build-tool-writeback @feature-gated - Scenario: Applying a Build draft can add Dify Tools to the Agent configuration - Given I am signed in as the default E2E admin - And Agent v2 Build chat Dify Tool writeback is available - And a basic configured Agent v2 test agent has been created via API - When I open the Agent v2 configure page - Then Agent v2 Build chat Dify Tool writeback should be available - - @build-unavailable-resources @feature-gated @stable-model - Scenario: Build chat reports unavailable Skill or Tool requests clearly - Given I am signed in as the default E2E admin - And Agent v2 Build chat unavailable Skill and Tool recovery is available - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API - When I open the Agent v2 configure page - Then Agent v2 Build chat unavailable Skill and Tool recovery should be available diff --git a/e2e/features/agent-v2/files.feature b/e2e/features/agent-v2/files.feature index 233e57dc69e..cc770c66674 100644 --- a/e2e/features/agent-v2/files.feature +++ b/e2e/features/agent-v2/files.feature @@ -36,22 +36,6 @@ Feature: Agent v2 files When I refresh the current page Then I should see the special-name Agent v2 file in the Files section - @files-limits @feature-gated - Scenario: Unsupported Agent v2 file formats show a clear rejection reason - Given I am signed in as the default E2E admin - And Agent v2 unsupported file format rejection is available - And a basic configured Agent v2 test agent has been created via API - When I open the Agent v2 configure page - Then Agent v2 unsupported file format rejection should be available - - @files-limits @feature-gated - Scenario: Oversized Agent v2 files show a clear rejection reason - Given I am signed in as the default E2E admin - And Agent v2 oversized file rejection is available - And a basic configured Agent v2 test agent has been created via API - When I open the Agent v2 configure page - Then Agent v2 oversized file rejection should be available - @core @files-limits Scenario: Dropping multiple Agent v2 files at once is rejected Given I am signed in as the default E2E admin diff --git a/e2e/features/agent-v2/output-variables.feature b/e2e/features/agent-v2/output-variables.feature index 9410ff656ff..2ca7ce1e761 100644 --- a/e2e/features/agent-v2/output-variables.feature +++ b/e2e/features/agent-v2/output-variables.feature @@ -44,14 +44,3 @@ Feature: Agent v2 output variables When I refresh the current page And I open the Agent v2 workflow node panel Then the Agent v2 workflow node task should reference the renamed file output - - @output-reference-delete @feature-gated @stable-model - Scenario: Workflow Agent v2 prompt output reference deletion remains explicit - Given I am signed in as the default E2E admin - And Agent v2 workflow task output reference deletion consistency is available - And the Agent Builder stable chat model is available - And a workflow app with an Agent v2 node has been created via API - When I open the app from the app list - And I open the Agent v2 workflow node panel - And I insert a file output reference from the Agent v2 workflow node task editor - Then Agent v2 workflow task output reference deletion consistency should be available diff --git a/e2e/features/apps/share-app.feature b/e2e/features/apps/share-app.feature index 1c707306ef4..265599ecd16 100644 --- a/e2e/features/apps/share-app.feature +++ b/e2e/features/apps/share-app.feature @@ -1,6 +1,7 @@ -@apps @authenticated @core +@apps @core Feature: Share app publicly + @authenticated Scenario: Enable public share for a published workflow app Given I am signed in as the default E2E admin And a "workflow" app has been created via API diff --git a/e2e/features/apps/workflow-run-publish.feature b/e2e/features/apps/workflow-run.feature similarity index 61% rename from e2e/features/apps/workflow-run-publish.feature rename to e2e/features/apps/workflow-run.feature index 8640a7490b5..aedcdc3360c 100644 --- a/e2e/features/apps/workflow-run-publish.feature +++ b/e2e/features/apps/workflow-run.feature @@ -1,13 +1,10 @@ @apps @authenticated @core @mode-matrix -Feature: Workflow run and publish +Feature: Workflow run - Scenario: Run and publish a minimal workflow app + Scenario: Run a minimal workflow app Given I am signed in as the default E2E admin And a "workflow" app has been created via API And a minimal runnable workflow draft has been synced When I open the app from the app list And I run the workflow Then the workflow run should succeed - When I open the publish panel - And I publish the app - Then the app should be marked as published diff --git a/e2e/features/auth/sign-out.feature b/e2e/features/auth/sign-out.feature index 4446beaf766..e6b276faa64 100644 --- a/e2e/features/auth/sign-out.feature +++ b/e2e/features/auth/sign-out.feature @@ -1,12 +1,5 @@ @auth @authenticated @core Feature: Sign out - Scenario: Sign out from the apps console - Given I am signed in as the default E2E admin - When I open the apps console - And I open the account menu - And I sign out - Then I should be on the sign-in page - Scenario: Redirect back to sign-in when reopening the apps console after signing out Given I am signed in as the default E2E admin When I open the apps console diff --git a/e2e/features/step-definitions/accessibility/keyboard-navigation.steps.ts b/e2e/features/step-definitions/accessibility/keyboard-navigation.steps.ts new file mode 100644 index 00000000000..824c98ac130 --- /dev/null +++ b/e2e/features/step-definitions/accessibility/keyboard-navigation.steps.ts @@ -0,0 +1,83 @@ +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { adminCredentials } from '../../../fixtures/auth' +import { e2eBrowser } from '../../../test-env' + +const getAccountMenuTrigger = (world: DifyWorld) => + world.getPage().getByRole('button', { name: 'Account' }) + +const getCreateAppMenuTrigger = (world: DifyWorld) => + world.getPage().getByRole('main').getByRole('button', { name: 'Create', exact: true }) + +When( + 'I focus and activate the skip navigation link with the keyboard', + async function (this: DifyWorld) { + const page = this.getPage() + const skipLink = page.getByRole('link', { name: 'Skip to main content' }) + const nextClickableItemKey = e2eBrowser === 'webkit' ? 'Alt+Tab' : 'Tab' + + await page.keyboard.press(nextClickableItemKey) + await expect(skipLink).toBeFocused() + await page.keyboard.press('Enter') + }, +) + +Then('the console main content should have keyboard focus', async function (this: DifyWorld) { + await expect(this.getPage().getByRole('main')).toBeFocused() +}) + +When('I complete the sign-in form using only the keyboard', async function (this: DifyWorld) { + const page = this.getPage() + const email = page.getByLabel('Email address') + const password = page.getByLabel('Password', { exact: true }) + const showPassword = page.getByRole('button', { name: 'Show password' }) + const submit = page.getByRole('button', { name: 'Sign in' }) + + await expect(email).toBeVisible() + for (let tabPresses = 0; tabPresses < 10; tabPresses += 1) { + await page.keyboard.press('Tab') + if (await email.evaluate((element) => element === document.activeElement)) break + } + await expect(email).toBeFocused() + await page.keyboard.insertText(adminCredentials.email) + + await page.keyboard.press('Tab') + await expect(password).toBeFocused() + await page.keyboard.insertText(adminCredentials.password) + + await page.keyboard.press('Tab') + await expect(showPassword).toBeFocused() + await page.keyboard.press('Tab') + await expect(submit).toBeFocused() + await page.keyboard.press('Enter') +}) + +When('I open and close the account menu using the keyboard', async function (this: DifyWorld) { + const page = this.getPage() + const trigger = getAccountMenuTrigger(this) + + await expect(trigger).toBeEnabled() + await trigger.press('Enter') + await expect(trigger).toHaveAttribute('aria-expanded', 'true') + await page.keyboard.press('Escape') + await expect(trigger).toHaveAttribute('aria-expanded', 'false') +}) + +Then('the account menu trigger should regain keyboard focus', async function (this: DifyWorld) { + await expect(getAccountMenuTrigger(this)).toBeFocused() +}) + +When('I open and close the create app menu using the keyboard', async function (this: DifyWorld) { + const page = this.getPage() + const trigger = getCreateAppMenuTrigger(this) + + await trigger.press('Enter') + await expect(trigger).toHaveAttribute('aria-expanded', 'true') + await page.keyboard.press('Escape') + await expect(trigger).toHaveAttribute('aria-expanded', 'false') +}) + +Then('the create app menu trigger should regain keyboard focus', async function (this: DifyWorld) { + await expect(getCreateAppMenuTrigger(this)).toBeFocused() +}) diff --git a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts index 9252a063af9..32a58440429 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts @@ -144,13 +144,6 @@ Then( }, ) -When('I close Agent v2 API key management', async function (this: DifyWorld) { - const apiKeyDialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) - - await apiKeyDialog.getByLabel('Close').click() - await expect(apiKeyDialog).not.toBeVisible() -}) - When('I open the Agent v2 API Reference', async function (this: DifyWorld) { const page = this.getPage() const apiReferenceLink = page.getByRole('link', { name: 'API Reference' }) diff --git a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts index 12e3260d957..dabe1269416 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts @@ -1,16 +1,20 @@ import type { Page } from '@playwright/test' import type { DifyWorld } from '../../support/world' -import { Given, Then, When } from '@cucumber/cucumber' +import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { getAgentComposerDraft } from '../../agent-v2/support/agent' import { agentBuilderExpectedTokens } from '../../agent-v2/support/agent-builder-resources' -import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' import { getCurrentAgentId, getDialog, getWebAppCard } from './access-point-helpers' const WEB_APP_RUNTIME_RESPONSE_STEP_TIMEOUT_MS = 180_000 const getWebAppMessageInput = (webAppPage: Page) => webAppPage.getByPlaceholder(/^Talk to /).last() +const recordComposerDraftSnapshot = async (world: DifyWorld) => { + const draft = await getAgentComposerDraft(getCurrentAgentId(world)) + world.agentBuilder.accessPoint.composerDraftSnapshot = JSON.stringify(draft.agent_soul ?? {}) +} + Then('I should see the Agent v2 Web app access URL', async function (this: DifyWorld) { const webAppCard = getWebAppCard(this) @@ -20,13 +24,8 @@ Then('I should see the Agent v2 Web app access URL', async function (this: DifyW await expect(webAppCard.getByRole('link', { name: 'Launch' })).toBeVisible() }) -Then('I record the current Agent v2 orchestration draft', async function (this: DifyWorld) { - const draft = await getAgentComposerDraft(getCurrentAgentId(this)) - - this.agentBuilder.accessPoint.composerDraftSnapshot = JSON.stringify(draft.agent_soul ?? {}) -}) - When('I copy the Agent v2 Web app access URL', async function (this: DifyWorld) { + await recordComposerDraftSnapshot(this) await getWebAppCard(this).getByLabel('Copy access URL').click() }) @@ -35,6 +34,7 @@ Then('the Agent v2 Web app access URL should show it was copied', async function }) When('I launch the Agent v2 Web app', async function (this: DifyWorld) { + await recordComposerDraftSnapshot(this) const launchLink = getWebAppCard(this).getByRole('link', { name: 'Launch' }) const href = await launchLink.getAttribute('href') if (!href) throw new Error('Agent v2 Web app Launch link does not expose an href.') @@ -125,6 +125,7 @@ When('I close the Agent v2 Web app', async function (this: DifyWorld) { }) When('I open Agent v2 Embedded configuration', async function (this: DifyWorld) { + await recordComposerDraftSnapshot(this) await getWebAppCard(this).getByRole('button', { name: 'Embedded' }).click() }) @@ -137,6 +138,7 @@ Then('I should see the Agent v2 Embedded configuration dialog', async function ( }) When('I open Agent v2 Web app customization', async function (this: DifyWorld) { + await recordComposerDraftSnapshot(this) await getWebAppCard(this).getByRole('button', { name: 'Custom Frontend' }).click() }) @@ -149,6 +151,7 @@ Then('I should see the Agent v2 Web app customization dialog', async function (t }) When('I open Agent v2 Web app settings', async function (this: DifyWorld) { + await recordComposerDraftSnapshot(this) await getWebAppCard(this).getByRole('button', { name: 'Branding' }).click() }) @@ -172,66 +175,3 @@ Then( expect(JSON.stringify(draft.agent_soul ?? {})).toBe(snapshot) }, ) - -Given( - 'Agent v2 disabled Web app public unavailable state is available', - async function (this: DifyWorld) { - return skipBlockedPrecondition( - this, - 'Disabled Agent v2 Web app public URL does not expose a stable user-visible unavailable state; the current route redirects to Web app sign-in.', - { - owner: 'product', - remediation: - 'Define and implement the disabled public Web app UX before enabling this scenario.', - }, - ) - }, -) - -When('I open the disabled Agent v2 Web app URL', async function (this: DifyWorld) { - const webAppURL = this.agentBuilder.accessPoint.webAppURL - if (!webAppURL) throw new Error('No Agent v2 Web app URL was recorded.') - if (!this.context) throw new Error('Playwright browser context has not been initialized.') - - const webAppPage = await this.context.newPage() - await webAppPage.goto(webAppURL) - - this.agentBuilder.accessPoint.webAppPage = webAppPage -}) - -Then( - 'the disabled Agent v2 Web app should show an unavailable state', - async function (this: DifyWorld) { - const webAppPage = this.agentBuilder.accessPoint.webAppPage - if (!webAppPage) throw new Error('No Agent v2 Web app page was opened.') - - await expect(webAppPage.getByText(/app is unavailable|site is disabled/i)).toBeVisible({ - timeout: 30_000, - }) - await webAppPage.close() - this.agentBuilder.accessPoint.webAppPage = undefined - }, -) - -When('I open the restored Agent v2 Web app URL', async function (this: DifyWorld) { - const webAppURL = this.agentBuilder.accessPoint.webAppURL - if (!webAppURL) throw new Error('No Agent v2 Web app URL was recorded.') - if (!this.context) throw new Error('Playwright browser context has not been initialized.') - - const webAppPage = await this.context.newPage() - await webAppPage.goto(webAppURL) - - this.agentBuilder.accessPoint.webAppPage = webAppPage -}) - -Then( - 'the restored Agent v2 Web app should not show an unavailable state', - async function (this: DifyWorld) { - const webAppPage = this.agentBuilder.accessPoint.webAppPage - if (!webAppPage) throw new Error('No Agent v2 Web app page was opened.') - - await expect(webAppPage.getByText(/app is unavailable|site is disabled/i)).not.toBeVisible() - await webAppPage.close() - this.agentBuilder.accessPoint.webAppPage = undefined - }, -) diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index cdc4d01fbbb..61858c02dfb 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -3,7 +3,7 @@ import type { AccessSurfaceName } from './access-point-helpers' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { setAgentApiAccess, setAgentSiteAccessAndGetURL } from '../../agent-v2/support/access-point' -import { getAgentAccessPath, publishAgentWithPublishableDraft } from '../../agent-v2/support/agent' +import { publishAgentWithPublishableDraft } from '../../agent-v2/support/agent' import { getAccessRegion, getAccessSurfaceCard, @@ -31,10 +31,6 @@ Given( }, ) -When('I open the Agent v2 Access Point page', async function (this: DifyWorld) { - await this.getPage().goto(getAgentAccessPath(getCurrentAgentId(this))) -}) - When( 'I open the preseeded Agent v2 Access Point page for {string} from the Agent Roster', async function (this: DifyWorld, agentName: string) { diff --git a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts index b36f463f501..9d9d33044d0 100644 --- a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts +++ b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts @@ -1,21 +1,30 @@ import type { DifyWorld } from '../../support/world' import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { openAgentAdvancedSettings } from './configure-helpers' When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) { const page = this.getPage() const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + const trigger = advancedSettings + .getByRole('heading', { name: 'Advanced Settings' }) + .getByRole('button') - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(trigger).toHaveAttribute('aria-expanded', 'false') + await trigger.click() + await expect(trigger).toHaveAttribute('aria-expanded', 'true') await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() }) When('I collapse Agent v2 Advanced Settings', async function (this: DifyWorld) { const page = this.getPage() const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + const trigger = advancedSettings + .getByRole('heading', { name: 'Advanced Settings' }) + .getByRole('button') - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(trigger).toHaveAttribute('aria-expanded', 'true') + await trigger.click() + await expect(trigger).toHaveAttribute('aria-expanded', 'false') await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).not.toBeVisible() }) @@ -36,7 +45,7 @@ Then( Then( 'I should see the supported Agent v2 Advanced Settings entries', async function (this: DifyWorld) { - const advancedSettings = await openAgentAdvancedSettings(this.getPage()) + const advancedSettings = this.getPage().getByRole('region', { name: 'Advanced Settings' }) const envEditor = advancedSettings.getByRole('region', { name: 'Env Editor' }) await expect(envEditor).toBeVisible() diff --git a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts index fdb03d61fdf..3615dde0c1b 100644 --- a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts +++ b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts @@ -1,6 +1,6 @@ import type { PostAgentByAgentIdCopyResponse } from '@dify/contracts/api/console/agent/types.gen' import type { DifyWorld } from '../../support/world' -import { Given, Then, When } from '@cucumber/cucumber' +import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { createE2EResourceName } from '../../../support/naming' import { getAgentComposerDraft, getTestAgent } from '../../agent-v2/support/agent' @@ -10,12 +10,7 @@ import { agentBuilderPreseededResources, } from '../../agent-v2/support/agent-builder-resources' import { normalAgentPrompt } from '../../agent-v2/support/agent-soul' -import { - asArray, - asRecord, - asString, - skipBlockedPrecondition, -} from '../../agent-v2/support/preflight/common' +import { asArray, asRecord, asString } from '../../agent-v2/support/preflight/common' import { agentBuilderTestMaterials } from '../../agent-v2/support/test-materials' import { expectProviderToolActionVisible, @@ -78,12 +73,7 @@ When( const copyName = createE2EResourceName('Agent', 'copy') await page.goto('/agents') - const card = page - .locator('article') - .filter({ - has: page.getByRole('link', { name: agentName }), - }) - .first() + const card = page.getByRole('article', { name: agentName, exact: true }) await expect(card).toBeVisible({ timeout: 30_000 }) await card.hover() @@ -268,26 +258,6 @@ Then('I should see the Agent v2 tool state fixture tools', async function (this: ) }) -async function skipToolCredentialErrorState(world: DifyWorld) { - return skipBlockedPrecondition( - world, - 'Agent v2 Tool credential error state is not covered: the current fixture only proves usable and not-authorized tool states.', - { - owner: 'seed/product', - remediation: - 'Define a stable invalid credential fixture and the expected user-visible error label before enabling this scenario.', - }, - ) -} - -Given('Agent v2 Tool credential error state is available', async function (this: DifyWorld) { - return skipToolCredentialErrorState(this) -}) - -Then('Agent v2 Tool credential error state should be available', async function (this: DifyWorld) { - return skipToolCredentialErrorState(this) -}) - Then('I should see the Agent v2 dual retrieval fixture settings', async function (this: DifyWorld) { const page = this.getPage() const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts index 62f414ee0f4..fd9b2cdce4e 100644 --- a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -22,13 +22,10 @@ import { updatedAgentPrompt, updatedAgentSoulConfig, } from '../../agent-v2/support/agent-soul' -import { asArray, asRecord, skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' -import { hasToolEntry } from '../../agent-v2/support/preflight/tools' import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath, } from '../../agent-v2/support/test-materials' -import { getPreseededToolContract } from '../../agent-v2/support/tools' import { expectAgentModelRequiredFeedback, getAgentEnvVariableValue, @@ -286,54 +283,6 @@ When('I apply the Agent v2 Build draft via API', async function (this: DifyWorld await applyAgentBuildDraft(getCurrentAgentId(this)) }) -async function skipBuildDraftToolWriteback(world: DifyWorld) { - return skipBlockedPrecondition( - world, - 'Build chat Dify Tool writeback is not available: finalize/config mutation paths do not support tools; current passing coverage only verifies API-seeded Build draft apply for files, skills, and env.', - { - owner: 'product', - remediation: 'Define and implement Build draft Tool writeback before enabling this scenario.', - }, - ) -} - -Given('Agent v2 Build chat Dify Tool writeback is available', async function (this: DifyWorld) { - return skipBuildDraftToolWriteback(this) -}) - -Then( - 'Agent v2 Build chat Dify Tool writeback should be available', - async function (this: DifyWorld) { - return skipBuildDraftToolWriteback(this) - }, -) - -async function skipBuildDraftUnavailableResourceRecovery(world: DifyWorld) { - return skipBlockedPrecondition( - world, - 'Build chat unavailable Skill/Tool recovery is not covered: the product needs a stable user-visible failure state and deterministic request fixture before this can be automated.', - { - owner: 'product/seed', - remediation: - 'Define the unavailable-resource UX contract, then seed a stable model-backed prompt that requests a missing Skill and Tool without mutating the saved Agent config.', - }, - ) -} - -Given( - 'Agent v2 Build chat unavailable Skill and Tool recovery is available', - async function (this: DifyWorld) { - return skipBuildDraftUnavailableResourceRecovery(this) - }, -) - -Then( - 'Agent v2 Build chat unavailable Skill and Tool recovery should be available', - async function (this: DifyWorld) { - return skipBuildDraftUnavailableResourceRecovery(this) - }, -) - Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) { const page = this.getPage() @@ -438,26 +387,6 @@ Then( }, ) -Then( - 'the normal Agent v2 draft should not include the e2e-summary-skill Skill', - async function (this: DifyWorld) { - await expect - .poll( - async () => { - const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul - - return ( - agentSoul?.config_skills?.some( - (skill) => skill.name === agentBuilderPreseededResources.summarySkill, - ) ?? false - ) - }, - { timeout: 30_000 }, - ) - .toBe(false) - }, -) - Then( 'the normal Agent v2 draft should not include the generated build note', async function (this: DifyWorld) { @@ -471,26 +400,6 @@ Then( }, ) -Then( - 'the normal Agent v2 draft should not include the Agent Builder JSON Replace tool', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - const tool = getPreseededToolContract(this, agentBuilderPreseededResources.jsonReplaceTool) - - await expect - .poll( - async () => { - const draft = await getAgentComposerDraft(agentId) - const tools = asArray(asRecord(draft.agent_soul?.tools).dify_tools) - - return hasToolEntry(tools, tool) - }, - { timeout: 30_000 }, - ) - .toBe(false) - }, -) - Then( 'the Agent v2 draft should include the supported Build draft config', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/agent-v2/configure-helpers.ts b/e2e/features/step-definitions/agent-v2/configure-helpers.ts index 2b5771417e7..79ec578c64e 100644 --- a/e2e/features/step-definitions/agent-v2/configure-helpers.ts +++ b/e2e/features/step-definitions/agent-v2/configure-helpers.ts @@ -161,16 +161,68 @@ export const uploadSummaryConfigSkillForBuildDraft = async (world: DifyWorld) => export const openAgentAdvancedSettings = async (page: ReturnType) => { const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + const trigger = advancedSettings + .getByRole('heading', { name: 'Advanced Settings' }) + .getByRole('button') const envEditorHeading = advancedSettings.getByRole('heading', { name: 'Env Editor' }) - if (!(await envEditorHeading.isVisible().catch(() => false))) - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - + await expect(trigger).toHaveAttribute('aria-expanded', 'false') + await trigger.click() + await expect(trigger).toHaveAttribute('aria-expanded', 'true') await expect(envEditorHeading).toBeVisible() return advancedSettings } +const findAgentEnvVariableRows = async (advancedSettings: Locator, key: string) => { + const matchingRows: Locator[] = [] + + for (const row of await advancedSettings.getByRole('row').all()) { + const keyInput = row.getByRole('textbox', { name: 'Key' }) + if ((await keyInput.count()) === 1 && (await keyInput.inputValue()) === key) + matchingRows.push(row) + } + + return matchingRows +} + +export const getAgentEnvVariableRow = async (advancedSettings: Locator, key: string) => { + let matchingRows: Locator[] = [] + + await expect + .poll( + async () => { + matchingRows = await findAgentEnvVariableRows(advancedSettings, key) + return matchingRows.length + }, + { timeout: 30_000 }, + ) + .toBeGreaterThan(0) + + return matchingRows[0]! +} + +export const expectAgentEnvVariableAbsent = async (advancedSettings: Locator, key: string) => { + await expect + .poll(async () => (await findAgentEnvVariableRows(advancedSettings, key)).length) + .toBe(0) +} + +export const expectAgentEnvVariableRows = async ( + advancedSettings: Locator, + key: string, + value: string, +) => { + await getAgentEnvVariableRow(advancedSettings, key) + const variableRows = await findAgentEnvVariableRows(advancedSettings, key) + + for (const variableRow of variableRows) { + await expect(variableRow.getByRole('textbox', { name: 'Key' })).toHaveValue(key) + await expect(variableRow.getByRole('textbox', { name: 'Value' })).toHaveValue(value) + await expect(variableRow.getByText('Plain', { exact: true })).toBeVisible() + } +} + export const expectAgentEnvVariableVisible = async ( world: DifyWorld, key: string, @@ -178,44 +230,13 @@ export const expectAgentEnvVariableVisible = async ( ) => { const advancedSettings = await openAgentAdvancedSettings(world.getPage()) - await expect - .poll( - async () => { - const text = await advancedSettings.textContent() - const inputValues = await advancedSettings - .getByRole('textbox') - .evaluateAll((inputs) => inputs.map((input) => (input as HTMLInputElement).value)) - - return { - hasKey: inputValues.includes(key) || !!text?.includes(key), - hasValue: inputValues.includes(value) || !!text?.includes(value), - } - }, - { timeout: 30_000 }, - ) - .toEqual({ - hasKey: true, - hasValue: true, - }) - await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() + await expectAgentEnvVariableRows(advancedSettings, key, value) } export const expectAgentEnvVariableHidden = async (world: DifyWorld, key: string) => { const advancedSettings = await openAgentAdvancedSettings(world.getPage()) - await expect - .poll( - async () => { - const text = await advancedSettings.textContent() - const inputValues = await advancedSettings - .getByRole('textbox') - .evaluateAll((inputs) => inputs.map((input) => (input as HTMLInputElement).value)) - - return inputValues.includes(key) || !!text?.includes(key) - }, - { timeout: 30_000 }, - ) - .toBe(false) + await expectAgentEnvVariableAbsent(advancedSettings, key) } export const expectNormalAgentPromptDraft = async (world: DifyWorld) => { @@ -236,9 +257,11 @@ export const expectProviderToolActionVisible = async ( name: tool.providerName, }) await expect(provider).toBeVisible() + await expect(provider).toHaveAttribute('aria-expanded', 'false') + await provider.click() + await expect(provider).toHaveAttribute('aria-expanded', 'true') const action = toolsSection.getByText(tool.actionName, { exact: true }) - if (!(await action.isVisible())) await provider.click() await expect(action).toBeVisible() return { action, tool } diff --git a/e2e/features/step-definitions/agent-v2/env-editor.steps.ts b/e2e/features/step-definitions/agent-v2/env-editor.steps.ts index d304e96d5d8..38573073441 100644 --- a/e2e/features/step-definitions/agent-v2/env-editor.steps.ts +++ b/e2e/features/step-definitions/agent-v2/env-editor.steps.ts @@ -5,8 +5,11 @@ import { getAgentComposerDraft } from '../../agent-v2/support/agent' import { agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources' import { getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials' import { + expectAgentEnvVariableAbsent, expectAgentEnvVariableHidden, + expectAgentEnvVariableRows, expectAgentEnvVariableVisible, + getAgentEnvVariableRow, getAgentEnvVariables, getAgentEnvVariableValue, getCurrentAgentId, @@ -48,13 +51,16 @@ When( const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) await advancedSettings.getByRole('button', { name: 'Add environment variable' }).click() - await advancedSettings + const newVariableRow = await getAgentEnvVariableRow(advancedSettings, '') + await newVariableRow .getByRole('textbox', { name: 'Key' }) - .last() .fill(agentBuilderFixedInputs.envModeKey) - await advancedSettings + const savedVariableRow = await getAgentEnvVariableRow( + advancedSettings, + agentBuilderFixedInputs.envModeKey, + ) + await savedVariableRow .getByRole('textbox', { name: 'Value' }) - .last() .fill(agentBuilderFixedInputs.envModeValue) await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) }, @@ -231,22 +237,16 @@ Then( async function (this: DifyWorld) { const advancedSettings = await openAgentAdvancedSettings(this.getPage()) - await expect - .poll( - async () => - advancedSettings - .getByRole('textbox') - .evaluateAll((inputs) => inputs.map((input) => (input as HTMLInputElement).value)), - { timeout: 30_000 }, - ) - .toEqual( - expect.arrayContaining([ - agentBuilderFixedInputs.envPlainKey, - agentBuilderFixedInputs.envPlainValue, - agentBuilderFixedInputs.envModeKey, - agentBuilderFixedInputs.envModeValue, - ]), - ) + await expectAgentEnvVariableRows( + advancedSettings, + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + ) + await expectAgentEnvVariableRows( + advancedSettings, + agentBuilderFixedInputs.envModeKey, + agentBuilderFixedInputs.envModeValue, + ) await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) }, ) @@ -256,29 +256,12 @@ Then( async function (this: DifyWorld) { const advancedSettings = await openAgentAdvancedSettings(this.getPage()) - await expect - .poll( - async () => - advancedSettings - .getByRole('textbox') - .evaluateAll((inputs) => inputs.map((input) => (input as HTMLInputElement).value)), - { timeout: 30_000 }, - ) - .toEqual( - expect.arrayContaining([ - agentBuilderFixedInputs.envModeKey, - agentBuilderFixedInputs.envModeValue, - ]), - ) - await expect - .poll( - async () => - advancedSettings - .getByRole('textbox') - .evaluateAll((inputs) => inputs.map((input) => (input as HTMLInputElement).value)), - { timeout: 30_000 }, - ) - .not.toContain(agentBuilderFixedInputs.envPlainKey) + await expectAgentEnvVariableRows( + advancedSettings, + agentBuilderFixedInputs.envModeKey, + agentBuilderFixedInputs.envModeValue, + ) + await expectAgentEnvVariableAbsent(advancedSettings, agentBuilderFixedInputs.envPlainKey) await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(1) }, ) @@ -288,14 +271,18 @@ Then( async function (this: DifyWorld) { const page = this.getPage() const advancedSettings = await openAgentAdvancedSettings(page) - - await expect(advancedSettings.getByRole('textbox', { name: 'Key' })).toHaveValue( + const variableRow = await getAgentEnvVariableRow( + advancedSettings, agentBuilderFixedInputs.envPlainKey, ) - await expect(advancedSettings.getByRole('textbox', { name: 'Value' })).toHaveValue( + + await expect(variableRow.getByRole('textbox', { name: 'Key' })).toHaveValue( + agentBuilderFixedInputs.envPlainKey, + ) + await expect(variableRow.getByRole('textbox', { name: 'Value' })).toHaveValue( agentBuilderFixedInputs.envPlainValue, ) - await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() + await expect(variableRow.getByText('Plain', { exact: true })).toBeVisible() await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() }, ) @@ -324,22 +311,16 @@ Then( const page = this.getPage() const advancedSettings = await openAgentAdvancedSettings(page) - await expect - .poll( - async () => - advancedSettings - .getByRole('textbox') - .evaluateAll((inputs) => inputs.map((input) => (input as HTMLInputElement).value)), - { timeout: 30_000 }, - ) - .toEqual( - expect.arrayContaining([ - agentBuilderFixedInputs.envPlainKey, - agentBuilderFixedInputs.envPlainValue, - agentBuilderFixedInputs.envAfterInvalidImportKey, - agentBuilderFixedInputs.envAfterInvalidImportValue, - ]), - ) + await expectAgentEnvVariableRows( + advancedSettings, + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + ) + await expectAgentEnvVariableRows( + advancedSettings, + agentBuilderFixedInputs.envAfterInvalidImportKey, + agentBuilderFixedInputs.envAfterInvalidImportValue, + ) await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() }, ) diff --git a/e2e/features/step-definitions/agent-v2/files.steps.ts b/e2e/features/step-definitions/agent-v2/files.steps.ts index 52399c31c72..e47b5a7fa65 100644 --- a/e2e/features/step-definitions/agent-v2/files.steps.ts +++ b/e2e/features/step-definitions/agent-v2/files.steps.ts @@ -1,7 +1,6 @@ import type { DifyWorld } from '../../support/world' -import { Given, Then, When } from '@cucumber/cucumber' +import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' import { agentBuilderTestMaterials } from '../../agent-v2/support/test-materials' import { expectAgentConfigFileHidden, @@ -124,45 +123,3 @@ Then( await expectAgentConfigFileSaved(this, 'specialFilename') }, ) - -async function skipUnsupportedFileFormatRejection(world: DifyWorld) { - return skipBlockedPrecondition( - world, - 'Agent v2 unsupported file format rejection is not stable: default upload configuration allows arbitrary extensions unless UPLOAD_FILE_EXTENSION_BLACKLIST is seeded.', - { - owner: 'product/seed', - remediation: - 'Define Agent config file type restrictions or seed UPLOAD_FILE_EXTENSION_BLACKLIST before enabling this scenario.', - }, - ) -} - -Given('Agent v2 unsupported file format rejection is available', async function (this: DifyWorld) { - return skipUnsupportedFileFormatRejection(this) -}) - -Then( - 'Agent v2 unsupported file format rejection should be available', - async function (this: DifyWorld) { - return skipUnsupportedFileFormatRejection(this) - }, -) - -async function skipOversizedFileRejection(world: DifyWorld) { - return skipBlockedPrecondition( - world, - 'Agent v2 oversized file rejection lacks a clear user-visible reason: the current upload dialog collapses upload and commit failures into a generic failure toast.', - { - owner: 'product', - remediation: 'Expose a stable user-visible file-size error before enabling this scenario.', - }, - ) -} - -Given('Agent v2 oversized file rejection is available', async function (this: DifyWorld) { - return skipOversizedFileRejection(this) -}) - -Then('Agent v2 oversized file rejection should be available', async function (this: DifyWorld) { - return skipOversizedFileRejection(this) -}) diff --git a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts index c55e5598fab..790ac481543 100644 --- a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts +++ b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts @@ -1,10 +1,9 @@ import type { DataTable } from '@cucumber/cucumber' import type { DeclaredOutputConfig } from '@dify/contracts/api/console/apps/types.gen' import type { AgentV2WorkflowOutputVariable, DifyWorld } from '../../support/world' -import { Given, Then, When } from '@cucumber/cucumber' +import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { getWorkflowDraft } from '../../../support/api' -import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' const agentV2WorkflowNodeId = 'agent-v2' const taskFileOutputName = 'e2e_report.pdf' @@ -51,11 +50,12 @@ const waitForWorkflowDraftSave = (world: DifyWorld, appId: string) => const openWorkflowOutputVariablesPanel = async (world: DifyWorld) => { const page = world.getPage() + const outputVariablesButton = page.getByRole('button', { name: 'Output Variables' }) const newOutputButton = page.getByRole('button', { name: 'New output' }) - if (!(await newOutputButton.isVisible().catch(() => false))) - await page.getByRole('button', { name: 'Output Variables' }).click() - + await expect(outputVariablesButton).toHaveAttribute('aria-expanded', 'false') + await outputVariablesButton.click() + await expect(outputVariablesButton).toHaveAttribute('aria-expanded', 'true') await expect(newOutputButton).toBeVisible() } @@ -345,29 +345,3 @@ async function expectAgentTaskOutputReference( await expect(page.getByText('file', { exact: true })).toBeVisible() if (unexpectedName) await expect(page.getByText(unexpectedName, { exact: true })).toHaveCount(0) } - -async function skipWorkflowTaskOutputReferenceDeletionConsistency(world: DifyWorld) { - return skipBlockedPrecondition( - world, - 'Agent v2 workflow task output deletion consistency is not available: deleting an output from the list currently leaves the Prompt token without a stable user-visible invalid-reference state.', - { - owner: 'product', - remediation: - 'Define whether deletion should sync the Prompt token, block deletion, or expose an invalid-reference state before enabling this scenario.', - }, - ) -} - -Given( - 'Agent v2 workflow task output reference deletion consistency is available', - async function (this: DifyWorld) { - return skipWorkflowTaskOutputReferenceDeletionConsistency(this) - }, -) - -Then( - 'Agent v2 workflow task output reference deletion consistency should be available', - async function (this: DifyWorld) { - return skipWorkflowTaskOutputReferenceDeletionConsistency(this) - }, -) diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index 89e3d2515d2..5f5747cd5ab 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -17,7 +17,6 @@ import { } from '../../agent-v2/support/preflight/agents' import { skipMissingIndexingPreseededDataset, - skipMissingPreseededDataset, skipMissingReadyPreseededDataset, } from '../../agent-v2/support/preflight/datasets' import { @@ -81,16 +80,6 @@ Given( }, ) -Given( - 'the Agent Builder preseeded dataset {string} is available', - async function (this: DifyWorld, resourceName: string) { - const resource = await skipMissingPreseededDataset(this, resourceName) - if (resource === 'skipped') return resource - - this.agentBuilder.preflight.preseededResources[resourceName] = resource - }, -) - Given( 'the Agent Builder preseeded dataset {string} is indexed and ready', async function (this: DifyWorld, resourceName: string) { diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts index 89075a2fbda..cbe375c180a 100644 --- a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -36,8 +36,7 @@ Given( When('I open the Agent v2 workflow node panel', async function (this: DifyWorld) { const page = this.getPage() - const workflowCanvas = page.locator('#workflow-container') - const agentNode = workflowCanvas.getByRole('button', { name: 'Agent' }).first() + const agentNode = page.getByRole('button', { name: 'Agent', exact: true }) await expect(agentNode).toBeVisible({ timeout: 30_000 }) await agentNode.click() diff --git a/e2e/features/step-definitions/apps/create-app.steps.ts b/e2e/features/step-definitions/apps/create-app.steps.ts index d6fd5bd3729..ee5824c1074 100644 --- a/e2e/features/step-definitions/apps/create-app.steps.ts +++ b/e2e/features/step-definitions/apps/create-app.steps.ts @@ -16,8 +16,8 @@ When('I enter a unique E2E app name', async function (this: DifyWorld) { When('I confirm app creation', async function (this: DifyWorld) { const createButton = this.getPage() + .getByRole('dialog') .getByRole('button', { name: /^Create(?:\s|$)/ }) - .last() await expect(createButton).toBeEnabled() await createButton.click() @@ -25,12 +25,9 @@ When('I confirm app creation', async function (this: DifyWorld) { When('I select the {string} app type', async function (this: DifyWorld, appType: string) { const dialog = this.getPage().getByRole('dialog') - // The modal defaults to ADVANCED_CHAT, so the preview panel immediately renders - //

Chatflow

alongside the card's
Chatflow
. - // locator('div').getByText(...) would still match the

because getByText - // searches inside each div for any descendant. Use :text-is() instead, which - // targets only
elements whose own normalised text equals appType exactly. - const appTypeCard = dialog.locator(`div:text-is("${appType}")`) + const appTypeCard = dialog.getByRole('button', { + name: new RegExp(`^${appType.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`), + }) await expect(appTypeCard).toBeVisible() await appTypeCard.click() diff --git a/e2e/features/step-definitions/apps/duplicate-app.steps.ts b/e2e/features/step-definitions/apps/duplicate-app.steps.ts index bc5fd2ddee0..b347ddb0216 100644 --- a/e2e/features/step-definitions/apps/duplicate-app.steps.ts +++ b/e2e/features/step-definitions/apps/duplicate-app.steps.ts @@ -17,14 +17,9 @@ When('I open the options menu for the last created E2E app', async function (thi const page = this.getPage() const appLink = page.getByRole('link', { name: appName, exact: true }) - const appCard = page - .locator('div') - .filter({ has: appLink }) - .filter({ has: page.getByRole('button', { name: 'More' }) }) - .last() await expect(appLink).toBeVisible() - await appCard.hover() - await appCard.getByRole('button', { name: 'More' }).click() + await appLink.hover() + await page.getByRole('button', { name: `More actions for ${appName}`, exact: true }).click() }) When('I click {string} in the app options menu', async function (this: DifyWorld, label: string) { diff --git a/e2e/features/step-definitions/apps/publish-app.steps.ts b/e2e/features/step-definitions/apps/publish-app.steps.ts index c426bc4c5a1..aca00b220e4 100644 --- a/e2e/features/step-definitions/apps/publish-app.steps.ts +++ b/e2e/features/step-definitions/apps/publish-app.steps.ts @@ -3,7 +3,7 @@ import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' When('I open the publish panel', async function (this: DifyWorld) { - await this.getPage().getByRole('button', { name: 'Publish' }).first().click() + await this.getPage().getByRole('button', { name: 'Publish', exact: true }).click() }) When('I publish the app', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/apps/share-app.steps.ts b/e2e/features/step-definitions/apps/share-app.steps.ts index 2a99151f720..b216b518696 100644 --- a/e2e/features/step-definitions/apps/share-app.steps.ts +++ b/e2e/features/step-definitions/apps/share-app.steps.ts @@ -21,12 +21,17 @@ When('I enable the Web App share', async function (this: DifyWorld) { } await page.getByRole('button', { name: new RegExp(escapeRegExp(appName)) }).click() - await expect(page.getByRole('switch').first()).toBeEnabled({ timeout: 15_000 }) - await page.getByRole('switch').first().click() + const webAppCard = page.getByRole('region', { name: 'Web App' }) + const webAppSwitch = webAppCard.getByRole('switch', { name: 'Web App' }) + await expect(webAppSwitch).toBeEnabled({ timeout: 15_000 }) + await webAppSwitch.click() }) Then('the Web App should be in service', async function (this: DifyWorld) { - await expect(this.getPage().getByText('In Service').first()).toBeVisible({ timeout: 10_000 }) + const webAppCard = this.getPage().getByRole('region', { name: 'Web App' }) + await expect(webAppCard.getByText('In Service', { exact: true })).toBeVisible({ + timeout: 10_000, + }) }) Given('a workflow app has been published and shared via API', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/apps/workflow-run.steps.ts b/e2e/features/step-definitions/apps/workflow-run.steps.ts index c225591d691..d9cf64bb9c3 100644 --- a/e2e/features/step-definitions/apps/workflow-run.steps.ts +++ b/e2e/features/step-definitions/apps/workflow-run.steps.ts @@ -21,5 +21,7 @@ When('I run the workflow', async function (this: DifyWorld) { Then('the workflow run should succeed', async function (this: DifyWorld) { const page = this.getPage() await page.getByText('DETAIL', { exact: true }).click() - await expect(page.getByText('SUCCESS', { exact: true }).first()).toBeVisible({ timeout: 55_000 }) + await expect(page.getByRole('status').getByText('SUCCESS', { exact: true })).toBeVisible({ + timeout: 55_000, + }) }) diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index ebafbb9892a..5077834a6ea 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -6,14 +6,14 @@ import { mkdir, writeFile } from 'node:fs/promises' import path from 'node:path' import { fileURLToPath } from 'node:url' import { After, AfterAll, Before, BeforeAll, setDefaultTimeout, Status } from '@cucumber/cucumber' -import { chromium } from '@playwright/test' +import { chromium, webkit } from '@playwright/test' import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtures/auth' import { deleteTestApp } from '../../support/api' -import { runCleanupTasks } from '../../support/cleanup' +import { runCleanupTasks, shouldFailForCleanupErrors } from '../../support/cleanup' import { deleteTestDataset } from '../../support/datasets' import { getVoiceInputTestMaterialPath } from '../../support/test-materials' import { deleteBuiltinToolCredential } from '../../support/tools' -import { baseURL, cucumberHeadless, cucumberSlowMo } from '../../test-env' +import { baseURL, cucumberHeadless, cucumberSlowMo, e2eBrowser } from '../../test-env' import { deleteTestAgent } from '../agent-v2/support/agent' import { deleteAgentConfigFile, @@ -91,12 +91,13 @@ const captureDiagnosticPage = async ( BeforeAll({ timeout: AUTH_BOOTSTRAP_TIMEOUT_MS }, async () => { await mkdir(artifactsDir, { recursive: true }) - browser = await chromium.launch({ + const browserType = e2eBrowser === 'webkit' ? webkit : chromium + browser = await browserType.launch({ headless: cucumberHeadless, slowMo: cucumberSlowMo, }) - console.warn(`[e2e] session cache bootstrap against ${baseURL}`) + console.warn(`[e2e] ${e2eBrowser} session cache bootstrap against ${baseURL}`) await ensureAuthenticatedState(browser, baseURL) }) @@ -120,6 +121,9 @@ Before(async function (this: DifyWorld, { pickle }) { const scenarioTags = pickle.tags.map((tag) => tag.name) const isMicrophoneScenario = scenarioTags.includes('@microphone') const isUnauthenticatedScenario = scenarioTags.includes('@unauthenticated') + if (isMicrophoneScenario && e2eBrowser !== 'chromium') + throw new Error('Microphone scenarios require E2E_BROWSER=chromium.') + const scenarioBrowser = isMicrophoneScenario ? await getMicrophoneBrowser() : browser if (isUnauthenticatedScenario) await this.startUnauthenticatedSession(scenarioBrowser) @@ -151,7 +155,7 @@ After( const message = `Cleanup errors:\n${closeErrors.join('\n')}` this.attach(message, 'text/plain') - if (result?.status === Status.PASSED) throw new Error(message) + if (shouldFailForCleanupErrors(result?.status)) throw new Error(message) }, ) @@ -196,7 +200,7 @@ After( const message = `Cleanup errors:\n${cleanupErrors.join('\n')}` this.attach(message, 'text/plain') - if (result?.status === Status.PASSED) throw new Error(message) + if (shouldFailForCleanupErrors(result?.status)) throw new Error(message) }, ) diff --git a/e2e/package.json b/e2e/package.json index 0f0e9f71553..f6e1b3bd099 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -9,7 +9,7 @@ "e2e:full": "tsx ./scripts/run-cucumber.ts --full", "e2e:full:headed": "tsx ./scripts/run-cucumber.ts --full --headed", "e2e:headed": "tsx ./scripts/run-cucumber.ts --headed", - "e2e:install": "playwright install --with-deps chromium", + "e2e:install": "playwright install --with-deps chromium webkit", "e2e:middleware:down": "tsx ./scripts/setup.ts middleware-down", "e2e:middleware:up": "tsx ./scripts/setup.ts middleware-up", "e2e:reset": "tsx ./scripts/setup.ts reset", diff --git a/e2e/scripts/env.ts b/e2e/scripts/env.ts index 5d33516426f..0302265259c 100644 --- a/e2e/scripts/env.ts +++ b/e2e/scripts/env.ts @@ -81,6 +81,7 @@ export const validateE2eEnv = () => E2E_AGENT_DECISION_MODEL_TYPE: process.env.E2E_AGENT_DECISION_MODEL_TYPE, E2E_API_URL: process.env.E2E_API_URL, E2E_BASE_URL: process.env.E2E_BASE_URL, + E2E_BROWSER: process.env.E2E_BROWSER, E2E_BROKEN_MODEL_NAME: process.env.E2E_BROKEN_MODEL_NAME, E2E_BROKEN_MODEL_PROVIDER: process.env.E2E_BROKEN_MODEL_PROVIDER, E2E_BROKEN_MODEL_TYPE: process.env.E2E_BROKEN_MODEL_TYPE, @@ -123,6 +124,7 @@ export const validateE2eEnv = () => E2E_AGENT_DECISION_MODEL_TYPE: z.string().min(1).optional(), E2E_API_URL: z.url().optional(), E2E_BASE_URL: z.url().optional(), + E2E_BROWSER: z.enum(['chromium', 'webkit']).optional(), E2E_BROKEN_MODEL_NAME: z.string().min(1).optional(), E2E_BROKEN_MODEL_PROVIDER: z.string().min(1).optional(), E2E_BROKEN_MODEL_TYPE: z.string().min(1).optional(), diff --git a/e2e/scripts/run-cucumber.ts b/e2e/scripts/run-cucumber.ts index 1b26926c53f..93c05e10d15 100644 --- a/e2e/scripts/run-cucumber.ts +++ b/e2e/scripts/run-cucumber.ts @@ -1,6 +1,13 @@ import type { ManagedProcess } from '../support/process' import { mkdir, readFile, rm } from 'node:fs/promises' import path from 'node:path' +import { runCleanupTasks } from '../support/cleanup' +import { + assertCucumberReport, + formatCucumberReportSummary, + getCucumberReportGate, + readCucumberReportSummary, +} from '../support/cucumber-report' import { startLoggedProcess, stopManagedProcess, waitForUrl } from '../support/process' import { startWebServer, stopWebServer } from '../support/web-server' import { apiURL, baseURL, reuseExistingWebServer } from '../test-env' @@ -150,19 +157,17 @@ const main = async () => { const cleanup = async () => { if (!cleanupPromise) { cleanupPromise = (async () => { - await stopWebServer() - await stopManagedProcess(celeryProcess) - await stopManagedProcess(apiProcess) - await stopManagedProcess(difyAgentProcess) - await stopManagedProcess(shellctlProcess) + const cleanupErrors = await runCleanupTasks([ + { label: 'Stop web server', run: stopWebServer }, + { label: 'Stop celery worker', run: () => stopManagedProcess(celeryProcess) }, + { label: 'Stop API server', run: () => stopManagedProcess(apiProcess) }, + { label: 'Stop agent backend', run: () => stopManagedProcess(difyAgentProcess) }, + { label: 'Stop shellctl sandbox', run: () => stopManagedProcess(shellctlProcess) }, + ...(startMiddlewareForRun ? [{ label: 'Stop middleware', run: stopMiddleware }] : []), + ]) - if (startMiddlewareForRun) { - try { - await stopMiddleware() - } catch { - // Cleanup should continue even if middleware shutdown fails. - } - } + if (cleanupErrors.length > 0) + throw new Error(`E2E teardown errors:\n${cleanupErrors.join('\n')}`) })() } @@ -170,9 +175,13 @@ const main = async () => { } const onTerminate = () => { - void cleanup().finally(() => { - process.exit(1) - }) + void cleanup() + .catch((error) => { + console.error(error instanceof Error ? error.message : String(error)) + }) + .finally(() => { + process.exit(1) + }) } process.once('SIGINT', onTerminate) @@ -265,6 +274,16 @@ const main = async () => { env: cucumberEnv, }) + const reportGate = getCucumberReportGate(cucumberEnv) + if (reportGate) { + const reportPath = path.join(cucumberReportDir, 'report.json') + const reportSummary = await readCucumberReportSummary(reportPath) + console.warn( + `[e2e] cucumber report ${reportGate.profile}: ${formatCucumberReportSummary(reportSummary)}`, + ) + assertCucumberReport(reportSummary, reportGate) + } + process.exitCode = result.exitCode } finally { process.off('SIGINT', onTerminate) diff --git a/e2e/scripts/setup.ts b/e2e/scripts/setup.ts index 4bf42ca0a91..c8478290f19 100644 --- a/e2e/scripts/setup.ts +++ b/e2e/scripts/setup.ts @@ -55,9 +55,13 @@ const middlewareDataPaths = [ const e2eStatePaths = [ path.join(e2eDir, '.auth'), path.join(e2eDir, 'cucumber-report'), + path.join(e2eDir, 'cucumber-report-non-external'), + path.join(e2eDir, 'cucumber-report-webkit'), path.join(e2eDir, '.logs'), path.join(e2eDir, '.logs-non-external'), + path.join(e2eDir, '.logs-webkit'), path.join(e2eDir, 'playwright-report'), + path.join(e2eDir, 'seed-report'), path.join(e2eDir, 'test-results'), ] diff --git a/e2e/support/cleanup.ts b/e2e/support/cleanup.ts index 147c59a1f79..6bb85ab90ba 100644 --- a/e2e/support/cleanup.ts +++ b/e2e/support/cleanup.ts @@ -3,6 +3,9 @@ export type CleanupTask = { run: () => Promise | void } +export const shouldFailForCleanupErrors = (status: string | undefined) => + status === 'PASSED' || status === 'SKIPPED' + export async function runCleanupTasks(tasks: CleanupTask[]): Promise { const errors: string[] = [] diff --git a/e2e/support/cucumber-report.ts b/e2e/support/cucumber-report.ts new file mode 100644 index 00000000000..4fd582b9f76 --- /dev/null +++ b/e2e/support/cucumber-report.ts @@ -0,0 +1,279 @@ +import { Buffer } from 'node:buffer' +import { readFile } from 'node:fs/promises' + +type CucumberEmbedding = { + data?: string + mime_type?: string +} + +type CucumberStep = { + embeddings?: CucumberEmbedding[] + hidden?: boolean + result?: { + status?: string + } +} + +type CucumberScenario = { + name?: string + steps?: CucumberStep[] + tags?: { name?: string }[] + type?: string +} + +export type CucumberReport = { + elements?: CucumberScenario[] + uri?: string +}[] + +export type CucumberReportSummary = { + blockedScenarios: { + name: string + tags: string[] + uri: string + }[] + blockedSkipped: number + failed: number + other: number + passed: number + selected: number + skipped: number + unexpectedSkipped: number +} + +export type CucumberReportGate = { + allowedBlockedScenarios: Record + maxSkipped: number + maxUnexpectedSkipped: number + minPassed: number + minSelected: number + profile: string +} + +const reportGateProfiles = { + core: { + allowedBlockedScenarios: { + 'features/agent-v2/access-point.feature': ['Workflow access shows the referencing workflow'], + 'features/agent-v2/advanced-settings.feature': [ + 'Content Moderation keyword preset replies are saved and restored', + ], + 'features/agent-v2/agent-edit.feature': [ + 'Saved orchestration sections are visible on the Agent Edit page', + 'Duplicated Agent inherits configuration without changing the original Agent', + 'Tool states are visible on the Agent Edit page', + 'Dual Knowledge Retrieval settings are visible on the Agent Edit page', + 'Agent Edit opens the same Agent in Agent Console', + ], + 'features/agent-v2/configure-persistence.feature': [ + 'Selecting a stable model in Configure persists after refresh', + 'Persisted Agent v2 instructions remain visible after refresh', + ], + 'features/agent-v2/knowledge.feature': [ + 'Agent decide Knowledge Retrieval settings are saved and restored', + 'Custom query Knowledge Retrieval settings are saved and restored', + 'Removing Knowledge Retrieval clears the saved dataset reference', + ], + 'features/agent-v2/output-variables.feature': [ + 'Workflow Agent v2 output variables persist after refresh', + 'Workflow Agent v2 nested object output variables persist after refresh', + 'Workflow Agent v2 prompt output reference stays synced when renamed', + ], + 'features/agent-v2/preflight.feature': [ + 'Stable chat model is available', + 'Default speech-to-text model is available', + 'Agent-decision chat model is available', + 'Broken chat model is available for recovery scenarios', + 'JSON Replace tool is available', + 'Tavily Search tool is available', + 'Agent knowledge base is available', + 'Indexing knowledge base is available', + 'Full config Agent is available', + 'Full config Agent includes the summary Skill', + 'Full config Agent includes core fixture configuration', + 'Content Moderation Settings is enabled', + 'Tool states Agent is available', + 'Tool states Agent includes tool state fixture configuration', + 'OAuth2 tool Agent includes credential fixture configuration', + 'Dual retrieval Agent is available', + 'Dual retrieval Agent includes dual retrieval fixture configuration', + 'Published Web app Agent exposes Web app access', + 'Backend API-enabled Agent is available', + 'Backend API-enabled Agent exposes API access with a key', + 'Workflow reference Agent is available', + 'Reference workflow is available', + 'Workflow reference Agent is used by the reference workflow', + ], + 'features/agent-v2/publish.feature': [ + 'Publish a configured Agent v2 draft', + 'Publish action follows unpublished changes', + 'Published Agent v2 version remains isolated from draft edits', + 'Restoring a published Agent v2 version shows the restored configuration in Builder', + ], + 'features/agent-v2/tools.feature': [ + 'JSON Replace tool is saved after adding it from the Tools selector', + 'OAuth2 tool credentials stay authorized after Configure autosaves', + ], + }, + maxSkipped: 44, + maxUnexpectedSkipped: 0, + minPassed: 65, + minSelected: 109, + }, + external: { + allowedBlockedScenarios: {}, + maxSkipped: 0, + maxUnexpectedSkipped: 0, + minPassed: 11, + minSelected: 11, + }, + 'webkit-browser-smoke': { + allowedBlockedScenarios: {}, + maxSkipped: 0, + maxUnexpectedSkipped: 0, + minPassed: 4, + minSelected: 4, + }, +} satisfies Record> + +export const getCucumberReportGate = (env: NodeJS.ProcessEnv): CucumberReportGate | undefined => { + const profile = env.E2E_CUCUMBER_REPORT_PROFILE?.trim() + if (!profile) return undefined + + const gate = reportGateProfiles[profile as keyof typeof reportGateProfiles] + if (!gate) throw new Error(`Unknown Cucumber report gate profile "${profile}".`) + + return { + ...gate, + allowedBlockedScenarios: Object.fromEntries( + Object.entries(gate.allowedBlockedScenarios).map(([uri, names]) => [uri, [...names]]), + ), + profile, + } +} + +const failureStatuses = new Set(['ambiguous', 'failed', 'pending', 'undefined', 'unknown']) + +const hasBlockedPrecondition = (steps: CucumberStep[]) => + steps + .filter((step) => !step.hidden && step.result?.status?.toLowerCase() === 'skipped') + .flatMap((step) => step.embeddings || []) + .filter((embedding) => embedding.mime_type === 'text/plain' && embedding.data) + .some((embedding) => { + const contents = Buffer.from(embedding.data || '', 'base64').toString('utf8') + return contents.startsWith('Blocked precondition:') + }) + +export const summarizeCucumberReport = (report: CucumberReport): CucumberReportSummary => { + const summary: CucumberReportSummary = { + blockedScenarios: [], + blockedSkipped: 0, + failed: 0, + other: 0, + passed: 0, + selected: 0, + skipped: 0, + unexpectedSkipped: 0, + } + + for (const feature of report) { + for (const scenario of feature.elements || []) { + if (scenario.type !== 'scenario') continue + + summary.selected += 1 + const steps = scenario.steps || [] + const statuses = steps + .map((step) => step.result?.status?.toLowerCase()) + .filter((status): status is string => Boolean(status)) + + if (statuses.some((status) => failureStatuses.has(status))) { + summary.failed += 1 + continue + } + + if (steps.length === 0 || statuses.length !== steps.length) { + summary.other += 1 + continue + } + + if (statuses.includes('skipped')) { + summary.skipped += 1 + if (hasBlockedPrecondition(steps)) { + summary.blockedSkipped += 1 + summary.blockedScenarios.push({ + name: scenario.name || '', + tags: (scenario.tags || []).flatMap((tag) => (tag.name ? [tag.name] : [])), + uri: feature.uri || '', + }) + } else summary.unexpectedSkipped += 1 + continue + } + + if (statuses.length > 0 && statuses.every((status) => status === 'passed')) { + summary.passed += 1 + continue + } + + summary.other += 1 + } + } + + return summary +} + +export const readCucumberReportSummary = async (reportPath: string) => { + const contents = await readFile(reportPath, 'utf8') + const report = JSON.parse(contents) as CucumberReport + return summarizeCucumberReport(report) +} + +export const assertCucumberReport = (summary: CucumberReportSummary, gate: CucumberReportGate) => { + const errors: string[] = [] + const allowedBlockedScenarios = new Set( + Object.entries(gate.allowedBlockedScenarios).flatMap(([uri, names]) => + names.map((name) => `${uri}\0${name}`), + ), + ) + const disallowedBlockedScenarios = summary.blockedScenarios.filter( + (scenario) => !allowedBlockedScenarios.has(`${scenario.uri}\0${scenario.name}`), + ) + + if (summary.selected < gate.minSelected) + errors.push(`selected scenarios ${summary.selected} is below minimum ${gate.minSelected}`) + if (summary.passed < gate.minPassed) + errors.push(`passed scenarios ${summary.passed} is below minimum ${gate.minPassed}`) + if (summary.skipped > gate.maxSkipped) + errors.push(`skipped scenarios ${summary.skipped} exceeds maximum ${gate.maxSkipped}`) + if (summary.unexpectedSkipped > gate.maxUnexpectedSkipped) { + errors.push( + `unexpected skipped scenarios ${summary.unexpectedSkipped} exceeds maximum ${gate.maxUnexpectedSkipped}`, + ) + } + if (disallowedBlockedScenarios.length > 0) { + errors.push( + `blocked scenarios not present in the checked-in allowlist: ${disallowedBlockedScenarios + .map((scenario) => `${scenario.uri}: ${scenario.name}`) + .join(', ')}`, + ) + } + if (summary.failed > 0) errors.push(`failed scenarios ${summary.failed} exceeds maximum 0`) + if (summary.other > 0) errors.push(`unclassified scenarios ${summary.other} exceeds maximum 0`) + + if (errors.length > 0) + throw new Error( + [ + `Cucumber report gate "${gate.profile}" failed:`, + ...errors.map((error) => `- ${error}`), + ].join('\n'), + ) +} + +export const formatCucumberReportSummary = (summary: CucumberReportSummary) => + [ + `selected=${summary.selected}`, + `passed=${summary.passed}`, + `skipped=${summary.skipped}`, + `blockedSkipped=${summary.blockedSkipped}`, + `unexpectedSkipped=${summary.unexpectedSkipped}`, + `failed=${summary.failed}`, + `other=${summary.other}`, + ].join(' ') diff --git a/e2e/test-env.ts b/e2e/test-env.ts index c0afc2a8c1f..0af395909b9 100644 --- a/e2e/test-env.ts +++ b/e2e/test-env.ts @@ -2,11 +2,22 @@ export const defaultBaseURL = 'http://127.0.0.1:3000' export const defaultApiURL = 'http://127.0.0.1:5001' export const defaultLocale = 'en-US' +export const supportedE2EBrowsers = ['chromium', 'webkit'] as const +export type E2EBrowser = (typeof supportedE2EBrowsers)[number] + +export const resolveE2EBrowser = (value: string | undefined): E2EBrowser => { + if (value === undefined) return 'chromium' + if (supportedE2EBrowsers.some((browser) => browser === value)) return value as E2EBrowser + + throw new Error(`Unsupported E2E browser "${value}".`) +} + export const baseURL = process.env.E2E_BASE_URL || defaultBaseURL export const apiURL = process.env.E2E_API_URL || defaultApiURL export const cucumberHeadless = process.env.CUCUMBER_HEADLESS !== '0' export const cucumberSlowMo = Number(process.env.E2E_SLOW_MO || 0) +export const e2eBrowser = resolveE2EBrowser(process.env.E2E_BROWSER) export const reuseExistingWebServer = process.env.E2E_REUSE_WEB_SERVER ? process.env.E2E_REUSE_WEB_SERVER !== '0' : !process.env.CI diff --git a/e2e/tests/cleanup.test.ts b/e2e/tests/cleanup.test.ts index 30eb0a37b73..486cb676f38 100644 --- a/e2e/tests/cleanup.test.ts +++ b/e2e/tests/cleanup.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest' -import { runCleanupTasks } from '../support/cleanup' +import { runCleanupTasks, shouldFailForCleanupErrors } from '../support/cleanup' describe('runCleanupTasks', () => { it('runs every task in order', async () => { @@ -47,3 +47,16 @@ describe('runCleanupTasks', () => { expect(errors).toEqual(['sync cleanup: sync failure', 'async cleanup: async failure']) }) }) + +describe('shouldFailForCleanupErrors', () => { + it.each(['PASSED', 'SKIPPED'])('fails a %s scenario when cleanup fails', (status) => { + expect(shouldFailForCleanupErrors(status)).toBe(true) + }) + + it.each(['FAILED', 'AMBIGUOUS', 'PENDING', 'UNDEFINED', 'UNKNOWN'])( + 'preserves an existing %s result when cleanup fails', + (status) => { + expect(shouldFailForCleanupErrors(status)).toBe(false) + }, + ) +}) diff --git a/e2e/tests/cucumber-report.test.ts b/e2e/tests/cucumber-report.test.ts new file mode 100644 index 00000000000..473ade1b814 --- /dev/null +++ b/e2e/tests/cucumber-report.test.ts @@ -0,0 +1,214 @@ +import { Buffer } from 'node:buffer' +import { describe, expect, it } from 'vitest' +import { + assertCucumberReport, + getCucumberReportGate, + summarizeCucumberReport, +} from '../support/cucumber-report' + +const step = (status?: string, options: { blockedReason?: string; hidden?: boolean } = {}) => ({ + ...(options.blockedReason + ? { + embeddings: [ + { + data: Buffer.from(`Blocked precondition: ${options.blockedReason}`).toString('base64'), + mime_type: 'text/plain', + }, + ], + } + : {}), + ...(options.hidden ? { hidden: true } : {}), + ...(status ? { result: { status } } : {}), +}) + +const scenario = (name: string, steps: ReturnType[], tags: string[] = []) => ({ + name, + steps, + tags: tags.map((tag) => ({ name: tag })), + type: 'scenario', +}) + +describe('summarizeCucumberReport', () => { + it('separates passed, explicitly blocked, and unexpected skipped scenarios', () => { + const summary = summarizeCucumberReport([ + { + elements: [ + scenario('passes', [step('passed')]), + scenario( + 'blocked', + [step('passed'), step('skipped', { blockedReason: 'fixture' })], + ['@fixture'], + ), + scenario('unexpected skip', [step('skipped')]), + ], + uri: 'features/example.feature', + }, + ]) + + expect(summary).toEqual({ + blockedScenarios: [ + { + name: 'blocked', + tags: ['@fixture'], + uri: 'features/example.feature', + }, + ], + blockedSkipped: 1, + failed: 0, + other: 0, + passed: 1, + selected: 3, + skipped: 2, + unexpectedSkipped: 1, + }) + }) + + it('treats a failed hook as a failed scenario', () => { + const summary = summarizeCucumberReport([ + { + elements: [scenario('cleanup fails', [step('passed'), step('failed', { hidden: true })])], + uri: 'features/example.feature', + }, + ]) + + expect(summary.failed).toBe(1) + expect(summary.passed).toBe(0) + }) + + it('classifies a scenario with a missing step status as unclassified', () => { + const summary = summarizeCucumberReport([ + { + elements: [scenario('missing status', [step('passed', { hidden: true }), step()])], + uri: 'features/example.feature', + }, + ]) + + expect(summary.other).toBe(1) + expect(summary.passed).toBe(0) + }) +}) + +describe('assertCucumberReport', () => { + it('accepts an explicitly blocked readiness report within configured limits', () => { + const summary = { + blockedScenarios: [ + { name: 'fixture blocked', tags: ['@fixture'], uri: 'features/example.feature' }, + { name: 'preflight blocked', tags: ['@preflight'], uri: 'features/example.feature' }, + ], + blockedSkipped: 2, + failed: 0, + other: 0, + passed: 3, + selected: 5, + skipped: 2, + unexpectedSkipped: 0, + } + + expect(() => + assertCucumberReport(summary, { + allowedBlockedScenarios: { + 'features/example.feature': ['fixture blocked', 'preflight blocked'], + }, + maxSkipped: 2, + maxUnexpectedSkipped: 0, + minPassed: 3, + minSelected: 5, + profile: 'core', + }), + ).not.toThrow() + }) + + it('rejects zero coverage, all-skipped coverage, and unexplained skips', () => { + const summary = { + blockedScenarios: [ + { name: 'fixture blocked', tags: ['@fixture'], uri: 'features/example.feature' }, + ], + blockedSkipped: 1, + failed: 0, + other: 0, + passed: 0, + selected: 2, + skipped: 2, + unexpectedSkipped: 1, + } + + expect(() => + assertCucumberReport(summary, { + allowedBlockedScenarios: { + 'features/example.feature': ['fixture blocked'], + }, + maxSkipped: 2, + maxUnexpectedSkipped: 0, + minPassed: 1, + minSelected: 3, + profile: 'external', + }), + ).toThrow( + [ + 'Cucumber report gate "external" failed:', + '- selected scenarios 2 is below minimum 3', + '- passed scenarios 0 is below minimum 1', + '- unexpected skipped scenarios 1 exceeds maximum 0', + ].join('\n'), + ) + }) + + it('rejects a blocked scenario that replaces an allowed scenario with the same dependency tag', () => { + const summary = { + blockedScenarios: [ + { name: 'core regression', tags: ['@fixture'], uri: 'features/example.feature' }, + ], + blockedSkipped: 1, + failed: 0, + other: 0, + passed: 1, + selected: 2, + skipped: 1, + unexpectedSkipped: 0, + } + + expect(() => + assertCucumberReport(summary, { + allowedBlockedScenarios: { + 'features/example.feature': ['fixture blocked'], + }, + maxSkipped: 1, + maxUnexpectedSkipped: 0, + minPassed: 1, + minSelected: 2, + profile: 'core', + }), + ).toThrow( + 'blocked scenarios not present in the checked-in allowlist: features/example.feature: core regression', + ) + }) +}) + +describe('getCucumberReportGate', () => { + it('returns no gate unless a profile is configured', () => { + expect(getCucumberReportGate({})).toBeUndefined() + }) + + it('returns the checked-in gate for a known profile', () => { + expect( + getCucumberReportGate({ + E2E_CUCUMBER_REPORT_PROFILE: 'webkit-browser-smoke', + }), + ).toEqual({ + allowedBlockedScenarios: {}, + maxSkipped: 0, + maxUnexpectedSkipped: 0, + minPassed: 4, + minSelected: 4, + profile: 'webkit-browser-smoke', + }) + }) + + it('rejects an unknown profile instead of silently weakening the gate', () => { + expect(() => + getCucumberReportGate({ + E2E_CUCUMBER_REPORT_PROFILE: 'custom', + }), + ).toThrow('Unknown Cucumber report gate profile "custom".') + }) +}) diff --git a/e2e/tests/test-env.test.ts b/e2e/tests/test-env.test.ts new file mode 100644 index 00000000000..ae3c530d83a --- /dev/null +++ b/e2e/tests/test-env.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest' +import { resolveE2EBrowser } from '../test-env' + +describe('resolveE2EBrowser', () => { + it('uses Chromium by default', () => { + expect(resolveE2EBrowser(undefined)).toBe('chromium') + }) + + it.each(['chromium', 'webkit'] as const)('accepts %s', (browser) => { + expect(resolveE2EBrowser(browser)).toBe(browser) + }) + + it('rejects unsupported browsers', () => { + expect(() => resolveE2EBrowser('firefox')).toThrow('Unsupported E2E browser "firefox".') + }) +}) diff --git a/web/__tests__/apps/app-card-operations-flow.test.tsx b/web/__tests__/apps/app-card-operations-flow.test.tsx index e0322e58f10..b38ecb92b74 100644 --- a/web/__tests__/apps/app-card-operations-flow.test.tsx +++ b/web/__tests__/apps/app-card-operations-flow.test.tsx @@ -10,7 +10,8 @@ * - Access mode icons */ import type { App } from '@/types/app' -import { fireEvent, screen, waitFor } from '@testing-library/react' +import { screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' import { beforeEach, describe, expect, it, vi } from 'vitest' import { renderWithSystemFeatures } from '@/__tests__/utils/mock-system-features' import { AppCard } from '@/app/components/apps/app-card' @@ -278,8 +279,14 @@ const renderAppCard = (app?: Partial) => { }) } -const openOperationsMenu = () => { - fireEvent.click(screen.getByRole('button', { name: 'common.operation.more' })) +const openOperationsMenu = async (appName = 'Test Chat App') => { + const user = userEvent.setup() + await user.click( + screen.getByRole('button', { + name: `common.operation.moreActionsFor:{"name":"${appName}"}`, + }), + ) + return user } describe('App Card Operations Flow', () => { @@ -334,15 +341,15 @@ describe('App Card Operations Flow', () => { it('should show delete confirmation and call API on confirm', async () => { renderAppCard({ id: 'app-to-delete', name: 'Deletable App' }) - openOperationsMenu() - fireEvent.click(await screen.findByText('common.operation.delete')) + const user = await openOperationsMenu('Deletable App') + await user.click(await screen.findByRole('menuitem', { name: 'common.operation.delete' })) await waitFor(() => { expect(screen.getByText('app.deleteAppConfirmTitle')).toBeInTheDocument() }) - fireEvent.change(screen.getByRole('textbox'), { target: { value: 'Deletable App' } }) - fireEvent.click(screen.getByRole('button', { name: 'common.operation.confirm' })) + await user.type(screen.getByRole('textbox'), 'Deletable App') + await user.click(screen.getByRole('button', { name: 'common.operation.confirm' })) await waitFor(() => { expect(mockDeleteAppMutation).toHaveBeenCalledWith('app-to-delete') @@ -355,9 +362,9 @@ describe('App Card Operations Flow', () => { it('should open edit modal and call updateAppInfo on confirm', async () => { renderAppCard({ id: 'app-edit', name: 'Editable App' }) - openOperationsMenu() - fireEvent.click(await screen.findByText('app.editApp')) - fireEvent.click(await screen.findByTestId('confirm-edit')) + const user = await openOperationsMenu('Editable App') + await user.click(await screen.findByRole('menuitem', { name: 'app.editApp' })) + await user.click(await screen.findByRole('button', { name: 'Confirm' })) await waitFor(() => { expect(updateAppInfo).toHaveBeenCalledWith( @@ -375,8 +382,8 @@ describe('App Card Operations Flow', () => { it('should call exportAppConfig for completion apps', async () => { renderAppCard({ id: 'app-export', mode: AppModeEnum.COMPLETION, name: 'Export App' }) - openOperationsMenu() - fireEvent.click(await screen.findByText('app.export')) + const user = await openOperationsMenu('Export App') + await user.click(await screen.findByRole('menuitem', { name: 'app.export' })) await waitFor(() => { expect(exportAppConfig).toHaveBeenCalledWith( @@ -392,7 +399,9 @@ describe('App Card Operations Flow', () => { renderAppCard({ name: 'Readonly App', created_by: 'another-user', permission_keys: [] }) expect( - screen.queryByRole('button', { name: 'common.operation.more' }), + screen.queryByRole('button', { + name: /common\.operation\.moreActionsFor/, + }), ).not.toBeInTheDocument() }) }) @@ -402,21 +411,16 @@ describe('App Card Operations Flow', () => { it('should show switch option for chat mode apps', async () => { renderAppCard({ id: 'app-switch', mode: AppModeEnum.CHAT }) - openOperationsMenu() - - await waitFor(() => { - expect(screen.queryByText('app.switch')).toBeInTheDocument() - }) + await openOperationsMenu() + expect(await screen.findByRole('menuitem', { name: 'app.switch' })).toBeVisible() }) it('should not show switch option for workflow apps', async () => { renderAppCard({ id: 'app-wf', mode: AppModeEnum.WORKFLOW, name: 'WF App' }) - openOperationsMenu() - - await waitFor(() => { - expect(screen.queryByText('app.switch')).not.toBeInTheDocument() - }) + await openOperationsMenu('WF App') + expect(await screen.findByRole('menu')).toBeVisible() + expect(screen.queryByRole('menuitem', { name: 'app.switch' })).not.toBeInTheDocument() }) }) }) diff --git a/web/app/components/app/overview/__tests__/app-card.spec.tsx b/web/app/components/app/overview/__tests__/app-card.spec.tsx index a6ecc0d6829..f3b2f79ed08 100644 --- a/web/app/components/app/overview/__tests__/app-card.spec.tsx +++ b/web/app/components/app/overview/__tests__/app-card.spec.tsx @@ -1,6 +1,6 @@ import type { ReactElement } from 'react' import type { AppDetailResponse } from '@/models/app' -import { fireEvent, screen, waitFor } from '@testing-library/react' +import { fireEvent, screen, waitFor, within } from '@testing-library/react' import { renderWithSystemFeatures } from '@/__tests__/utils/mock-system-features' import { InputVarType } from '@/app/components/workflow/types' import { AccessMode } from '@/models/access-control' @@ -181,6 +181,20 @@ describe('AppCard', () => { ) }) + it('should expose the web app controls as a named region', () => { + render() + + const webAppCard = screen.getByRole('region', { + name: /(?:^|\.)overview\.appInfo\.title(?=$|:)/, + }) + + expect( + within(webAppCard).getByRole('switch', { + name: /(?:^|\.)overview\.appInfo\.title(?=$|:)/, + }), + ).toBeInTheDocument() + }) + it('should open the workflow web app directly when launch is clicked even with hidden inputs', () => { mockWorkflow = { graph: { diff --git a/web/app/components/app/overview/app-card.tsx b/web/app/components/app/overview/app-card.tsx index b96a15ae417..f7f6703f7a3 100644 --- a/web/app/components/app/overview/app-card.tsx +++ b/web/app/components/app/overview/app-card.tsx @@ -292,6 +292,8 @@ function AppCard({ return (
) : ( { expect(screen.queryByRole('link', { name: 'Preview Only App' })).not.toBeInTheDocument() expect(screen.queryByRole('button', { name: 'app.studio.starApp' })).not.toBeInTheDocument() expect( - screen.queryByRole('button', { name: 'common.operation.more' }), + screen.queryByRole('button', { + name: /common\.operation\.moreActionsFor/, + }), ).not.toBeInTheDocument() fireEvent.click(tagSelector) @@ -617,7 +619,9 @@ describe('AppCard', () => { ).not.toBeInTheDocument() expect(screen.queryByRole('button', { name: 'app.studio.starApp' })).not.toBeInTheDocument() expect( - screen.queryByRole('button', { name: 'common.operation.more' }), + screen.queryByRole('button', { + name: /common\.operation\.moreActionsFor/, + }), ).not.toBeInTheDocument() fireEvent.click(card) diff --git a/web/app/components/apps/app-card.tsx b/web/app/components/apps/app-card.tsx index c4bd32479d4..61df5be1490 100644 --- a/web/app/components/apps/app-card.tsx +++ b/web/app/components/apps/app-card.tsx @@ -643,7 +643,10 @@ export function AppCardActionBar({ app, onRefresh }: AppCardActionBarProps) { onOpenChange={setIsOperationsMenuOpen} > $['operation.more'], { ns: 'common' })} + aria-label={t(($) => $['operation.moreActionsFor'], { + ns: 'common', + name: app.name, + })} className={cn( 'flex h-8 w-8 cursor-pointer items-center justify-center rounded-lg focus-visible:ring-2 focus-visible:ring-state-accent-solid focus-visible:outline-hidden', isOperationsMenuOpen ? 'bg-state-base-hover' : 'hover:bg-state-base-hover', @@ -1271,7 +1274,10 @@ export function AppCard({ onOpenChange={setIsOperationsMenuOpen} > $['operation.more'], { ns: 'common' })} + aria-label={t(($) => $['operation.moreActionsFor'], { + ns: 'common', + name: app.name, + })} className={cn( 'flex h-8 w-8 cursor-pointer items-center justify-center rounded-lg focus-visible:ring-2 focus-visible:ring-state-accent-solid focus-visible:outline-hidden', isOperationsMenuOpen ? 'bg-state-base-hover' : 'hover:bg-state-base-hover', diff --git a/web/app/components/header/account-dropdown/__tests__/index.spec.tsx b/web/app/components/header/account-dropdown/__tests__/index.spec.tsx index 0d67357e949..cf38adc01b3 100644 --- a/web/app/components/header/account-dropdown/__tests__/index.spec.tsx +++ b/web/app/components/header/account-dropdown/__tests__/index.spec.tsx @@ -3,6 +3,7 @@ import type { AppContextStateMockState } from '@/__tests__/utils/mock-app-contex import type { ModalContextState } from '@/context/modal-context' import type { ProviderContextState } from '@/context/provider-context' import { fireEvent, screen, waitFor } from '@testing-library/react' +import { renderToString } from 'react-dom/server' import { renderWithSystemFeatures } from '@/__tests__/utils/mock-system-features' import { Plan } from '@/app/components/billing/type' import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants' @@ -255,6 +256,13 @@ describe('AccountDropdown', () => { expect(screen.getByRole('button', { name: 'common.account.account' })).toBeInTheDocument() }) + it('should keep the account trigger disabled in server-rendered markup', () => { + const container = document.createElement('div') + container.innerHTML = renderToString() + + expect(container.querySelector('button[aria-label="common.account.account"]')).toBeDisabled() + }) + it('should show EDU badge for education accounts', () => { // Arrange vi.mocked(useProviderContext).mockReturnValue({ diff --git a/web/app/components/header/account-dropdown/index.tsx b/web/app/components/header/account-dropdown/index.tsx index 635b95d5055..d9c81a13b04 100644 --- a/web/app/components/header/account-dropdown/index.tsx +++ b/web/app/components/header/account-dropdown/index.tsx @@ -9,7 +9,7 @@ import { DropdownMenuTrigger, } from '@langgenius/dify-ui/dropdown-menu' import { useAtomValue } from 'jotai' -import { useState } from 'react' +import { useState, useSyncExternalStore } from 'react' import { useTranslation } from 'react-i18next' import { resetUser } from '@/app/components/base/amplitude/utils' import { @@ -33,10 +33,19 @@ type AccountDropdownProps = { const mainNavMenuPopupClassName = 'w-60 max-w-80 overflow-hidden bg-components-panel-bg-blur! p-0! backdrop-blur-[5px]' +const subscribeHydrationState = () => () => {} +const getHydrationSnapshot = () => false +const getServerHydrationSnapshot = () => true + export default function AppSelector({ trigger, variant = 'default' }: AccountDropdownProps = {}) { const router = useRouter() const [aboutVisible, setAboutVisible] = useState(false) const [isAccountMenuOpen, setIsAccountMenuOpen] = useState(false) + const isHydrating = useSyncExternalStore( + subscribeHydrationState, + getHydrationSnapshot, + getServerHydrationSnapshot, + ) const { t } = useTranslation() const userProfile = useAtomValue(userProfileAtom) const langGeniusVersionInfo = useAtomValue(langGeniusVersionInfoAtom) @@ -64,6 +73,7 @@ export default function AppSelector({ trigger, variant = 'default' }: AccountDro {trigger ? ( $['account.account'], { ns: 'common' }), @@ -71,9 +81,10 @@ export default function AppSelector({ trigger, variant = 'default' }: AccountDro /> ) : ( $['account.account'], { ns: 'common' })} className={cn( - 'inline-flex items-center rounded-[20px] p-0.5 hover:bg-background-default-dodge', + 'inline-flex items-center rounded-[20px] p-0.5 hover:bg-background-default-dodge disabled:cursor-default disabled:hover:bg-transparent', isAccountMenuOpen && 'bg-background-default-dodge', )} > diff --git a/web/app/components/main-nav/components/account-section.tsx b/web/app/components/main-nav/components/account-section.tsx index 3dac4faacdb..f17cfd05842 100644 --- a/web/app/components/main-nav/components/account-section.tsx +++ b/web/app/components/main-nav/components/account-section.tsx @@ -22,7 +22,7 @@ const AccountSection = ({ compact = false }: AccountSectionProps) => { aria-label={ariaLabel} title={userProfile.name} className={cn( - 'flex min-w-0 shrink items-center rounded-full text-left text-components-main-nav-text transition-colors hover:bg-state-base-hover focus-visible:inset-ring-2 focus-visible:inset-ring-state-accent-solid focus-visible:outline-hidden', + 'flex min-w-0 shrink items-center rounded-full text-left text-components-main-nav-text transition-colors hover:bg-state-base-hover focus-visible:inset-ring-2 focus-visible:inset-ring-state-accent-solid focus-visible:outline-hidden disabled:cursor-default disabled:hover:bg-transparent', compact ? 'justify-center p-1' : 'max-w-[180px] gap-3 py-1 pr-4 pl-1', isOpen && 'bg-state-base-hover', )} diff --git a/web/app/components/workflow/run/__tests__/status.spec.tsx b/web/app/components/workflow/run/__tests__/status.spec.tsx index 677faf162e3..a7ee5f55ebc 100644 --- a/web/app/components/workflow/run/__tests__/status.spec.tsx +++ b/web/app/components/workflow/run/__tests__/status.spec.tsx @@ -96,7 +96,7 @@ describe('Status', () => { it('renders succeeded metadata values', () => { render() - expect(screen.getByText('SUCCESS')).toBeInTheDocument() + expect(screen.getByRole('status')).toHaveTextContent('SUCCESS') expect(screen.getByText('1.234s')).toBeInTheDocument() expect(screen.getByText('8 Tokens')).toBeInTheDocument() }) diff --git a/web/app/components/workflow/run/status-container.tsx b/web/app/components/workflow/run/status-container.tsx index 5e3d40eefdc..4ef510d9d2b 100644 --- a/web/app/components/workflow/run/status-container.tsx +++ b/web/app/components/workflow/run/status-container.tsx @@ -14,6 +14,7 @@ const StatusContainer: FC = ({ status, children }) => { return (
- + {editable ? ( $['agentDetail.configure.advancedSettings.envEditor.keyColumn'])} @@ -236,7 +239,7 @@ function EnvEditorRow({ )} - + {editable && !shouldMaskValue ? ( $['agentDetail.configure.advancedSettings.envEditor.valueColumn'])} @@ -275,11 +278,11 @@ function EnvEditorRow({ )} {showScope && ( - + )} - + {editable && (