From b20e4971db726bf3b48ed13d63d7b4fd85fb9fc7 Mon Sep 17 00:00:00 2001 From: Brendan Allan <14191578+Brendonovich@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:28:23 +0800 Subject: [PATCH] test(app): run CI e2e against production builds (#45422) --- .github/workflows/test.yml | 4 +- packages/app/README.md | 32 +++++++-- .../app/e2e/performance/playwright.config.ts | 14 ++-- .../regression/cross-server-tab-close.spec.ts | 10 +-- .../remote-session-settings.spec.ts | 4 +- .../e2e/regression/remote-tab-busy.spec.ts | 4 +- .../regression/tab-navigate-mousedown.spec.ts | 4 +- packages/app/e2e/utils/mock-server.ts | 2 +- packages/app/package.json | 1 + packages/app/playwright.config.ts | 35 ++++++---- .../test-browser/playwright-config.test.ts | 70 +++++++++++++++++++ 11 files changed, 144 insertions(+), 36 deletions(-) create mode 100644 packages/app/test-browser/playwright-config.test.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 696adb46457..be239ff80e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -237,9 +237,9 @@ jobs: working-directory: packages/app run: bunx playwright install chromium - - name: Run app e2e tests + - name: Run app e2e tests against production build if: env.E2E_ENABLED == 'true' - run: bun --cwd packages/app test:e2e:local + run: bun --cwd packages/app test:e2e:built env: CI: true timeout-minutes: 30 diff --git a/packages/app/README.md b/packages/app/README.md index 304e272cd0a..c8011e88ec5 100644 --- a/packages/app/README.md +++ b/packages/app/README.md @@ -31,7 +31,7 @@ Your app is ready to be deployed! ## E2E Testing -Playwright starts the Vite dev server automatically via `webServer`, and UI tests expect an opencode backend at `localhost:4096` by default. +Locally, Playwright starts the Vite dev server automatically via `webServer`, or reuses one already running at the configured address. The browser suite uses isolated API fixtures rather than a live opencode backend. ```bash bunx playwright install chromium @@ -39,11 +39,35 @@ bun run test:e2e:local bun run test:e2e:local -- --grep "settings" ``` +CI builds the app once and runs the same suite against Vite preview, serving production assets from `dist`. Managed built runs never reuse an existing server, so a running dev server cannot silently replace the production build. To run this mode locally: + +```bash +bun run test:e2e:built +bun run test:e2e:built -- --grep "settings" +``` + +To test an already-running dev server without starting or building a server: + +```bash +PLAYWRIGHT_BASE_URL=http://127.0.0.1:4444 bun run test:e2e +``` + +For an already-running production build, also set `PLAYWRIGHT_BUILD=1` so the fixture API uses the app's origin: + +```bash +PLAYWRIGHT_BUILD=1 PLAYWRIGHT_BASE_URL=http://127.0.0.1:4444 bun run test:e2e +``` + +External targets must use HTTP because fixture URLs use HTTP. `PLAYWRIGHT_BASE_URL` skips server startup and building in either mode. + +Compiled CLI startup and service lifecycle coverage runs separately in CI via `packages/cli/script/service-smoke.ts`. + Environment options: -- `PLAYWRIGHT_SERVER_HOST` / `PLAYWRIGHT_SERVER_PORT` (backend address, default: `localhost:4096`) -- `PLAYWRIGHT_PORT` (Vite dev server port, default: `3000`) -- `PLAYWRIGHT_BASE_URL` (override base URL, default: `http://localhost:`) +- `PLAYWRIGHT_BUILD=1` (build and preview locally; always enabled when `CI` is set) +- `PLAYWRIGHT_SERVER_HOST` / `PLAYWRIGHT_SERVER_PORT` (dev fixture API address, default: `127.0.0.1:4096`; built runs use the app's origin, matching production) +- `PLAYWRIGHT_PORT` (managed dev or preview server port, default: `3000`) +- `PLAYWRIGHT_BASE_URL` (use an externally managed app instead of starting a server; otherwise defaults to `http://127.0.0.1:`) ## Deployment diff --git a/packages/app/e2e/performance/playwright.config.ts b/packages/app/e2e/performance/playwright.config.ts index f70d4bc80c9..b2752275a8e 100644 --- a/packages/app/e2e/performance/playwright.config.ts +++ b/packages/app/e2e/performance/playwright.config.ts @@ -1,7 +1,7 @@ import config from "../../playwright.config" const port = Number(process.env.PLAYWRIGHT_PORT ?? 3000) -process.env.PLAYWRIGHT_SERVER_PORT = String(port) +process.env.PLAYWRIGHT_SERVER_PORT = new URL(process.env.PLAYWRIGHT_BASE_URL ?? `http://127.0.0.1:${port}`).port || "80" process.env.OPENCODE_PERFORMANCE_RUN_ID ??= `${new Date().toISOString().replace(/[:.]/g, "-")}-${process.pid}` export default { @@ -12,9 +12,11 @@ export default { fullyParallel: false, workers: 1, reporter: [["html", { outputFolder: "../playwright-report/performance", open: "never" }], ["line"]], - webServer: { - ...config.webServer, - command: `bun run build && bun run serve -- --host 0.0.0.0 --port ${port} --strictPort`, - reuseExistingServer: false, - }, + webServer: config.webServer + ? { + ...config.webServer, + command: `bun run build && bun run serve -- --host 0.0.0.0 --port ${port} --strictPort`, + reuseExistingServer: false, + } + : undefined, } diff --git a/packages/app/e2e/regression/cross-server-tab-close.spec.ts b/packages/app/e2e/regression/cross-server-tab-close.spec.ts index d8ecab26282..29c26a4bb07 100644 --- a/packages/app/e2e/regression/cross-server-tab-close.spec.ts +++ b/packages/app/e2e/regression/cross-server-tab-close.spec.ts @@ -3,7 +3,7 @@ import { base64Encode } from "@opencode-ai/util/encode" import { currentSession } from "../utils/mock-server" import { installSseTransport } from "../utils/sse-transport" -const serverA = "http://127.0.0.1:4096" +const serverA = `http://${process.env.PLAYWRIGHT_SERVER_HOST ?? "127.0.0.1"}:${process.env.PLAYWRIGHT_SERVER_PORT ?? "4096"}` const serverB = "http://127.0.0.1:4097" const sessionA = session("ses_server_a", "C:/server-a", "Server A session") const sessionB = session("ses_server_b", "/home/server-b", "Server B session") @@ -12,17 +12,17 @@ test("closing the active server's last tab opens the remaining server tab", asyn const requests: string[] = [] await mockServers(page, requests) await page.addInitScript( - ({ serverB, sessionA, sessionB }) => { + ({ serverA, serverB, sessionA, sessionB }) => { localStorage.setItem("opencode.global.dat:server", JSON.stringify({ list: [serverB] })) localStorage.setItem( "opencode.window.browser.dat:tabs", JSON.stringify([ - { type: "session", server: "http://127.0.0.1:4096", sessionId: sessionA }, + { type: "session", server: serverA, sessionId: sessionA }, { type: "session", server: serverB, sessionId: sessionB }, ]), ) }, - { serverB, sessionA: sessionA.id, sessionB: sessionB.id }, + { serverA, serverB, sessionA: sessionA.id, sessionB: sessionB.id }, ) const hrefA = `/server/${base64Encode(serverA)}/session/${sessionA.id}` @@ -55,7 +55,7 @@ function session(id: string, directory: string, title: string) { async function mockServers(page: Page, requests: string[]) { await installSseTransport(page, { server: serverA }) await installSseTransport(page, { server: serverB }) - await page.route("**/*", async (route) => { + await page.route("**/api/**", async (route) => { const url = new URL(route.request().url()) if (url.origin !== serverA && url.origin !== serverB) return route.fallback() requests.push(url.toString()) diff --git a/packages/app/e2e/regression/remote-session-settings.spec.ts b/packages/app/e2e/regression/remote-session-settings.spec.ts index c137f53d78d..35296f93e5e 100644 --- a/packages/app/e2e/regression/remote-session-settings.spec.ts +++ b/packages/app/e2e/regression/remote-session-settings.spec.ts @@ -3,7 +3,7 @@ import { expect, test, type Page, type Route } from "@playwright/test" import { installSseTransport } from "../utils/sse-transport" import { currentSession } from "../utils/mock-server" -const serverA = `http://127.0.0.1:${process.env.PLAYWRIGHT_SERVER_PORT ?? "4096"}` +const serverA = `http://${process.env.PLAYWRIGHT_SERVER_HOST ?? "127.0.0.1"}:${process.env.PLAYWRIGHT_SERVER_PORT ?? "4096"}` const serverB = "http://127.0.0.1:4097" const directoryA = "C:/server-a" const directoryB = "/home/server-b" @@ -308,7 +308,7 @@ async function mockServers( permissionResponses: PermissionResponse[] = [], options: MockServerOptions = {}, ) { - await page.route("**/*", async (route) => { + await page.route("**/api/**", async (route) => { const url = new URL(route.request().url()) if (url.origin !== serverA && url.origin !== serverB) return route.fallback() const remote = url.origin === serverB diff --git a/packages/app/e2e/regression/remote-tab-busy.spec.ts b/packages/app/e2e/regression/remote-tab-busy.spec.ts index 9afc695eacb..edd5537ee02 100644 --- a/packages/app/e2e/regression/remote-tab-busy.spec.ts +++ b/packages/app/e2e/regression/remote-tab-busy.spec.ts @@ -2,7 +2,7 @@ import { expect, test, type Page, type Route } from "@playwright/test" import { base64Encode } from "@opencode-ai/util/encode" import { currentSession } from "../utils/mock-server" -const serverA = "http://127.0.0.1:4096" +const serverA = `http://${process.env.PLAYWRIGHT_SERVER_HOST ?? "127.0.0.1"}:${process.env.PLAYWRIGHT_SERVER_PORT ?? "4096"}` const serverB = "http://127.0.0.1:4097" const sessionA = session("ses_server_a", "C:/server-a", "Server A session") const sessionB = session("ses_server_b", "/home/server-b", "Server B session") @@ -51,7 +51,7 @@ function session(id: string, directory: string, title: string) { } async function mockServers(page: Page) { - await page.route("**/*", async (route) => { + await page.route("**/api/**", async (route) => { const url = new URL(route.request().url()) if (url.origin !== serverA && url.origin !== serverB) return route.fallback() const current = url.origin === serverA ? sessionA : sessionB diff --git a/packages/app/e2e/regression/tab-navigate-mousedown.spec.ts b/packages/app/e2e/regression/tab-navigate-mousedown.spec.ts index f78801b8fd5..fdd3f75478e 100644 --- a/packages/app/e2e/regression/tab-navigate-mousedown.spec.ts +++ b/packages/app/e2e/regression/tab-navigate-mousedown.spec.ts @@ -2,7 +2,7 @@ import { expect, test, type Page, type Route } from "@playwright/test" import { base64Encode } from "@opencode-ai/util/encode" import { currentSession } from "../utils/mock-server" -const server = "http://127.0.0.1:4096" +const server = `http://${process.env.PLAYWRIGHT_SERVER_HOST ?? "127.0.0.1"}:${process.env.PLAYWRIGHT_SERVER_PORT ?? "4096"}` const sessionA = session("ses_tab_a", "Tab A session") const sessionB = session("ses_tab_b", "Tab B session") const sessionC = session("ses_tab_c", "Tab C session") @@ -238,7 +238,7 @@ function session(id: string, title: string) { async function mockServer(page: Page) { const sessions = [sessionA, sessionB, sessionC] - await page.route("**/*", async (route) => { + await page.route("**/api/**", async (route) => { const url = new URL(route.request().url()) if (url.origin !== server) return route.fallback() if (url.pathname === `/api/session/${unresolvedSessionID}`) return new Promise(() => {}) diff --git a/packages/app/e2e/utils/mock-server.ts b/packages/app/e2e/utils/mock-server.ts index 6f9ff810987..c3bccda0974 100644 --- a/packages/app/e2e/utils/mock-server.ts +++ b/packages/app/e2e/utils/mock-server.ts @@ -144,7 +144,7 @@ export async function mockOpenCodeServer(page: Page, config: MockServerConfig) { ) page.on("close", () => void transport.dispose()) - await page.route("**/*", async (route) => { + await page.route("**/api/**", async (route) => { const url = new URL(route.request().url()) const appPort = new URL( process.env.PLAYWRIGHT_BASE_URL ?? `http://127.0.0.1:${process.env.PLAYWRIGHT_PORT ?? "3000"}`, diff --git a/packages/app/package.json b/packages/app/package.json index 9a6f7ef3492..0c0f106b57b 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -25,6 +25,7 @@ "test:browser": "bun test --conditions=browser --preload ./happydom.ts ./test-browser", "test:unit:watch": "bun test --conditions=solid --watch --preload ./happydom.ts ./src", "test:e2e": "playwright test", + "test:e2e:built": "PLAYWRIGHT_BUILD=1 playwright test", "test:e2e:local": "playwright test", "test:components": "playwright test --config playwright.components.config.ts", "test:components:ui": "playwright test --config playwright.components.config.ts --ui", diff --git a/packages/app/playwright.config.ts b/packages/app/playwright.config.ts index 03e6915f5e4..6fe3ab39f2d 100644 --- a/packages/app/playwright.config.ts +++ b/packages/app/playwright.config.ts @@ -2,10 +2,19 @@ import { defineConfig, devices } from "@playwright/test" const port = Number(process.env.PLAYWRIGHT_PORT ?? 3000) const baseURL = process.env.PLAYWRIGHT_BASE_URL ?? `http://127.0.0.1:${port}` +const url = new URL(baseURL) +if (url.protocol !== "http:") throw new Error("E2E fixtures require an http:// app URL") +const built = !!process.env.CI || process.env.PLAYWRIGHT_BUILD === "1" +// Production connects to its own origin, so fixture URLs must match the preview server. +if (built) { + process.env.PLAYWRIGHT_SERVER_HOST = url.hostname + process.env.PLAYWRIGHT_SERVER_PORT = url.port || "80" +} const serverHost = process.env.PLAYWRIGHT_SERVER_HOST ?? "127.0.0.1" const serverPort = process.env.PLAYWRIGHT_SERVER_PORT ?? "4096" -const command = `bun run dev -- --host 0.0.0.0 --port ${port}` -const reuse = !process.env.CI +const command = built + ? `bun run build && bun run serve -- --host 127.0.0.1 --port ${port} --strictPort` + : `bun run dev -- --host 127.0.0.1 --port ${port} --strictPort` const workers = Number(process.env.PLAYWRIGHT_WORKERS ?? (process.env.CI ? 5 : 0)) || undefined export default defineConfig({ testDir: "./e2e", @@ -23,16 +32,18 @@ export default defineConfig({ retries: process.env.CI ? 2 : 0, workers, reporter: [["html", { outputFolder: "e2e/playwright-report", open: "never" }], ["line"]], - webServer: { - command, - url: baseURL, - reuseExistingServer: reuse, - timeout: 120_000, - env: { - VITE_OPENCODE_SERVER_HOST: serverHost, - VITE_OPENCODE_SERVER_PORT: serverPort, - }, - }, + webServer: process.env.PLAYWRIGHT_BASE_URL + ? undefined + : { + command, + url: baseURL, + reuseExistingServer: !built, + timeout: 120_000, + env: { + VITE_OPENCODE_SERVER_HOST: serverHost, + VITE_OPENCODE_SERVER_PORT: serverPort, + }, + }, use: { baseURL, trace: "on-first-retry", diff --git a/packages/app/test-browser/playwright-config.test.ts b/packages/app/test-browser/playwright-config.test.ts new file mode 100644 index 00000000000..f8d55479dc8 --- /dev/null +++ b/packages/app/test-browser/playwright-config.test.ts @@ -0,0 +1,70 @@ +import { expect, test } from "bun:test" +import path from "node:path" + +test.each([ + { name: "local development", ci: "", build: "", built: false }, + { name: "local production build", ci: "", build: "1", built: true }, + { name: "CI production build", ci: "true", build: "", built: true }, + { name: "CI cannot opt into development", ci: "true", build: "0", built: true }, +])("Playwright uses $name", ({ ci, build, built }) => { + const result = Bun.spawnSync({ + cmd: [process.execPath, "-e", 'import config from "./playwright.config.ts"; console.log(JSON.stringify(config))'], + cwd: path.resolve(import.meta.dirname, ".."), + env: { + ...process.env, + CI: ci, + PLAYWRIGHT_BUILD: build, + PLAYWRIGHT_BASE_URL: undefined, + PLAYWRIGHT_PORT: "4321", + PLAYWRIGHT_SERVER_HOST: "127.0.0.2", + PLAYWRIGHT_SERVER_PORT: "4322", + }, + }) + expect(result.exitCode).toBe(0) + const config = JSON.parse(result.stdout.toString()) + expect(config.use.baseURL).toBe("http://127.0.0.1:4321") + expect(config.webServer.url).toBe(config.use.baseURL) + expect(config.webServer.command).toBe( + built + ? "bun run build && bun run serve -- --host 127.0.0.1 --port 4321 --strictPort" + : "bun run dev -- --host 127.0.0.1 --port 4321 --strictPort", + ) + expect(config.webServer.reuseExistingServer).toBe(!built) + expect(config.webServer.env).toEqual({ + VITE_OPENCODE_SERVER_HOST: built ? "127.0.0.1" : "127.0.0.2", + VITE_OPENCODE_SERVER_PORT: built ? "4321" : "4322", + }) +}) + +test.each([ + "./playwright.config.ts", + "./e2e/performance/playwright.config.ts", + "./e2e/performance/playwright.uncapped.config.ts", + "./e2e/performance/timeline-stability/playwright.config.ts", +])("%s leaves an explicit external app unmanaged", (file) => { + const result = Bun.spawnSync({ + cmd: [ + process.execPath, + "-e", + `import config from ${JSON.stringify(file)}; console.log(JSON.stringify({ ...config, fixtureHost: process.env.PLAYWRIGHT_SERVER_HOST, fixturePort: process.env.PLAYWRIGHT_SERVER_PORT }))`, + ], + cwd: path.resolve(import.meta.dirname, ".."), + env: { ...process.env, CI: "true", PLAYWRIGHT_BASE_URL: "http://127.0.0.1:4444" }, + }) + expect(result.exitCode).toBe(0) + const config = JSON.parse(result.stdout.toString()) + expect(config.webServer).toBeUndefined() + expect(config.use.baseURL).toBe("http://127.0.0.1:4444") + expect(config.fixtureHost).toBe("127.0.0.1") + expect(config.fixturePort).toBe("4444") +}) + +test("Playwright rejects HTTPS targets unsupported by the API fixtures", () => { + const result = Bun.spawnSync({ + cmd: [process.execPath, "-e", 'import "./playwright.config.ts"'], + cwd: path.resolve(import.meta.dirname, ".."), + env: { ...process.env, CI: "true", PLAYWRIGHT_BASE_URL: "https://e2e.example.com" }, + }) + expect(result.exitCode).not.toBe(0) + expect(result.stderr.toString()).toContain("E2E fixtures require an http:// app URL") +})