diff --git a/playwright.benchmark.config.ts b/playwright.benchmark.config.ts index 16e18c53b..c4ad1b3e4 100644 --- a/playwright.benchmark.config.ts +++ b/playwright.benchmark.config.ts @@ -29,6 +29,12 @@ export default defineConfig({ wait: { stdout: /ready\s+built in/i }, reuseExistingServer: !process.env.CI, timeout: 10 * 60 * 1000 + }, + { + command: 'npm start --workspace=packages/bruno-tests', + url: 'http://localhost:8081/ping', + reuseExistingServer: !process.env.CI, + timeout: 10 * 60 * 1000 } ], diff --git a/tests/benchmarks/runner/runner-allocation.bench.ts b/tests/benchmarks/runner/runner-allocation.bench.ts new file mode 100644 index 000000000..4f002b424 --- /dev/null +++ b/tests/benchmarks/runner/runner-allocation.bench.ts @@ -0,0 +1,164 @@ +// TODO - (chirag): +// This test runs on our testbench and we need a stable collection to benchmark. +// Have a discussion with the QA team (Abhishek) to come up with some collection. +// The numbers coming out of this collection becomes less dependable as the collection evolves. +import { test, expect } from '../../../playwright'; +import { type Page, type ElectronApplication, type CDPSession } from '@playwright/test'; +import { + openCollection, + closeAllCollections, + selectEnvironment, + openRunnerTab, + buildRunnerLocators, + buildCommonLocators +} from '../../utils/page'; +import { + RendererMemorySampler, + MainMemorySampler, + type RendererPhaseStats, + type MainPhaseStats, + type StatsByPhase +} from '../utils/memory'; +import { writeResults, buildResultEntry, type ResultEntry } from '../utils/results'; +import { formatMib } from '../utils/format'; +import * as path from 'node:path'; +import * as fs from 'node:fs'; + +const SOURCE_COLLECTION_DIR = path.join(process.cwd(), 'packages', 'bruno-tests', 'collection'); +const COLLECTION_NAME = 'bruno-testbench'; +const ENVIRONMENT = 'Local'; +const ITERATIONS = 1; +const RUNS = 3; +const RUN_COMPLETION_TIMEOUT = 15 * 60 * 1000; +const REPORT_TIMEOUT = 5 * 60 * 1000; + +const PHASE_RUN = 'collection-run'; + +interface RunMetrics { + renderer: StatsByPhase; + main: StatsByPhase; + rendererRetainedBytes: number; +} + +const copyCollection = (destDir: string) => { + fs.cpSync(SOURCE_COLLECTION_DIR, destDir, { + recursive: true, + filter: (src) => !src.split(path.sep).includes('node_modules') + }); +}; + +const openTestbench = async (page: Page, electronApp: ElectronApplication, dir: string) => { + await test.step(`Open the ${COLLECTION_NAME} collection`, async () => { + const locators = buildCommonLocators(page); + await electronApp.evaluate(({ dialog }, dir) => { + dialog.showOpenDialog = async () => ({ canceled: false, filePaths: [dir] }); + }, dir); + await locators.plusMenu.button().click(); + await locators.dropdown.tippyItem('Open collection').click(); + await expect(locators.sidebar.collection(COLLECTION_NAME)).toBeVisible({ timeout: 10000 }); + await openCollection(page, COLLECTION_NAME); + }); +}; + +const measureOneRun = async ( + page: Page, + electronApp: ElectronApplication, + cdp: CDPSession +): Promise => { + const runner = buildRunnerLocators(page); + + await test.step('Reset to a clean runner state', async () => { + const resetVisible = await runner.resetButton().isVisible({ timeout: 1000 }).catch(() => false); + if (resetVisible) { + await runner.resetButton().click(); + } + await runner.runCollectionButton().waitFor({ state: 'visible', timeout: 10000 }); + }); + + return test.step('Run the collection and sample renderer + main heap', async () => { + const rendererSampler = await RendererMemorySampler.start(cdp, 200, PHASE_RUN); + const mainSampler = await MainMemorySampler.start(electronApp, 100, PHASE_RUN); + + await runner.runCollectionButton().click(); + await runner.runAgainButton().waitFor({ timeout: RUN_COMPLETION_TIMEOUT }); + + const renderer = await rendererSampler.terminate(); + const main = await mainSampler.terminate(); + const rendererRetainedBytes = await RendererMemorySampler.measureRetainedBytes(cdp); + + return { renderer, main, rendererRetainedBytes }; + }); +}; + +test.describe('Benchmark: Runner allocation', () => { + const runs: RunMetrics[] = []; + let measuredPhases: string[] = [PHASE_RUN]; + + test('renderer + main heap while running the bruno-testbench collection', async ({ page, electronApp, createTmpDir }) => { + test.setTimeout(RUNS * (RUN_COMPLETION_TIMEOUT + REPORT_TIMEOUT) + 5 * 60 * 1000); + + const cdp = await page.context().newCDPSession(page); + + measuredPhases = [PHASE_RUN]; + + const collectionDir = await test.step('Copy the collection to a temp dir', async () => { + const dir = await createTmpDir('bruno-testbench'); + copyCollection(dir); + return dir; + }); + + await openTestbench(page, electronApp, collectionDir); + await openRunnerTab(page, COLLECTION_NAME); + await selectEnvironment(page, ENVIRONMENT); + + for (let i = 0; i < RUNS; i++) { + await test.step(`Run ${i + 1} of ${RUNS} (${ITERATIONS} iterations)`, async () => { + const m = await measureOneRun(page, electronApp, cdp); + runs.push(m); + for (const phase of measuredPhases) { + console.log( + `[BENCHMARK] run ${i + 1}/${RUNS} (${ITERATIONS} iterations) — ${phase}: ` + + `renderer peak ${formatMib(m.renderer[phase].peakHeapUsedBytes)} · ` + + `main peak rss ${formatMib(m.main[phase].peakRssBytes)} / external ${formatMib(m.main[phase].peakExternalBytes)}` + ); + } + console.log(`[BENCHMARK] run ${i + 1}/${RUNS} — renderer retained ${formatMib(m.rendererRetainedBytes)}`); + }); + } + + await closeAllCollections(page); + }); + + test.afterAll(async () => { + if (!runs.length) return; + const resultsDir = path.join(process.cwd(), 'tests', 'benchmarks', 'results'); + fs.mkdirSync(resultsDir, { recursive: true }); + const outputPath = path.join(resultsDir, 'runner-allocation.json'); + + const meta = { collection: COLLECTION_NAME, iterations: ITERATIONS }; + const entries: Record = { + 'renderer-retained-heap': buildResultEntry( + runs.map((r) => r.rendererRetainedBytes), + { metric: 'renderer-retained-heap', ...meta } + ) + }; + + for (const phase of measuredPhases) { + entries[`renderer-peak-heap:${phase}`] = buildResultEntry( + runs.map((r) => r.renderer[phase].peakHeapUsedBytes), + { metric: 'renderer-peak-heap', phase, ...meta } + ); + entries[`main-peak-rss:${phase}`] = buildResultEntry( + runs.map((r) => r.main[phase].peakRssBytes), + { metric: 'main-peak-rss', phase, ...meta } + ); + entries[`main-peak-external:${phase}`] = buildResultEntry( + runs.map((r) => r.main[phase].peakExternalBytes), + { metric: 'main-peak-external', phase, ...meta } + ); + } + + writeResults(outputPath, { name: 'Runner Allocation', unit: 'bytes', direction: 'smaller' }, entries); + console.log(`[BENCHMARK] Results written to ${outputPath}`); + }); +}); diff --git a/tests/benchmarks/utils/format.ts b/tests/benchmarks/utils/format.ts new file mode 100644 index 000000000..10a0d7cd5 --- /dev/null +++ b/tests/benchmarks/utils/format.ts @@ -0,0 +1,7 @@ +// TODO (chirag): use this in the mount test in another PR +export const round = (value: number, decimals = 0): number => { + const factor = 10 ** decimals; + return Math.round(value * factor) / factor; +}; + +export const formatMib = (bytes: number): string => `${round(bytes / 1024 ** 2, 1)} MiB`; diff --git a/tests/benchmarks/utils/memory.ts b/tests/benchmarks/utils/memory.ts new file mode 100644 index 000000000..daa128f34 --- /dev/null +++ b/tests/benchmarks/utils/memory.ts @@ -0,0 +1,170 @@ +import { type CDPSession, type ElectronApplication } from '@playwright/test'; + +export const DEFAULT_PHASE = 'begin'; + +type PhaseBuckets = Record; + +export type StatsByPhase = Record; + +export interface PhasedMemorySampler { + startPhase(name: string): Promise; + terminate(): Promise>; +} + +export type RendererMemorySample = number; + +export interface RendererPhaseStats { + peakHeapUsedBytes: number; + count: number; +} + +export interface MainMemorySample { + rss: number; + heapUsed: number; + external: number; + arrayBuffers: number; +} + +export interface MainPhaseStats { + peakRssBytes: number; + peakHeapUsedBytes: number; + peakExternalBytes: number; + peakArrayBuffersBytes: number; + count: number; +} + +interface MainSamplingGlobals { + __benchMainMem: PhaseBuckets; + __benchMainMemPhase: string | null; + __benchMainMemTimer: ReturnType | null; +} + +const statsPerPhase = ( + buckets: PhaseBuckets, + statsOf: (samples: TSample[]) => TStats +): StatsByPhase => + Object.fromEntries(Object.entries(buckets).map(([name, samples]) => [name, statsOf(samples)])); + +const rendererStatsOf = (samples: RendererMemorySample[]): RendererPhaseStats => ({ + peakHeapUsedBytes: samples.reduce((mx, used) => Math.max(mx, used), 0), + count: samples.length +}); + +const mainStatsOf = (samples: MainMemorySample[]): MainPhaseStats => { + const peak = (key: keyof MainMemorySample) => samples.reduce((mx, s) => Math.max(mx, s[key] || 0), 0); + return { + peakRssBytes: peak('rss'), + peakHeapUsedBytes: peak('heapUsed'), + peakExternalBytes: peak('external'), + peakArrayBuffersBytes: peak('arrayBuffers'), + count: samples.length + }; +}; + +const readHeapUsedBytes = async (cdp: CDPSession): Promise => { + const { metrics } = (await cdp.send('Performance.getMetrics')) as { + metrics: { name: string; value: number }[]; + }; + const used = metrics.find((m) => m.name === 'JSHeapUsedSize'); + return used ? used.value : 0; +}; + +export class RendererMemorySampler implements PhasedMemorySampler { + private readonly buckets = new Map(); + private current: RendererMemorySample[]; + private running = true; + private loop!: Promise; + + private constructor( + private readonly cdp: CDPSession, + private readonly intervalMs: number, + initialPhase: string + ) { + this.current = []; + this.buckets.set(initialPhase, this.current); + } + + static async start( + cdp: CDPSession, + intervalMs = 200, + initialPhase = DEFAULT_PHASE + ): Promise { + await cdp.send('Performance.enable'); + const sampler = new RendererMemorySampler(cdp, intervalMs, initialPhase); + sampler.loop = sampler.poll(); + return sampler; + } + + static async measureRetainedBytes(cdp: CDPSession): Promise { + await cdp.send('HeapProfiler.enable'); + await cdp.send('HeapProfiler.collectGarbage'); + return readHeapUsedBytes(cdp); + } + + async startPhase(name: string): Promise { + this.current = []; + this.buckets.set(name, this.current); + } + + async terminate(): Promise> { + this.running = false; + await this.loop; + return statsPerPhase(Object.fromEntries(this.buckets), rendererStatsOf); + } + + private async poll(): Promise { + while (this.running) { + this.current.push(await readHeapUsedBytes(this.cdp)); + await new Promise((resolve) => setTimeout(resolve, this.intervalMs)); + } + } +} + +export class MainMemorySampler implements PhasedMemorySampler { + private constructor(private readonly electronApp: ElectronApplication) {} + + static async start( + electronApp: ElectronApplication, + intervalMs = 100, + initialPhase = DEFAULT_PHASE + ): Promise { + await electronApp.evaluate((_electron, { interval, phase }) => { + const g = globalThis as unknown as MainSamplingGlobals; + g.__benchMainMem = { [phase]: [] }; + g.__benchMainMemPhase = phase; + g.__benchMainMemTimer = setInterval(() => { + const { rss, heapUsed, external, arrayBuffers } = process.memoryUsage(); + const currentPhase = g.__benchMainMemPhase; + if (currentPhase) { + g.__benchMainMem[currentPhase].push({ rss, heapUsed, external, arrayBuffers }); + } + }, interval); + }, { interval: intervalMs, phase: initialPhase }); + + return new MainMemorySampler(electronApp); + } + + async startPhase(name: string): Promise { + await this.electronApp.evaluate((_electron, phase) => { + const g = globalThis as unknown as MainSamplingGlobals; + g.__benchMainMem[phase] = []; + g.__benchMainMemPhase = phase; + }, name); + } + + async terminate(): Promise> { + const buckets = await this.electronApp.evaluate(() => { + const g = globalThis as unknown as MainSamplingGlobals; + if (g.__benchMainMemTimer) { + clearInterval(g.__benchMainMemTimer); + } + const collected = g.__benchMainMem || {}; + g.__benchMainMem = {}; + g.__benchMainMemPhase = null; + g.__benchMainMemTimer = null; + return collected; + }); + + return statsPerPhase(buckets, mainStatsOf); + } +}