test(benchmark): added benchmark test suite for collection runner (#9126)

2026-10-12 02:24:01 +00:00 · 2026-09-02 13:59:25 +05:30
parent 77f0b57c94
commit 089f8ba24d
4 changed files with 347 additions and 0 deletions
@@ -29,6 +29,12 @@ export default defineConfig({
      wait: { stdout: /ready\s+built in/i },
      reuseExistingServer: !process.env.CI,
      timeout: 10 * 60 * 1000
+    },
+    {
+      command: 'npm start --workspace=packages/bruno-tests',
+      url: 'http://localhost:8081/ping',
+      reuseExistingServer: !process.env.CI,
+      timeout: 10 * 60 * 1000
    }
  ],

@@ -0,0 +1,164 @@
+// TODO - (chirag):
+// This test runs on our testbench and we need a stable collection to benchmark.
+// Have a discussion with the QA team (Abhishek) to come up with some collection.
+// The numbers coming out of this collection becomes less dependable as the collection evolves.
+import { test, expect } from '../../../playwright';
+import { type Page, type ElectronApplication, type CDPSession } from '@playwright/test';
+import {
+  openCollection,
+  closeAllCollections,
+  selectEnvironment,
+  openRunnerTab,
+  buildRunnerLocators,
+  buildCommonLocators
+} from '../../utils/page';
+import {
+  RendererMemorySampler,
+  MainMemorySampler,
+  type RendererPhaseStats,
+  type MainPhaseStats,
+  type StatsByPhase
+} from '../utils/memory';
+import { writeResults, buildResultEntry, type ResultEntry } from '../utils/results';
+import { formatMib } from '../utils/format';
+import * as path from 'node:path';
+import * as fs from 'node:fs';
+
+const SOURCE_COLLECTION_DIR = path.join(process.cwd(), 'packages', 'bruno-tests', 'collection');
+const COLLECTION_NAME = 'bruno-testbench';
+const ENVIRONMENT = 'Local';
+const ITERATIONS = 1;
+const RUNS = 3;
+const RUN_COMPLETION_TIMEOUT = 15 * 60 * 1000;
+const REPORT_TIMEOUT = 5 * 60 * 1000;
+
+const PHASE_RUN = 'collection-run';
+
+interface RunMetrics {
+  renderer: StatsByPhase<RendererPhaseStats>;
+  main: StatsByPhase<MainPhaseStats>;
+  rendererRetainedBytes: number;
+}
+
+const copyCollection = (destDir: string) => {
+  fs.cpSync(SOURCE_COLLECTION_DIR, destDir, {
+    recursive: true,
+    filter: (src) => !src.split(path.sep).includes('node_modules')
+  });
+};
+
+const openTestbench = async (page: Page, electronApp: ElectronApplication, dir: string) => {
+  await test.step(`Open the ${COLLECTION_NAME} collection`, async () => {
+    const locators = buildCommonLocators(page);
+    await electronApp.evaluate(({ dialog }, dir) => {
+      dialog.showOpenDialog = async () => ({ canceled: false, filePaths: [dir] });
+    }, dir);
+    await locators.plusMenu.button().click();
+    await locators.dropdown.tippyItem('Open collection').click();
+    await expect(locators.sidebar.collection(COLLECTION_NAME)).toBeVisible({ timeout: 10000 });
+    await openCollection(page, COLLECTION_NAME);
+  });
+};
+
+const measureOneRun = async (
+  page: Page,
+  electronApp: ElectronApplication,
+  cdp: CDPSession
+): Promise<RunMetrics> => {
+  const runner = buildRunnerLocators(page);
+
+  await test.step('Reset to a clean runner state', async () => {
+    const resetVisible = await runner.resetButton().isVisible({ timeout: 1000 }).catch(() => false);
+    if (resetVisible) {
+      await runner.resetButton().click();
+    }
+    await runner.runCollectionButton().waitFor({ state: 'visible', timeout: 10000 });
+  });
+
+  return test.step('Run the collection and sample renderer + main heap', async () => {
+    const rendererSampler = await RendererMemorySampler.start(cdp, 200, PHASE_RUN);
+    const mainSampler = await MainMemorySampler.start(electronApp, 100, PHASE_RUN);
+
+    await runner.runCollectionButton().click();
+    await runner.runAgainButton().waitFor({ timeout: RUN_COMPLETION_TIMEOUT });
+
+    const renderer = await rendererSampler.terminate();
+    const main = await mainSampler.terminate();
+    const rendererRetainedBytes = await RendererMemorySampler.measureRetainedBytes(cdp);
+
+    return { renderer, main, rendererRetainedBytes };
+  });
+};
+
+test.describe('Benchmark: Runner allocation', () => {
+  const runs: RunMetrics[] = [];
+  let measuredPhases: string[] = [PHASE_RUN];
+
+  test('renderer + main heap while running the bruno-testbench collection', async ({ page, electronApp, createTmpDir }) => {
+    test.setTimeout(RUNS * (RUN_COMPLETION_TIMEOUT + REPORT_TIMEOUT) + 5 * 60 * 1000);
+
+    const cdp = await page.context().newCDPSession(page);
+
+    measuredPhases = [PHASE_RUN];
+
+    const collectionDir = await test.step('Copy the collection to a temp dir', async () => {
+      const dir = await createTmpDir('bruno-testbench');
+      copyCollection(dir);
+      return dir;
+    });
+
+    await openTestbench(page, electronApp, collectionDir);
+    await openRunnerTab(page, COLLECTION_NAME);
+    await selectEnvironment(page, ENVIRONMENT);
+
+    for (let i = 0; i < RUNS; i++) {
+      await test.step(`Run ${i + 1} of ${RUNS} (${ITERATIONS} iterations)`, async () => {
+        const m = await measureOneRun(page, electronApp, cdp);
+        runs.push(m);
+        for (const phase of measuredPhases) {
+          console.log(
+            `[BENCHMARK] run ${i + 1}/${RUNS} (${ITERATIONS} iterations) — ${phase}: `
+            + `renderer peak ${formatMib(m.renderer[phase].peakHeapUsedBytes)} · `
+            + `main peak rss ${formatMib(m.main[phase].peakRssBytes)} / external ${formatMib(m.main[phase].peakExternalBytes)}`
+          );
+        }
+        console.log(`[BENCHMARK] run ${i + 1}/${RUNS} — renderer retained ${formatMib(m.rendererRetainedBytes)}`);
+      });
+    }
+
+    await closeAllCollections(page);
+  });
+
+  test.afterAll(async () => {
+    if (!runs.length) return;
+    const resultsDir = path.join(process.cwd(), 'tests', 'benchmarks', 'results');
+    fs.mkdirSync(resultsDir, { recursive: true });
+    const outputPath = path.join(resultsDir, 'runner-allocation.json');
+
+    const meta = { collection: COLLECTION_NAME, iterations: ITERATIONS };
+    const entries: Record<string, ResultEntry> = {
+      'renderer-retained-heap': buildResultEntry(
+        runs.map((r) => r.rendererRetainedBytes),
+        { metric: 'renderer-retained-heap', ...meta }
+      )
+    };
+
+    for (const phase of measuredPhases) {
+      entries[`renderer-peak-heap:${phase}`] = buildResultEntry(
+        runs.map((r) => r.renderer[phase].peakHeapUsedBytes),
+        { metric: 'renderer-peak-heap', phase, ...meta }
+      );
+      entries[`main-peak-rss:${phase}`] = buildResultEntry(
+        runs.map((r) => r.main[phase].peakRssBytes),
+        { metric: 'main-peak-rss', phase, ...meta }
+      );
+      entries[`main-peak-external:${phase}`] = buildResultEntry(
+        runs.map((r) => r.main[phase].peakExternalBytes),
+        { metric: 'main-peak-external', phase, ...meta }
+      );
+    }
+
+    writeResults(outputPath, { name: 'Runner Allocation', unit: 'bytes', direction: 'smaller' }, entries);
+    console.log(`[BENCHMARK] Results written to ${outputPath}`);
+  });
+});
@@ -0,0 +1,7 @@
+// TODO (chirag): use this in the mount test in another PR
+export const round = (value: number, decimals = 0): number => {
+  const factor = 10 ** decimals;
+  return Math.round(value * factor) / factor;
+};
+
+export const formatMib = (bytes: number): string => `${round(bytes / 1024 ** 2, 1)} MiB`;
@@ -0,0 +1,170 @@
+import { type CDPSession, type ElectronApplication } from '@playwright/test';
+
+export const DEFAULT_PHASE = 'begin';
+
+type PhaseBuckets<TSample> = Record<string, TSample[]>;
+
+export type StatsByPhase<TStats> = Record<string, TStats>;
+
+export interface PhasedMemorySampler<TStats> {
+  startPhase(name: string): Promise<void>;
+  terminate(): Promise<StatsByPhase<TStats>>;
+}
+
+export type RendererMemorySample = number;
+
+export interface RendererPhaseStats {
+  peakHeapUsedBytes: number;
+  count: number;
+}
+
+export interface MainMemorySample {
+  rss: number;
+  heapUsed: number;
+  external: number;
+  arrayBuffers: number;
+}
+
+export interface MainPhaseStats {
+  peakRssBytes: number;
+  peakHeapUsedBytes: number;
+  peakExternalBytes: number;
+  peakArrayBuffersBytes: number;
+  count: number;
+}
+
+interface MainSamplingGlobals {
+  __benchMainMem: PhaseBuckets<MainMemorySample>;
+  __benchMainMemPhase: string | null;
+  __benchMainMemTimer: ReturnType<typeof setInterval> | null;
+}
+
+const statsPerPhase = <TSample, TStats>(
+  buckets: PhaseBuckets<TSample>,
+  statsOf: (samples: TSample[]) => TStats
+): StatsByPhase<TStats> =>
+  Object.fromEntries(Object.entries(buckets).map(([name, samples]) => [name, statsOf(samples)]));
+
+const rendererStatsOf = (samples: RendererMemorySample[]): RendererPhaseStats => ({
+  peakHeapUsedBytes: samples.reduce((mx, used) => Math.max(mx, used), 0),
+  count: samples.length
+});
+
+const mainStatsOf = (samples: MainMemorySample[]): MainPhaseStats => {
+  const peak = (key: keyof MainMemorySample) => samples.reduce((mx, s) => Math.max(mx, s[key] || 0), 0);
+  return {
+    peakRssBytes: peak('rss'),
+    peakHeapUsedBytes: peak('heapUsed'),
+    peakExternalBytes: peak('external'),
+    peakArrayBuffersBytes: peak('arrayBuffers'),
+    count: samples.length
+  };
+};
+
+const readHeapUsedBytes = async (cdp: CDPSession): Promise<number> => {
+  const { metrics } = (await cdp.send('Performance.getMetrics')) as {
+    metrics: { name: string; value: number }[];
+  };
+  const used = metrics.find((m) => m.name === 'JSHeapUsedSize');
+  return used ? used.value : 0;
+};
+
+export class RendererMemorySampler implements PhasedMemorySampler<RendererPhaseStats> {
+  private readonly buckets = new Map<string, RendererMemorySample[]>();
+  private current: RendererMemorySample[];
+  private running = true;
+  private loop!: Promise<void>;
+
+  private constructor(
+    private readonly cdp: CDPSession,
+    private readonly intervalMs: number,
+    initialPhase: string
+  ) {
+    this.current = [];
+    this.buckets.set(initialPhase, this.current);
+  }
+
+  static async start(
+    cdp: CDPSession,
+    intervalMs = 200,
+    initialPhase = DEFAULT_PHASE
+  ): Promise<RendererMemorySampler> {
+    await cdp.send('Performance.enable');
+    const sampler = new RendererMemorySampler(cdp, intervalMs, initialPhase);
+    sampler.loop = sampler.poll();
+    return sampler;
+  }
+
+  static async measureRetainedBytes(cdp: CDPSession): Promise<number> {
+    await cdp.send('HeapProfiler.enable');
+    await cdp.send('HeapProfiler.collectGarbage');
+    return readHeapUsedBytes(cdp);
+  }
+
+  async startPhase(name: string): Promise<void> {
+    this.current = [];
+    this.buckets.set(name, this.current);
+  }
+
+  async terminate(): Promise<StatsByPhase<RendererPhaseStats>> {
+    this.running = false;
+    await this.loop;
+    return statsPerPhase(Object.fromEntries(this.buckets), rendererStatsOf);
+  }
+
+  private async poll(): Promise<void> {
+    while (this.running) {
+      this.current.push(await readHeapUsedBytes(this.cdp));
+      await new Promise((resolve) => setTimeout(resolve, this.intervalMs));
+    }
+  }
+}
+
+export class MainMemorySampler implements PhasedMemorySampler<MainPhaseStats> {
+  private constructor(private readonly electronApp: ElectronApplication) {}
+
+  static async start(
+    electronApp: ElectronApplication,
+    intervalMs = 100,
+    initialPhase = DEFAULT_PHASE
+  ): Promise<MainMemorySampler> {
+    await electronApp.evaluate((_electron, { interval, phase }) => {
+      const g = globalThis as unknown as MainSamplingGlobals;
+      g.__benchMainMem = { [phase]: [] };
+      g.__benchMainMemPhase = phase;
+      g.__benchMainMemTimer = setInterval(() => {
+        const { rss, heapUsed, external, arrayBuffers } = process.memoryUsage();
+        const currentPhase = g.__benchMainMemPhase;
+        if (currentPhase) {
+          g.__benchMainMem[currentPhase].push({ rss, heapUsed, external, arrayBuffers });
+        }
+      }, interval);
+    }, { interval: intervalMs, phase: initialPhase });
+
+    return new MainMemorySampler(electronApp);
+  }
+
+  async startPhase(name: string): Promise<void> {
+    await this.electronApp.evaluate((_electron, phase) => {
+      const g = globalThis as unknown as MainSamplingGlobals;
+      g.__benchMainMem[phase] = [];
+      g.__benchMainMemPhase = phase;
+    }, name);
+  }
+
+  async terminate(): Promise<StatsByPhase<MainPhaseStats>> {
+    const buckets = await this.electronApp.evaluate(() => {
+      const g = globalThis as unknown as MainSamplingGlobals;
+      if (g.__benchMainMemTimer) {
+        clearInterval(g.__benchMainMemTimer);
+      }
+      const collected = g.__benchMainMem || {};
+      g.__benchMainMem = {};
+      g.__benchMainMemPhase = null;
+      g.__benchMainMemTimer = null;
+      return collected;
+    });
+
+    return statsPerPhase(buckets, mainStatsOf);
+  }
+}