diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000..cce4f033cf --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,89 @@ +# GitHub Action to run the benchmarks of the game engine (GDJS/tests/benchmarks) +# on the changes made by a pull request, compared to its base branch. +# On master, each change is compared to the previous commit, so the results +# (in the summary of the runs, and as artifacts) track the evolution over time. + +name: Benchmarks + +on: + pull_request: + paths: + - "GDJS/**" + - "Extensions/**" + push: + branches: + - master + paths: + - "GDJS/**" + - "Extensions/**" + workflow_dispatch: + +concurrency: + group: benchmarks-${{ github.ref }} + cancel-in-progress: true + +jobs: + compare-benchmarks: + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + # To comment the results on the pull request. + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + # The base version is built from the history. + fetch-depth: 0 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: "npm" + cache-dependency-path: | + GDJS/package-lock.json + GDJS/tests/package-lock.json + + - name: Install GDJS dependencies + run: npm ci + working-directory: GDJS + + - name: Install GDJS tests dependencies + run: npm ci + working-directory: GDJS/tests + + - name: Compare the benchmarks with the base version + run: npm run benchmark:compare -- --base-ref="$BASE_REF" --markdown=benchmark-results.md --json=benchmark-results.json + working-directory: GDJS/tests + env: + BASE_REF: ${{ github.event_name == 'pull_request' && format('origin/{0}', github.base_ref) || 'HEAD^' }} + + - name: Show the results in the summary + run: cat benchmark-results.md >> "$GITHUB_STEP_SUMMARY" + working-directory: GDJS/tests + + - uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ github.sha }} + path: GDJS/tests/benchmark-results.json + retention-days: 90 + + # Pull requests from forks have no permission to comment: their results + # are in the summary of the run. + - name: Comment the results on the pull request + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + # Don't fail the build if commenting is not possible. + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + working-directory: GDJS/tests + # Update the previous comment (found with the marker starting the + # report) rather than adding a new one at each push. + run: | + COMMENT_ID=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PULL_REQUEST_NUMBER/comments" --paginate --jq '.[] | select(.body | startswith("")) | .id' | head -n 1) + if [ -n "$COMMENT_ID" ]; then + gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$COMMENT_ID" -F body=@benchmark-results.md + else + gh pr comment "$PULL_REQUEST_NUMBER" --body-file benchmark-results.md + fi diff --git a/.gitignore b/.gitignore index 5ac5441044..b0a710d5c9 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ .Trashes Thumbs.db .claude +/GDJS/tests/.benchmark-runtimes diff --git a/Extensions/PlatformBehavior/benchmarks/platformerobjectruntimebehavior.benchmark.js b/Extensions/PlatformBehavior/benchmarks/platformerobjectruntimebehavior.benchmark.js index a8ce2113a9..4ddc0b7fab 100644 --- a/Extensions/PlatformBehavior/benchmarks/platformerobjectruntimebehavior.benchmark.js +++ b/Extensions/PlatformBehavior/benchmarks/platformerobjectruntimebehavior.benchmark.js @@ -2,7 +2,7 @@ describe('gdjs.PlatformerObjectRuntimeBehavior Benchmark', function () { let runtimeScene; let objects; const duplicateCount = 60; - const stepCount = 6000; + const stepCount = 1000; beforeEach(function () { runtimeScene = makePlatformerTestRuntimeScene(); @@ -30,6 +30,8 @@ describe('gdjs.PlatformerObjectRuntimeBehavior Benchmark', function () { roundCoordinates: true, }, ], + variables: [], + effects: [], }); object.getWidth = function () { return 10; @@ -50,29 +52,34 @@ describe('gdjs.PlatformerObjectRuntimeBehavior Benchmark', function () { }); it('benchmark', function () { - this.timeout(30000); + this.timeout(60000); - for (let b = 0; b < 10; ++b) { - const benchmarkSuite = makeBenchmarkSuite({ - benchmarksCount: 1, - iterationsCount: stepCount, - }); - benchmarkSuite.add('jump in loop', (t) => { - for (let i = 0; i < duplicateCount; ++i) { - const object = objects[i]; - if (t % 60 == i % 60) { - object.getBehavior('auto1').simulateJumpKey(); - } - if (t + (i % 61) < 31) { - object.getBehavior('auto1').simulateRightKey(); - } - if (t + (i % 61) >= 31) { - object.getBehavior('auto1').simulateLeftKey(); - } + // Many short batches: the median is less sensitive to a slow batch + // (garbage collection...). + const benchmarkSuite = makeBenchmarkSuite({ + benchmarksCount: 60, + iterationsCount: stepCount, + }); + // Counted across batches: the objects move the same way whatever the + // batch size. + let frameIndex = 0; + benchmarkSuite.add('platformer objects jumping in loop', () => { + const t = frameIndex++; + for (let i = 0; i < duplicateCount; ++i) { + const object = objects[i]; + if (t % 60 == i % 60) { + object.getBehavior('auto1').simulateJumpKey(); } - runtimeScene.renderAndStep(1000 / 60); - }); - console.log(benchmarkSuite.run()); - } + // Go right then left, to stay on the platforms: the same work is + // measured by every batch. + if ((t + i) % 60 < 30) { + object.getBehavior('auto1').simulateRightKey(); + } else { + object.getBehavior('auto1').simulateLeftKey(); + } + } + runtimeScene.renderAndStep(1000 / 60); + }); + console.log(benchmarkSuite.run()); }); }); diff --git a/GDJS/Runtime/gameplay-tests/gameplay-test-runner.ts b/GDJS/Runtime/gameplay-tests/gameplay-test-runner.ts index a6cfb30d54..090ba4bb39 100644 --- a/GDJS/Runtime/gameplay-tests/gameplay-test-runner.ts +++ b/GDJS/Runtime/gameplay-tests/gameplay-test-runner.ts @@ -361,6 +361,29 @@ namespace gdjs { jsHeapUsedMb?: number; }; + /** + * The measures of `harness.benchmark`: the time of each measured frame + * (game logic and rendering), and counters that don't depend on the + * machine, to compare two versions of the engine running the same + * scenario. + */ + export type GameplayTestBenchmarkResult = { + name: string; + /** The time of each measured frame, in order. */ + frameTimesMs: Array; + medianFrameTimeMs: number; + /** The profiled sections (see `GameplayTestProfilingResult`), for + * the last 600 measured frames at most. */ + sections: Array<{ name: string; avgTimeMs: number; maxTimeMs: number }>; + /** WebGL draw calls (2D and 3D) per frame, or null without renderer. */ + drawCallsPerFrame: number | null; + objectsCount: integer; + /** A hash of the position and angle of every object at the end: two + * versions of the engine running the same scenario must end on the + * same state. */ + stateChecksum: string; + }; + /** * The outcome of `lookTowardWithMouseDelta`: whether the aim succeeded, * the remaining aim error, and whether the game responded to the mouse @@ -442,6 +465,8 @@ namespace gdjs { screenshotsTakenCount: integer; /** The `stopProfiling()` summaries captured during the run. */ profiles: Array; + /** The `benchmark()` measures captured during the run. */ + benchmarks: Array; performance: { avgStepMs: number; worstStepMs: number; @@ -476,6 +501,9 @@ namespace gdjs { const MAX_PROFILING_TIMELINE_ENTRIES = 120; const MAX_PROFILING_WORST_FRAMES = 5; const MAX_PROFILES_PER_RESULT = 5; + const MAX_BENCHMARKS_PER_RESULT = 20; + const DEFAULT_BENCHMARK_WARMUP_FRAMES = 60; + const DEFAULT_BENCHMARK_FRAMES = 300; /** The scene change cause declared by the harness (see * `SceneStack.runWithSceneChangeCause`). */ const GAMEPLAY_TEST_SCENE_CHANGE_CAUSE = 'gameplayTest'; @@ -757,6 +785,10 @@ namespace gdjs { /** The profiling summaries captured during the run (attached to the * result as `profiles`). */ _profiles: Array = []; + _benchmarks: Array = []; + /** The `Math.random` replaced by `setRandomSeed`, restored at the end + * of the run. */ + _originalMathRandom: (() => number) | null = null; _timeoutMs: number; _loadingTimeoutMs: number; /** Time spent waiting for loading (game boot, scene assets) so far: @@ -1290,17 +1322,17 @@ namespace gdjs { /** * Step a single game frame (game logic + rendering) with a fixed - * time delta. + * time delta, and return the time it took. */ - _stepSingleFrame(dtMs: float): void { + _stepSingleFrame(dtMs: float): float { this._checkGuards(); - const stepStartTimeMs = Date.now(); + const stepStartTimeMs = performance.now(); this._runtimeGame.getSceneStack().step(dtMs); + const stepTimeMs = performance.now() - stepStartTimeMs; this._callOnFrameEnded(); this._framesExecuted++; this._gameTimeMs += dtMs; - const stepTimeMs = Date.now() - stepStartTimeMs; - this._lastFrameStepTimeMs = stepStartTimeMs + stepTimeMs; + this._lastFrameStepTimeMs = Date.now(); this._totalStepTimeMs += stepTimeMs; if (stepTimeMs > this._worstStepTimeMs) { this._worstStepTimeMs = stepTimeMs; @@ -1311,6 +1343,7 @@ namespace gdjs { this._lastProgressTimeMs = Date.now(); this._onProgress(this._framesExecuted); } + return stepTimeMs; } /** @@ -3622,6 +3655,162 @@ namespace gdjs { return profile; } + /** + * Replace `Math.random` by a generator seeded with the given number + * until the end of the test, so that the game makes the same random + * choices at every run. + */ + setRandomSeed(seed: integer): void { + if (!this._originalMathRandom) { + this._originalMathRandom = Math.random; + } + // Mulberry32, a small and fast pseudo-random generator. + let state = seed >>> 0; + Math.random = () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + } + + _restoreMathRandom(): void { + if (this._originalMathRandom) { + Math.random = this._originalMathRandom; + this._originalMathRandom = null; + } + } + + /** + * Step `warmupFrames` frames (not measured: the JavaScript engine + * optimizes the code meanwhile), then measure each of the next + * `frames` frames, the profiled sections, the draw calls and a + * checksum of the final state. + * Frames never wait for an animation frame (which renders the game + * once more, and can be delayed by the browser), so that two runs of + * the same scenario do the same work. + */ + async benchmark( + name: string, + options?: { warmupFrames?: integer; frames?: integer; dtMs?: float } + ): Promise { + const warmupFrames = + options && options.warmupFrames !== undefined + ? options.warmupFrames + : DEFAULT_BENCHMARK_WARMUP_FRAMES; + const frames = (options && options.frames) || DEFAULT_BENCHMARK_FRAMES; + const dtMs = (options && options.dtMs) || DEFAULT_FRAME_DT_MS; + const frameTimesMs: Array = []; + const stepFramesWithoutRendering = async ( + frameCount: integer, + isMeasured: boolean + ) => { + for (let i = 0; i < frameCount; i++) { + const stepTimeMs = this._stepSingleFrame(dtMs); + if (isMeasured) { + frameTimesMs.push(Math.round(stepTimeMs * 1000) / 1000); + } + await this._waitForGameSceneSwitchIfNeeded(); + if (Date.now() - this._lastYieldTimeMs >= YIELD_BUDGET_MS) { + await new Promise((resolve) => setTimeout(resolve, 0)); + this._lastYieldTimeMs = Date.now(); + } + } + }; + await stepFramesWithoutRendering(warmupFrames, false); + + // Available when the browser is launched with `--js-flags=--expose-gc`. + const collectGarbage = (globalThis as any).gc; + if (typeof collectGarbage === 'function') collectGarbage(); + + const drawCallsCounter = this._countDrawCalls(); + this.startProfiling(); + try { + await stepFramesWithoutRendering(frames, true); + } finally { + if (drawCallsCounter) drawCallsCounter.uninstall(); + } + const profile = this.stopProfiling(); + + const sortedFrameTimesMs = [...frameTimesMs].sort((a, b) => a - b); + const benchmarkResult: GameplayTestBenchmarkResult = { + name, + frameTimesMs, + medianFrameTimeMs: + sortedFrameTimesMs[Math.floor(sortedFrameTimesMs.length / 2)] || 0, + sections: profile ? profile.sections : [], + drawCallsPerFrame: drawCallsCounter + ? Math.round((drawCallsCounter.getCount() / frames) * 100) / 100 + : null, + objectsCount: + this._getCurrentScene().getAdhocListOfAllInstances().length, + stateChecksum: this._computeStateChecksum(), + }; + this._benchmarks.push(benchmarkResult); + if (this._benchmarks.length > MAX_BENCHMARKS_PER_RESULT) { + this._benchmarks.shift(); + } + return benchmarkResult; + } + + /** + * Count the WebGL draw calls, made by PixiJS and Three.js (which + * share the same WebGL context), until `uninstall` is called. + */ + private _countDrawCalls(): { + getCount: () => integer; + uninstall: () => void; + } | null { + const pixiRenderer = this._runtimeGame.getRenderer().getPIXIRenderer(); + const gl: any = pixiRenderer ? pixiRenderer.gl : null; + if (!gl) return null; + let count = 0; + const originalMethods: { [methodName: string]: Function } = {}; + for (const methodName of [ + 'drawArrays', + 'drawElements', + 'drawArraysInstanced', + 'drawElementsInstanced', + ]) { + const originalMethod = gl[methodName]; + if (!originalMethod) continue; + originalMethods[methodName] = originalMethod; + gl[methodName] = function () { + count++; + return originalMethod.apply(this, arguments); + }; + } + return { + getCount: () => count, + uninstall: () => { + for (const methodName in originalMethods) { + gl[methodName] = originalMethods[methodName]; + } + }, + }; + } + + private _computeStateChecksum(): string { + const instances = [ + ...this._getCurrentScene().getAdhocListOfAllInstances(), + ].sort((a, b) => a.id - b.id); + // FNV-1a hash of the rounded values. + let hash = 0x811c9dc5; + const addText = (text: string) => { + for (let i = 0; i < text.length; i++) { + hash = Math.imul(hash ^ text.charCodeAt(i), 16777619); + } + }; + for (const instance of instances) { + addText(instance.getName()); + addText(instance.getX().toFixed(2)); + addText(instance.getY().toFixed(2)); + addText(instance.getAngle().toFixed(2)); + } + return (hash >>> 0).toString(16); + } + /** * Record a console log in the test result (also shown in the * browser console). Used by the `console` given to the script. @@ -3772,6 +3961,7 @@ namespace gdjs { screenshots: this._screenshots, screenshotsTakenCount: this._screenshotsTakenCount, profiles: this._profiles, + benchmarks: this._benchmarks, performance: this._framesExecuted > 0 ? { @@ -3779,7 +3969,7 @@ namespace gdjs { Math.round( (this._totalStepTimeMs / this._framesExecuted) * 100 ) / 100, - worstStepMs: this._worstStepTimeMs, + worstStepMs: Math.round(this._worstStepTimeMs * 100) / 100, } : null, }; @@ -4179,6 +4369,7 @@ namespace gdjs { // Ignore errors during cleanup. } inputManager.onFrameEnded = originalOnFrameEnded; + harness._restoreMathRandom(); harness._uninstallPageVisibilityTracking(); harness._uninstallPointerLockShim(); harness._uninstallSoundLog(); diff --git a/GDJS/tests/README.md b/GDJS/tests/README.md index 00f8010090..981e13af75 100644 --- a/GDJS/tests/README.md +++ b/GDJS/tests/README.md @@ -27,6 +27,18 @@ Tests are launched using Chrome. You need Chrome installed to run them. You can Tests are located in the **tests** folder for the game engine, or directly in the folder of the tested extensions. +### Benchmarks + +Benchmarks are in the **benchmarks** folder: micro benchmarks of the game engine functions, and scene benchmarks (in **benchmarks/scenes**) running scenes like a game with the gameplay test harness (`harness.benchmark`). + +To compare the speed of your changes with the version your branch started from: + +```bash +npm run benchmark:compare -- --base-ref=origin/master --grep="Scene benchmarks" # --grep is optional +``` + +Both versions are built and the benchmarks are run alternately on each one (3 times by default, set with `--rounds`). A change is only reported when larger than 5% and measured between every round of one version and every round of the other, as speed varies between runs of a browser (by up to ~15% on a shared machine). The same comparison runs on pull requests changing the game engine (see `.github/workflows/benchmarks.yml`). + ### Games in the _games_ folder Games contained in the _games_ folder are mainly here to be launched manually to check that a particular feature is working. Read the comments in the events to see what is the expected behavior, or compare with the native platform if you can. diff --git a/GDJS/tests/benchmarks/collisions.js b/GDJS/tests/benchmarks/collisions.js new file mode 100644 index 0000000000..3e67346f0e --- /dev/null +++ b/GDJS/tests/benchmarks/collisions.js @@ -0,0 +1,69 @@ +// @ts-check + +describe('gdjs.evtTools.object.hitBoxesCollisionTest', function () { + const runtimeGame = gdjs.getPixiRuntimeGame(); + const runtimeScene = new gdjs.RuntimeScene(runtimeGame); + + /** + * Objects of 32x32 pixels spread on an area growing with their number, so + * that each object overlaps about the same number of others. + * @param {string} name + * @param {number} count + * @param {() => number} random + */ + const createObjects = (name, count, random) => { + const areaSize = Math.sqrt(count) * 64; + const objects = []; + for (let i = 0; i < count; i++) { + const object = new gdjs.TestRuntimeObject(runtimeScene, { + name, + type: '', + behaviors: [], + effects: [], + variables: [], + }); + object.setCustomWidthAndHeight(32, 32); + object.setPosition(random() * areaSize, random() * areaSize); + objects.push(object); + } + return objects; + }; + + for (const objectsCount of [10, 100, 500, 1000]) { + it(`benchmark ${objectsCount} objects colliding with ${objectsCount} other objects`, function () { + this.timeout(60000); + const random = makeSeededRandom(objectsCount); + const allEnemies = createObjects('Enemy', objectsCount, random); + const allBullets = createObjects('Bullet', objectsCount, random); + + // Like the code generated for events, lists are refilled before each + // condition, as the condition removes the objects not picked. + const enemies = []; + const bullets = []; + const enemiesLists = Hashtable.newFrom({ Enemy: enemies }); + const bulletsLists = Hashtable.newFrom({ Bullet: bullets }); + + const benchmarkSuite = makeBenchmarkSuite({ + benchmarksCount: 30, + // About the same duration for each objects count, with a brute-force + // test of every pair of objects. + iterationsCount: Math.ceil(1000000 / (objectsCount * objectsCount)), + }); + benchmarkSuite.add( + `hitBoxesCollisionTest of ${objectsCount} objects with ${objectsCount} objects`, + () => { + gdjs.copyArray(allEnemies, enemies); + gdjs.copyArray(allBullets, bullets); + gdjs.evtTools.object.hitBoxesCollisionTest( + enemiesLists, + bulletsLists, + false, + runtimeScene, + false + ); + } + ); + console.log(benchmarkSuite.run()); + }); + } +}); diff --git a/GDJS/tests/benchmarks/init.js b/GDJS/tests/benchmarks/init.js index 1a21840a4a..19f7758a30 100644 --- a/GDJS/tests/benchmarks/init.js +++ b/GDJS/tests/benchmarks/init.js @@ -1,47 +1,104 @@ +// @ts-check + +/** + * @typedef {{ + * name: string, + * samplesMs: Array, + * counters?: {[counterName: string]: number | string | null}, + * }} BenchmarkResult + */ + +/** + * Send a benchmark result to the karma reporter writing the results file + * read by `compare-benchmarks.js`. `samplesMs` are the measures to compare, + * `counters` are values that must not depend on the machine (draw calls, + * objects count...). + * @param {BenchmarkResult} benchmarkResult + */ +const reportBenchmarkResult = (benchmarkResult) => { + // @ts-ignore - Karma API to send information to the reporters. + if (window.__karma__) window.__karma__.info({ benchmarkResult }); +}; + +/** + * A pseudo-random generator, so that benchmarks always create the same + * objects at the same positions. + * @param {number} seed + * @returns {() => number} A function returning a number in [0, 1[. + */ +const makeSeededRandom = (seed) => { + // Mulberry32, a small and fast pseudo-random generator. + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +}; + /** * Helper allowing to run a benchmark of the time spent the execute a certain * number of iterations of one or more functions. * - * Note that this could surely be replaced by a more robust solution like - * Benchmark.js + * The first tenth of the batches are a warm-up (not measured), so that the + * JavaScript engine optimizes the functions before they are measured. * - * @param {*} options + * @param {{benchmarksCount?: number, iterationsCount?: number}} options */ -let makeBenchmarkSuite = (options = {}) => { +const makeBenchmarkSuite = (options = {}) => { + /** @type {{[benchmarkName: string]: Array}} */ const benchmarkTimings = {}; - const benchmarksCount = options.benchmarksCount || 1000; + const benchmarksCount = options.benchmarksCount || 200; const iterationsCount = options.iterationsCount || 100000; + const warmUpBenchmarksCount = Math.ceil(benchmarksCount / 10); + /** @type {Array<{title: string, fn: (i: number) => void}>} */ const testCases = []; - const suite = {}; - suite.add = (title, fn) => { - testCases.push({ title, fn }); - return suite; - }; - suite.run = () => { - for ( - let benchmarkIndex = 0; - benchmarkIndex < benchmarksCount; - benchmarkIndex++ - ) { - testCases.forEach(testCase => { - const description = testCase.title + '(' + iterationsCount + 'x)'; - const start = performance.now(); - for (let i = 0; i < iterationsCount; i++) { - testCase.fn(i); - } - benchmarkTimings[description] = benchmarkTimings[description] || []; - benchmarkTimings[description].push(performance.now() - start); - }); - } + const suite = { + /** + * @param {string} title + * @param {(i: number) => void} fn + */ + add: (title, fn) => { + testCases.push({ title, fn }); + return suite; + }, + run: () => { + for ( + let benchmarkIndex = -warmUpBenchmarksCount; + benchmarkIndex < benchmarksCount; + benchmarkIndex++ + ) { + testCases.forEach((testCase) => { + const description = testCase.title + '(' + iterationsCount + 'x)'; + const start = performance.now(); + for (let i = 0; i < iterationsCount; i++) { + testCase.fn(i); + } + const duration = performance.now() - start; + if (benchmarkIndex < 0) return; + benchmarkTimings[description] = benchmarkTimings[description] || []; + benchmarkTimings[description].push(duration); + }); + } - const results = {}; - for (let benchmarkName in benchmarkTimings) { - results[benchmarkName] = - benchmarkTimings[benchmarkName].reduce((sum, value) => sum + value, 0) / - benchmarksCount; - } - return results; + /** @type {{[benchmarkName: string]: number}} */ + const results = {}; + for (const benchmarkName in benchmarkTimings) { + reportBenchmarkResult({ + name: benchmarkName, + samplesMs: benchmarkTimings[benchmarkName], + }); + results[benchmarkName] = + benchmarkTimings[benchmarkName].reduce( + (sum, value) => sum + value, + 0 + ) / benchmarksCount; + } + return results; + }, }; return suite; }; diff --git a/GDJS/tests/benchmarks/polygon.js b/GDJS/tests/benchmarks/polygon.js index 91a54af022..52504d6dfd 100644 --- a/GDJS/tests/benchmarks/polygon.js +++ b/GDJS/tests/benchmarks/polygon.js @@ -9,7 +9,7 @@ describe('gdjs.Polygon', function() { const benchmarkSuite = makeBenchmarkSuite({ benchmarksCount: 20, - iterationsCount: 30000, + iterationsCount: 120000, }); benchmarkSuite .add('collisionTest between two overlapping rectangles', i => { diff --git a/GDJS/tests/benchmarks/runtimeobject.js b/GDJS/tests/benchmarks/runtimeobject.js index 7cc42d2eb8..1227e5d48b 100644 --- a/GDJS/tests/benchmarks/runtimeobject.js +++ b/GDJS/tests/benchmarks/runtimeobject.js @@ -11,7 +11,7 @@ describe('gdjs.RuntimeObject', function() { const benchmarkSuite = makeBenchmarkSuite({ benchmarksCount: 60, - iterationsCount: 60000, + iterationsCount: 180000, }); benchmarkSuite .add('getAABB of a non rotated, default center', (i) => { @@ -37,7 +37,7 @@ describe('gdjs.RuntimeObject', function() { const benchmarkSuite = makeBenchmarkSuite({ benchmarksCount: 60, - iterationsCount: 60000, + iterationsCount: 180000, }); benchmarkSuite .add('getAABB of a non rotated, non default center', (i) => { diff --git a/GDJS/tests/benchmarks/scenes/scenes.js b/GDJS/tests/benchmarks/scenes/scenes.js new file mode 100644 index 0000000000..4c6b66f67e --- /dev/null +++ b/GDJS/tests/benchmarks/scenes/scenes.js @@ -0,0 +1,715 @@ +// @ts-check + +/** + * Scene benchmarks: scenes run like in a game (objects, behaviors, events + * and rendering), measured by the gameplay test harness (`harness.benchmark`). + * Events are written like the code generated for them. + */ +describe('Scene benchmarks', function () { + const imageResourceName = 'base/GDJS/tests/tests-utils/assets/64x64.jpg'; + // A small resolution and small objects: on machines without GPU, pixels + // are drawn by the CPU, which is slow and is not the work of the engine + // being measured (it does not depend on the size of objects). + const gameWidth = 320; + const gameHeight = 240; + + /** @param {number} ms */ + const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + + /** + * @param {gdjs.RuntimeScene} runtimeScene + * @param {string} objectName + */ + const getObjects = (runtimeScene, objectName) => + runtimeScene.getObjects(objectName) || []; + + /** + * @param {string} name + * @param {Array} behaviors + * @returns {gdjs.SpriteObjectData} + */ + const createSpriteObjectData = (name, behaviors = []) => ({ + name, + type: 'Sprite', + variables: [], + behaviors, + effects: [], + updateIfNotVisible: false, + animations: [ + { + name: 'Idle', + useMultipleDirections: false, + directions: [ + { + timeBetweenFrames: 0, + looping: false, + sprites: [ + { + image: imageResourceName, + points: [], + originPoint: { name: 'origine', x: 0, y: 0 }, + centerPoint: { name: 'centre', x: 0, y: 0, automatic: true }, + hasCustomCollisionMask: false, + customCollisionMask: [], + }, + ], + }, + ], + }, + ], + }); + + /** + * @param {string} name + * @returns {gdjs.TextObjectData} + */ + const createTextObjectData = (name) => ({ + name, + type: 'TextObject::Text', + variables: [], + behaviors: [], + effects: [], + content: { + characterSize: 12, + font: '', + bold: false, + italic: false, + underlined: false, + color: '255;255;255', + text: 'Score: 0', + textAlignment: 'left', + verticalTextAlignment: 'top', + lineHeight: 0, + isOutlineEnabled: false, + outlineThickness: 0, + outlineColor: '0;0;0', + isShadowEnabled: false, + shadowColor: '0;0;0', + shadowOpacity: 0, + shadowDistance: 0, + shadowAngle: 0, + shadowBlurRadius: 0, + }, + }); + + /** + * @param {string} name + * @param {Array} behaviors + * @returns {gdjs.Cube3DObjectData} + */ + const createCubeObjectData = (name, behaviors) => + /** @type {gdjs.Cube3DObjectData} */ ({ + name, + type: 'Scene3D::Cube3DObject', + variables: [], + behaviors, + effects: [], + content: { + width: 16, + height: 16, + depth: 16, + enableTextureTransparency: false, + facesOrientation: 'Y', + frontFaceResourceName: imageResourceName, + backFaceResourceName: imageResourceName, + backFaceUpThroughWhichAxisRotation: 'X', + leftFaceResourceName: imageResourceName, + rightFaceResourceName: imageResourceName, + topFaceResourceName: imageResourceName, + bottomFaceResourceName: imageResourceName, + frontFaceResourceRepeat: false, + backFaceResourceRepeat: false, + leftFaceResourceRepeat: false, + rightFaceResourceRepeat: false, + topFaceResourceRepeat: false, + bottomFaceResourceRepeat: false, + tileScale: 1, + frontFaceVisible: true, + backFaceVisible: true, + leftFaceVisible: true, + rightFaceVisible: true, + topFaceVisible: true, + bottomFaceVisible: true, + tint: '255;255;255', + isCastingShadow: false, + isReceivingShadow: false, + materialType: 'Basic', + }, + }); + + /** + * @param {string} name + * @param {{x: number, y: number, z?: number, width: number, height: number, depth?: number}} instance + * @returns {InstanceData} + */ + const createInstance = (name, { x, y, z, width, height, depth }) => ({ + persistentUuid: '', + layer: '', + name, + x, + y, + z, + angle: 0, + zOrder: 0, + customSize: true, + width, + height, + depth, + numberProperties: [], + stringProperties: [], + initialVariables: [], + }); + + /** + * @param {string} name + * @param {number} count + * @param {{width: number, height: number, areaWidth?: number, areaHeight?: number, minY?: number}} options + * @param {() => number} random + */ + const createRandomInstances = (name, count, options, random) => { + const areaWidth = options.areaWidth || gameWidth; + const areaHeight = options.areaHeight || gameHeight; + const instances = []; + for (let i = 0; i < count; i++) { + instances.push( + createInstance(name, { + x: random() * (areaWidth - options.width), + y: (options.minY || 0) + random() * (areaHeight - options.height), + width: options.width, + height: options.height, + }) + ); + } + return instances; + }; + + /** + * Run the scene with the gameplay test harness, and report the measures of + * `harness.benchmark`. + * @param {{ + * name: string, + * objects: Array, + * instances: Array, + * behaviorsSharedData?: Array, + * eventsFunction?: (runtimeScene: gdjs.RuntimeScene) => void, + * renderingType?: '' | '2d' | '3d' | '2d+3d', + * warmupFrames?: number, + * frames?: number, + * }} scene + */ + const runSceneBenchmark = async ({ + name, + objects, + instances, + behaviorsSharedData, + eventsFunction, + renderingType, + warmupFrames, + frames, + }) => { + // The physics engines are loaded asynchronously, and needed even by games + // not using them (to dispose them). + for (let i = 0; i < 400 && !(window.Box2D && window.Jolt); i++) { + await delay(5); + } + const mangledName = 'SceneBenchmark' + name.replace(/[^A-Za-z0-9]/g, ''); + // Where the events of a scene are found (see `setEventsGeneratedCodeFunction`). + // @ts-ignore + gdjs[mangledName + 'Code'] = { func: eventsFunction || (() => {}) }; + const runtimeGame = new gdjs.RuntimeGame( + gdjs.createProjectData({ + layouts: [ + { + r: 0, + v: 0, + b: 0, + mangledName, + name, + stopSoundsOnStartup: false, + title: '', + behaviorsSharedData: behaviorsSharedData || [], + objects, + objectsGroups: [], + instances, + layers: [ + { + name: '', + renderingType, + visibility: true, + cameras: [], + effects: [], + ambientLightColorR: 200, + ambientLightColorG: 200, + ambientLightColorB: 200, + isLightingLayer: false, + followBaseLayerCamera: false, + }, + ], + variables: [], + usedResources: [], + renderer3DWorldScale: 100, + uiSettings: { + grid: false, + gridType: 'rectangular', + gridWidth: 10, + gridHeight: 10, + gridOffsetX: 0, + gridOffsetY: 0, + gridColor: 0, + gridAlpha: 1, + snap: false, + }, + }, + ], + resources: { + resources: [ + { + kind: 'image', + name: imageResourceName, + metadata: '', + file: imageResourceName, + userAdded: true, + }, + ], + }, + propertiesOverrides: { + windowWidth: gameWidth, + windowHeight: gameHeight, + }, + }) + ); + // The assets of the scene are loaded by `harness.goToScene`. + runtimeGame.getRenderer().createStandardCanvas(document.body); + + const result = await gdjs.gameplayTests.runGameplayTest(runtimeGame, { + testName: name, + timeoutMs: 120000, + maxScreenshots: 0, + source: [ + 'harness.setRandomSeed(1);', + `await harness.goToScene(${JSON.stringify(name)});`, + `await harness.benchmark(${JSON.stringify(name)}, ${JSON.stringify({ + warmupFrames: warmupFrames || 30, + frames: frames || 200, + })});`, + ].join('\n'), + }); + runtimeGame.dispose(true); + // @ts-ignore + delete gdjs[mangledName + 'Code']; + + expect(result.errors).to.eql([]); + expect(result.status).to.be('passed'); + const benchmarkResult = result.benchmarks[0]; + reportBenchmarkResult({ + name: 'Scene: ' + name, + samplesMs: benchmarkResult.frameTimesMs, + counters: { + drawCallsPerFrame: benchmarkResult.drawCallsPerFrame, + objectsCount: benchmarkResult.objectsCount, + stateChecksum: benchmarkResult.stateChecksum, + }, + }); + console.log( + `${name}: ${benchmarkResult.medianFrameTimeMs}ms per frame (median), ` + + `${benchmarkResult.drawCallsPerFrame} draw calls per frame, ` + + `${benchmarkResult.objectsCount} objects. Slowest sections: ` + + benchmarkResult.sections + .slice(0, 4) + .map((section) => `${section.name} ${section.avgTimeMs}ms`) + .join(', ') + ); + }; + + it('benchmark static sprites', async function () { + this.timeout(120000); + await runSceneBenchmark({ + name: 'Static sprites (10000)', + objects: [createSpriteObjectData('Tile')], + instances: createRandomInstances( + 'Tile', + 10000, + { width: 4, height: 4 }, + makeSeededRandom(1) + ), + }); + }); + + it('benchmark moving and rotating sprites', async function () { + this.timeout(120000); + await runSceneBenchmark({ + name: 'Moving and rotating sprites (5000)', + objects: [createSpriteObjectData('Ship')], + instances: createRandomInstances( + 'Ship', + 5000, + { width: 4, height: 4 }, + makeSeededRandom(1) + ), + eventsFunction: (runtimeScene) => { + for (const ship of getObjects(runtimeScene, 'Ship')) { + ship.rotate(90); + ship.addPolarForce(ship.getAngle(), 60, 0); + ship.setPosition( + (ship.getX() + gameWidth) % gameWidth, + (ship.getY() + gameHeight) % gameHeight + ); + } + }, + }); + }); + + it('benchmark collisions and separation', async function () { + this.timeout(120000); + const random = makeSeededRandom(1); + const area = { width: 10, height: 10, areaWidth: 640, areaHeight: 480 }; + const enemies = []; + const obstacles = []; + const enemiesLists = Hashtable.newFrom({ Enemy: enemies }); + const obstaclesLists = Hashtable.newFrom({ Obstacle: obstacles }); + await runSceneBenchmark({ + name: 'Collisions and separation (500 vs 500)', + objects: [ + createSpriteObjectData('Enemy'), + createSpriteObjectData('Obstacle'), + ], + instances: [ + ...createRandomInstances('Enemy', 500, area, random), + ...createRandomInstances('Obstacle', 500, area, random), + ], + eventsFunction: (runtimeScene) => { + gdjs.copyArray(getObjects(runtimeScene, 'Enemy'), enemies); + for (const enemy of enemies) { + enemy.addForceTowardPosition(320, 240, 40, 0); + } + gdjs.copyArray(getObjects(runtimeScene, 'Obstacle'), obstacles); + if ( + gdjs.evtTools.object.hitBoxesCollisionTest( + enemiesLists, + obstaclesLists, + false, + runtimeScene, + false + ) + ) { + for (const enemy of enemies) { + enemy.separateFromObjectsList(obstaclesLists, false); + } + } + }, + warmupFrames: 30, + frames: 120, + }); + }); + + it('benchmark physics 2D bodies', async function () { + this.timeout(120000); + const physicsBehavior = { + name: 'Physics2', + type: 'Physics2::Physics2Behavior', + bodyType: 'Dynamic', + bullet: false, + fixedRotation: false, + canSleep: true, + shape: 'Box', + shapeDimensionA: 0, + shapeDimensionB: 0, + shapeOffsetX: 0, + shapeOffsetY: 0, + polygonOrigin: 'Center', + vertices: [], + density: 1, + friction: 0.3, + restitution: 0.1, + linearDamping: 0, + angularDamping: 0.1, + gravityScale: 1, + layers: 1, + masks: 1, + }; + await runSceneBenchmark({ + name: 'Physics 2D (500 bodies)', + objects: [ + createSpriteObjectData('Box', [physicsBehavior]), + createSpriteObjectData('Ground', [ + { ...physicsBehavior, bodyType: 'Static' }, + ]), + ], + instances: [ + ...createRandomInstances( + 'Box', + 500, + { width: 8, height: 8, areaHeight: 400, minY: -400 }, + makeSeededRandom(1) + ), + createInstance('Ground', { + x: 0, + y: gameHeight - 16, + width: gameWidth, + height: 16, + }), + ], + behaviorsSharedData: [ + { + name: 'Physics2', + type: 'Physics2::Physics2Behavior', + gravityX: 0, + gravityY: 9.8, + scaleX: 100, + scaleY: 100, + }, + ], + }); + }); + + it('benchmark physics 3D bodies', async function () { + this.timeout(120000); + const physicsBehavior = { + name: 'Physics3D', + type: 'Physics3D::Physics3DBehavior', + bodyType: 'Dynamic', + bullet: false, + fixedRotation: false, + shape: 'Box', + meshShapeResourceName: '', + shapeOrientation: 'Z', + shapeDimensionA: 0, + shapeDimensionB: 0, + shapeDimensionC: 0, + shapeOffsetX: 0, + shapeOffsetY: 0, + shapeOffsetZ: 0, + massCenterOffsetX: 0, + massCenterOffsetY: 0, + massCenterOffsetZ: 0, + massOverride: 0, + density: 1, + friction: 0.3, + restitution: 0.1, + linearDamping: 0.1, + angularDamping: 0.1, + gravityScale: 1, + layers: (1 << 4) | (1 << 0), + masks: (1 << 4) | (1 << 0), + }; + const random = makeSeededRandom(1); + const boxes = []; + for (let i = 0; i < 500; i++) { + boxes.push( + createInstance('Box', { + x: random() * gameWidth, + y: random() * gameHeight, + z: 50 + random() * 800, + width: 16, + height: 16, + depth: 16, + }) + ); + } + await runSceneBenchmark({ + name: 'Physics 3D (500 bodies)', + objects: [ + createCubeObjectData('Box', [physicsBehavior]), + createCubeObjectData('Ground', [ + { ...physicsBehavior, bodyType: 'Static' }, + ]), + ], + instances: [ + ...boxes, + createInstance('Ground', { + x: -100, + y: -100, + z: -16, + width: gameWidth + 200, + height: gameHeight + 200, + depth: 16, + }), + ], + renderingType: '3d', + behaviorsSharedData: [ + { + name: 'Physics3D', + type: 'Physics3D::Physics3DBehavior', + gravityX: 0, + gravityY: 0, + gravityZ: -9.8, + worldScale: 100, + }, + ], + }); + }); + + it('benchmark platformer characters', async function () { + this.timeout(120000); + const platforms = []; + for (let x = 0; x < gameWidth; x += 32) { + platforms.push( + createInstance('Platform', { + x, + y: gameHeight - 16, + width: 32, + height: 16, + }) + ); + } + for (let x = 16; x < gameWidth; x += 96) { + platforms.push( + createInstance('Platform', { + x, + y: gameHeight - 80, + width: 48, + height: 8, + }) + ); + } + await runSceneBenchmark({ + name: 'Platformer characters (300)', + objects: [ + createSpriteObjectData('Player', [ + { + type: 'PlatformBehavior::PlatformerObjectBehavior', + name: 'PlatformerObject', + gravity: 1000, + maxFallingSpeed: 700, + acceleration: 1500, + deceleration: 1500, + maxSpeed: 150, + jumpSpeed: 400, + canGrabPlatforms: false, + ignoreDefaultControls: true, + slopeMaxAngle: 60, + jumpSustainTime: 0.2, + useLegacyTrajectory: false, + useRepeatedJump: false, + }, + ]), + createSpriteObjectData('Platform', [ + { + type: 'PlatformBehavior::PlatformBehavior', + name: 'Platform', + platformType: 'NormalPlatform', + canBeGrabbed: false, + yGrabOffset: 0, + }, + ]), + ], + instances: [ + ...createRandomInstances( + 'Player', + 300, + { width: 8, height: 16, areaHeight: gameHeight - 100 }, + makeSeededRandom(1) + ), + ...platforms, + ], + eventsFunction: (runtimeScene) => { + const timeFromStart = runtimeScene.getTimeManager().getTimeFromStart(); + const isGoingRight = Math.floor(timeFromStart / 2000) % 2 === 0; + const isJumping = Math.floor(timeFromStart / 500) % 3 === 0; + for (const player of getObjects(runtimeScene, 'Player')) { + const platformer = + /** @type {gdjs.PlatformerObjectRuntimeBehavior} */ ( + player.getBehavior('PlatformerObject') + ); + if (isGoingRight) platformer.simulateRightKey(); + else platformer.simulateLeftKey(); + if (isJumping && player.id % 2 === 0) platformer.simulateJumpKey(); + if (player.getY() > gameHeight) player.setPosition(player.getX(), 0); + } + }, + }); + }); + + it('benchmark text updates', async function () { + this.timeout(120000); + await runSceneBenchmark({ + name: 'Text updates (1000)', + objects: [createTextObjectData('Score')], + instances: createRandomInstances( + 'Score', + 1000, + { width: 60, height: 12 }, + makeSeededRandom(1) + ), + eventsFunction: (runtimeScene) => { + const score = Math.floor( + runtimeScene.getTimeManager().getTimeFromStart() / 10 + ); + const texts = getObjects(runtimeScene, 'Score'); + for (let i = 0; i < texts.length; i++) { + /** @type {gdjs.TextRuntimeObject} */ (texts[i]).setText( + 'Score: ' + gdjs.evtTools.common.toString(score + i) + ); + } + }, + warmupFrames: 10, + frames: 40, + }); + }); + + it('benchmark objects creation and deletion', async function () { + this.timeout(120000); + await runSceneBenchmark({ + name: 'Objects creation and deletion (100 per frame)', + objects: [createSpriteObjectData('Bullet')], + instances: [], + eventsFunction: (runtimeScene) => { + for (let i = 0; i < 100; i++) { + const bullet = runtimeScene.createObject('Bullet'); + if (bullet) { + bullet.setPosition( + Math.random() * gameWidth, + Math.random() * gameHeight + ); + bullet.setWidth(8); + bullet.setHeight(8); + } + } + const bullets = getObjects(runtimeScene, 'Bullet'); + if (bullets.length > 2000) { + for (const bullet of bullets.slice(0, 100)) { + bullet.deleteFromScene(); + } + } + }, + }); + }); + + it('benchmark heavy events', async function () { + this.timeout(120000); + await runSceneBenchmark({ + name: 'Heavy events (variables, loops, strings)', + objects: [createSpriteObjectData('Unit')], + instances: createRandomInstances( + 'Unit', + 1000, + { width: 8, height: 8 }, + makeSeededRandom(1) + ), + eventsFunction: (runtimeScene) => { + const data = runtimeScene.getVariables().get('Data'); + for (let i = 0; i < 5000; i++) { + const item = data.getChild('Item' + gdjs.evtTools.common.toString(i)); + item.setNumber(item.getAsNumber() * 0.99 + Math.sin(i)); + } + const log = runtimeScene.getVariables().get('Log'); + log.pushValue( + 'Frame ' + gdjs.evtTools.common.toString(data.getChildrenCount()) + ); + if (log.getChildrenCount() > 100) log.removeAtIndex(0); + for (const unit of getObjects(runtimeScene, 'Unit')) { + const health = unit.getVariables().get('Health'); + health.add(1); + if (health.getAsNumber() > 100) { + health.setNumber(0); + unit.setX((unit.getX() + 1) % gameWidth); + } + unit + .getVariables() + .get('Name') + .setString('Unit ' + gdjs.evtTools.common.toString(unit.id)); + } + }, + }); + }); +}); diff --git a/GDJS/tests/benchmarks/variablescontainer.js b/GDJS/tests/benchmarks/variablescontainer.js index 1183dc442e..3ecbd1dcd2 100644 --- a/GDJS/tests/benchmarks/variablescontainer.js +++ b/GDJS/tests/benchmarks/variablescontainer.js @@ -3,7 +3,7 @@ describe('gdjs.VariablesContainer', function() { this.timeout(20000); var container = new gdjs.VariablesContainer(); - const benchmarkSuite = makeBenchmarkSuite(); + const benchmarkSuite = makeBenchmarkSuite({ iterationsCount: 400000 }); benchmarkSuite .add('get', () => { container.get('Var1'); diff --git a/GDJS/tests/compare-benchmarks.js b/GDJS/tests/compare-benchmarks.js new file mode 100644 index 0000000000..41bd3dc574 --- /dev/null +++ b/GDJS/tests/compare-benchmarks.js @@ -0,0 +1,354 @@ +// @ts-check +/** + * Compare the benchmarks (`GDJS/tests/benchmarks`) of the current version of + * the game engine with a base version (by default, where the branch started + * from `origin/master`). + * + * Both versions are built, then the benchmarks are run alternately on each + * version, on the same machine: the differences of speed between machines + * (or a machine slowing down during the run) don't affect the comparison. + * + * Usage: node compare-benchmarks.js [--base-ref=origin/master] [--rounds=3] + * [--grep=] [--markdown=] [--json=] + * [--browser=ChromeHeadlessBenchmark] + */ +const { execFileSync, spawnSync } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const args = require('minimist')(process.argv.slice(2), { + string: ['base-ref', 'grep', 'markdown', 'json', 'browser'], + default: { + 'base-ref': 'origin/master', + rounds: 3, + browser: 'ChromeHeadlessBenchmark', + }, +}); + +/** A change smaller than this is never reported as significant. */ +const MINIMUM_SIGNIFICANT_CHANGE = 0.05; + +const repositoryPath = path.join(__dirname, '../..'); +// Relative to the repository root, as expected by `karma.conf.js`. +const builtRuntimesPath = 'GDJS/tests/.benchmark-runtimes'; +const resultsPath = path.join(repositoryPath, builtRuntimesPath, 'results'); + +/** + * @typedef {{ + * name: string, + * samplesMs: Array, + * counters?: {[counterName: string]: number | string | null}, + * }} BenchmarkResult + * @typedef {'base' | 'head'} Version + */ + +/** + * @param {Array} gitArgs + * @param {string=} cwd + */ +const git = (gitArgs, cwd = repositoryPath) => + execFileSync('git', gitArgs, { cwd, encoding: 'utf8' }).trim(); + +/** + * @param {string} gdjsPath The `GDJS` folder of the version to build. + * @param {Version} version + */ +const buildRuntime = (gdjsPath, version) => { + const outPath = path.join(repositoryPath, builtRuntimesPath, version); + fs.rmSync(outPath, { recursive: true, force: true }); + execFileSync('node', ['scripts/build.js', '--out', outPath], { + cwd: gdjsPath, + stdio: 'inherit', + }); +}; + +/** @param {string} baseCommit */ +const buildBaseRuntime = (baseCommit) => { + const worktreePath = fs.mkdtempSync( + path.join(os.tmpdir(), 'gdjs-benchmark-base-') + ); + git(['worktree', 'add', '--detach', worktreePath, baseCommit]); + try { + // The dependencies of the current version are reused (they rarely change + // and are only used to build). + fs.symlinkSync( + path.join(repositoryPath, 'GDJS/node_modules'), + path.join(worktreePath, 'GDJS/node_modules'), + 'dir' + ); + buildRuntime(path.join(worktreePath, 'GDJS'), 'base'); + } finally { + git(['worktree', 'remove', '--force', worktreePath]); + } +}; + +/** + * @param {Version} version + * @param {number} round + * @returns {Array} + */ +const runBenchmarks = (version, round) => { + const resultsFile = path.join(resultsPath, `${version}-${round}.json`); + console.log(`\nā±ļø Running the benchmarks on ${version} (round ${round})...`); + const { status } = spawnSync( + 'npx', + [ + 'karma', + 'start', + '--single-run', + `--browsers=${args.browser}`, + '--benchmarksOnly', + `--gdjsRuntimePath=${builtRuntimesPath}/${version}`, + `--benchmarkResultsFile=${resultsFile}`, + ...(args.grep ? [`--grep=${args.grep}`] : []), + ], + { cwd: __dirname, stdio: 'inherit' } + ); + if (status !== 0) { + console.warn( + `āš ļø Some benchmarks failed on ${version} (round ${round}): they are reported as missing.` + ); + } + return fs.existsSync(resultsFile) + ? JSON.parse(fs.readFileSync(resultsFile, 'utf8')) + : []; +}; + +/** @param {Array} values */ +const median = (values) => { + const sortedValues = [...values].sort((a, b) => a - b); + const middle = Math.floor(sortedValues.length / 2); + return sortedValues.length % 2 + ? sortedValues[middle] + : (sortedValues[middle - 1] + sortedValues[middle]) / 2; +}; + +/** @param {number} change */ +const formatChange = (change) => + (change > 0 ? '+' : '') + (change * 100).toFixed(1) + '%'; + +/** @param {number} timeMs */ +const formatTime = (timeMs) => + timeMs >= 10 ? timeMs.toFixed(1) + 'ms' : timeMs.toFixed(3) + 'ms'; + +/** + * Counters don't depend on the machine: compare them exactly. A counter + * differing between rounds of the same version is not deterministic. + * @param {Array} baseResults + * @param {Array} headResults + * @returns {Array<{name: string, base: string, head: string}>} + */ +const compareCounters = (baseResults, headResults) => { + const counterNames = new Set( + [...baseResults, ...headResults].flatMap((result) => + Object.keys(result.counters || {}) + ) + ); + return [...counterNames].map((counterName) => { + /** @param {Array} results */ + const formatCounter = (results) => { + const values = [ + ...new Set( + results.map((result) => String((result.counters || {})[counterName])) + ), + ]; + if (!values.length) return '-'; + return values.length === 1 + ? values[0] + : values.join(' / ') + ' (not deterministic)'; + }; + return { + name: counterName, + base: formatCounter(baseResults), + head: formatCounter(headResults), + }; + }); +}; + +/** + * @param {{[version: string]: Array>}} resultsByVersionAndRound + */ +const compareResults = (resultsByVersionAndRound) => { + /** @type {Array} */ + const benchmarkNames = []; + for (const version of ['base', 'head']) { + for (const roundResults of resultsByVersionAndRound[version]) { + for (const { name } of roundResults) { + if (!benchmarkNames.includes(name)) benchmarkNames.push(name); + } + } + } + + return benchmarkNames.map((name) => { + /** @param {Version} version */ + const getRoundResults = (version) => + resultsByVersionAndRound[version] + .map((roundResults) => + roundResults.find((result) => result.name === name) + ) + .filter(Boolean); + const baseResults = getRoundResults('base'); + const headResults = getRoundResults('head'); + // Each round is summarized by its median: robust to a few slow samples + // (garbage collection, the machine doing something else...). + const baseMediansMs = baseResults.map((result) => median(result.samplesMs)); + const headMediansMs = headResults.map((result) => median(result.samplesMs)); + if (!baseMediansMs.length || !headMediansMs.length) { + return { + name, + baseMs: baseMediansMs.length ? median(baseMediansMs) : null, + headMs: headMediansMs.length ? median(headMediansMs) : null, + change: null, + changeRange: null, + verdict: baseMediansMs.length ? 'removed or failing' : 'new', + counters: compareCounters(baseResults, headResults), + }; + } + + const baseMs = median(baseMediansMs); + const headMs = median(headMediansMs); + const change = headMs / baseMs - 1; + // The changes measured between every round of the base and every round + // of the head: the change is significant only if all agree. + const roundChanges = baseMediansMs.flatMap((baseMedianMs) => + headMediansMs.map((headMedianMs) => headMedianMs / baseMedianMs - 1) + ); + const changeRange = [Math.min(...roundChanges), Math.max(...roundChanges)]; + const isSignificant = + Math.abs(change) >= MINIMUM_SIGNIFICANT_CHANGE && + (changeRange[0] > 0 || changeRange[1] < 0); + + return { + name, + baseMs, + headMs, + change, + changeRange, + verdict: !isSignificant ? 'unchanged' : change < 0 ? 'faster' : 'slower', + counters: compareCounters(baseResults, headResults), + }; + }); +}; + +/** + * @param {ReturnType} comparisons + * @param {{baseCommit: string, headCommit: string, rounds: number}} context + */ +const makeMarkdownReport = ( + comparisons, + { baseCommit, headCommit, rounds } +) => { + const verdictEmojis = { + faster: 'šŸš€ faster', + slower: '🐢 slower', + unchanged: 'ā‰ˆ', + new: 'šŸ†•', + 'removed or failing': 'āš ļø removed or failing', + }; + const lines = [ + '', + '### ā±ļø Game engine benchmarks', + '', + `Comparing \`${headCommit.slice(0, 8)}\` to its base \`${baseCommit.slice(0, 8)}\` ` + + '(on a pull request, the version tested is the pull request merged into its base), ' + + `run alternately ${rounds} times on the same machine (${os.cpus()[0].model}, ` + + `${os.cpus().length} cores, rendering without GPU).`, + '', + '| Benchmark | Base | This version | Change | Range between rounds | |', + '|---|---:|---:|---:|---:|---|', + ...comparisons.map( + ({ name, baseMs, headMs, change, changeRange, verdict }) => + `| ${name} | ${baseMs === null ? '-' : formatTime(baseMs)} | ` + + `${headMs === null ? '-' : formatTime(headMs)} | ` + + `${change === null ? '-' : formatChange(change)} | ` + + `${changeRange === null ? '-' : changeRange.map(formatChange).join(' to ')} | ` + + `${verdictEmojis[verdict]} |` + ), + '', + 'Times are the median duration of a batch of iterations (micro benchmarks) or of a frame (scene benchmarks). ' + + `A change is reported only when larger than ${MINIMUM_SIGNIFICANT_CHANGE * 100}% ` + + 'and measured by every pair of rounds.', + ]; + + const changedCounters = comparisons.flatMap(({ name, counters }) => + counters + .filter( + (counter) => + (counter.base !== '-' && + counter.head !== '-' && + counter.base !== counter.head) || + counter.base.includes('not deterministic') || + counter.head.includes('not deterministic') + ) + .map((counter) => ({ benchmarkName: name, ...counter })) + ); + const comparedCountersBenchmarksCount = comparisons.filter(({ counters }) => + counters.some((counter) => counter.base !== '-' && counter.head !== '-') + ).length; + lines.push( + '', + changedCounters.length + ? 'āš ļø **Counters changed or not deterministic** (draw calls, objects count or final state of a scene):' + : comparedCountersBenchmarksCount + ? `āœ… The counters (draw calls, objects count and final state) of the ${comparedCountersBenchmarksCount} scene benchmarks run on both versions are unchanged.` + : 'ā„¹ļø No scene benchmark ran on both versions: their counters (draw calls, objects count and final state) could not be compared.' + ); + if (changedCounters.length) { + lines.push( + '', + '| Benchmark | Counter | Base | This version |', + '|---|---|---|---|', + ...changedCounters.map( + ({ benchmarkName, name, base, head }) => + `| ${benchmarkName} | ${name} | ${base} | ${head} |` + ) + ); + } + return lines.join('\n') + '\n'; +}; + +const main = () => { + const rounds = Number(args.rounds); + const headCommit = git(['rev-parse', 'HEAD']); + const baseCommit = git(['merge-base', args['base-ref'], 'HEAD']); + fs.rmSync(resultsPath, { recursive: true, force: true }); + fs.mkdirSync(resultsPath, { recursive: true }); + + console.log( + `šŸ—ļø Building the game engine of this version (${headCommit})...` + ); + buildRuntime(path.join(repositoryPath, 'GDJS'), 'head'); + console.log( + `šŸ—ļø Building the game engine of the base version (${baseCommit})...` + ); + buildBaseRuntime(baseCommit); + + /** @type {{[version: string]: Array>}} */ + const resultsByVersionAndRound = { base: [], head: [] }; + for (let round = 1; round <= rounds; round++) { + // Alternate which version runs first, so that a machine getting slower + // (or faster) during the run affects both versions the same way. + /** @type {Array} */ + const versions = round % 2 ? ['base', 'head'] : ['head', 'base']; + for (const version of versions) { + resultsByVersionAndRound[version].push(runBenchmarks(version, round)); + } + } + + const comparisons = compareResults(resultsByVersionAndRound); + const markdownReport = makeMarkdownReport(comparisons, { + baseCommit, + headCommit, + rounds, + }); + console.log('\n' + markdownReport); + if (args.markdown) fs.writeFileSync(args.markdown, markdownReport); + if (args.json) { + fs.writeFileSync( + args.json, + JSON.stringify({ baseCommit, headCommit, rounds, comparisons }, null, 2) + ); + } +}; + +main(); diff --git a/GDJS/tests/karma.conf.js b/GDJS/tests/karma.conf.js index 9fc148c4fa..765121530d 100644 --- a/GDJS/tests/karma.conf.js +++ b/GDJS/tests/karma.conf.js @@ -1,4 +1,30 @@ +const fs = require('fs'); + +/** + * Write the benchmark results sent by the browser (see `reportBenchmarkResult` + * in `benchmarks/init.js`) to the file given with `--benchmarkResultsFile`. + */ +function BenchmarkResultsReporter(config) { + const benchmarkResults = []; + this.onBrowserInfo = (browser, info) => { + if (info && info.benchmarkResult) + benchmarkResults.push(info.benchmarkResult); + }; + this.onRunComplete = () => { + fs.writeFileSync( + config.benchmarkResultsFile, + JSON.stringify(benchmarkResults, null, 2) + ); + }; +} +BenchmarkResultsReporter.$inject = ['config']; + module.exports = function (config) { + // The built game engine to test, relative to the repository root. Changed + // to compare the benchmarks of two versions (see `compare-benchmarks.js`). + const gdjsRuntimePath = + config.gdjsRuntimePath || 'newIDE/app/resources/GDJS/Runtime'; + const testFiles = [ './Extensions/TweenBehavior/tests/CubicBezierEasingTestCases.js', './Extensions/**/tests/**.spec.js', @@ -7,8 +33,11 @@ module.exports = function (config) { const benchmarkFiles = [ './GDJS/tests/benchmarks/init.js', - './Extensions/**/benchmark/**.benchmark.js', - './GDJS/tests/benchmarks/**/*.js', + './Extensions/**/benchmarks/**.benchmark.js', + './GDJS/tests/benchmarks/*.js', + // Last, so that the micro benchmarks are not affected by what the scenes + // leave behind (memory to collect, rendering to finish...). + './GDJS/tests/benchmarks/scenes/*.js', ]; config.set({ @@ -34,6 +63,23 @@ module.exports = function (config) { '--use-angle=swiftshader', ], }, + // Same, with the garbage collector exposed so that benchmarks can run + // it before measuring. + ChromeHeadlessBenchmark: { + base: 'ChromeHeadless', + flags: [ + '--no-sandbox', + '--disable-gpu', + '--use-gl=swiftshader', + '--enable-webgl', + '--use-angle=swiftshader', + '--js-flags=--expose-gc', + // Never slow down the timers used to yield between frames. + '--disable-background-timer-throttling', + '--disable-renderer-backgrounding', + '--disable-backgrounding-occluded-windows', + ], + }, }, plugins: [ require('karma-chrome-launcher'), @@ -41,8 +87,13 @@ module.exports = function (config) { require('karma-firefox-launcher'), require('karma-mocha'), require('karma-sinon'), + { 'reporter:benchmark-results': ['type', BenchmarkResultsReporter] }, ], + reporters: config.benchmarkResultsFile + ? ['dots', 'benchmark-results'] + : ['progress'], client: { + args: config.grep ? ['--grep', config.grep] : [], mocha: { reporter: 'html', timeout: 10000, // Give a bit more time for CIs (the default 2s can be too low sometimes, as a real browser is involved). @@ -53,190 +104,183 @@ module.exports = function (config) { '/base/tests-utils/': '/base/GDJS/tests/tests-utils/', // Jolt is loaded with a dynamic `import('./jolt-physics.wasm.js')`, // which is resolved relatively to the page, not to the script. - '/jolt-physics.wasm.js': - '/base/newIDE/app/resources/GDJS/Runtime/Extensions/Physics3DBehavior/jolt-physics.wasm.js', - '/jolt-physics.wasm.wasm': - '/base/newIDE/app/resources/GDJS/Runtime/Extensions/Physics3DBehavior/jolt-physics.wasm.wasm', + '/jolt-physics.wasm.js': `/base/${gdjsRuntimePath}/Extensions/Physics3DBehavior/jolt-physics.wasm.js`, + '/jolt-physics.wasm.wasm': `/base/${gdjsRuntimePath}/Extensions/Physics3DBehavior/jolt-physics.wasm.wasm`, }, files: [ './GDJS/tests/node_modules/expect.js/index.js', //GDJS game engine files: (Order is important) - './newIDE/app/resources/GDJS/Runtime/libs/jshashtable.js', - './newIDE/app/resources/GDJS/Runtime/logger.js', - './newIDE/app/resources/GDJS/Runtime/gd.js', - './newIDE/app/resources/GDJS/Runtime/AsyncTasksManager.js', - './newIDE/app/resources/GDJS/Runtime/libs/rbush.js', - './newIDE/app/resources/GDJS/Runtime/pixi-renderers/pixi.js', - './newIDE/app/resources/GDJS/Runtime/pixi-renderers/three.js', - './newIDE/app/resources/GDJS/Runtime/pixi-renderers/*.js', - './newIDE/app/resources/GDJS/Runtime/howler-sound-manager/howler.min.js', - './newIDE/app/resources/GDJS/Runtime/howler-sound-manager/howler-sound-manager.js', - './newIDE/app/resources/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver.js', - './newIDE/app/resources/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver-font-manager.js', - './newIDE/app/resources/GDJS/Runtime/Model3DManager.js', - './newIDE/app/resources/GDJS/Runtime/jsonmanager.js', - './newIDE/app/resources/GDJS/Runtime/ResourceLoader.js', - './newIDE/app/resources/GDJS/Runtime/ResourceCache.js', - './newIDE/app/resources/GDJS/Runtime/timemanager.js', - './newIDE/app/resources/GDJS/Runtime/polygon.js', - './newIDE/app/resources/GDJS/Runtime/runtimeobject.js', - './newIDE/app/resources/GDJS/Runtime/RuntimeInstanceContainer.js', - './newIDE/app/resources/GDJS/Runtime/runtimescene.js', - './newIDE/app/resources/GDJS/Runtime/scenestack.js', - './newIDE/app/resources/GDJS/Runtime/profiler.js', - './newIDE/app/resources/GDJS/Runtime/force.js', - './newIDE/app/resources/GDJS/Runtime/RuntimeLayer.js', - './newIDE/app/resources/GDJS/Runtime/layer.js', - './newIDE/app/resources/GDJS/Runtime/RuntimeCustomObjectLayer.js', - './newIDE/app/resources/GDJS/Runtime/timer.js', - './newIDE/app/resources/GDJS/Runtime/inputmanager.js', - './newIDE/app/resources/GDJS/Runtime/capturemanager.js', - './newIDE/app/resources/GDJS/Runtime/named-easings-manager.js', - './newIDE/app/resources/GDJS/Runtime/runtimegame.js', - './newIDE/app/resources/GDJS/Runtime/runtimewatermark.js', - './newIDE/app/resources/GDJS/Runtime/variable.js', - './newIDE/app/resources/GDJS/Runtime/variablescontainer.js', - './newIDE/app/resources/GDJS/Runtime/oncetriggers.js', - './newIDE/app/resources/GDJS/Runtime/runtimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/SpriteAnimator.js', - './newIDE/app/resources/GDJS/Runtime/spriteruntimeobject.js', - './newIDE/app/resources/GDJS/Runtime/CustomRuntimeObject.js', - './newIDE/app/resources/GDJS/Runtime/CustomRuntimeObject2D.js', - './newIDE/app/resources/GDJS/Runtime/CustomRuntimeObjectInstanceContainer.js', - './newIDE/app/resources/GDJS/Runtime/events-tools/commontools.js', - './newIDE/app/resources/GDJS/Runtime/events-tools/runtimescenetools.js', - './newIDE/app/resources/GDJS/Runtime/events-tools/inputtools.js', - './newIDE/app/resources/GDJS/Runtime/events-tools/networktools.js', - './newIDE/app/resources/GDJS/Runtime/events-tools/objecttools.js', - './newIDE/app/resources/GDJS/Runtime/events-tools/cameratools.js', - './newIDE/app/resources/GDJS/Runtime/events-tools/soundtools.js', - './newIDE/app/resources/GDJS/Runtime/events-tools/storagetools.js', - './newIDE/app/resources/GDJS/Runtime/indexeddb.js', - './newIDE/app/resources/GDJS/Runtime/events-tools/stringtools.js', - './newIDE/app/resources/GDJS/Runtime/events-tools/windowtools.js', - './newIDE/app/resources/GDJS/Runtime/debugger-client/abstract-debugger-client.js', - './newIDE/app/resources/GDJS/Runtime/debugger-client/hot-reloader.js', - './newIDE/app/resources/GDJS/Runtime/gameplay-tests/gameplay-test-runner.js', - './newIDE/app/resources/GDJS/Runtime/affinetransformation.js', + `${gdjsRuntimePath}/libs/jshashtable.js`, + `${gdjsRuntimePath}/logger.js`, + `${gdjsRuntimePath}/gd.js`, + `${gdjsRuntimePath}/AsyncTasksManager.js`, + `${gdjsRuntimePath}/libs/rbush.js`, + `${gdjsRuntimePath}/pixi-renderers/pixi.js`, + `${gdjsRuntimePath}/pixi-renderers/three.js`, + `${gdjsRuntimePath}/pixi-renderers/*.js`, + `${gdjsRuntimePath}/howler-sound-manager/howler.min.js`, + `${gdjsRuntimePath}/howler-sound-manager/howler-sound-manager.js`, + `${gdjsRuntimePath}/fontfaceobserver-font-manager/fontfaceobserver.js`, + `${gdjsRuntimePath}/fontfaceobserver-font-manager/fontfaceobserver-font-manager.js`, + `${gdjsRuntimePath}/Model3DManager.js`, + `${gdjsRuntimePath}/jsonmanager.js`, + `${gdjsRuntimePath}/ResourceLoader.js`, + `${gdjsRuntimePath}/ResourceCache.js`, + `${gdjsRuntimePath}/timemanager.js`, + `${gdjsRuntimePath}/polygon.js`, + `${gdjsRuntimePath}/runtimeobject.js`, + `${gdjsRuntimePath}/RuntimeInstanceContainer.js`, + `${gdjsRuntimePath}/runtimescene.js`, + `${gdjsRuntimePath}/scenestack.js`, + `${gdjsRuntimePath}/profiler.js`, + `${gdjsRuntimePath}/force.js`, + `${gdjsRuntimePath}/RuntimeLayer.js`, + `${gdjsRuntimePath}/layer.js`, + `${gdjsRuntimePath}/RuntimeCustomObjectLayer.js`, + `${gdjsRuntimePath}/timer.js`, + `${gdjsRuntimePath}/inputmanager.js`, + `${gdjsRuntimePath}/capturemanager.js`, + `${gdjsRuntimePath}/named-easings-manager.js`, + `${gdjsRuntimePath}/runtimegame.js`, + `${gdjsRuntimePath}/runtimewatermark.js`, + `${gdjsRuntimePath}/variable.js`, + `${gdjsRuntimePath}/variablescontainer.js`, + `${gdjsRuntimePath}/oncetriggers.js`, + `${gdjsRuntimePath}/runtimebehavior.js`, + `${gdjsRuntimePath}/SpriteAnimator.js`, + `${gdjsRuntimePath}/spriteruntimeobject.js`, + `${gdjsRuntimePath}/CustomRuntimeObject.js`, + `${gdjsRuntimePath}/CustomRuntimeObject2D.js`, + `${gdjsRuntimePath}/CustomRuntimeObjectInstanceContainer.js`, + `${gdjsRuntimePath}/events-tools/commontools.js`, + `${gdjsRuntimePath}/events-tools/runtimescenetools.js`, + `${gdjsRuntimePath}/events-tools/inputtools.js`, + `${gdjsRuntimePath}/events-tools/networktools.js`, + `${gdjsRuntimePath}/events-tools/objecttools.js`, + `${gdjsRuntimePath}/events-tools/cameratools.js`, + `${gdjsRuntimePath}/events-tools/soundtools.js`, + `${gdjsRuntimePath}/events-tools/storagetools.js`, + `${gdjsRuntimePath}/indexeddb.js`, + `${gdjsRuntimePath}/events-tools/stringtools.js`, + `${gdjsRuntimePath}/events-tools/windowtools.js`, + `${gdjsRuntimePath}/debugger-client/abstract-debugger-client.js`, + `${gdjsRuntimePath}/debugger-client/hot-reloader.js`, + `${gdjsRuntimePath}/gameplay-tests/gameplay-test-runner.js`, + `${gdjsRuntimePath}/affinetransformation.js`, //Extensions: - './newIDE/app/resources/GDJS/Runtime/Extensions/DraggableBehavior/draggableruntimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/AnchorBehavior/anchorruntimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/PlatformBehavior/platformerobjectruntimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/PlatformBehavior/platformruntimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/LinkedObjects/linkedobjects.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Inventory/inventory.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Inventory/inventorytools.js', + `${gdjsRuntimePath}/Extensions/DraggableBehavior/draggableruntimebehavior.js`, + `${gdjsRuntimePath}/Extensions/AnchorBehavior/anchorruntimebehavior.js`, + `${gdjsRuntimePath}/Extensions/PlatformBehavior/platformerobjectruntimebehavior.js`, + `${gdjsRuntimePath}/Extensions/PlatformBehavior/platformruntimebehavior.js`, + `${gdjsRuntimePath}/Extensions/LinkedObjects/linkedobjects.js`, + `${gdjsRuntimePath}/Extensions/Inventory/inventory.js`, + `${gdjsRuntimePath}/Extensions/Inventory/inventorytools.js`, { - pattern: - './newIDE/app/resources/GDJS/Runtime/Extensions/Physics2Behavior/Box2D_v2.3.1_min.wasm.wasm', + pattern: `${gdjsRuntimePath}/Extensions/Physics2Behavior/Box2D_v2.3.1_min.wasm.wasm`, watched: true, included: false, served: true, nocache: false, }, - './newIDE/app/resources/GDJS/Runtime/Extensions/Physics2Behavior/Box2D_v2.3.1_min.wasm.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Physics2Behavior/physics2runtimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Physics2Behavior/physics2tools.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Leaderboards/leaderboardstools.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/PlayerAuthentication/playerauthenticationtools.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/PlayerAuthentication/playerauthenticationcomponents.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Multiplayer/messageManager.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Multiplayer/multiplayerVariablesManager.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Multiplayer/multiplayertools.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Multiplayer/multiplayercomponents.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Multiplayer/multiplayerobjectruntimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Lighting/lightruntimeobject.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Lighting/lightruntimeobject-pixi-renderer.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Lighting/lightobstacleruntimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/PathfindingBehavior/PathTools.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/PathfindingBehavior/pathfindingobstacleruntimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/PathfindingBehavior/pathfindingruntimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/NavMeshPathfinding/A_recast-navigation-generators.js', + `${gdjsRuntimePath}/Extensions/Physics2Behavior/Box2D_v2.3.1_min.wasm.js`, + `${gdjsRuntimePath}/Extensions/Physics2Behavior/physics2runtimebehavior.js`, + `${gdjsRuntimePath}/Extensions/Physics2Behavior/physics2tools.js`, + `${gdjsRuntimePath}/Extensions/Leaderboards/leaderboardstools.js`, + `${gdjsRuntimePath}/Extensions/PlayerAuthentication/playerauthenticationtools.js`, + `${gdjsRuntimePath}/Extensions/PlayerAuthentication/playerauthenticationcomponents.js`, + `${gdjsRuntimePath}/Extensions/Multiplayer/messageManager.js`, + `${gdjsRuntimePath}/Extensions/Multiplayer/multiplayerVariablesManager.js`, + `${gdjsRuntimePath}/Extensions/Multiplayer/multiplayertools.js`, + `${gdjsRuntimePath}/Extensions/Multiplayer/multiplayercomponents.js`, + `${gdjsRuntimePath}/Extensions/Multiplayer/multiplayerobjectruntimebehavior.js`, + `${gdjsRuntimePath}/Extensions/Lighting/lightruntimeobject.js`, + `${gdjsRuntimePath}/Extensions/Lighting/lightruntimeobject-pixi-renderer.js`, + `${gdjsRuntimePath}/Extensions/Lighting/lightobstacleruntimebehavior.js`, + `${gdjsRuntimePath}/Extensions/PathfindingBehavior/PathTools.js`, + `${gdjsRuntimePath}/Extensions/PathfindingBehavior/pathfindingobstacleruntimebehavior.js`, + `${gdjsRuntimePath}/Extensions/PathfindingBehavior/pathfindingruntimebehavior.js`, + `${gdjsRuntimePath}/Extensions/NavMeshPathfinding/A_recast-navigation-generators.js`, { - pattern: - './newIDE/app/resources/GDJS/Runtime/Extensions/NavMeshPathfinding/recast-navigation.wasm.js', + pattern: `${gdjsRuntimePath}/Extensions/NavMeshPathfinding/recast-navigation.wasm.js`, watched: true, included: false, served: true, nocache: false, }, { - pattern: - './newIDE/app/resources/GDJS/Runtime/Extensions/NavMeshPathfinding/recast-navigation.wasm.wasm', + pattern: `${gdjsRuntimePath}/Extensions/NavMeshPathfinding/recast-navigation.wasm.wasm`, watched: true, included: false, served: true, nocache: false, }, - './newIDE/app/resources/GDJS/Runtime/Extensions/NavMeshPathfinding/NavMeshObstacleRuntimeBehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/NavMeshPathfinding/NavMeshCharacterRuntimeBehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/PrimitiveDrawing/shapepainterruntimeobject.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/PrimitiveDrawing/shapepainterruntimeobject-pixi-renderer.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TextInput/textinputruntimeobject.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TextInput/textinputruntimeobject-pixi-renderer.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TextObject/textruntimeobject.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TextObject/textruntimeobject-pixi-renderer.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/3D/A_RuntimeObject3D.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/3D/A_RuntimeObject3DRenderer.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/3D/Cube3DRuntimeObject.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/3D/Cube3DRuntimeObjectPixiRenderer.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/3D/CustomRuntimeObject3D.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/3D/CustomRuntimeObject3DRenderer.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/3D/BloomEffect.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/3D/BrightnessAndContrastEffect.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/3D/DepthOfFieldEffect.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/3D/N8AOEffect.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/3D/DirectionalLight.js', + `${gdjsRuntimePath}/Extensions/NavMeshPathfinding/NavMeshObstacleRuntimeBehavior.js`, + `${gdjsRuntimePath}/Extensions/NavMeshPathfinding/NavMeshCharacterRuntimeBehavior.js`, + `${gdjsRuntimePath}/Extensions/PrimitiveDrawing/shapepainterruntimeobject.js`, + `${gdjsRuntimePath}/Extensions/PrimitiveDrawing/shapepainterruntimeobject-pixi-renderer.js`, + `${gdjsRuntimePath}/Extensions/TextInput/textinputruntimeobject.js`, + `${gdjsRuntimePath}/Extensions/TextInput/textinputruntimeobject-pixi-renderer.js`, + `${gdjsRuntimePath}/Extensions/TextObject/textruntimeobject.js`, + `${gdjsRuntimePath}/Extensions/TextObject/textruntimeobject-pixi-renderer.js`, + `${gdjsRuntimePath}/Extensions/3D/A_RuntimeObject3D.js`, + `${gdjsRuntimePath}/Extensions/3D/A_RuntimeObject3DRenderer.js`, + `${gdjsRuntimePath}/Extensions/3D/Cube3DRuntimeObject.js`, + `${gdjsRuntimePath}/Extensions/3D/Cube3DRuntimeObjectPixiRenderer.js`, + `${gdjsRuntimePath}/Extensions/3D/CustomRuntimeObject3D.js`, + `${gdjsRuntimePath}/Extensions/3D/CustomRuntimeObject3DRenderer.js`, + `${gdjsRuntimePath}/Extensions/3D/BloomEffect.js`, + `${gdjsRuntimePath}/Extensions/3D/BrightnessAndContrastEffect.js`, + `${gdjsRuntimePath}/Extensions/3D/DepthOfFieldEffect.js`, + `${gdjsRuntimePath}/Extensions/3D/N8AOEffect.js`, + `${gdjsRuntimePath}/Extensions/3D/DirectionalLight.js`, { - pattern: - './newIDE/app/resources/GDJS/Runtime/Extensions/Physics3DBehavior/jolt-physics.wasm.js', + pattern: `${gdjsRuntimePath}/Extensions/Physics3DBehavior/jolt-physics.wasm.js`, watched: true, included: false, served: true, nocache: false, }, { - pattern: - './newIDE/app/resources/GDJS/Runtime/Extensions/Physics3DBehavior/jolt-physics.wasm.wasm', + pattern: `${gdjsRuntimePath}/Extensions/Physics3DBehavior/jolt-physics.wasm.wasm`, watched: true, included: false, served: true, nocache: false, }, - './newIDE/app/resources/GDJS/Runtime/Extensions/Physics3DBehavior/Physics3DRuntimeBehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Physics3DBehavior/Physics3DTools.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Physics3DBehavior/PhysicsCharacter3DRuntimeBehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TopDownMovementBehavior/topdownmovementruntimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TweenBehavior/TweenManager.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TweenBehavior/tweentools.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TweenBehavior/tweenruntimebehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Firebase/A_firebasejs/*.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Firebase/B_firebasetools/*.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Effects/outline-pixi-filter.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Effects/pixi-filters/filter-outline.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Effects/blur-pixi-filter.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Effects/kawase-blur-pixi-filter.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Effects/pixi-filters/filter-kawase-blur.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TileMap/tilemapcollisionmaskruntimeobject.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TileMap/TileMapRuntimeManager.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TileMap/TileMapBehavior.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TileMap/AbstractTileMapRuntimeObject.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TileMap/tilemapruntimeobject-pixi-renderer.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TileMap/tilemapruntimeobject.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TileMap/collision/TileMapCollisionMaskRenderer.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TileMap/collision/TransformedTileMap.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TileMap/helper/TileMapHelper.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/TileMap/pako/dist/pako.min.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Spine/spine-pixi-v7/A_spine-pixi-v7-pre.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Spine/spine-pixi-v7/B_spine-pixi-v7.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Spine/spine-pixi-v7/C_spine-pixi-v7-post.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Spine/spineruntimeobject.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Spine/spineruntimeobject-pixi-renderer.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/Spine/managers/*.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/SaveState/SaveStateTools.js', - './newIDE/app/resources/GDJS/Runtime/Extensions/SaveState/SaveConfigurationRuntimeBehavior.js', + `${gdjsRuntimePath}/Extensions/Physics3DBehavior/Physics3DRuntimeBehavior.js`, + `${gdjsRuntimePath}/Extensions/Physics3DBehavior/Physics3DTools.js`, + `${gdjsRuntimePath}/Extensions/Physics3DBehavior/PhysicsCharacter3DRuntimeBehavior.js`, + `${gdjsRuntimePath}/Extensions/TopDownMovementBehavior/topdownmovementruntimebehavior.js`, + `${gdjsRuntimePath}/Extensions/TweenBehavior/TweenManager.js`, + `${gdjsRuntimePath}/Extensions/TweenBehavior/tweentools.js`, + `${gdjsRuntimePath}/Extensions/TweenBehavior/tweenruntimebehavior.js`, + `${gdjsRuntimePath}/Extensions/Firebase/A_firebasejs/*.js`, + `${gdjsRuntimePath}/Extensions/Firebase/B_firebasetools/*.js`, + `${gdjsRuntimePath}/Extensions/Effects/outline-pixi-filter.js`, + `${gdjsRuntimePath}/Extensions/Effects/pixi-filters/filter-outline.js`, + `${gdjsRuntimePath}/Extensions/Effects/blur-pixi-filter.js`, + `${gdjsRuntimePath}/Extensions/Effects/kawase-blur-pixi-filter.js`, + `${gdjsRuntimePath}/Extensions/Effects/pixi-filters/filter-kawase-blur.js`, + `${gdjsRuntimePath}/Extensions/TileMap/tilemapcollisionmaskruntimeobject.js`, + `${gdjsRuntimePath}/Extensions/TileMap/TileMapRuntimeManager.js`, + `${gdjsRuntimePath}/Extensions/TileMap/TileMapBehavior.js`, + `${gdjsRuntimePath}/Extensions/TileMap/AbstractTileMapRuntimeObject.js`, + `${gdjsRuntimePath}/Extensions/TileMap/tilemapruntimeobject-pixi-renderer.js`, + `${gdjsRuntimePath}/Extensions/TileMap/tilemapruntimeobject.js`, + `${gdjsRuntimePath}/Extensions/TileMap/collision/TileMapCollisionMaskRenderer.js`, + `${gdjsRuntimePath}/Extensions/TileMap/collision/TransformedTileMap.js`, + `${gdjsRuntimePath}/Extensions/TileMap/helper/TileMapHelper.js`, + `${gdjsRuntimePath}/Extensions/TileMap/pako/dist/pako.min.js`, + `${gdjsRuntimePath}/Extensions/Spine/spine-pixi-v7/A_spine-pixi-v7-pre.js`, + `${gdjsRuntimePath}/Extensions/Spine/spine-pixi-v7/B_spine-pixi-v7.js`, + `${gdjsRuntimePath}/Extensions/Spine/spine-pixi-v7/C_spine-pixi-v7-post.js`, + `${gdjsRuntimePath}/Extensions/Spine/spineruntimeobject.js`, + `${gdjsRuntimePath}/Extensions/Spine/spineruntimeobject-pixi-renderer.js`, + `${gdjsRuntimePath}/Extensions/Spine/managers/*.js`, + `${gdjsRuntimePath}/Extensions/SaveState/SaveStateTools.js`, + `${gdjsRuntimePath}/Extensions/SaveState/SaveConfigurationRuntimeBehavior.js`, // Test extensions: './GDJS/tests/tests/Extensions/**.js', @@ -252,7 +296,7 @@ module.exports = function (config) { // Source maps { - pattern: './newIDE/app/resources/GDJS/Runtime/**/*.map', + pattern: `${gdjsRuntimePath}/**/*.map`, watched: false, included: false, served: true, @@ -282,8 +326,10 @@ module.exports = function (config) { nocache: false, }, - ...testFiles, - ...(config.enableBenchmarks ? benchmarkFiles : []), + ...(config.benchmarksOnly ? [] : testFiles), + ...(config.enableBenchmarks || config.benchmarksOnly + ? benchmarkFiles + : []), ], }); }; diff --git a/GDJS/tests/package.json b/GDJS/tests/package.json index f45b29b503..6af1abfef2 100644 --- a/GDJS/tests/package.json +++ b/GDJS/tests/package.json @@ -10,6 +10,7 @@ "test:chromeHeadlessNoSandbox": "karma start --browsers ChromeHeadlessNoSandbox --single-run", "test-benchmark": "karma start --browsers ChromeHeadless --single-run --enableBenchmarks", "test-benchmark:watch": "karma start --browsers ChromeHeadless --enableBenchmarks", + "benchmark:compare": "node compare-benchmarks.js", "test:firefox": "karma start --browsers Firefox --single-run", "test:firefox:watch": "karma start --browsers Firefox", "test:chrome": "karma start --browsers Chrome --single-run", diff --git a/GDJS/tests/tests/gameplaytestharness.js b/GDJS/tests/tests/gameplaytestharness.js index 42d51215c8..7099c7374d 100644 --- a/GDJS/tests/tests/gameplaytestharness.js +++ b/GDJS/tests/tests/gameplaytestharness.js @@ -978,6 +978,62 @@ describe('gdjs.gameplayTests', () => { expect(result.status).to.be('passed'); }); + it('benchmarks frames and reports the measures in the result', async () => { + const runtimeGame = makeRuntimeGame(); + const result = await runTestScript( + runtimeGame, + ` + await harness.goToScene('Scene 1'); + harness.spawn('MyObject', 10, 20); + const benchmark = await harness.benchmark('Test benchmark', { + warmupFrames: 2, + frames: 5, + }); + harness.assert(benchmark.frameTimesMs.length === 5, '5 frames are measured'); + harness.assert( + benchmark.medianFrameTimeMs >= 0, + 'The median frame time is returned' + ); + harness.assert(benchmark.objectsCount === 1, 'The objects are counted'); + harness.assert( + benchmark.drawCallsPerFrame === null, + 'No draw calls are counted without renderer' + ); + ` + ); + + expect(result.status).to.be('passed'); + expect(result.framesExecuted).to.be(1 + 2 + 5); + expect(result.benchmarks.length).to.be(1); + expect(result.benchmarks[0].name).to.be('Test benchmark'); + expect(result.benchmarks[0].stateChecksum).to.be.a('string'); + }); + + it('seeds Math.random until the end of the test', async () => { + const originalMathRandom = Math.random; + const runTestScriptPickingRandomNumbers = () => + runTestScript( + makeRuntimeGame(), + ` + harness.setRandomSeed(42); + await harness.goToScene('Scene 1'); + console.log('Random numbers: ' + [Math.random(), Math.random()]); + ` + ); + /** @param {gdjs.gameplayTests.GameplayTestResult} result */ + const getRandomNumbersLog = (result) => + result.consoleLogs.find((log) => + log.message.startsWith('Random numbers: ') + ); + const result1 = await runTestScriptPickingRandomNumbers(); + const result2 = await runTestScriptPickingRandomNumbers(); + + expect(result1.status).to.be('passed'); + expect(getRandomNumbersLog(result1)).to.be.ok(); + expect(getRandomNumbersLog(result1)).to.eql(getRandomNumbersLog(result2)); + expect(Math.random).to.be(originalMathRandom); + }); + it('reports an aim result object with the mouse responsiveness', async () => { const runtimeGame = makeRuntimeGame(); const result = await runTestScript(