From d57a3960ed4c59e5e3bf4a5175322e62e084ed58 Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Thu, 21 May 2026 16:46:14 +0200 Subject: [PATCH] Add extra CI checking an export using the CLI options of GDevelop (#8632) - Done daily and after every builds on master. Only show in developer changelog --- .circleci/config.yml | 141 ++++++++- .../app/scripts/test-portable-cli-export.js | 286 ++++++++++++++++++ 2 files changed, 426 insertions(+), 1 deletion(-) create mode 100644 newIDE/app/scripts/test-portable-cli-export.js diff --git a/.circleci/config.yml b/.circleci/config.yml index b9e6ddf815..364fb64b8f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -171,6 +171,25 @@ jobs: - store_artifacts: path: newIDE/electron-app/dist + # Stage just the x64 portable zip for the downstream + # cli-html5-export-linux job (avoid sharing the much larger + # .AppImage/.deb installers via the workspace). + - run: + name: Stage portable zip for downstream cli-html5-export-linux job + command: | + mkdir -p build-linux-workspace + for f in newIDE/electron-app/dist/gdevelop-*.zip; do + case "$f" in + *arm64*) ;; + *) cp "$f" build-linux-workspace/ ;; + esac + done + ls -lh build-linux-workspace + - persist_to_workspace: + root: build-linux-workspace + paths: + - . + # Upload artifacts (AWS) - run: name: Deploy to S3 (specific commit) @@ -600,6 +619,96 @@ jobs: - store_artifacts: path: newIDE/electron-app/dist + # End-to-end test that runs the GDevelop CLI to export an example game + # to HTML5, using either the portable Linux build just produced by + # build-linux (use-local-build=true) or the latest one downloaded from S3. + run-cli-html5-export-linux: + parameters: + use-local-build: + description: When true, attach the workspace produced by build-linux and use its portable zip instead of downloading the latest one from S3. + type: boolean + default: false + resource_class: medium + docker: + - image: cimg/node:24.13.0 + + working_directory: ~/GDevelop + + steps: + - checkout + + - when: + condition: << parameters.use-local-build >> + steps: + - attach_workspace: + at: ~/GDevelop/build-linux-workspace + + # Runtime libraries required by Electron (headless still needs nss / + # alsa to even start up). + - run: + name: Install Electron runtime dependencies + command: | + sudo apt-get update + sudo apt-get install -y \ + libnss3 \ + libasound2t64 \ + libgbm1 \ + libgtk-3-0 \ + libxss1 \ + libatk-bridge2.0-0 \ + libdrm2 \ + libxkbcommon0 \ + xvfb + + # The script reuses the helpers from `newIDE/app/scripts` which + # depend on axios, minimist, shelljs and adm-zip. Rather than running + # a full `npm install` of `newIDE/app` (huge), we install just the + # four packages we need into a scratch directory and expose it via + # NODE_PATH so the script and its `./lib/*` helpers can `require` them. + - run: + name: Install minimal script deps (axios, minimist, shelljs, adm-zip) + command: | + mkdir -p cli-html5-export-deps + cd cli-html5-export-deps + npm init -y > /dev/null + npm install --no-audit --no-fund --silent \ + axios@^0.18.1 \ + minimist@^1.2.5 \ + shelljs@^0.8.4 \ + adm-zip@^0.5.10 + + - run: + name: Run CLI HTML5 export + # Wrap with xvfb-run as a safety net: even though the editor hides + # its BrowserWindow in --run-command mode, Electron itself still + # needs a usable display for some GPU/graphics initialization on + # headless Linux runners. + no_output_timeout: 20m + command: | + export NODE_PATH="$PWD/cli-html5-export-deps/node_modules" + if [ "<< parameters.use-local-build >>" = "true" ]; then + ZIP_PATH=$(ls ~/GDevelop/build-linux-workspace/gdevelop-*.zip | grep -v -- '-arm64' | head -1) + if [ -z "$ZIP_PATH" ]; then + echo "No portable zip found in build-linux workspace." >&2 + exit 1 + fi + echo "Using locally built portable zip: $ZIP_PATH" + EXTRA_ARGS="--zipPath=$ZIP_PATH" + else + echo "No local build available; will download the latest portable zip from S3." + EXTRA_ARGS="--branch=master" + fi + xvfb-run -a --server-args="-screen 0 1280x800x24" \ + node newIDE/app/scripts/test-portable-cli-export.js \ + $EXTRA_ARGS \ + --example=3d-platformer \ + --workDir=./cli-html5-export-work \ + --artifactsDir=./cli-html5-export-artifacts + + - store_artifacts: + path: ./cli-html5-export-artifacts + destination: cli-html5-export-artifacts + workflows: gdevelop_js-wasm-extra-checks: jobs: @@ -611,10 +720,13 @@ workflows: - master - stable - /experimental-build.*/ + tests: + jobs: + - build-gdcore-tests-sanitizers + builds: jobs: - build-gdevelop_js-wasm-only - - build-gdcore-tests-sanitizers - build-macos: # The macOS version builds by itself GDevelop.js # (so we verify we can build it on macOS). @@ -635,6 +747,19 @@ workflows: - master - stable - /experimental-build.*/ + # Right after build-linux, run the CLI HTML5 export against the + # portable zip it just produced (via the attached workspace). + - run-cli-html5-export-linux: + name: cli-html5-export-linux + use-local-build: true + requires: + - build-linux + filters: + branches: + only: + - master + - stable + - /experimental-build.*/ - build-windows: requires: - build-gdevelop_js-wasm-only @@ -653,3 +778,17 @@ workflows: - master - stable - /experimental-build.*/ + + # Daily end-to-end test of the GDevelop CLI HTML5 export. Runs every day + # at 16:00 UTC (4pm UTC) against the latest portable build uploaded to S3 + # for master. + daily-cli-html5-export-linux: + triggers: + - schedule: + cron: "0 16 * * *" + filters: + branches: + only: + - master + jobs: + - run-cli-html5-export-linux diff --git a/newIDE/app/scripts/test-portable-cli-export.js b/newIDE/app/scripts/test-portable-cli-export.js new file mode 100644 index 0000000000..b2cf12cdcc --- /dev/null +++ b/newIDE/app/scripts/test-portable-cli-export.js @@ -0,0 +1,286 @@ +// @ts-check +/** + * End-to-end smoke test for the GDevelop "portable" build + CLI export. + * + * What it does: + * 1. Reads the current GDevelop version from + * `newIDE/electron-app/app/package.json` (the source of truth, same + * as `make-version-metadata.js`). + * 2. Gets the matching Linux portable zip, either from a local path + * (via --zipPath, e.g. the artifact a build-linux job just produced) + * or by downloading it from S3. + * 3. Extracts it with adm-zip. + * 4. Sparse-clones an example game from the GDevelop-examples repository + * (defaults to "3d-platformer"). + * 5. Runs the extracted GDevelop binary in CLI mode with + * `--run-command EXPORT_HTML5_EXTERNAL` to export the game to HTML5. + * 6. Verifies the exported HTML5 game looks valid (e.g. `index.html` + * exists) and zips it into the artifacts folder. + * + * Usage: + * node test-portable-cli-export.js [--branch=master] [--example=3d-platformer] + * [--zipPath=/path/to/gdevelop-X.Y.Z.zip] + * [--workDir=./.portable-cli-test] + * [--artifactsDir=./portable-cli-artifacts] + */ + +const fs = require('fs'); +const path = require('path'); +const { spawn } = require('child_process'); +const shell = require('shelljs'); +const AdmZip = require('adm-zip'); +const args = require('minimist')(process.argv.slice(2)); +const { downloadLocalFile } = require('./lib/DownloadLocalFile'); +const { retryIfFailed } = require('./lib/RetryIfFailed'); + +const electronAppPackageJson = require('../../electron-app/app/package.json'); + +const branch = args['branch'] || 'master'; +const exampleSlug = args['example'] || '3d-platformer'; +const workDir = path.resolve(args['workDir'] || './.portable-cli-test'); +const artifactsDir = path.resolve( + args['artifactsDir'] || './portable-cli-artifacts' +); +// If set, use this local zip instead of downloading from S3 (useful when +// running the smoke test right after a build-linux job that already produced +// the portable zip on disk). +const localZipPath = args['zipPath'] ? path.resolve(args['zipPath']) : null; + +const version = electronAppPackageJson.version; +const pathToArtifacts = `https://gdevelop-releases.s3.amazonaws.com/${branch}/latest`; +const portableZipName = `gdevelop-${version}.zip`; +const portableZipUrl = `${pathToArtifacts}/${portableZipName}`; + +const EXAMPLES_REPO = 'https://github.com/GDevelopApp/GDevelop-examples.git'; +const EXAMPLES_BRANCH = 'main'; +const CLI_TIMEOUT_MS = 10 * 60 * 1000; + +/** + * @param {string} msg + * @returns {never} + */ +const fail = msg => { + throw new Error(msg); +}; + +(async () => { + shell.echo(`ℹ️ GDevelop version (from electron-app/app/package.json): ${version}`); + shell.echo(`ℹ️ Branch: ${branch} — example: ${exampleSlug}`); + shell.echo(`ℹ️ Work dir: ${workDir}`); + shell.echo(`ℹ️ Artifacts dir: ${artifactsDir}`); + + shell.mkdir('-p', workDir); + shell.mkdir('-p', artifactsDir); + + // 1. Get the Linux portable zip — either from a local path passed in via + // --zipPath (e.g. when chained after a build-linux job) or by + // downloading the latest version from S3. + let portableZipPath; + if (localZipPath) { + if (!fs.existsSync(localZipPath)) { + fail(`Local --zipPath does not exist: ${localZipPath}`); + } + portableZipPath = localZipPath; + shell.echo( + `📦 Using local portable zip: ${portableZipPath} (${formatBytes( + fs.statSync(portableZipPath).size + )})` + ); + } else { + portableZipPath = path.join(workDir, portableZipName); + shell.echo(`🌐 Downloading ${portableZipUrl} ...`); + await retryIfFailed( + { times: 3, backoff: { initialDelay: 2000, factor: 2 } }, + () => downloadLocalFile(portableZipUrl, portableZipPath) + ); + shell.echo( + `✅ Downloaded ${portableZipName} (${formatBytes( + fs.statSync(portableZipPath).size + )})` + ); + } + + // 2. Extract the portable zip. + const extractedDir = path.join(workDir, `gdevelop-${version}`); + shell.rm('-rf', extractedDir); + shell.mkdir('-p', extractedDir); + shell.echo(`📂 Extracting portable zip to ${extractedDir} ...`); + new AdmZip(portableZipPath).extractAllTo(extractedDir, /*overwrite=*/ true); + + // 3. Locate the GDevelop executable inside the extracted dir. + const gdevelopBinary = findExecutable(extractedDir); + if (!gdevelopBinary) { + fail( + `Could not find a GDevelop executable inside ${extractedDir}. ` + + `Contents: ${shell.ls(extractedDir).join(', ')}` + ); + } + shell.echo(`✅ Found GDevelop binary: ${gdevelopBinary}`); + shell.chmod('+x', gdevelopBinary); + + // 4. Fetch the example game from GDevelop-examples (sparse clone — the + // full repo is large so we only check out the one example folder). + const exampleParentDir = path.join(workDir, 'example'); + shell.rm('-rf', exampleParentDir); + shell.mkdir('-p', exampleParentDir); + shell.echo(`🌐 Sparse-cloning ${exampleSlug} from GDevelop-examples ...`); + cloneExampleSparse(EXAMPLES_REPO, EXAMPLES_BRANCH, exampleSlug, exampleParentDir); + const exampleDir = path.join(exampleParentDir, 'examples', exampleSlug); + const gameJsonPath = path.join(exampleDir, `${exampleSlug}.json`); + if (!fs.existsSync(gameJsonPath)) { + fail( + `Example game project not found at ${gameJsonPath} after sparse clone.` + ); + } + shell.echo(`✅ Example game project: ${gameJsonPath}`); + + // 5. Run the GDevelop CLI export. + shell.echo('🚀 Running GDevelop CLI HTML5 export ...'); + await runCliExport(gdevelopBinary, gameJsonPath); + + // 6. Verify the export. + const exportedBuildDir = path.join(exampleDir, 'build'); + const exportedIndex = path.join(exportedBuildDir, 'index.html'); + if (!fs.existsSync(exportedIndex)) { + fail( + `Export did not produce an index.html at ${exportedIndex}. ` + + `Build dir contents: ${ + fs.existsSync(exportedBuildDir) + ? shell.ls(exportedBuildDir).join(', ') + : '(missing)' + }` + ); + } + const indexSize = fs.statSync(exportedIndex).size; + if (indexSize < 200) { + fail(`Exported index.html is suspiciously small (${indexSize} bytes).`); + } + shell.echo(`✅ index.html exported (${formatBytes(indexSize)})`); + + // 7. Zip the exported HTML5 game into the artifacts folder so the CI job + // can surface a single downloadable archive via store_artifacts. + const artifactZipPath = path.join( + artifactsDir, + `${exampleSlug}-html5-${version}.zip` + ); + shell.rm('-f', artifactZipPath); + const outZip = new AdmZip(); + outZip.addLocalFolder(exportedBuildDir); + outZip.writeZip(artifactZipPath); + shell.echo( + `✅ Exported game zipped to ${artifactZipPath} (${formatBytes( + fs.statSync(artifactZipPath).size + )})` + ); + + shell.echo('🎉 Portable CLI HTML5 export smoke test succeeded.'); +})().catch(err => { + shell.echo(`❌ ${err && err.stack ? err.stack : err}`); + shell.exit(1); +}); + +// --------------------------------------------------------------------------- +// Helpers + +/** + * Search the extracted portable build for the Electron executable. + * On Linux electron-builder names it after the `name` field in package.json + * ("gdevelop"). We just look for the first executable file with that name. + * @param {string} root + * @returns {string|null} + */ +function findExecutable(root) { + const candidates = ['gdevelop', 'GDevelop']; + /** @type {string[]} */ + const stack = [root]; + while (stack.length) { + const dir = stack.pop(); + if (!dir) continue; + let entries; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch (e) { + continue; + } + for (const e of entries) { + const full = path.join(dir, e.name); + if (e.isDirectory()) { + stack.push(full); + } else if (e.isFile() && candidates.includes(e.name)) { + return full; + } + } + } + return null; +} + +/** + * Sparse-clone a single example folder from the GDevelop-examples repo to + * avoid pulling the multi-GB full history. + * @param {string} repoUrl + * @param {string} repoBranch + * @param {string} slug + * @param {string} dest + */ +function cloneExampleSparse(repoUrl, repoBranch, slug, dest) { + const sh = cmd => { + const r = shell.exec(cmd, { cwd: dest, silent: false }); + if (r.code !== 0) { + throw new Error(`Command failed (${r.code}): ${cmd}`); + } + }; + sh('git init -q'); + sh(`git remote add origin ${repoUrl}`); + sh('git config core.sparseCheckout true'); + sh('git sparse-checkout init --cone'); + sh(`git sparse-checkout set examples/${slug}`); + sh(`git fetch --depth=1 origin ${repoBranch}`); + sh('git checkout FETCH_HEAD'); +} + +/** + * Run the GDevelop CLI export. Resolves on success, rejects on non-zero + * exit. Times out (and SIGKILLs) after `CLI_TIMEOUT_MS`. + * @param {string} gdevelopBinary + * @param {string} gameJsonPath + * @returns {Promise} + */ +function runCliExport(gdevelopBinary, gameJsonPath) { + return new Promise((resolve, reject) => { + const cliArgs = [ + '--no-sandbox', + '--disable-update-check', + '--run-command', + 'EXPORT_HTML5_EXTERNAL', + gameJsonPath, + ]; + shell.echo(`▶ ${gdevelopBinary} ${cliArgs.join(' ')}`); + const child = spawn(gdevelopBinary, cliArgs, { stdio: 'inherit' }); + + const timer = setTimeout(() => { + shell.echo('❌ CLI export timed out, killing process.'); + child.kill('SIGKILL'); + }, CLI_TIMEOUT_MS); + + child.on('error', err => { + clearTimeout(timer); + reject(err); + }); + child.on('exit', (code, signal) => { + clearTimeout(timer); + if (code === 0) resolve(); + else + reject( + new Error( + `GDevelop CLI exited with code=${code} signal=${signal || 'none'}` + ) + ); + }); + }); +} + +function formatBytes(n) { + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KiB`; + return `${(n / 1024 / 1024).toFixed(1)} MiB`; +}