Add automatic addition of extensions, examples and assets commits in the changelog (#3051)

* Also ensure scripts are type checked.

Don't show in changelog

.semaphore/semaphore.yml

@@ -50,6 +50,7 @@ blocks:
             - cd newIDE/app
             - npm run postinstall
             - npm run flow
+            - npm run check-script-types
             - cd ../..
         - name: GDJS typing and documentation generation
           commands:

.travis.yml

@@ -102,6 +102,7 @@ script:
   - npm test
   - npm run flow
   - npm run check-format
+  - npm run check-script-types
   - cd ../..
 # GDJS tests:
   - cd GDJS

newIDE/app/scripts/extract-changelog.js

@@ -1,17 +1,43 @@
 // @ts-check
 /**
- * This is a simple script getting the commits since the last tag (which
+ * This is a script getting the commits since the last tag (which
  * corresponds to a GitHub release). Each commit is filtered to:
  * - hide trivial stuff
  * - hide those containing "Don't mention in changelog"/"Don't show in changelog".
  * - categorize as fix or improvement
  *
  * Authors nickname are fetched from GitHub.
+ *
+ * Commits from the extensions/assets/examples repository are also
+ * used to build the complete changelog.
  */
 
 const shell = require('shelljs');
 const child = require('child_process');
 const { default: axios } = require('axios');
+const path = require('path');
+const args = require('minimist')(process.argv.slice(2));
+
+if (!args['extensionsGitPath']) {
+  shell.echo(
+    '⚠️ You should pass --extensionsGitPath pointing to the git directory of GDevelop-extensions.'
+  );
+}
+const extensionsGitPath = args['extensionsGitPath'];
+
+if (!args['assetsGitPath']) {
+  shell.echo(
+    '⚠️ You should pass --assetsGitPath pointing to the git directory of GDevelop-assets.'
+  );
+}
+const assetsGitPath = args['assetsGitPath'];
+
+if (!args['examplesGitPath']) {
+  shell.echo(
+    '⚠️ You should pass --examplesGitPath pointing to the git directory of GDevelop-examples.'
+  );
+}
+const examplesGitPath = args['examplesGitPath'];
 
 /** @typedef {{sha: string, message: string, authorEmail: string}} GitRawCommitInfo */
 /** @typedef {{
@@ -23,6 +49,7 @@ const { default: axios } = require('axios');
   hidden: boolean
 }} GitEnrichedCommitInfo */
 
+/** Helpers to read information from a git repository. */
 const getGitTools = repoPath => {
   /**
    * @param {string} options
@@ -36,11 +63,14 @@ const getGitTools = repoPath => {
       )
       .toString('utf-8');
 
-    return output.split('---COMMITDELIMITER---\n').map(commit => {
-      const [message, authorEmail, sha] = commit.split('---DELIMITER---');
+    return output
+      .split('---COMMITDELIMITER---\n')
+      .map(commit => {
+        const [message, authorEmail, sha] = commit.split('---DELIMITER---');
 
-      return { sha, message, authorEmail };
-    });
+        return { sha, message, authorEmail };
+      })
+      .filter(commit => Boolean(commit.sha));
   };
 
   return {
@@ -52,14 +82,14 @@ const getGitTools = repoPath => {
     },
     getTagIsoDate: tag => {
       return child
-        .execSync(`git tag -l ${tag} --format='%(creatordate:short)'`, {
+        .execSync(`git tag -l ${tag} --format='%(creatordate:iso8601)'`, {
           cwd: repoPath,
         })
         .toString('utf-8')
         .trim();
     },
     extractCommitsSinceTag: tag => extractCommits(`${tag}..HEAD`),
-    extractCommitsSinceDate: date => extractCommits(`--after=${date}`),
+    extractCommitsSinceDate: date => extractCommits(`--after="${date}"`),
   };
 };
 
@@ -71,7 +101,6 @@ const enrichCommits = rawCommits => {
   return (
     rawCommits
       // Clean commits
-      .filter(commit => Boolean(commit.sha))
       .map(commit => ({
         message: commit.message.trim(),
         authorEmail: commit.authorEmail.trim(),
@@ -106,6 +135,8 @@ const enrichCommits = rawCommits => {
           lowerCaseMessage.indexOf('fix typo') === 0 ||
           lowerCaseMessage.includes('add files forgotten in last commit') ||
           lowerCaseMessage.indexOf('apply review') === 0 ||
+          lowerCaseMessage.indexOf('compress images') === 0 ||
+          lowerCaseMessage.indexOf('update extensions-registry.json') === 0 ||
           lowerCaseMessage.includes('package-lock.json') ||
           lowerCaseMessage.includes('yarn.lock');
         const isFix = lowerCaseMessage.indexOf('fix') === 0;
@@ -125,12 +156,13 @@ const enrichCommits = rawCommits => {
   );
 };
 
+let authorEmailsToNicknames = {};
+
 /**
  * @param {GitEnrichedCommitInfo[]} commits
  * @return {Promise<GitEnrichedCommitInfo[]>}
  */
 const findAuthorNicknameInCommits = async commits => {
-  let authorEmailsToNicknames = {};
   let lastGithubCall = 0;
 
   /** @returns {Promise<void>} */
@@ -196,18 +228,18 @@ const findAuthorNicknameInCommits = async commits => {
 };
 
 /**
- * @param {GitEnrichedCommitInfo} commit
+ * @param {{commit: GitEnrichedCommitInfo, includeAuthor: boolean}} options
  * @returns {string}
  */
-const formatCommitMessage = commit => {
-  const includeAuthor = commit.authorNickname !== '4ian';
-  const author = includeAuthor
-    ? `(Thanks ${
-        commit.authorNickname
-          ? '@' + commit.authorNickname
-          : 'TODO:' + commit.authorEmail
-      }!)`
-    : '';
+const formatCommitMessage = ({ commit, includeAuthor }) => {
+  const author =
+    includeAuthor && commit.authorNickname !== '4ian'
+      ? `(Thanks ${
+          commit.authorNickname
+            ? '@' + commit.authorNickname
+            : 'TODO:' + commit.authorEmail
+        }!)`
+      : '';
 
   const ignoreRestRegex = /(Don't|Do not) (show|mention) (details|the rest )in (the )?changelog/i;
   const foundIgnoreRest = commit.message.match(ignoreRestRegex);
@@ -217,10 +249,10 @@ const formatCommitMessage = commit => {
       ? commit.message.substr(0, foundIgnoreRest.index)
       : commit.message;
 
-  const prNumberCleanedMessage = cleanedMessage.replace(
-    /(\(#[1-9][0-9]*\))/,
-    ''
-  );
+  const prNumberCleanedMessage = cleanedMessage
+    .replace(/(\(#[1-9][0-9]*\))/, '')
+    .replace(/fix #[1-9][0-9]*,*/gi, '')
+    .replace(/closes #[1-9][0-9]*,*/gi, '');
 
   const devCleanedMessage = prNumberCleanedMessage.replace(
     /only show in developer changelog/i,
@@ -251,13 +283,24 @@ const formatHiddenCommitMessage = commit => {
 };
 
 /**
- * @param {{hiddenCommits: GitEnrichedCommitInfo[], improvementsCommits: GitEnrichedCommitInfo[], fixCommits: GitEnrichedCommitInfo[], devCommits: GitEnrichedCommitInfo[]}} commits
+ * @param {{
+ *   hiddenCommits: GitEnrichedCommitInfo[],
+ *   improvementsCommits: GitEnrichedCommitInfo[],
+ *   fixCommits: GitEnrichedCommitInfo[],
+ *   devCommits: GitEnrichedCommitInfo[],
+ *   extensionsCommits: GitEnrichedCommitInfo[] | null,
+ *   assetsCommits: GitEnrichedCommitInfo[] | null,
+ *   examplesCommits: GitEnrichedCommitInfo[] | null,
+ * }} commits
  */
 const outputChangelog = ({
   hiddenCommits,
   improvementsCommits,
   fixCommits,
   devCommits,
+  extensionsCommits,
+  assetsCommits,
+  examplesCommits,
 }) => {
   shell.echo(
     `ℹ️ Hidden these commits: \n${hiddenCommits
@@ -268,30 +311,65 @@ const outputChangelog = ({
   shell.echo(`\n✅ The generated changelog is:\n`);
 
   shell.echo(`\n## 💝 Improvements\n`);
-  shell.echo(improvementsCommits.map(formatCommitMessage).join('\n'));
+  shell.echo(
+    improvementsCommits
+      .map(commit => formatCommitMessage({ commit, includeAuthor: true }))
+      .join('\n')
+  );
+
+  shell.echo(`\n## ⚙️ Extensions, 🎨 assets and 🕹 examples\n`);
+  shell.echo(
+    extensionsCommits
+      ? extensionsCommits
+          .map(commit => formatCommitMessage({ commit, includeAuthor: false }))
+          .join('\n')
+      : 'TODO: Add extensions commits here.'
+  );
+  shell.echo(
+    assetsCommits
+      ? assetsCommits
+          .map(commit => formatCommitMessage({ commit, includeAuthor: false }))
+          .join('\n')
+      : 'TODO: Add assets commits here.'
+  );
+  shell.echo(
+    examplesCommits
+      ? examplesCommits
+          .map(commit => formatCommitMessage({ commit, includeAuthor: false }))
+          .join('\n')
+      : 'TODO: Add examples commits here.'
+  );
 
   shell.echo(`\n## 🐛 Bug fixes\n`);
-  shell.echo(fixCommits.map(formatCommitMessage).join('\n'));
+  shell.echo(
+    fixCommits
+      .map(commit => formatCommitMessage({ commit, includeAuthor: true }))
+      .join('\n')
+  );
 
   if (devCommits.length > 0) {
     shell.echo(`\n### 🛠 Internal changes (for developers)\n`);
-    shell.echo(devCommits.map(formatCommitMessage).join('\n'));
+    shell.echo(
+      devCommits
+        .map(commit => formatCommitMessage({ commit, includeAuthor: true }))
+        .join('\n')
+    );
   }
 };
 
 (async () => {
   const gdevelopRepoGitTools = getGitTools('.');
   const lastTag = gdevelopRepoGitTools.getLastTag();
+  // Uncomment if you want to test the result of the script with an older release:
+  // const lastTag = 'v5.0.0-beta115';
   shell.echo(`ℹ️ Last tag is ${lastTag}`);
 
   const lastTagDate = gdevelopRepoGitTools.getTagIsoDate(lastTag);
   shell.echo(`ℹ️ Date of tag is ${lastTagDate}`);
 
   const rawCommits = gdevelopRepoGitTools.extractCommitsSinceTag(lastTag);
-
   const commits = enrichCommits(rawCommits);
   const commitsWithAuthors = await findAuthorNicknameInCommits(commits);
-
   const hiddenCommits = commitsWithAuthors.filter(commit => commit.hidden);
   const displayedCommits = commitsWithAuthors.filter(commit => !commit.hidden);
   const devCommits = displayedCommits.filter(commit => commit.forDev);
@@ -299,10 +377,50 @@ const outputChangelog = ({
   const improvementsCommits = displayedCommits.filter(
     commit => !commit.isFix && !commit.forDev
   );
+
+  /** @type {GitEnrichedCommitInfo[] | null} */
+  let assetsCommits = null;
+  if (assetsGitPath) {
+    const assetsRepoGitTools = getGitTools(assetsGitPath);
+    const assetsRawCommits = assetsRepoGitTools.extractCommitsSinceDate(
+      lastTagDate
+    );
+    assetsCommits = enrichCommits(assetsRawCommits).filter(
+      commit => !commit.hidden
+    );
+  }
+
+  /** @type {GitEnrichedCommitInfo[] | null} */
+  let extensionsCommits = null;
+  if (extensionsGitPath) {
+    const extensionsRepoGitTools = getGitTools(extensionsGitPath);
+    const extensionsRawCommits = extensionsRepoGitTools.extractCommitsSinceDate(
+      lastTagDate
+    );
+    extensionsCommits = enrichCommits(extensionsRawCommits).filter(
+      commit => !commit.hidden
+    );
+  }
+
+  /** @type {GitEnrichedCommitInfo[] | null} */
+  let examplesCommits = null;
+  if (examplesGitPath) {
+    const examplesRepoGitTools = getGitTools(examplesGitPath);
+    const examplesRawCommits = examplesRepoGitTools.extractCommitsSinceDate(
+      lastTagDate
+    );
+    examplesCommits = enrichCommits(examplesRawCommits).filter(
+      commit => !commit.hidden
+    );
+  }
+
   outputChangelog({
     hiddenCommits,
     improvementsCommits,
     fixCommits,
     devCommits,
+    extensionsCommits,
+    assetsCommits,
+    examplesCommits,
   });
 })();

newIDE/app/scripts/extract-reference-document.js

@@ -794,6 +794,7 @@ const rawTextsToString = rawTexts =>
 
 initializeGDevelopJs().then(async gd => {
   try {
+    // @ts-ignore - not passing onFindGDJS - is it still useful?
     const loadingResults = await makeExtensionsLoader({
       gd,
       objectsEditorService: null,