!2 pick gen abc plugin change to js

Merge pull request !2 from zrg/cherry_pick_dts_monthly0816
2026-07-21 06:05:27 -04:00 · 2022-09-25 06:10:38 +00:00
parent 45410390fe 2ed20e6a37
commit aacea7c2e7
2 changed files with 202 additions and 106 deletions
@@ -26,22 +26,15 @@ const ES2ABC = 'es2abc';
 function js2abcByWorkers(jsonInput, cmd) {
  const inputPaths = JSON.parse(jsonInput);
  for (let i = 0; i < inputPaths.length; ++i) {
-    let input = inputPaths[i].path;
-    let singleCmd = `${cmd} "${input}"`;
+    // for matching debug info mechanism
+    const input = inputPaths[i].path.replace(/\.temp\.js$/, "_.js");
+    const cacheOutputPath = inputPaths[i].cacheOutputPath;
+    const cacheAbcFilePath = cacheOutputPath.replace(/\.temp\.js$/, ".abc");
+    const singleCmd = `${cmd} "${cacheOutputPath}" -o "${cacheAbcFilePath}" --source-file "${input}"`;
    try {
      childProcess.execSync(singleCmd);
    } catch (e) {
-      console.error(red, `ERROR Failed to convert file ${input} to abc `, reset);
-      process.exit(FAIL);
-    }
-
-    const abcFile = input.replace(/\.js$/, '.abc');
-    if (fs.existsSync(abcFile)) {
-      const abcFileNew = abcFile.replace(/_.abc$/, '.abc');
-      fs.copyFileSync(abcFile, abcFileNew);
-      fs.unlinkSync(abcFile);
-    } else {
-      console.error(red, `ERROR ${abcFile} is lost`, reset);
+      console.debug(red, `ERROR Failed to convert file ${input} to abc `, reset);
      process.exit(FAIL);
    }
  }
@@ -50,9 +43,10 @@ function js2abcByWorkers(jsonInput, cmd) {
 function es2abcByWorkers(jsonInput, cmd) {
  const inputPaths = JSON.parse(jsonInput);
  for (let i = 0; i < inputPaths.length; ++i) {
-    const input = inputPaths[i].path;
-    const abcFile = input.replace(/_.js$/, '.abc');
-    const singleCmd = `${cmd} "${input}" --output "${abcFile}"`;
+    const input = inputPaths[i].path.replace(/\.temp\.js$/, "_.js");
+    const cacheOutputPath = inputPaths[i].cacheOutputPath;
+    const cacheAbcFilePath = cacheOutputPath.replace(/\.temp\.js$/, ".abc");
+    const singleCmd = `${cmd} "${cacheOutputPath}" --output "${cacheAbcFilePath}" --source-file "${input}"`;
    console.debug('gen abc cmd is: ', singleCmd, ' ,file size is:', inputPaths[i].size, ' byte');
    try {
      childProcess.execSync(singleCmd);
@@ -67,11 +61,11 @@ function es2abcByWorkers(jsonInput, cmd) {

 if (cluster.isWorker && process.env["inputs"] !== undefined && process.env["cmd"] !== undefined) {
  if (process.env.panda === TS2ABC) {
-    js2abcByWorkers(process.env['inputs'], process.env['cmd']);
+    js2abcByWorkers(process.env["inputs"], process.env["cmd"]);
  } else if (process.env.panda === ES2ABC  || process.env.panda === 'undefined' || process.env.panda === undefined) {
-    es2abcByWorkers(process.env['inputs'], process.env['cmd']);
+    es2abcByWorkers(process.env["inputs"], process.env["cmd"]);
  } else {
-    console.error(red, `ERROR please set panda module`, reset);
+    logger.error(red, `ERROR please set panda module`, reset);
    process.exit(FAIL);
  }
  process.exit(SUCCESS);
@@ -18,6 +18,8 @@ const path = require('path');
 const cluster = require('cluster');
 const process = require('process');
 const crypto = require('crypto');
+const events = require('events');
+const os = require('os');

 const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' +
              '  var define = globalObjects.define;' + '\n' +
@@ -34,7 +36,6 @@ const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' +
              '  (function(global) {' + '\n' +
              '    "use strict";' + '\n';
 const last = '\n' + '})(this.__appProto__);' + '\n' + '})';
-const firstFileEXT = '_.js';
 const genAbcScript = 'gen-abc.js';
 let output;
 let isWin = false;
@@ -54,9 +55,12 @@ const reset = '\u001b[39m';
 const blue = '\u001b[34m';
 const hashFile = 'gen_hash.json';
 const ARK = '/ark/';
-let delayCount = 0;
+const NODE_MODULES = 'node_modules';
+const TEMPORARY = 'temporary';
 const TS2ABC = 'ts2abc';
 const ES2ABC = 'es2abc';
+let previewCount = 0;
+let compileCount = 0;

 class GenAbcPlugin {
  constructor(output_, arkDir_, nodeJs_, workerFile_, isDebug_) {
@@ -77,9 +81,18 @@ class GenAbcPlugin {
      return;
    }

+    if (!checkNodeModules()) {
+      process.exitCode = FAIL;
+      return;
+    }
+
+    // for preview mode max listeners
+    events.EventEmitter.defaultMaxListeners = 100;
+
    compiler.hooks.emit.tap('GenAbcPlugin', (compilation) => {
      const assets = compilation.assets;
      const keys = Object.keys(assets);
+      buildPathInfo = output;
      keys.forEach(key => {
        // choice *.js
        if (output && path.extname(key) === '.js') {
@@ -90,17 +103,21 @@ class GenAbcPlugin {
          if (key === 'commons.js' || key === 'vendors.js' || !checkWorksFile(key, workerFile)) {
            newContent = `\n\n\n\n\n\n\n\n\n\n\n\n\n\n` + newContent;
          }
-          const keyPath = key.replace(/\.js$/, firstFileEXT)
-          writeFileSync(newContent, path.resolve(output, keyPath), key, true);
+          const keyPath = key.replace(/\.js$/, ".temp.js");
+          writeFileSync(newContent, output, keyPath, key, true);
        } else if (output && path.extname(key) === '.json' &&
          process.env.DEVICE_LEVEL === 'card' && !checkI18n(key)) {
-          writeFileSync(assets[key].source(), path.resolve(output, key), key, false);
+          writeFileSync(assets[key].source(), output, key, key, false);
        }
      })
    });
    compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => {
+      if (intermediateJsBundle.length === 0) {
+        return;
+      }
      buildPathInfo = output;
-      judgeWorkersToGenAbc(invokeWorkerToGenAbc);
+      previewCount++;
+      invokeWorkerToGenAbc();
    });
  }
 }
@@ -133,20 +150,36 @@ function checkWorksFile(assetPath, workerFile) {
  return true;
 }

-function writeFileSync(inputString, output, jsBundleFile, isToBin) {
-    const parent = path.join(output, '..');
+function writeFileSync(inputString, buildPath, keyPath, jsBundleFile, isToBin) {
+    let output = path.resolve(buildPath, keyPath);
+    let parent = path.join(output, '..');
    if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) {
        mkDir(parent);
    }
-    fs.writeFileSync(output, inputString);
    if (!isToBin) {
+      fs.writeFileSync(output, inputString);
      return;
    }
-    if (fs.existsSync(output)) {
-      let fileSize = fs.statSync(output).size;
-      intermediateJsBundle.push({path: output, size: fileSize});
+    let cacheOutputPath = "";
+    if (process.env.cachePath) {
+      let buildDirArr = buildPathInfo.split(path.sep);
+      let abilityDir = buildDirArr[buildDirArr.length - 1];
+      cacheOutputPath = path.join(process.env.cachePath, TEMPORARY, abilityDir, keyPath);
    } else {
-      console.error(red, `ERROR Failed to convert file ${jsBundleFile} to bin. ${output} is lost`, reset);
+      cacheOutputPath = output;
+    }
+    parent = path.join(cacheOutputPath, '..');
+    if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) {
+      mkDir(parent);
+    }
+    fs.writeFileSync(cacheOutputPath, inputString);
+    if (fs.existsSync(cacheOutputPath)) {
+      let fileSize = fs.statSync(cacheOutputPath).size;
+      output = toUnixPath(output);
+      cacheOutputPath = toUnixPath(cacheOutputPath);
+      intermediateJsBundle.push({path: output, size: fileSize, cacheOutputPath: cacheOutputPath});
+    } else {
+      console.debug(red, `ERROR Failed to convert file ${jsBundleFile} to bin. ${output} is lost`, reset);
      process.exitCode = FAIL;
    }
 }
@@ -197,38 +230,22 @@ function splitJsBundlesBySize(bundleArray, groupNumber) {
 }

 function invokeWorkerToGenAbc() {
-  let param = '';
+  if (process.env.isPreview === "true") {
+    process.exitCode = SUCCESS;
+  }
  let cmdPrefix = '';
-
+  let maxWorkerNumber = 3;
+  const abcArgs = initAbcEnv();
  if (process.env.panda === TS2ABC) {
-    if (isDebug) {
-      param += ' --debug';
-    }
-
-    let js2abc = path.join(arkDir, 'build', 'src', 'index.js');
-    if (isWin) {
-      js2abc = path.join(arkDir, 'build-win', 'src', 'index.js');
-    } else if (isMac) {
-      js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js');
-    }
-    cmdPrefix = `${nodeJs} --expose-gc "${js2abc}" ${param} `;
+    cmdPrefix = `${nodeJs} ${abcArgs.join(' ')}`;
  } else if (process.env.panda === ES2ABC  || process.env.panda === 'undefined' || process.env.panda === undefined) {
-    if (isDebug) {
-      param += ' --debug-info';
-    }
-    let es2abc = path.join(arkDir, 'build', 'bin', 'es2abc');
-    if (isWin) {
-      es2abc = path.join(arkDir, 'build-win', 'bin', 'es2abc.exe');
-    } else if (isMac) {
-      es2abc = path.join(arkDir, 'build-mac', 'bin', 'es2abc');
-    }
-    cmdPrefix = `"${es2abc}" ${param} `;
+    maxWorkerNumber = os.cpus().length;
+    cmdPrefix = `${abcArgs.join(' ')}`;
  } else {
-    console.error(red, `ERROR please set panda module`, reset);
+    console.debug(red, `ERROR please set panda module`, reset);
  }

  filterIntermediateJsBundleByHashJson(buildPathInfo, intermediateJsBundle);
-  const maxWorkerNumber = 3;
  const splitedBundles = splitJsBundlesBySize(fileterIntermediateJsBundle, maxWorkerNumber);
  const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length;

@@ -256,6 +273,10 @@ function invokeWorkerToGenAbc() {
    }

    let count_ = 0;
+    if (process.env.isPreview === 'true') {
+      process.removeAllListeners("exit");
+      cluster.removeAllListeners("exit");
+    }
    cluster.on('exit', (worker, code, signal) => {
      if (code === FAIL || process.exitCode === FAIL) {
        process.exitCode = FAIL;
@@ -263,26 +284,32 @@ function invokeWorkerToGenAbc() {
      }
      count_++;
      if (count_ === workerNumber) {
-        writeHashJson();
-        clearGlobalInfo();
-        if (process.env.isPreview) {
+        // for preview of with incre compile
+        if (process.env.isPreview === "true" && compileCount < previewCount) {
+          compileCount++;
+          processExtraAssetForBundle();
          console.log(blue, 'COMPILE RESULT:SUCCESS ', reset);
+          if (compileCount >= previewCount) {
+            return;
+          }
+          invokeWorkerToGenAbc();
        }
      }
    });
    process.on('exit', (code) => {
-      intermediateJsBundle.forEach((item) => {
-        let input = item.path;
-        if (fs.existsSync(input)) {
-          fs.unlinkSync(input);
-        }
-      })
+      // for build options
+      processExtraAssetForBundle();
    });
+
+    // for preview of without incre compile
+    if (workerNumber === 0 && process.env.isPreview === "true") {
+      processExtraAssetForBundle();
+    }
  }
 }

 function clearGlobalInfo() {
-  if (!process.env.isPreview) {
+  if (process.env.isPreview !== "true") {
    intermediateJsBundle = [];
  }
  fileterIntermediateJsBundle = [];
@@ -306,7 +333,7 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) {
  }
  const hashFilePath = genHashJsonPath(buildPath);
  if (hashFilePath.length == 0) {
-    return ;
+    return;
  }
  let updateJsonObject = {};
  let jsonObject = {};
@@ -316,24 +343,20 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) {
    jsonObject = JSON.parse(jsonFile);
    fileterIntermediateJsBundle = [];
    for (let i = 0; i < inputPaths.length; ++i) {
-      let input = inputPaths[i].path;
-      let abcPath = input.replace(/_.js$/, '.abc');
-
-      if (!fs.existsSync(input)) {
-        console.error(red, `ERROR ${input} is lost`, reset);
+      const cacheOutputPath = inputPaths[i].cacheOutputPath;
+      const cacheAbcFilePath = cacheOutputPath.replace(/\.temp\.js$/, '.abc');
+      if (!fs.existsSync(cacheOutputPath)) {
+        console.debug(red, `ERROR ${cacheOutputPath} is lost`, reset);
        process.exitCode = FAIL;
        break;
      }

-      if (fs.existsSync(input) && fs.existsSync(abcPath)) {
-        const hashInputContentData = toHashData(input);
-        const hashAbcContentData = toHashData(abcPath);
-        if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) {
-          updateJsonObject[input] = hashInputContentData;
-          updateJsonObject[abcPath] = hashAbcContentData;
-          if (!process.env.isPreview) {
-            fs.unlinkSync(input);
-          }
+      if (fs.existsSync(cacheAbcFilePath)) {
+        const hashInputContentData = toHashData(cacheOutputPath);
+        const hashAbcContentData = toHashData(cacheAbcFilePath);
+        if (jsonObject[cacheOutputPath] === hashInputContentData && jsonObject[cacheAbcFilePath] === hashAbcContentData) {
+          updateJsonObject[cacheOutputPath] = hashInputContentData;
+          updateJsonObject[cacheAbcFilePath] = hashAbcContentData;
        } else {
          fileterIntermediateJsBundle.push(inputPaths[i]);
        }
@@ -348,28 +371,23 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) {

 function writeHashJson() {
  for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) {
-    let input = fileterIntermediateJsBundle[i].path;
-    let abcPath = input.replace(/_.js$/, '.abc');
-    if (!fs.existsSync(input) || !fs.existsSync(abcPath)) {
-      console.error(red, `ERROR ${input} is lost`, reset);
+    const cacheOutputPath = fileterIntermediateJsBundle[i].cacheOutputPath;
+    const cacheAbcFilePath = cacheOutputPath.replace(/\.temp\.js$/, '.abc');
+    if (!fs.existsSync(cacheOutputPath) || !fs.existsSync(cacheAbcFilePath)) {
+      console.debug(red, `ERROR ${cacheOutputPath} is lost`, reset);
      process.exitCode = FAIL;
      break;
    }
-    if (fs.existsSync(input) && fs.existsSync(abcPath)) {
-      const hashInputContentData = toHashData(input);
-      const hashAbcContentData = toHashData(abcPath);
-      hashJsonObject[input] = hashInputContentData;
-      hashJsonObject[abcPath] = hashAbcContentData;
-    }
-    if (!process.env.isPreview && fs.existsSync(input)) {
-      fs.unlinkSync(input);
-    }
+    const hashInputContentData = toHashData(cacheOutputPath);
+    const hashAbcContentData = toHashData(cacheAbcFilePath);
+    hashJsonObject[cacheOutputPath] = hashInputContentData;
+    hashJsonObject[cacheAbcFilePath] = hashAbcContentData;
  }
  const hashFilePath = genHashJsonPath(buildPathInfo);
  if (hashFilePath.length == 0) {
-    return ;
+    return;
  }
-  if (!process.env.isPreview || delayCount < 1) {
+  if (process.env.isPreview !== "true" || previewCount < 1) {
    fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject));
  }
 }
@@ -380,7 +398,14 @@ function genHashJsonPath(buildPath) {
    if (!fs.existsSync(process.env.cachePath) || !fs.statSync(process.env.cachePath).isDirectory()) {
      return '';
    }
-    return path.join(process.env.cachePath, hashFile);
+    let buildDirArr = buildPathInfo.split(path.sep);
+    let abilityDir = buildDirArr[buildDirArr.length - 1];
+    let hashJsonPath = path.join(process.env.cachePath, TEMPORARY, abilityDir, hashFile);
+    let parent = path.join(hashJsonPath, '..');
+    if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) {
+      mkDir(parent);
+    }
+    return hashJsonPath;
  } else if (buildPath.indexOf(ARK) >= 0) {
    const dataTmps = buildPath.split(ARK);
    const hashPath = path.join(dataTmps[0], ARK);
@@ -414,13 +439,90 @@ module.exports = {
  checkWorksFile: checkWorksFile
 }

-function judgeWorkersToGenAbc(callback) {
-  const workerNumber = Object.keys(cluster.workers).length;
-  if (workerNumber === 0) {
-    callback();
-    return ;
-  } else {
-    delayCount++;
-    setTimeout(judgeWorkersToGenAbc.bind(null, callback), 50);
+function copyFileCachePathToBuildPath() {
+  for (let i = 0; i < intermediateJsBundle.length; ++i) {
+    const abcFile = intermediateJsBundle[i].path.replace(/\.temp\.js$/, ".abc");
+    const cacheOutputPath = intermediateJsBundle[i].cacheOutputPath;
+    const cacheAbcFilePath = intermediateJsBundle[i].cacheOutputPath.replace(/\.temp\.js$/, ".abc");
+    if (!fs.existsSync(cacheAbcFilePath)) {
+      console.debug(red, `ERROR ${cacheAbcFilePath} is lost`, reset);
+      process.exitCode = FAIL;
+      break;
+    }
+    let parent = path.join(abcFile, '..');
+    if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) {
+      mkDir(parent);
+    }
+    // for preview mode, cache path and old abc file both exist, should copy abc file for updating
+    if (process.env.cachePath !== undefined) {
+      fs.copyFileSync(cacheAbcFilePath, abcFile);
+    }
+    if (process.env.cachePath === undefined && fs.existsSync(cacheOutputPath)) {
+      fs.unlinkSync(cacheOutputPath);
+    }
  }
 }
+
+function processExtraAssetForBundle() {
+  writeHashJson();
+  copyFileCachePathToBuildPath();
+  clearGlobalInfo();
+}
+
+function checkNodeModules() {
+  if (process.env.panda === TS2ABC) {
+    let arkEntryPath = path.join(arkDir, 'build');
+    if (isWin) {
+      arkEntryPath = path.join(arkDir, 'build-win');
+    } else if (isMac) {
+      arkEntryPath = path.join(arkDir, 'build-mac');
+    }
+    let nodeModulesPath = path.join(arkEntryPath, NODE_MODULES);
+    if (!(fs.existsSync(nodeModulesPath) && fs.statSync(nodeModulesPath).isDirectory())) {
+      console.error(red, `ERROR: node_modules for ark compiler not found.
+        Please make sure switch to non-root user before runing "npm install" for safity requirements and try re-run "npm install" under ${arkEntryPath}`, reset);
+      return false;
+    }
+  }
+
+  return true;
+}
+
+function initAbcEnv() {
+  let args = [];
+  if (process.env.panda === TS2ABC) {
+    let js2abc = path.join(arkDir, 'build', 'src', 'index.js');
+    if (isWin) {
+      js2abc = path.join(arkDir, 'build-win', 'src', 'index.js');
+    } else if (isMac) {
+      js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js');
+    }
+
+    js2abc = '"' + js2abc + '"';
+    args = [
+      '--expose-gc',
+      js2abc
+    ];
+    if (isDebug) {
+      args.push('--debug');
+    }
+  } else if (process.env.panda === ES2ABC  || process.env.panda === 'undefined' || process.env.panda === undefined) {
+    let es2abc = path.join(arkDir, 'build', 'bin', 'es2abc');
+    if (isWin) {
+      es2abc = path.join(arkDir, 'build-win', 'bin', 'es2abc.exe');
+    } else if (isMac) {
+      es2abc = path.join(arkDir, 'build-mac', 'bin', 'es2abc');
+    }
+
+    args = [
+      '"' + es2abc + '"'
+    ];
+    if (isDebug) {
+      args.push('--debug-info');
+    }
+  }  else {
+    console.debug(red, `ERROR: please set panda module`, reset);
+  }
+
+  return args;
+}