From 0f61ca48004e649f5c622b628dfeb3d4eac01db5 Mon Sep 17 00:00:00 2001 From: c00513733 Date: Mon, 28 Feb 2022 13:02:34 +0800 Subject: [PATCH] Generate abc by multi threads Signed-off-by: c00513733 --- compiler/src/gen_abc.ts | 44 +++++++++++++ compiler/src/gen_abc_plugin.ts | 109 ++++++++++++++++++++++----------- 2 files changed, 116 insertions(+), 37 deletions(-) create mode 100644 compiler/src/gen_abc.ts diff --git a/compiler/src/gen_abc.ts b/compiler/src/gen_abc.ts new file mode 100644 index 0000000..dd923af --- /dev/null +++ b/compiler/src/gen_abc.ts @@ -0,0 +1,44 @@ +import * as process from 'child_process'; +import * as fs from 'fs'; +import { logger } from './compile_info'; + +const {workerData, threadId} = require('worker_threads'); + +interface File { + path: string, + size: number +} +const red: string = '\u001b[31m'; +const reset: string = '\u001b[39m'; + +function js2abcByWorkers(inputPaths: File[], cmd: string): Promise { + for (let i = 0; i < inputPaths.length; ++i) { + let input = inputPaths[i].path; + let singleCmd = `${cmd} "${input}"`; + logger.debug("gen abc cmd is: ", singleCmd); + try { + process.execSync(singleCmd); + } catch (e) { + logger.error(red, `ETS:ERROR Failed to convert file ${input} to abc `, reset); + return; + } + + if (fs.existsSync(input)) { + fs.unlinkSync(input); + } + + const abcFile: string = input.replace(/\.js$/, '.abc'); + if (fs.existsSync(abcFile)) { + const abcFileNew: string = abcFile.replace(/_.abc$/, '.abc'); + fs.renameSync(abcFile, abcFileNew); + } else { + logger.error(red, `ETS:ERROR ${abcFile} is lost`, reset); + } + } +} + +logger.debug("worker data is: ", JSON.stringify(workerData)); +if (JSON.stringify(workerData) !== 'null') { + logger.debug("==>worker #", threadId, "started!"); + js2abcByWorkers(workerData.input, workerData.cmd); +} diff --git a/compiler/src/gen_abc_plugin.ts b/compiler/src/gen_abc_plugin.ts index b07844f..1565e11 100644 --- a/compiler/src/gen_abc_plugin.ts +++ b/compiler/src/gen_abc_plugin.ts @@ -13,14 +13,14 @@ * limitations under the License. */ -import * as process from 'child_process'; import * as fs from 'fs'; - import * as path from 'path'; import Compiler from 'webpack/lib/Compiler'; import { logger } from './compile_info'; +const {Worker, isMainThread} = require('worker_threads'); const firstFileEXT: string = '_.js'; +const genAbcScript = 'gen_abc.js'; let output: string; let isWin: boolean = false; let isMac: boolean = false; @@ -28,6 +28,12 @@ let isDebug: boolean = false; let arkDir: string; let nodeJs: string; +interface File { + path: string, + size: number +} +let intermediateJsBundle: Array = []; + const red: string = '\u001b[31m'; const reset: string = '\u001b[39m'; @@ -54,7 +60,7 @@ export class GenAbcPlugin { compiler.hooks.emit.tap('GenAbcPlugin', (compilation) => { Object.keys(compilation.assets).forEach(key => { - // choice *.js + // choose *.js if (output && path.extname(key) === '.js') { const newContent: string = compilation.assets[key].source(); const keyPath: string = key.replace(/\.js$/, firstFileEXT); @@ -62,6 +68,34 @@ export class GenAbcPlugin { } }); }); + + compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => { + let param: string = ''; + if (isDebug) { + param += ' --debug'; + } + + if (isMainThread) { + let js2abc: string = path.join(arkDir, 'build', 'src', 'index.js'); + if (isWin) { + js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); + } else if (isMac) { + js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); + } + let maxWorkerNumber = 3; + let splitedBundles = splitJsBundlesBySize(intermediateJsBundle, maxWorkerNumber); + let workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length; + let cmdPrefix: string = `${nodeJs} --expose-gc "${js2abc}" ${param} `; + let workers = []; + for (let i = 0; i < workerNumber; ++i) { + workers.push(new Worker(path.resolve(__dirname, genAbcScript), + {workerData: {input: splitedBundles[i], cmd: cmdPrefix} })); + workers[i].on('exit', () => { + logger.debug("worker ", i, "finished!"); + }); + } + } + }); } } @@ -72,7 +106,8 @@ function writeFileSync(inputString: string, output: string, jsBundleFile: string } fs.writeFileSync(output, inputString); if (fs.existsSync(output)) { - js2abcFirst(output); + let fileSize = fs.statSync(output).size; + intermediateJsBundle.push({path: output, size: fileSize}); } else { logger.error(red, `ETS:ERROR Failed to convert file ${jsBundleFile} to bin. ${output} is lost`, reset); } @@ -86,37 +121,37 @@ function mkDir(path_: string): void { fs.mkdirSync(path_); } -function js2abcFirst(inputPath: string): void { - let param: string = ''; - if (isDebug) { - param += '--debug'; - } - - let js2abc: string = path.join(arkDir, 'build', 'src', 'index.js'); - if (isWin) { - js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); - } else if (isMac) { - js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); - } - - const cmd: string = `${nodeJs} --expose-gc "${js2abc}" "${inputPath}" ${param}`; - - try { - process.execSync(cmd); - } catch (e) { - logger.error(red, `ETS:ERROR Failed to convert file ${inputPath} to abc `, reset); - return; - } - - if (fs.existsSync(inputPath)) { - fs.unlinkSync(inputPath); - } - - const abcFile: string = inputPath.replace(/\.js$/, '.abc'); - if (fs.existsSync(abcFile)) { - const abcFileNew: string = abcFile.replace(/_.abc$/, '.abc'); - fs.renameSync(abcFile, abcFileNew); - } else { - logger.error(red, `ETS:ERROR ${abcFile} is lost`, reset); - } +function getSmallestSizeGroup(groupSize: Map) { + let groupSizeArray = Array.from(groupSize); + groupSizeArray.sort(function(g1, g2) { + return g1[1] - g2[1]; // sort by value + }); + return groupSizeArray[0][0]; // return key +} + +function splitJsBundlesBySize(bundleArray: Array, groupNumber: number){ + let result = []; + if (bundleArray.length < groupNumber) { + result.push(bundleArray); + return result; + } + + bundleArray.sort(function(f1: File, f2: File) { + return f2.size - f1.size; + }); + let groupFileSize = new Map(); + for (let i = 0; i < groupNumber; ++i) { + result.push([]); + groupFileSize.set(i, 0); + } + + let index = 0; + while(index < bundleArray.length) { + let smallestGroup = getSmallestSizeGroup(groupFileSize); + result[smallestGroup].push(bundleArray[index]); + let sizeUpdate = groupFileSize.get(smallestGroup) + bundleArray[index].size; + groupFileSize.set(smallestGroup, sizeUpdate); + index++; + } + return result; }