!618 adapter module

Merge pull request !618 from zrg/temp_module
This commit is contained in:
openharmony_ci
2022-06-07 07:15:35 +00:00
committed by Gitee
10 changed files with 864 additions and 58 deletions
+15
View File
@@ -337,6 +337,20 @@ function hashProjectPath(projectPath) {
return process.env.hashProjectPath;
}
function loadModuleInfo(projectConfig, envArgs) {
if (projectConfig.aceBuildJson && fs.existsSync(projectConfig.aceBuildJson)) {
const buildJsonInfo = JSON.parse(fs.readFileSync(projectConfig.aceBuildJson).toString());
projectConfig.compileMode = buildJsonInfo.compileMode;
projectConfig.projectRootPath = buildJsonInfo.projectRootPath;
projectConfig.modulePathMap = buildJsonInfo.modulePathMap;
projectConfig.processTs = false;
projectConfig.buildArkMode = envArgs.buildMode;
if (buildJsonInfo.compileMode === 'esmodule') {
projectConfig.nodeModulesPath = buildJsonInfo.nodeModulesPath;
}
}
}
const globalProgram = {
program: null,
watchProgram: null
@@ -350,3 +364,4 @@ exports.resources = resources;
exports.loadWorker = loadWorker;
exports.abilityConfig = abilityConfig;
exports.readWorkerFile = readWorkerFile;
exports.loadModuleInfo = loadModuleInfo;
+3 -3
View File
@@ -23,10 +23,10 @@ const red: string = '\u001b[31m';
const reset: string = '\u001b[39m';
function js2abcByWorkers(jsonInput: string, cmd: string): Promise<void> {
const inputPaths = JSON.parse(jsonInput);
const inputPaths: any = JSON.parse(jsonInput);
for (let i = 0; i < inputPaths.length; ++i) {
const input = inputPaths[i].path;
const singleCmd = `${cmd} "${input}"`;
const input: string = inputPaths[i].path;
const singleCmd: any = `${cmd} "${input}"`;
logger.debug('gen abc cmd is: ', singleCmd, ' ,file size is:', inputPaths[i].size, ' byte');
try {
childProcess.execSync(singleCmd);
+457 -44
View File
@@ -19,10 +19,34 @@ import cluster from 'cluster';
import process from 'process';
import Compiler from 'webpack/lib/Compiler';
import { logger } from './compile_info';
import { toUnixPath, toHashData } from './utils';
import {
toUnixPath,
toHashData,
genTemporaryPath,
genBuildPath,
genAbcFileName,
mkdirsSync,
genSourceMapFileName,
checkNodeModulesFile
} from './utils';
import { projectConfig } from '../main';
import {
ESMODULE,
JSBUNDLE,
NODE_MODULES,
ENTRY_TXT,
EXTNAME_ETS,
EXTNAME_JS,
EXTNAME_TS,
EXTNAME_MJS,
EXTNAME_CJS,
EXTNAME_D_TS,
EXTNAME_ABC
} from './pre_define';
const firstFileEXT: string = '_.js';
const genAbcScript = 'gen_abc.js';
const genAbcScript: string = 'gen_abc.js';
const genModuleAbcScript: string = 'gen_module_abc.js';
let output: string;
let isWin: boolean = false;
let isMac: boolean = false;
@@ -36,13 +60,49 @@ interface File {
}
const intermediateJsBundle: Array<File> = [];
let fileterIntermediateJsBundle: Array<File> = [];
const moduleInfos: Array<ModuleInfo> = [];
let filterModuleInfos: Array<ModuleInfo> = [];
const commonJsModuleInfos: Array<ModuleInfo> = [];
const ESMModuleInfos: Array<ModuleInfo> = [];
const entryInfos: Map<string, EntryInfo> = new Map<string, EntryInfo>();
let hashJsonObject = {};
let buildPathInfo = '';
let moduleHashJsonObject = {};
let buildPathInfo: string = '';
const red: string = '\u001b[31m';
const reset: string = '\u001b[39m';
const hashFile = 'gen_hash.json';
const ARK = '/ark/';
const hashFile: string = 'gen_hash.json';
const ARK: string = '/ark/';
class ModuleInfo {
filePath: string;
tempFilePath: string;
buildFilePath: string;
abcFilePath: string;
isCommonJs: boolean;
constructor(filePath: string, tempFilePath: string, buildFilePath: string, abcFilePath: string, isCommonJs: boolean) {
this.filePath = filePath;
this.tempFilePath = tempFilePath;
this.buildFilePath = buildFilePath;
this.abcFilePath = abcFilePath;
this.isCommonJs = isCommonJs;
}
}
class EntryInfo {
npmInfo: string;
abcFileName: string;
buildPath: string;
entry: string;
constructor(npmInfo: string, abcFileName: string, buildPath: string, entry: string) {
this.npmInfo = npmInfo;
this.abcFileName = abcFileName;
this.buildPath = buildPath;
this.entry = entry;
}
}
export class GenAbcPlugin {
constructor(output_, arkDir_, nodeJs_, isDebug_) {
@@ -65,10 +125,34 @@ export class GenAbcPlugin {
}
}
compiler.hooks.compilation.tap('GenAbcPlugin', (compilation) => {
if (projectConfig.compileMode === JSBUNDLE || projectConfig.compileMode === undefined) {
return;
}
buildPathInfo = output;
compilation.hooks.finishModules.tap('finishModules', handleFinishModules.bind(this));
});
compiler.hooks.compilation.tap('GenAbcPlugin', (compilation) => {
compilation.hooks.processAssets.tap('processAssets', (assets) => {
if (projectConfig.compileMode === JSBUNDLE || projectConfig.compileMode === undefined) {
return;
}
Object.keys(compilation.assets).forEach(key => {
if (path.extname(key) === EXTNAME_JS) {
delete assets[key];
}
});
});
});
compiler.hooks.emit.tap('GenAbcPlugin', (compilation) => {
if (projectConfig.compileMode === ESMODULE) {
return;
}
Object.keys(compilation.assets).forEach(key => {
// choose *.js
if (output && path.extname(key) === '.js') {
if (output && path.extname(key) === EXTNAME_JS) {
const newContent: string = compilation.assets[key].source();
const keyPath: string = key.replace(/\.js$/, firstFileEXT);
writeFileSync(newContent, path.resolve(output, keyPath), key);
@@ -77,12 +161,163 @@ export class GenAbcPlugin {
});
compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => {
if (projectConfig.compileMode === ESMODULE) {
return;
}
buildPathInfo = output;
invokeWorkersToGenAbc();
});
}
}
function getEntryInfo(tempFilePath: string, resourceResolveData: any): void {
if (!resourceResolveData.descriptionFilePath) {
return;
}
const packageName: string = resourceResolveData.descriptionFileData['name'];
const packageJsonPath: string = resourceResolveData.descriptionFilePath;
let npmInfoPath: string = path.resolve(packageJsonPath, '..');
const fakeEntryPath: string = path.resolve(npmInfoPath, 'fake.js');
const tempFakeEntryPath: string = genTemporaryPath(fakeEntryPath, projectConfig.projectPath, process.env.cachePath);
const buildFakeEntryPath: string = genBuildPath(fakeEntryPath, projectConfig.projectPath, projectConfig.buildPath);
npmInfoPath = toUnixPath(path.resolve(tempFakeEntryPath, '..'));
const buildNpmInfoPath: string = toUnixPath(path.resolve(buildFakeEntryPath, '..'));
if (entryInfos.has(npmInfoPath)) {
return;
}
let abcFileName: string = genAbcFileName(tempFilePath);
const abcFilePaths: string[] = abcFileName.split(NODE_MODULES);
abcFileName = [NODE_MODULES, abcFilePaths[abcFilePaths.length - 1]].join(path.sep);
abcFileName = toUnixPath(abcFileName);
const packagePaths: string[] = tempFilePath.split(NODE_MODULES);
const entryPaths: string[] = packagePaths[packagePaths.length - 1].split(packageName);
let entry: string = toUnixPath(entryPaths[entryPaths.length - 1]);
if (entry.startsWith('/')) {
entry = entry.slice(1, entry.length);
}
const entryInfo: EntryInfo = new EntryInfo(npmInfoPath, abcFileName, buildNpmInfoPath, entry);
entryInfos.set(npmInfoPath, entryInfo);
}
function processNodeModulesFile(filePath: string, tempFilePath: string, buildFilePath: string, abcFilePath: string, nodeModulesFile: Array<string>, module: any): void {
getEntryInfo(tempFilePath, module.resourceResolveData);
const descriptionFileData: any = module.resourceResolveData.descriptionFileData;
if (descriptionFileData && descriptionFileData['type'] && descriptionFileData['type'] === 'module') {
const tempModuleInfo: ModuleInfo = new ModuleInfo(filePath, tempFilePath, buildFilePath, abcFilePath, false);
moduleInfos.push(tempModuleInfo);
nodeModulesFile.push(tempFilePath);
} else if (filePath.endsWith(EXTNAME_MJS)) {
const tempModuleInfo: ModuleInfo = new ModuleInfo(filePath, tempFilePath, buildFilePath, abcFilePath, false);
moduleInfos.push(tempModuleInfo);
nodeModulesFile.push(tempFilePath);
} else {
const tempModuleInfo: ModuleInfo = new ModuleInfo(filePath, tempFilePath, buildFilePath, abcFilePath, true);
moduleInfos.push(tempModuleInfo);
nodeModulesFile.push(tempFilePath);
}
return;
}
function processEtsModule(filePath: string, tempFilePath: string, buildFilePath: string, nodeModulesFile: Array<string>, module: any): void {
if (projectConfig.processTs === true) {
tempFilePath = tempFilePath.replace(/\.ets$/, EXTNAME_TS);
buildFilePath = buildFilePath.replace(/\.ets$/, EXTNAME_TS);
} else {
tempFilePath = tempFilePath.replace(/\.ets$/, EXTNAME_JS);
buildFilePath = buildFilePath.replace(/\.ets$/, EXTNAME_JS);
}
const abcFilePath: string = genAbcFileName(tempFilePath);
if (checkNodeModulesFile(filePath, projectConfig.projectPath)) {
processNodeModulesFile(filePath, tempFilePath, buildFilePath, abcFilePath, nodeModulesFile, module);
} else {
const tempModuleInfo: ModuleInfo = new ModuleInfo(filePath, tempFilePath, buildFilePath, abcFilePath, false);
moduleInfos.push(tempModuleInfo);
}
}
function processDtsModule(filePath: string, tempFilePath: string, buildFilePath: string, nodeModulesFile: Array<string>, module: any): void {
return;
}
function processTsModule(filePath: string, tempFilePath: string, buildFilePath: string, nodeModulesFile: Array<string>, module: any): void {
if (projectConfig.processTs === false) {
tempFilePath = tempFilePath.replace(/\.ts$/, EXTNAME_JS);
buildFilePath = buildFilePath.replace(/\.ts$/, EXTNAME_JS);
}
const abcFilePath: string = genAbcFileName(tempFilePath);
if (checkNodeModulesFile(filePath, projectConfig.projectPath)) {
processNodeModulesFile(filePath, tempFilePath, buildFilePath, abcFilePath, nodeModulesFile, module);
} else {
const tempModuleInfo: ModuleInfo = new ModuleInfo(filePath, tempFilePath, buildFilePath, abcFilePath, false);
moduleInfos.push(tempModuleInfo);
}
}
function processJsModule(filePath: string, tempFilePath: string, buildFilePath: string, nodeModulesFile: Array<string>, module: any): void {
const parent: string = path.join(tempFilePath, '..');
if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) {
mkDir(parent);
}
if (filePath.endsWith(EXTNAME_MJS) || filePath.endsWith(EXTNAME_CJS)) {
fs.copyFileSync(filePath, tempFilePath);
}
const abcFilePath: string = genAbcFileName(tempFilePath);
if (checkNodeModulesFile(filePath, projectConfig.projectPath)) {
processNodeModulesFile(filePath, tempFilePath, buildFilePath, abcFilePath, nodeModulesFile, module);
} else {
const tempModuleInfo: ModuleInfo = new ModuleInfo(filePath, tempFilePath, buildFilePath, abcFilePath, false);
moduleInfos.push(tempModuleInfo);
}
}
function handleFinishModules(modules, callback): void {
const nodeModulesFile: Array<string> = [];
modules.forEach(module => {
if (module !== undefined && module.resourceResolveData !== undefined) {
const filePath: string = module.resourceResolveData.path;
let tempFilePath = genTemporaryPath(filePath, projectConfig.projectPath, process.env.cachePath);
if (tempFilePath.length === 0) {
return;
}
let buildFilePath: string = genBuildPath(filePath, projectConfig.projectPath, projectConfig.buildPath);
tempFilePath = toUnixPath(tempFilePath);
buildFilePath = toUnixPath(buildFilePath);
if (filePath.endsWith(EXTNAME_ETS)) {
processEtsModule(filePath, tempFilePath, buildFilePath, nodeModulesFile, module);
} else if (filePath.endsWith(EXTNAME_D_TS)) {
processDtsModule(filePath, tempFilePath, buildFilePath, nodeModulesFile, module);
} else if (filePath.endsWith(EXTNAME_TS)) {
processTsModule(filePath, tempFilePath, buildFilePath, nodeModulesFile, module);
} else if (filePath.endsWith(EXTNAME_JS) || filePath.endsWith(EXTNAME_MJS) || filePath.endsWith(EXTNAME_CJS)) {
processJsModule(filePath, tempFilePath, buildFilePath, nodeModulesFile, module);
} else {
logger.error(red, `ETS:ERROR Cannot find resolve this file path: ${filePath}`, reset);
}
}
});
invokeWorkersModuleToGenAbc(moduleInfos);
processEntryToGenAbc(entryInfos);
}
function processEntryToGenAbc(entryInfos: Map<string, EntryInfo>): void {
for (const value of entryInfos.values()) {
const tempAbcFilePath: string = toUnixPath(path.resolve(value.npmInfo, ENTRY_TXT));
const buildAbcFilePath: string = toUnixPath(path.resolve(value.buildPath, ENTRY_TXT));
fs.writeFileSync(tempAbcFilePath, value.entry, 'utf-8');
if (!fs.existsSync(buildAbcFilePath)) {
const parent: string = path.join(buildAbcFilePath, '..');
if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) {
mkDir(parent);
}
}
fs.copyFileSync(tempAbcFilePath, buildAbcFilePath);
}
}
function writeFileSync(inputString: string, output: string, jsBundleFile: string): void {
const parent: string = path.join(output, '..');
if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) {
@@ -90,7 +325,7 @@ function writeFileSync(inputString: string, output: string, jsBundleFile: string
}
fs.writeFileSync(output, inputString);
if (fs.existsSync(output)) {
const fileSize = fs.statSync(output).size;
const fileSize: any = fs.statSync(output).size;
intermediateJsBundle.push({path: output, size: fileSize});
} else {
logger.error(red, `ETS:ERROR Failed to convert file ${jsBundleFile} to bin. ${output} is lost`, reset);
@@ -105,16 +340,16 @@ function mkDir(path_: string): void {
fs.mkdirSync(path_);
}
function getSmallestSizeGroup(groupSize: Map<number, number>) {
const groupSizeArray = Array.from(groupSize);
function getSmallestSizeGroup(groupSize: Map<number, number>): any {
const groupSizeArray: any = Array.from(groupSize);
groupSizeArray.sort(function(g1, g2) {
return g1[1] - g2[1]; // sort by size
});
return groupSizeArray[0][0];
}
function splitJsBundlesBySize(bundleArray: Array<File>, groupNumber: number) {
const result = [];
function splitJsBundlesBySize(bundleArray: Array<File>, groupNumber: number): any {
const result: any = [];
if (bundleArray.length < groupNumber) {
result.push(bundleArray);
return result;
@@ -123,7 +358,7 @@ function splitJsBundlesBySize(bundleArray: Array<File>, groupNumber: number) {
bundleArray.sort(function(f1: File, f2: File) {
return f2.size - f1.size;
});
const groupFileSize = new Map();
const groupFileSize: any = new Map();
for (let i = 0; i < groupNumber; ++i) {
result.push([]);
groupFileSize.set(i, 0);
@@ -131,16 +366,122 @@ function splitJsBundlesBySize(bundleArray: Array<File>, groupNumber: number) {
let index = 0;
while (index < bundleArray.length) {
const smallestGroup = getSmallestSizeGroup(groupFileSize);
const smallestGroup: any = getSmallestSizeGroup(groupFileSize);
result[smallestGroup].push(bundleArray[index]);
const sizeUpdate = groupFileSize.get(smallestGroup) + bundleArray[index].size;
const sizeUpdate: any = groupFileSize.get(smallestGroup) + bundleArray[index].size;
groupFileSize.set(smallestGroup, sizeUpdate);
index++;
}
return result;
}
function invokeWorkersToGenAbc() {
function invokeWorkersModuleToGenAbc(moduleInfos: Array<ModuleInfo>): void {
if (fs.existsSync(buildPathInfo)) {
fs.rmdirSync(buildPathInfo, { recursive: true});
}
if (fs.existsSync(projectConfig.nodeModulesPath)) {
fs.rmdirSync(projectConfig.nodeModulesPath, { recursive: true});
}
filterIntermediateModuleByHashJson(buildPathInfo, moduleInfos);
filterModuleInfos.forEach(moduleInfo => {
if (moduleInfo.isCommonJs) {
commonJsModuleInfos.push(moduleInfo);
} else {
ESMModuleInfos.push(moduleInfo);
}
});
invokeCluterModuleToAbc();
}
function initAbcEnv() : string[] {
let js2abc: string = path.join(arkDir, 'build', 'src', 'index.js');
if (isWin) {
js2abc = path.join(arkDir, 'build-win', 'src', 'index.js');
} else if (isMac) {
js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js');
}
const args: string[] = [
'--expose-gc',
js2abc
];
if (isDebug) {
args.push('--debug');
}
return args;
}
function invokeCluterModuleToAbc(): void {
const abcArgs: string[] = initAbcEnv();
const clusterNewApiVersion: number = 16;
const currentNodeVersion: number = parseInt(process.version.split('.')[0]);
const useNewApi: boolean = currentNodeVersion >= clusterNewApiVersion;
if (useNewApi && cluster.isPrimary || !useNewApi && cluster.isMaster) {
if (useNewApi) {
cluster.setupPrimary({
exec: path.resolve(__dirname, genModuleAbcScript)
});
} else {
cluster.setupMaster({
exec: path.resolve(__dirname, genModuleAbcScript)
});
}
if (commonJsModuleInfos.length > 0) {
const tempAbcArgs: string[] = abcArgs.slice(0);
tempAbcArgs.push('-c');
const commonJsCmdPrefix: any = `${nodeJs} ${tempAbcArgs.join(' ')}`;
const chunkSize: number = 50;
const splitedModules: any[] = splitModulesByNumber(commonJsModuleInfos, chunkSize);
const workerNumber: number = splitedModules.length;
for (let i = 0; i < workerNumber; i++) {
const workerData: any = {
'inputs': JSON.stringify(splitedModules[i]),
'cmd': commonJsCmdPrefix
};
cluster.fork(workerData);
}
}
if (ESMModuleInfos.length > 0) {
const tempAbcArgs: string[] = abcArgs.slice(0);
tempAbcArgs.push('-m');
const ESMCmdPrefix: any = `${nodeJs} ${tempAbcArgs.join(' ')}`;
const chunkSize: number = 50;
const splitedModules: any[] = splitModulesByNumber(ESMModuleInfos, chunkSize);
const workerNumber: number = splitedModules.length;
for (let i = 0; i < workerNumber; i++) {
const workerData: any = {
'inputs': JSON.stringify(splitedModules[i]),
'cmd': ESMCmdPrefix
};
cluster.fork(workerData);
}
}
cluster.on('exit', (worker, code, signal) => {
logger.debug(`worker ${worker.process.pid} finished`);
});
process.on('exit', (code) => {
writeModuleHashJson();
});
}
}
function splitModulesByNumber(moduleInfos: Array<ModuleInfo>, chunkSize: number): any[] {
const result: any[] = [];
for (let i = 0; i < moduleInfos.length; i += chunkSize) {
result.push(moduleInfos.slice(i, i + chunkSize));
}
return result;
}
function invokeWorkersToGenAbc(): void {
let param: string = '';
if (isDebug) {
param += ' --debug';
@@ -154,14 +495,14 @@ function invokeWorkersToGenAbc() {
}
filterIntermediateJsBundleByHashJson(buildPathInfo, intermediateJsBundle);
const maxWorkerNumber = 3;
const splitedBundles = splitJsBundlesBySize(fileterIntermediateJsBundle, maxWorkerNumber);
const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length;
const maxWorkerNumber: number = 3;
const splitedBundles: any[] = splitJsBundlesBySize(fileterIntermediateJsBundle, maxWorkerNumber);
const workerNumber: number = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length;
const cmdPrefix: string = `${nodeJs} --expose-gc "${js2abc}" ${param} `;
const clusterNewApiVersion = 16;
const currentNodeVersion = parseInt(process.version.split('.')[0]);
const useNewApi = currentNodeVersion >= clusterNewApiVersion;
const clusterNewApiVersion: number = 16;
const currentNodeVersion: number = parseInt(process.version.split('.')[0]);
const useNewApi: boolean = currentNodeVersion >= clusterNewApiVersion;
if (useNewApi && cluster.isPrimary || !useNewApi && cluster.isMaster) {
if (useNewApi) {
@@ -175,7 +516,7 @@ function invokeWorkersToGenAbc() {
}
for (let i = 0; i < workerNumber; ++i) {
const workerData = {
const workerData: any = {
'inputs': JSON.stringify(splitedBundles[i]),
'cmd': cmdPrefix
};
@@ -192,31 +533,103 @@ function invokeWorkersToGenAbc() {
}
}
function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: File[]) {
for (let i = 0; i < inputPaths.length; ++i) {
fileterIntermediateJsBundle.push(inputPaths[i]);
function filterIntermediateModuleByHashJson(buildPath: string, moduleInfos: Array<ModuleInfo>): void {
for (let i = 0; i < moduleInfos.length; ++i) {
filterModuleInfos.push(moduleInfos[i]);
}
const hashFilePath = genHashJsonPath(buildPath);
const hashFilePath: string = genHashJsonPath(buildPath);
if (hashFilePath.length === 0) {
return;
}
const updateJsonObject = {};
let jsonObject = {};
let jsonFile = '';
const updateJsonObject: any = {};
let jsonObject: any = {};
let jsonFile: string = '';
if (fs.existsSync(hashFilePath)) {
jsonFile = fs.readFileSync(hashFilePath).toString();
jsonObject = JSON.parse(jsonFile);
fileterIntermediateJsBundle = [];
for (let i = 0; i < inputPaths.length; ++i) {
const input = inputPaths[i].path;
const abcPath = input.replace(/_.js$/, '.abc');
filterModuleInfos = [];
for (let i = 0; i < moduleInfos.length; ++i) {
const input: string = moduleInfos[i].tempFilePath;
const abcPath: string = moduleInfos[i].abcFilePath;
if (!fs.existsSync(input)) {
logger.error(red, `ETS:ERROR ${input} is lost`, reset);
continue;
}
if (fs.existsSync(abcPath)) {
const hashInputContentData = toHashData(input);
const hashAbcContentData = toHashData(abcPath);
const hashInputContentData: any = toHashData(input);
const hashAbcContentData: any = toHashData(abcPath);
if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) {
updateJsonObject[input] = hashInputContentData;
updateJsonObject[abcPath] = hashAbcContentData;
mkdirsSync(path.dirname(moduleInfos[i].buildFilePath));
fs.copyFileSync(moduleInfos[i].tempFilePath, moduleInfos[i].buildFilePath);
fs.copyFileSync(genAbcFileName(moduleInfos[i].tempFilePath), genAbcFileName(moduleInfos[i].buildFilePath));
if (projectConfig.buildArkMode === 'debug' && fs.existsSync(genSourceMapFileName(moduleInfos[i].tempFilePath))) {
fs.copyFileSync(genSourceMapFileName(moduleInfos[i].tempFilePath), genSourceMapFileName(moduleInfos[i].buildFilePath));
}
} else {
filterModuleInfos.push(moduleInfos[i]);
}
} else {
filterModuleInfos.push(moduleInfos[i]);
}
}
}
moduleHashJsonObject = updateJsonObject;
}
function writeModuleHashJson(): void {
for (let i = 0; i < filterModuleInfos.length; ++i) {
const input: string = filterModuleInfos[i].tempFilePath;
const abcPath: string = filterModuleInfos[i].abcFilePath;
if (!fs.existsSync(input) || !fs.existsSync(abcPath)) {
logger.error(red, `ETS:ERROR ${input} is lost`, reset);
continue;
}
const hashInputContentData: any = toHashData(input);
const hashAbcContentData: any = toHashData(abcPath);
moduleHashJsonObject[input] = hashInputContentData;
moduleHashJsonObject[abcPath] = hashAbcContentData;
mkdirsSync(path.dirname(filterModuleInfos[i].buildFilePath));
fs.copyFileSync(filterModuleInfos[i].tempFilePath, filterModuleInfos[i].buildFilePath);
fs.copyFileSync(genAbcFileName(filterModuleInfos[i].tempFilePath), genAbcFileName(filterModuleInfos[i].buildFilePath));
if (projectConfig.buildArkMode === 'debug' && fs.existsSync(genSourceMapFileName(moduleInfos[i].tempFilePath))) {
fs.copyFileSync(genSourceMapFileName(moduleInfos[i].tempFilePath), genSourceMapFileName(moduleInfos[i].buildFilePath));
}
}
const hashFilePath: string = genHashJsonPath(buildPathInfo);
if (hashFilePath.length === 0) {
return;
}
fs.writeFileSync(hashFilePath, JSON.stringify(moduleHashJsonObject));
}
function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: File[]): void {
for (let i = 0; i < inputPaths.length; ++i) {
fileterIntermediateJsBundle.push(inputPaths[i]);
}
const hashFilePath: string = genHashJsonPath(buildPath);
if (hashFilePath.length === 0) {
return;
}
const updateJsonObject: any = {};
let jsonObject: any = {};
let jsonFile: string = '';
if (fs.existsSync(hashFilePath)) {
jsonFile = fs.readFileSync(hashFilePath).toString();
jsonObject = JSON.parse(jsonFile);
fileterIntermediateJsBundle = [];
for (let i = 0; i < inputPaths.length; ++i) {
const input: string = inputPaths[i].path;
const abcPath: string = input.replace(/_.js$/, EXTNAME_ABC);
if (!fs.existsSync(input)) {
logger.error(red, `ETS:ERROR ${input} is lost`, reset);
continue;
}
if (fs.existsSync(abcPath)) {
const hashInputContentData: any = toHashData(input);
const hashAbcContentData: any = toHashData(abcPath);
if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) {
updateJsonObject[input] = hashInputContentData;
updateJsonObject[abcPath] = hashAbcContentData;
@@ -233,28 +646,28 @@ function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: Fil
hashJsonObject = updateJsonObject;
}
function writeHashJson() {
function writeHashJson(): void {
for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) {
const input = fileterIntermediateJsBundle[i].path;
const abcPath = input.replace(/_.js$/, '.abc');
const input:string = fileterIntermediateJsBundle[i].path;
const abcPath: string = input.replace(/_.js$/, EXTNAME_ABC);
if (!fs.existsSync(input) || !fs.existsSync(abcPath)) {
logger.error(red, `ETS:ERROR ${input} is lost`, reset);
continue;
}
const hashInputContentData = toHashData(input);
const hashAbcContentData = toHashData(abcPath);
const hashInputContentData: any = toHashData(input);
const hashAbcContentData: any = toHashData(abcPath);
hashJsonObject[input] = hashInputContentData;
hashJsonObject[abcPath] = hashAbcContentData;
fs.unlinkSync(input);
}
const hashFilePath = genHashJsonPath(buildPathInfo);
const hashFilePath: string = genHashJsonPath(buildPathInfo);
if (hashFilePath.length === 0) {
return;
}
fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject));
}
function genHashJsonPath(buildPath: string) {
function genHashJsonPath(buildPath: string): void {
buildPath = toUnixPath(buildPath);
if (process.env.cachePath) {
if (!fs.existsSync(process.env.cachePath) || !fs.statSync(process.env.cachePath).isDirectory()) {
@@ -263,8 +676,8 @@ function genHashJsonPath(buildPath: string) {
}
return path.join(process.env.cachePath, hashFile);
} else if (buildPath.indexOf(ARK) >= 0) {
const dataTmps = buildPath.split(ARK);
const hashPath = path.join(dataTmps[0], ARK);
const dataTmps: string[] = buildPath.split(ARK);
const hashPath: string = path.join(dataTmps[0], ARK);
if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) {
logger.error(red, `ETS:ERROR hash path does not exist`, reset);
return '';
+50
View File
@@ -0,0 +1,50 @@
/*
* Copyright (c) 2022 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as childProcess from 'child_process';
import * as process from 'process';
import cluster from 'cluster';
import { logger } from './compile_info';
const red: string = '\u001b[31m';
const reset: string = '\u001b[39m';
function js2abcByWorkers(jsonInput: string, cmd: string): Promise<void> {
const inputPaths: any = JSON.parse(jsonInput);
const inputs: string[] = [];
for (let i = 0; i < inputPaths.length; ++i) {
const input: string = inputPaths[i].tempFilePath;
inputs.push('"' + input + '"');
}
const inputsStr: string = inputs.join(' ');
const singleCmd: any = `${cmd} ${inputsStr}`;
logger.debug('gen abc cmd is: ', singleCmd);
try {
childProcess.execSync(singleCmd);
} catch (e) {
logger.error(red, `ETS:ERROR Failed to convert file ${inputsStr} to abc `, reset);
return;
}
return;
}
logger.debug('worker data is: ', JSON.stringify(process.env));
logger.debug('gen_abc isWorker is: ', cluster.isWorker);
if (cluster.isWorker && process.env['inputs'] !== undefined && process.env['cmd'] !== undefined) {
logger.debug('==>worker #', cluster.worker.id, 'started!');
js2abcByWorkers(process.env['inputs'], process.env['cmd']);
process.exit();
}
+17
View File
@@ -235,3 +235,20 @@ export const COMPONENT_TOGGLE: string = 'Toggle';
export const TTOGGLE_CHECKBOX: string = 'Checkbox';
export const TOGGLE_SWITCH: string = 'Switch';
export const ESMODULE: string = 'esmodule';
export const JSBUNDLE: string = 'jsbundle';
export const ENTRY_TXT: string = 'entry.txt';
export const ARK: string = 'ark';
export const TEMPRARY: string = 'temprary';
export const MAIN: string = 'main';
export const AUXILIARY: string = 'auxiliary';
export const ZERO: string = '0';
export const ONE: string = '1';
export const EXTNAME_JS: string = '.js';
export const EXTNAME_TS: string = '.ts';
export const EXTNAME_JS_MAP: string = '.js.map';
export const EXTNAME_TS_MAP: string = '.ts.map';
export const EXTNAME_MJS: string = '.mjs';
export const EXTNAME_CJS: string = '.cjs';
export const EXTNAME_D_TS: string = '.d.ts';
export const EXTNAME_ABC: string = '.abc';
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright (c) 2022 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import ts from 'typescript';
import {
BUILD_ON
} from './pre_define';
import { writeFileSyncByNode } from './utils';
export function processJs(program: ts.Program, ut = false): Function {
return (context: ts.TransformationContext) => {
return (node: ts.SourceFile) => {
if (process.env.compiler === BUILD_ON) {
writeFileSyncByNode(node, false);
return node;
} else {
return node;
}
};
};
}
+14
View File
@@ -0,0 +1,14 @@
import { writeFileSyncByString } from './utils';
import { projectConfig } from '../main';
import {
ESMODULE,
ARK
} from './pre_define';
module.exports = function processjs2file(source: string): string {
if (projectConfig.compileMode === ESMODULE && projectConfig.processTs === false
&& process.env.compilerType && process.env.compilerType === ARK) {
writeFileSyncByString(this.resourcePath, source, false);
}
return source;
};
+17 -3
View File
@@ -42,14 +42,17 @@ import {
SET_CONTROLLER_CTR_TYPE,
SET_CONTROLLER_METHOD,
JS_DIALOG,
CUSTOM_DIALOG_CONTROLLER_BUILDER
CUSTOM_DIALOG_CONTROLLER_BUILDER,
ESMODULE,
ARK
} from './pre_define';
import {
componentInfo,
LogInfo,
LogType,
hasDecorator,
FileLog
FileLog,
writeFileSyncByNode
} from './utils';
import {
processComponentBlock,
@@ -67,7 +70,10 @@ import {
localStorageLinkCollection,
localStoragePropCollection
} from './validate_ui_syntax';
import { projectConfig, resources } from '../main';
import {
resources,
projectConfig
} from '../main';
import { createCustomComponentNewExpression, createViewCreate } from './process_component_member';
export const transformLog: FileLog = new FileLog();
@@ -82,6 +88,10 @@ export function processUISyntax(program: ts.Program, ut = false): Function {
if (process.env.compiler === BUILD_ON) {
if (!ut && (path.basename(node.fileName) === 'app.ets' || /\.ts$/.test(node.fileName))) {
node = ts.visitEachChild(node, processResourceNode, context);
if (projectConfig.compileMode === ESMODULE && projectConfig.processTs === true
&& process.env.compilerType && process.env.compilerType === ARK) {
writeFileSyncByNode(node, true);
}
return node;
}
transformLog.sourceFile = node;
@@ -97,6 +107,10 @@ export function processUISyntax(program: ts.Program, ut = false): Function {
});
node = ts.factory.updateSourceFile(node, statements);
INTERFACE_NODE_SET.clear();
if (projectConfig.compileMode === ESMODULE && projectConfig.processTs === true
&& process.env.compilerType && process.env.compilerType === ARK) {
writeFileSyncByNode(node, true);
}
return node;
} else {
return node;
+243 -4
View File
@@ -13,16 +13,38 @@
* limitations under the License.
*/
import ts from 'typescript';
import path from 'path';
import ts from 'typescript';
import fs from 'fs';
import { projectConfig } from '../main';
import { createHash } from 'crypto';
import { processSystemApi } from './validate_ui_syntax';
import {
NODE_MODULES,
TEMPRARY,
MAIN,
AUXILIARY,
ZERO,
ONE,
EXTNAME_JS,
EXTNAME_TS,
EXTNAME_MJS,
EXTNAME_CJS,
EXTNAME_ABC,
EXTNAME_ETS,
EXTNAME_TS_MAP,
EXTNAME_JS_MAP
} from './pre_define';
export enum LogType {
ERROR = 'ERROR',
WARN = 'WARN',
NOTE = 'NOTE'
}
export const TEMPORARYS: string = 'temporarys';
export const BUILD: string = 'build';
export const SRC_MAIN: string = 'src/main';
const TS_NOCHECK: string = '// @ts-nocheck';
export interface LogInfo {
type: LogType,
@@ -217,9 +239,9 @@ export function toUnixPath(data: string): string {
return data;
}
export function toHashData(path: string) {
const content = fs.readFileSync(path);
const hash = createHash('sha256');
export function toHashData(path: string): any {
const content: string = fs.readFileSync(path).toString();
const hash: any = createHash('sha256');
hash.update(content);
return hash.digest('hex');
}
@@ -234,3 +256,220 @@ export function writeFileSync(filePath: string, content: string): void {
fs.writeFileSync(filePath, content);
}
export function genTemporaryPath(filePath: string, projectPath: string, buildPath: string, toTsFile: boolean = true): string {
filePath = toUnixPath(filePath);
if (filePath.endsWith(EXTNAME_MJS)) {
filePath = filePath.replace(/\.mjs$/, EXTNAME_JS);
}
if (filePath.endsWith(EXTNAME_CJS)) {
filePath = filePath.replace(/\.cjs$/, EXTNAME_JS);
}
projectPath = toUnixPath(projectPath);
const hapPath: string = toUnixPath(projectConfig.projectRootPath);
const tempFilePath: string = filePath.replace(hapPath, '');
if (checkNodeModulesFile(filePath, projectPath)) {
const fakeNodeModulesPath: string = toUnixPath(path.resolve(projectConfig.projectRootPath, NODE_MODULES));
const dataTmps: string[] = tempFilePath.split(NODE_MODULES);
let output: string = '';
if (filePath.indexOf(fakeNodeModulesPath) === -1) {
const sufStr: string = dataTmps[dataTmps.length - 1];
output = path.join(buildPath, TEMPRARY, NODE_MODULES, MAIN, sufStr);
} else {
const sufStr: string = dataTmps[dataTmps.length - 1];
output = path.join(buildPath, TEMPRARY, NODE_MODULES, AUXILIARY, sufStr);
}
return output;
}
if (filePath.indexOf(projectPath) !== -1) {
const sufStr: string = filePath.replace(projectPath, '');
const output: string = path.join(buildPath, TEMPRARY, sufStr);
return output;
}
return '';
}
export function genBuildPath(filePath: string, projectPath: string, buildPath: string, toTsFile: boolean = true): string {
filePath = toUnixPath(filePath);
if (filePath.endsWith(EXTNAME_MJS)) {
filePath = filePath.replace(/\.mjs$/, EXTNAME_JS);
}
if (filePath.endsWith(EXTNAME_CJS)) {
filePath = filePath.replace(/\.cjs$/, EXTNAME_JS);
}
projectPath = toUnixPath(projectPath);
const hapPath: string = toUnixPath(projectConfig.projectRootPath);
const tempFilePath: string = filePath.replace(hapPath, '');
if (checkNodeModulesFile(filePath, projectPath)) {
filePath = toUnixPath(filePath);
const fakeNodeModulesPath: string = toUnixPath(path.resolve(projectConfig.projectRootPath, NODE_MODULES));
const dataTmps: string[] = tempFilePath.split(NODE_MODULES);
let output: string = '';
if (filePath.indexOf(fakeNodeModulesPath) === -1) {
const sufStr: string = dataTmps[dataTmps.length - 1];
output = path.join(projectConfig.nodeModulesPath, ZERO, sufStr);
} else {
const sufStr: string = dataTmps[dataTmps.length - 1];
output = path.join(projectConfig.nodeModulesPath, ONE, sufStr);
}
return output;
}
if (filePath.indexOf(projectPath) !== -1) {
const sufStr: string = filePath.replace(projectPath, '');
const output: string = path.join(buildPath, sufStr);
return output;
}
return '';
}
export function checkNodeModulesFile(filePath: string, projectPath: string): boolean {
filePath = toUnixPath(filePath);
projectPath = toUnixPath(projectPath);
const hapPath: string = toUnixPath(projectConfig.projectRootPath);
const tempFilePath: string = filePath.replace(hapPath, '');
if (tempFilePath.indexOf(NODE_MODULES) !== -1) {
const fakeNodeModulesPath: string = toUnixPath(path.resolve(projectConfig.projectRootPath, NODE_MODULES));
if (filePath.indexOf(fakeNodeModulesPath) !== -1) {
return true;
}
if (projectConfig.modulePathMap) {
for (const key in projectConfig.modulePathMap) {
const value: string = projectConfig.modulePathMap[key];
const fakeModuleNodeModulesPath: string = toUnixPath(path.resolve(value, NODE_MODULES));
if (filePath.indexOf(fakeModuleNodeModulesPath) !== -1) {
return true;
}
}
}
}
return false;
}
export function mkdirsSync(dirname: string): boolean {
if (fs.existsSync(dirname)) {
return true;
} else if (mkdirsSync(path.dirname(dirname))) {
fs.mkdirSync(dirname);
return true;
}
return false;
}
export function writeFileSyncByString(sourcePath: string, sourceCode: string, toTsFile: boolean): void {
const temporaryFile: string = genTemporaryPath(sourcePath, projectConfig.projectPath, process.env.cachePath, toTsFile);
if (temporaryFile.length === 0) {
return;
}
mkdirsSync(path.dirname(temporaryFile));
fs.writeFileSync(temporaryFile, sourceCode);
return;
}
export function writeFileSyncByNode(node: ts.SourceFile, toTsFile: boolean): void {
if (toTsFile) {
const newStatements: ts.Node[] = [];
const tsIgnoreNode: ts.Node = ts.factory.createExpressionStatement(ts.factory.createIdentifier(TS_NOCHECK));
newStatements.push(tsIgnoreNode);
if (node.statements && node.statements.length) {
newStatements.push(...node.statements);
}
node = ts.factory.updateSourceFile(node, newStatements);
}
const mixedInfo: {content: string, sourceMapContent: string} = genContentAndSourceMapInfo(node, toTsFile);
let temporaryFile: string = genTemporaryPath(node.fileName, projectConfig.projectPath, process.env.cachePath, toTsFile);
if (temporaryFile.length === 0) {
return;
}
let temporarySourceMapFile: string = '';
if (temporaryFile.endsWith(EXTNAME_ETS)) {
if (toTsFile) {
temporaryFile = temporaryFile.replace(/\.ets$/, EXTNAME_TS);
} else {
temporaryFile = temporaryFile.replace(/\.ets$/, EXTNAME_JS);
}
temporarySourceMapFile = genSourceMapFileName(temporaryFile);
} else {
if (!toTsFile) {
temporaryFile = temporaryFile.replace(/\.ts$/, EXTNAME_JS);
temporarySourceMapFile = genSourceMapFileName(temporaryFile);
}
}
mkdirsSync(path.dirname(temporaryFile));
fs.writeFileSync(temporaryFile, mixedInfo.content);
if (temporarySourceMapFile.length > 0 && projectConfig.buildArkMode === 'debug') {
fs.writeFileSync(temporarySourceMapFile, mixedInfo.sourceMapContent);
}
}
function genContentAndSourceMapInfo(node: ts.SourceFile, toTsFile: boolean): any {
const printer: ts.Printer = ts.createPrinter({ newLine: ts.NewLineKind.LineFeed });
const options: ts.CompilerOptions = {
sourceMap: true
};
const mapOpions: any = {
sourceMap: true,
inlineSourceMap: false,
inlineSources: false,
sourceRoot: '',
mapRoot: '',
extendedDiagnostics: false
};
const host: ts.CompilerHost = ts.createCompilerHost(options);
const fileName: string = node.fileName;
// @ts-ignore
const sourceMapGenerator: any = ts.createSourceMapGenerator(
host,
// @ts-ignore
ts.getBaseFileName(fileName),
'',
'',
mapOpions
);
// @ts-ignore
const writer: any = ts.createTextWriter(
// @ts-ignore
ts.getNewLineCharacter({newLine: ts.NewLineKind.LineFeed, removeComments: false}));
printer['writeFile'](node, writer, sourceMapGenerator);
const sourceMapJson: any = sourceMapGenerator.toJSON();
sourceMapJson['sources'] = [fileName];
const result: string = writer.getText();
let content: string = result;
content = processSystemApi(content, true);
if (toTsFile) {
content = result.replace(`${TS_NOCHECK};`, TS_NOCHECK);
}
const sourceMapContent: string = JSON.stringify(sourceMapJson);
return {
content: content,
sourceMapContent: sourceMapContent
};
}
export function genAbcFileName(temporaryFile: string): string {
let abcFile: string = temporaryFile;
if (temporaryFile.endsWith(EXTNAME_TS)) {
abcFile = temporaryFile.replace(/\.ts$/, EXTNAME_ABC);
} else {
abcFile = temporaryFile.replace(/\.js$/, EXTNAME_ABC);
}
return abcFile;
}
export function genSourceMapFileName(temporaryFile: string): string {
let abcFile: string = temporaryFile;
if (temporaryFile.endsWith(EXTNAME_TS)) {
abcFile = temporaryFile.replace(/\.ts$/, EXTNAME_TS_MAP);
} else {
abcFile = temporaryFile.replace(/\.js$/, EXTNAME_JS_MAP);
}
return abcFile;
}
+15 -4
View File
@@ -28,13 +28,14 @@ const {
resources,
loadWorker,
abilityConfig,
readWorkerFile
readWorkerFile,
loadModuleInfo
} = require('./main');
const { ResultStates } = require('./lib/compile_info');
const { processUISyntax } = require('./lib/process_ui_syntax');
const { IGNORE_ERROR_CODE } = require('./lib/utils');
const { BUILD_SHARE_PATH } = require('./lib/pre_define');
const { processJs } = require('./lib/process_js_ast');
process.env.watchMode = (process.env.watchMode && process.env.watchMode === 'true') || 'false';
function initConfig(config) {
@@ -78,10 +79,16 @@ function initConfig(config) {
transpileOnly: true,
configFile: path.resolve(__dirname, 'tsconfig.json'),
getCustomTransformers(program) {
return {
let transformerOperation = {
before: [processUISyntax(program)],
after: []
};
if (projectConfig.compileMode === 'esmodule' && projectConfig.processTs === false
&& process.env.compilerType && process.env.compilerType === 'ark') {
transformerOperation.after.push(processJs(program));
}
return transformerOperation;
},
ignoreDiagnostics: IGNORE_ERROR_CODE
}
@@ -92,6 +99,7 @@ function initConfig(config) {
{
test: /\.js$/,
use: [
{ loader: path.resolve(__dirname, 'lib/process_js_file.js')},
{ loader: path.resolve(__dirname, 'lib/process_system_module.js') }
]
}
@@ -306,14 +314,17 @@ module.exports = (env, argv) => {
const config = {};
setProjectConfig(env);
loadEntryObj(projectConfig);
loadModuleInfo(projectConfig, env);
setTsConfigFile();
initConfig(config);
const workerFile = readWorkerFile();
setOptimizationConfig(config, workerFile);
setCopyPluginConfig(config);
if (env.isPreview !== "true") {
loadWorker(projectConfig, workerFile);
if (env.compilerType && env.compilerType === 'ark') {
process.env.compilerType = 'ark';
let arkDir = path.join(__dirname, 'bin', 'ark');
if (env.arkFrontendDir) {
arkDir = env.arkFrontendDir;
@@ -360,4 +371,4 @@ module.exports = (env, argv) => {
}
config.output.library = projectConfig.hashProjectPath;
return config;
}
}