From 4a5258387705d9bae081cad9139a3b8bcefd818e Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Fri, 23 Sep 2022 10:56:59 +0800 Subject: [PATCH] Pick gen abc plugin change to js Signed-off-by: zhangrengao Change-Id: I73b7c9f1afd49375fc5ac469d22f1cdc165da83c --- ace-loader/src/gen-abc.js | 32 ++-- ace-loader/src/genAbc-plugin.js | 276 ++++++++++++++++++++++---------- 2 files changed, 202 insertions(+), 106 deletions(-) diff --git a/ace-loader/src/gen-abc.js b/ace-loader/src/gen-abc.js index 1bfdbcd..28261f4 100644 --- a/ace-loader/src/gen-abc.js +++ b/ace-loader/src/gen-abc.js @@ -26,22 +26,15 @@ const ES2ABC = 'es2abc'; function js2abcByWorkers(jsonInput, cmd) { const inputPaths = JSON.parse(jsonInput); for (let i = 0; i < inputPaths.length; ++i) { - let input = inputPaths[i].path; - let singleCmd = `${cmd} "${input}"`; + // for matching debug info mechanism + const input = inputPaths[i].path.replace(/\.temp\.js$/, "_.js"); + const cacheOutputPath = inputPaths[i].cacheOutputPath; + const cacheAbcFilePath = cacheOutputPath.replace(/\.temp\.js$/, ".abc"); + const singleCmd = `${cmd} "${cacheOutputPath}" -o "${cacheAbcFilePath}" --source-file "${input}"`; try { childProcess.execSync(singleCmd); } catch (e) { - console.error(red, `ERROR Failed to convert file ${input} to abc `, reset); - process.exit(FAIL); - } - - const abcFile = input.replace(/\.js$/, '.abc'); - if (fs.existsSync(abcFile)) { - const abcFileNew = abcFile.replace(/_.abc$/, '.abc'); - fs.copyFileSync(abcFile, abcFileNew); - fs.unlinkSync(abcFile); - } else { - console.error(red, `ERROR ${abcFile} is lost`, reset); + console.debug(red, `ERROR Failed to convert file ${input} to abc `, reset); process.exit(FAIL); } } @@ -50,9 +43,10 @@ function js2abcByWorkers(jsonInput, cmd) { function es2abcByWorkers(jsonInput, cmd) { const inputPaths = JSON.parse(jsonInput); for (let i = 0; i < inputPaths.length; ++i) { - const input = inputPaths[i].path; - const abcFile = input.replace(/_.js$/, '.abc'); - const singleCmd = `${cmd} "${input}" --output "${abcFile}"`; + const input = inputPaths[i].path.replace(/\.temp\.js$/, "_.js"); + const cacheOutputPath = inputPaths[i].cacheOutputPath; + const cacheAbcFilePath = cacheOutputPath.replace(/\.temp\.js$/, ".abc"); + const singleCmd = `${cmd} "${cacheOutputPath}" --output "${cacheAbcFilePath}" --source-file "${input}"`; console.debug('gen abc cmd is: ', singleCmd, ' ,file size is:', inputPaths[i].size, ' byte'); try { childProcess.execSync(singleCmd); @@ -67,11 +61,11 @@ function es2abcByWorkers(jsonInput, cmd) { if (cluster.isWorker && process.env["inputs"] !== undefined && process.env["cmd"] !== undefined) { if (process.env.panda === TS2ABC) { - js2abcByWorkers(process.env['inputs'], process.env['cmd']); + js2abcByWorkers(process.env["inputs"], process.env["cmd"]); } else if (process.env.panda === ES2ABC || process.env.panda === 'undefined' || process.env.panda === undefined) { - es2abcByWorkers(process.env['inputs'], process.env['cmd']); + es2abcByWorkers(process.env["inputs"], process.env["cmd"]); } else { - console.error(red, `ERROR please set panda module`, reset); + logger.error(red, `ERROR please set panda module`, reset); process.exit(FAIL); } process.exit(SUCCESS); diff --git a/ace-loader/src/genAbc-plugin.js b/ace-loader/src/genAbc-plugin.js index 5bc027f..f4c8eff 100644 --- a/ace-loader/src/genAbc-plugin.js +++ b/ace-loader/src/genAbc-plugin.js @@ -18,6 +18,8 @@ const path = require('path'); const cluster = require('cluster'); const process = require('process'); const crypto = require('crypto'); +const events = require('events'); +const os = require('os'); const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + ' var define = globalObjects.define;' + '\n' + @@ -34,7 +36,6 @@ const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + ' (function(global) {' + '\n' + ' "use strict";' + '\n'; const last = '\n' + '})(this.__appProto__);' + '\n' + '})'; -const firstFileEXT = '_.js'; const genAbcScript = 'gen-abc.js'; let output; let isWin = false; @@ -54,9 +55,12 @@ const reset = '\u001b[39m'; const blue = '\u001b[34m'; const hashFile = 'gen_hash.json'; const ARK = '/ark/'; -let delayCount = 0; +const NODE_MODULES = 'node_modules'; +const TEMPORARY = 'temporary'; const TS2ABC = 'ts2abc'; const ES2ABC = 'es2abc'; +let previewCount = 0; +let compileCount = 0; class GenAbcPlugin { constructor(output_, arkDir_, nodeJs_, workerFile_, isDebug_) { @@ -77,9 +81,18 @@ class GenAbcPlugin { return; } + if (!checkNodeModules()) { + process.exitCode = FAIL; + return; + } + + // for preview mode max listeners + events.EventEmitter.defaultMaxListeners = 100; + compiler.hooks.emit.tap('GenAbcPlugin', (compilation) => { const assets = compilation.assets; const keys = Object.keys(assets); + buildPathInfo = output; keys.forEach(key => { // choice *.js if (output && path.extname(key) === '.js') { @@ -90,17 +103,21 @@ class GenAbcPlugin { if (key === 'commons.js' || key === 'vendors.js' || !checkWorksFile(key, workerFile)) { newContent = `\n\n\n\n\n\n\n\n\n\n\n\n\n\n` + newContent; } - const keyPath = key.replace(/\.js$/, firstFileEXT) - writeFileSync(newContent, path.resolve(output, keyPath), key, true); + const keyPath = key.replace(/\.js$/, ".temp.js"); + writeFileSync(newContent, output, keyPath, key, true); } else if (output && path.extname(key) === '.json' && process.env.DEVICE_LEVEL === 'card' && !checkI18n(key)) { - writeFileSync(assets[key].source(), path.resolve(output, key), key, false); + writeFileSync(assets[key].source(), output, key, key, false); } }) }); compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => { + if (intermediateJsBundle.length === 0) { + return; + } buildPathInfo = output; - judgeWorkersToGenAbc(invokeWorkerToGenAbc); + previewCount++; + invokeWorkerToGenAbc(); }); } } @@ -133,20 +150,36 @@ function checkWorksFile(assetPath, workerFile) { return true; } -function writeFileSync(inputString, output, jsBundleFile, isToBin) { - const parent = path.join(output, '..'); +function writeFileSync(inputString, buildPath, keyPath, jsBundleFile, isToBin) { + let output = path.resolve(buildPath, keyPath); + let parent = path.join(output, '..'); if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) { mkDir(parent); } - fs.writeFileSync(output, inputString); if (!isToBin) { + fs.writeFileSync(output, inputString); return; } - if (fs.existsSync(output)) { - let fileSize = fs.statSync(output).size; - intermediateJsBundle.push({path: output, size: fileSize}); + let cacheOutputPath = ""; + if (process.env.cachePath) { + let buildDirArr = buildPathInfo.split(path.sep); + let abilityDir = buildDirArr[buildDirArr.length - 1]; + cacheOutputPath = path.join(process.env.cachePath, TEMPORARY, abilityDir, keyPath); + } else { + cacheOutputPath = output; + } + parent = path.join(cacheOutputPath, '..'); + if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) { + mkDir(parent); + } + fs.writeFileSync(cacheOutputPath, inputString); + if (fs.existsSync(cacheOutputPath)) { + let fileSize = fs.statSync(cacheOutputPath).size; + output = toUnixPath(output); + cacheOutputPath = toUnixPath(cacheOutputPath); + intermediateJsBundle.push({path: output, size: fileSize, cacheOutputPath: cacheOutputPath}); } else { - console.error(red, `ERROR Failed to convert file ${jsBundleFile} to bin. ${output} is lost`, reset); + console.debug(red, `ERROR Failed to convert file ${jsBundleFile} to bin. ${output} is lost`, reset); process.exitCode = FAIL; } } @@ -197,38 +230,22 @@ function splitJsBundlesBySize(bundleArray, groupNumber) { } function invokeWorkerToGenAbc() { - let param = ''; + if (process.env.isPreview === "true") { + process.exitCode = SUCCESS; + } let cmdPrefix = ''; - + let maxWorkerNumber = 3; + const abcArgs = initAbcEnv(); if (process.env.panda === TS2ABC) { - if (isDebug) { - param += ' --debug'; - } - - let js2abc = path.join(arkDir, 'build', 'src', 'index.js'); - if (isWin) { - js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); - } else if (isMac) { - js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); - } - cmdPrefix = `${nodeJs} --expose-gc "${js2abc}" ${param} `; + cmdPrefix = `${nodeJs} ${abcArgs.join(' ')}`; } else if (process.env.panda === ES2ABC || process.env.panda === 'undefined' || process.env.panda === undefined) { - if (isDebug) { - param += ' --debug-info'; - } - let es2abc = path.join(arkDir, 'build', 'bin', 'es2abc'); - if (isWin) { - es2abc = path.join(arkDir, 'build-win', 'bin', 'es2abc.exe'); - } else if (isMac) { - es2abc = path.join(arkDir, 'build-mac', 'bin', 'es2abc'); - } - cmdPrefix = `"${es2abc}" ${param} `; + maxWorkerNumber = os.cpus().length; + cmdPrefix = `${abcArgs.join(' ')}`; } else { - console.error(red, `ERROR please set panda module`, reset); + console.debug(red, `ERROR please set panda module`, reset); } filterIntermediateJsBundleByHashJson(buildPathInfo, intermediateJsBundle); - const maxWorkerNumber = 3; const splitedBundles = splitJsBundlesBySize(fileterIntermediateJsBundle, maxWorkerNumber); const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length; @@ -256,6 +273,10 @@ function invokeWorkerToGenAbc() { } let count_ = 0; + if (process.env.isPreview === 'true') { + process.removeAllListeners("exit"); + cluster.removeAllListeners("exit"); + } cluster.on('exit', (worker, code, signal) => { if (code === FAIL || process.exitCode === FAIL) { process.exitCode = FAIL; @@ -263,26 +284,32 @@ function invokeWorkerToGenAbc() { } count_++; if (count_ === workerNumber) { - writeHashJson(); - clearGlobalInfo(); - if (process.env.isPreview) { + // for preview of with incre compile + if (process.env.isPreview === "true" && compileCount < previewCount) { + compileCount++; + processExtraAssetForBundle(); console.log(blue, 'COMPILE RESULT:SUCCESS ', reset); + if (compileCount >= previewCount) { + return; + } + invokeWorkerToGenAbc(); } } }); process.on('exit', (code) => { - intermediateJsBundle.forEach((item) => { - let input = item.path; - if (fs.existsSync(input)) { - fs.unlinkSync(input); - } - }) + // for build options + processExtraAssetForBundle(); }); + + // for preview of without incre compile + if (workerNumber === 0 && process.env.isPreview === "true") { + processExtraAssetForBundle(); + } } } function clearGlobalInfo() { - if (!process.env.isPreview) { + if (process.env.isPreview !== "true") { intermediateJsBundle = []; } fileterIntermediateJsBundle = []; @@ -306,7 +333,7 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { } const hashFilePath = genHashJsonPath(buildPath); if (hashFilePath.length == 0) { - return ; + return; } let updateJsonObject = {}; let jsonObject = {}; @@ -316,24 +343,20 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { jsonObject = JSON.parse(jsonFile); fileterIntermediateJsBundle = []; for (let i = 0; i < inputPaths.length; ++i) { - let input = inputPaths[i].path; - let abcPath = input.replace(/_.js$/, '.abc'); - - if (!fs.existsSync(input)) { - console.error(red, `ERROR ${input} is lost`, reset); + const cacheOutputPath = inputPaths[i].cacheOutputPath; + const cacheAbcFilePath = cacheOutputPath.replace(/\.temp\.js$/, '.abc'); + if (!fs.existsSync(cacheOutputPath)) { + console.debug(red, `ERROR ${cacheOutputPath} is lost`, reset); process.exitCode = FAIL; break; } - if (fs.existsSync(input) && fs.existsSync(abcPath)) { - const hashInputContentData = toHashData(input); - const hashAbcContentData = toHashData(abcPath); - if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { - updateJsonObject[input] = hashInputContentData; - updateJsonObject[abcPath] = hashAbcContentData; - if (!process.env.isPreview) { - fs.unlinkSync(input); - } + if (fs.existsSync(cacheAbcFilePath)) { + const hashInputContentData = toHashData(cacheOutputPath); + const hashAbcContentData = toHashData(cacheAbcFilePath); + if (jsonObject[cacheOutputPath] === hashInputContentData && jsonObject[cacheAbcFilePath] === hashAbcContentData) { + updateJsonObject[cacheOutputPath] = hashInputContentData; + updateJsonObject[cacheAbcFilePath] = hashAbcContentData; } else { fileterIntermediateJsBundle.push(inputPaths[i]); } @@ -348,28 +371,23 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { function writeHashJson() { for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { - let input = fileterIntermediateJsBundle[i].path; - let abcPath = input.replace(/_.js$/, '.abc'); - if (!fs.existsSync(input) || !fs.existsSync(abcPath)) { - console.error(red, `ERROR ${input} is lost`, reset); + const cacheOutputPath = fileterIntermediateJsBundle[i].cacheOutputPath; + const cacheAbcFilePath = cacheOutputPath.replace(/\.temp\.js$/, '.abc'); + if (!fs.existsSync(cacheOutputPath) || !fs.existsSync(cacheAbcFilePath)) { + console.debug(red, `ERROR ${cacheOutputPath} is lost`, reset); process.exitCode = FAIL; break; } - if (fs.existsSync(input) && fs.existsSync(abcPath)) { - const hashInputContentData = toHashData(input); - const hashAbcContentData = toHashData(abcPath); - hashJsonObject[input] = hashInputContentData; - hashJsonObject[abcPath] = hashAbcContentData; - } - if (!process.env.isPreview && fs.existsSync(input)) { - fs.unlinkSync(input); - } + const hashInputContentData = toHashData(cacheOutputPath); + const hashAbcContentData = toHashData(cacheAbcFilePath); + hashJsonObject[cacheOutputPath] = hashInputContentData; + hashJsonObject[cacheAbcFilePath] = hashAbcContentData; } const hashFilePath = genHashJsonPath(buildPathInfo); if (hashFilePath.length == 0) { - return ; + return; } - if (!process.env.isPreview || delayCount < 1) { + if (process.env.isPreview !== "true" || previewCount < 1) { fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); } } @@ -380,7 +398,14 @@ function genHashJsonPath(buildPath) { if (!fs.existsSync(process.env.cachePath) || !fs.statSync(process.env.cachePath).isDirectory()) { return ''; } - return path.join(process.env.cachePath, hashFile); + let buildDirArr = buildPathInfo.split(path.sep); + let abilityDir = buildDirArr[buildDirArr.length - 1]; + let hashJsonPath = path.join(process.env.cachePath, TEMPORARY, abilityDir, hashFile); + let parent = path.join(hashJsonPath, '..'); + if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) { + mkDir(parent); + } + return hashJsonPath; } else if (buildPath.indexOf(ARK) >= 0) { const dataTmps = buildPath.split(ARK); const hashPath = path.join(dataTmps[0], ARK); @@ -414,13 +439,90 @@ module.exports = { checkWorksFile: checkWorksFile } -function judgeWorkersToGenAbc(callback) { - const workerNumber = Object.keys(cluster.workers).length; - if (workerNumber === 0) { - callback(); - return ; - } else { - delayCount++; - setTimeout(judgeWorkersToGenAbc.bind(null, callback), 50); +function copyFileCachePathToBuildPath() { + for (let i = 0; i < intermediateJsBundle.length; ++i) { + const abcFile = intermediateJsBundle[i].path.replace(/\.temp\.js$/, ".abc"); + const cacheOutputPath = intermediateJsBundle[i].cacheOutputPath; + const cacheAbcFilePath = intermediateJsBundle[i].cacheOutputPath.replace(/\.temp\.js$/, ".abc"); + if (!fs.existsSync(cacheAbcFilePath)) { + console.debug(red, `ERROR ${cacheAbcFilePath} is lost`, reset); + process.exitCode = FAIL; + break; + } + let parent = path.join(abcFile, '..'); + if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) { + mkDir(parent); + } + // for preview mode, cache path and old abc file both exist, should copy abc file for updating + if (process.env.cachePath !== undefined) { + fs.copyFileSync(cacheAbcFilePath, abcFile); + } + if (process.env.cachePath === undefined && fs.existsSync(cacheOutputPath)) { + fs.unlinkSync(cacheOutputPath); + } + } +} + +function processExtraAssetForBundle() { + writeHashJson(); + copyFileCachePathToBuildPath(); + clearGlobalInfo(); +} + +function checkNodeModules() { + if (process.env.panda === TS2ABC) { + let arkEntryPath = path.join(arkDir, 'build'); + if (isWin) { + arkEntryPath = path.join(arkDir, 'build-win'); + } else if (isMac) { + arkEntryPath = path.join(arkDir, 'build-mac'); + } + let nodeModulesPath = path.join(arkEntryPath, NODE_MODULES); + if (!(fs.existsSync(nodeModulesPath) && fs.statSync(nodeModulesPath).isDirectory())) { + console.error(red, `ERROR: node_modules for ark compiler not found. + Please make sure switch to non-root user before runing "npm install" for safity requirements and try re-run "npm install" under ${arkEntryPath}`, reset); + return false; + } } + + return true; } + +function initAbcEnv() { + let args = []; + if (process.env.panda === TS2ABC) { + let js2abc = path.join(arkDir, 'build', 'src', 'index.js'); + if (isWin) { + js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); + } else if (isMac) { + js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); + } + + js2abc = '"' + js2abc + '"'; + args = [ + '--expose-gc', + js2abc + ]; + if (isDebug) { + args.push('--debug'); + } + } else if (process.env.panda === ES2ABC || process.env.panda === 'undefined' || process.env.panda === undefined) { + let es2abc = path.join(arkDir, 'build', 'bin', 'es2abc'); + if (isWin) { + es2abc = path.join(arkDir, 'build-win', 'bin', 'es2abc.exe'); + } else if (isMac) { + es2abc = path.join(arkDir, 'build-mac', 'bin', 'es2abc'); + } + + args = [ + '"' + es2abc + '"' + ]; + if (isDebug) { + args.push('--debug-info'); + } + } else { + console.debug(red, `ERROR: please set panda module`, reset); + } + + return args; +} \ No newline at end of file -- Gitee