From 78007d988b732476656d0cdc93e3ad52e0544fee Mon Sep 17 00:00:00 2001 From: hufeng Date: Wed, 22 May 2024 11:33:50 +0800 Subject: [PATCH 1/4] Fix tsc fileName dismatching rollupId when generating sourceMap in case-nonsensetive platform Signed-off-by: hufeng Change-Id: I0ca7c8c75ba6a8c299c5aca4e0a14b2ca5b135fe --- compiler/src/ark_utils.ts | 112 ++++++++++++------ .../ark_compiler/module/module_source_file.ts | 2 +- compiler/src/fast_build/ark_compiler/utils.ts | 5 +- compiler/src/process_module_files.ts | 14 ++- .../test/ark_compiler_ut/ark_utils.test.ts | 4 +- 5 files changed, 92 insertions(+), 45 deletions(-) diff --git a/compiler/src/ark_utils.ts b/compiler/src/ark_utils.ts index 346685c3d..87173207b 100644 --- a/compiler/src/ark_utils.ts +++ b/compiler/src/ark_utils.ts @@ -280,6 +280,11 @@ export function getBuildModeInLowerCase(projectConfig: Object): string { return (compileToolIsRollUp() ? projectConfig.buildMode : projectConfig.buildArkMode).toLowerCase(); } +/** + * This Api only used by wepack compiling process - js-loader + * @param sourcePath The path in build cache dir + * @param sourceCode The intermediate js source code + */ export function writeFileSyncByString(sourcePath: string, sourceCode: string, projectConfig: Object, logger: Object): void { const filePath: string = genTemporaryPath(sourcePath, projectConfig.projectPath, process.env.cachePath, projectConfig); if (filePath.length === 0) { @@ -292,7 +297,8 @@ export function writeFileSyncByString(sourcePath: string, sourceCode: string, pr fs.writeFileSync(filePath, sourceCode); return; } - writeObfuscatedSourceCode(sourceCode, filePath, logger, projectConfig); + writeObfuscatedSourceCode({content: sourceCode, buildFilePath: filePath, relativeSourceFilePath: ''}, + logger, projectConfig); } if (/\.json$/.test(sourcePath)) { fs.writeFileSync(filePath, sourceCode); @@ -453,41 +459,62 @@ function replaceRelativeDependency(item: string, moduleRequest: string, sourcePa return item; } -export async function writeObfuscatedSourceCode(content: string, filePath: string, logger: Object, projectConfig: Object, - relativeSourceFilePath: string = '', rollupNewSourceMaps: Object = {}, sourcePath?: string): Promise { +interface ModuleInfo { + content: string, + /** + * the path in build cache dir + */ + buildFilePath: string, + /** + * the `originSourceFilePath` relative to project root dir. + */ + relativeSourceFilePath: string, + /** + * the origin source file path will be set with rollup moduleId when obfuscate intermediate js source code, + * whereas be set with tsc node.fileName when obfuscate intermediate ts source code. + */ + originSourceFilePath?: string, + rollupModuleId?: string +} + +export async function writeObfuscatedSourceCode(moduleInfo: ModuleInfo, logger: Object, + projectConfig: Object, rollupNewSourceMaps: Object = {}): Promise { if (compileToolIsRollUp() && projectConfig.arkObfuscator) { MemoryUtils.tryGC(); - performancePrinter?.filesPrinter?.startEvent(filePath); - await writeArkguardObfuscatedSourceCode(content, filePath, logger, projectConfig, relativeSourceFilePath, rollupNewSourceMaps, sourcePath); - performancePrinter?.filesPrinter?.endEvent(filePath, undefined, true); + performancePrinter?.filesPrinter?.startEvent(moduleInfo.buildFilePath); + await writeArkguardObfuscatedSourceCode(moduleInfo, logger, projectConfig, rollupNewSourceMaps); + performancePrinter?.filesPrinter?.endEvent(moduleInfo.buildFilePath, undefined, true); MemoryUtils.tryGC(); return; } - mkdirsSync(path.dirname(filePath)); + mkdirsSync(path.dirname(moduleInfo.buildFilePath)); if (!compileToolIsRollUp()) { - await writeMinimizedSourceCode(content, filePath, logger, projectConfig.compileHar); + await writeMinimizedSourceCode(moduleInfo.content, moduleInfo.buildFilePath, logger, projectConfig.compileHar); return; } - sourcePath = toUnixPath(sourcePath); - let genFileInHar: GeneratedFileInHar = harFilesRecord.get(sourcePath); + moduleInfo.originSourceFilePath = toUnixPath(moduleInfo.originSourceFilePath); + let genFileInHar: GeneratedFileInHar = harFilesRecord.get(moduleInfo.originSourceFilePath); if (!genFileInHar) { - genFileInHar = { sourcePath: sourcePath }; + genFileInHar = { sourcePath: moduleInfo.originSourceFilePath }; } if (!genFileInHar.sourceCachePath) { - genFileInHar.sourceCachePath = toUnixPath(filePath); + genFileInHar.sourceCachePath = toUnixPath(moduleInfo.buildFilePath); } - harFilesRecord.set(sourcePath, genFileInHar); + harFilesRecord.set(moduleInfo.originSourceFilePath, genFileInHar); - fs.writeFileSync(filePath, content); + fs.writeFileSync(moduleInfo.buildFilePath, moduleInfo.content); } - -export async function writeArkguardObfuscatedSourceCode(content: string, filePath: string, logger: Object, projectConfig: Object, - relativeSourceFilePath: string = '', rollupNewSourceMaps: Object = {}, originalFilePath: string): Promise { +/** + * This Api only be used by rollup compiling process & only be + * exported for unit test. + */ +export async function writeArkguardObfuscatedSourceCode(moduleInfo: ModuleInfo, logger: Object, + projectConfig: Object, rollupNewSourceMaps: Object = {}): Promise { const arkObfuscator = projectConfig.arkObfuscator; - const isDeclaration = (/\.d\.e?ts$/).test(filePath); + const isDeclaration = (/\.d\.e?ts$/).test(moduleInfo.buildFilePath); const packageDir = projectConfig.packageDir; const projectRootPath = projectConfig.projectRootPath; const useNormalized = projectConfig.useNormalizedOHMUrl; @@ -496,16 +523,16 @@ export async function writeArkguardObfuscatedSourceCode(content: string, filePat const sourceMapGeneratorInstance = SourceMapGenerator.getInstance(); let previousStageSourceMap: sourceMap.RawSourceMap | undefined = undefined; - const selectedFilePath = sourceMapGeneratorInstance.isNewSourceMaps() ? originalFilePath : relativeSourceFilePath; - if (relativeSourceFilePath.length > 0 && !isDeclaration) { + if (moduleInfo.relativeSourceFilePath.length > 0 && !isDeclaration) { + const selectedFilePath = sourceMapGeneratorInstance.isNewSourceMaps() ? moduleInfo.rollupModuleId! : moduleInfo.relativeSourceFilePath; previousStageSourceMap = sourceMapGeneratorInstance.getSpecifySourceMap(rollupNewSourceMaps, selectedFilePath) as sourceMap.RawSourceMap; } let historyNameCache = new Map(); if (nameCacheObj) { - let namecachePath = relativeSourceFilePath; + let namecachePath = moduleInfo.relativeSourceFilePath; if (isDeclaration) { - namecachePath = harFilesRecord.get(originalFilePath).sourceCachePath; + namecachePath = harFilesRecord.get(moduleInfo.originSourceFilePath).sourceCachePath; } let identifierCache = nameCacheObj[namecachePath]?.[IDENTIFIER_CACHE]; deleteLineInfoForNameString(historyNameCache, identifierCache); @@ -520,31 +547,32 @@ export async function writeArkguardObfuscatedSourceCode(content: string, filePat useTsHar: boolean } = { packageDir, projectRootPath, localPackageSet, useNormalized, useTsHar }; try { - mixedInfo = await arkObfuscator.obfuscate(content, filePath, previousStageSourceMap, - historyNameCache, originalFilePath, projectInfo); + mixedInfo = await arkObfuscator.obfuscate(moduleInfo.content, moduleInfo.buildFilePath, previousStageSourceMap, + historyNameCache, moduleInfo.originSourceFilePath, projectInfo); } catch (err) { - logger.error(red, `ArkTS:INTERNAL ERROR: Failed to obfuscate file '${relativeSourceFilePath}' with arkguard. ${err}`); + logger.error(red, `ArkTS:INTERNAL ERROR: Failed to obfuscate file '${moduleInfo.relativeSourceFilePath}' with arkguard. ${err}`); } if (mixedInfo.sourceMap && !isDeclaration) { - mixedInfo.sourceMap.sources = [relativeSourceFilePath]; - sourceMapGeneratorInstance.fillSourceMapPackageInfo(originalFilePath, mixedInfo.sourceMap); + const selectedFilePath = sourceMapGeneratorInstance.isNewSourceMaps() ? moduleInfo.rollupModuleId! : moduleInfo.relativeSourceFilePath; + mixedInfo.sourceMap.sources = [moduleInfo.relativeSourceFilePath]; + sourceMapGeneratorInstance.fillSourceMapPackageInfo(moduleInfo.rollupModuleId!, mixedInfo.sourceMap); sourceMapGeneratorInstance.updateSpecifySourceMap(rollupNewSourceMaps, selectedFilePath, mixedInfo.sourceMap); } if (mixedInfo.nameCache && !isDeclaration) { - let obfName: string = relativeSourceFilePath; - let isOhModule = isPackageModulesFile(originalFilePath, projectConfig); + let obfName: string = moduleInfo.relativeSourceFilePath; + let isOhModule = isPackageModulesFile(moduleInfo.originSourceFilePath, projectConfig); if (projectConfig.obfuscationMergedObConfig?.options.enableFileNameObfuscation && !isOhModule) { - obfName = mangleFilePath(relativeSourceFilePath); + obfName = mangleFilePath(moduleInfo.relativeSourceFilePath); } mixedInfo.nameCache["obfName"] = obfName; - nameCacheObj[relativeSourceFilePath] = mixedInfo.nameCache; + nameCacheObj[moduleInfo.relativeSourceFilePath] = mixedInfo.nameCache; } - const newFilePath: string = tryMangleFileName(filePath, projectConfig, originalFilePath); - if (newFilePath !== filePath && !isDeclaration) { - sourceMapGeneratorInstance.saveKeyMappingForObfFileName(originalFilePath); + const newFilePath: string = tryMangleFileName(moduleInfo.buildFilePath, projectConfig, moduleInfo.originSourceFilePath); + if (newFilePath !== moduleInfo.buildFilePath && !isDeclaration) { + sourceMapGeneratorInstance.saveKeyMappingForObfFileName(moduleInfo.rollupModuleId!); } mkdirsSync(path.dirname(newFilePath)); fs.writeFileSync(newFilePath, mixedInfo.content ?? ''); @@ -577,8 +605,12 @@ export async function mangleDeclarationFileName(logger: Object, projectConfig: O for (const [sourcePath, genFilesInHar] of harFilesRecord) { if (genFilesInHar.originalDeclarationCachePath && genFilesInHar.originalDeclarationContent) { let relativeSourceFilePath = toUnixPath(genFilesInHar.originalDeclarationCachePath).replace(toUnixPath(projectConfig.projectRootPath) + '/', ''); - await writeObfuscatedSourceCode(genFilesInHar.originalDeclarationContent, genFilesInHar.originalDeclarationCachePath, logger, projectConfig, - relativeSourceFilePath, {}, sourcePath); + await writeObfuscatedSourceCode({ + content: genFilesInHar.originalDeclarationContent, + buildFilePath: genFilesInHar.originalDeclarationCachePath, + relativeSourceFilePath: relativeSourceFilePath, + originSourceFilePath: sourcePath + }, logger, projectConfig, {}); } } } @@ -654,6 +686,11 @@ export function getPackageInfo(configFile: string): Array { return [bundleName, moduleName]; } +/** + * This Api only used by wepack compiling process - result_process + * @param sourcePath The path in build cache dir + * @param sourceContent The intermediate js source code + */ export function generateSourceFilesToTemporary(sourcePath: string, sourceContent: string, sourceMap: Object, projectConfig: Object, logger: Object): void { let jsFilePath: string = genTemporaryPath(sourcePath, projectConfig.projectPath, process.env.cachePath, projectConfig); @@ -682,7 +719,8 @@ export function generateSourceFilesToTemporary(sourcePath: string, sourceContent return; } - writeObfuscatedSourceCode(sourceContent, jsFilePath, logger, projectConfig); + writeObfuscatedSourceCode({content: sourceContent, buildFilePath: jsFilePath, relativeSourceFilePath: ''}, + logger, projectConfig); } export function genAbcFileName(temporaryFile: string): string { diff --git a/compiler/src/fast_build/ark_compiler/module/module_source_file.ts b/compiler/src/fast_build/ark_compiler/module/module_source_file.ts index b337c1d3d..c7ebd02a8 100644 --- a/compiler/src/fast_build/ark_compiler/module/module_source_file.ts +++ b/compiler/src/fast_build/ark_compiler/module/module_source_file.ts @@ -300,7 +300,7 @@ export class ModuleSourceFile { private async writeSourceFile(parentEvent: Object): Promise { if (this.isSourceNode && !isJsSourceFile(this.moduleId)) { - await writeFileSyncByNode(this.source, ModuleSourceFile.projectConfig, parentEvent, ModuleSourceFile.logger); + await writeFileSyncByNode(this.source, ModuleSourceFile.projectConfig, this.moduleId, parentEvent, ModuleSourceFile.logger); } else { await writeFileContentToTempDir(this.moduleId, this.source, ModuleSourceFile.projectConfig, ModuleSourceFile.logger, parentEvent); } diff --git a/compiler/src/fast_build/ark_compiler/utils.ts b/compiler/src/fast_build/ark_compiler/utils.ts index 96346588a..061384e7e 100644 --- a/compiler/src/fast_build/ark_compiler/utils.ts +++ b/compiler/src/fast_build/ark_compiler/utils.ts @@ -163,8 +163,9 @@ async function writeFileContent(sourceFilePath: string, filePath: string, conten if (projectConfig.compileHar || !isDebug(projectConfig)) { const relativeSourceFilePath: string = toUnixPath(sourceFilePath).replace(toUnixPath(projectConfig.projectRootPath) + '/', ''); - await writeObfuscatedSourceCode(content, filePath, logger, projectConfig, relativeSourceFilePath, - SourceMapGenerator.getInstance().getSourceMaps(), sourceFilePath); + await writeObfuscatedSourceCode({content: content, buildFilePath: filePath, + relativeSourceFilePath: relativeSourceFilePath, originSourceFilePath: sourceFilePath}, + logger, projectConfig, SourceMapGenerator.getInstance().getSourceMaps()); return; } mkdirsSync(path.dirname(filePath)); diff --git a/compiler/src/process_module_files.ts b/compiler/src/process_module_files.ts index 9774a1c3d..fad8011db 100644 --- a/compiler/src/process_module_files.ts +++ b/compiler/src/process_module_files.ts @@ -40,7 +40,7 @@ import { isAotMode, isDebug } from './fast_build/ark_compiler/utils'; export const SRC_MAIN: string = 'src/main'; -export async function writeFileSyncByNode(node: ts.SourceFile, projectConfig: Object, parentEvent?: Object, logger?: Object): Promise { +export async function writeFileSyncByNode(node: ts.SourceFile, projectConfig: Object, moduleId?: string , parentEvent?: Object, logger?: Object): Promise { const eventWriteFileSyncByNode = createAndStartEvent(parentEvent, 'write file sync by node'); const eventGenContentAndSourceMapInfo = createAndStartEvent(eventWriteFileSyncByNode, 'generate content and source map information'); const mixedInfo: { content: string, sourceMapJson: ts.RawSourceMap } = genContentAndSourceMapInfo(node, projectConfig); @@ -59,8 +59,8 @@ export async function writeFileSyncByNode(node: ts.SourceFile, projectConfig: Ob let relativeSourceFilePath = toUnixPath(node.fileName).replace(toUnixPath(projectConfig.projectRootPath) + '/', ''); let sourceMaps: Object; if (process.env.compileTool === 'rollup') { - const key = sourceMapGenerator.isNewSourceMaps() ? node.fileName : relativeSourceFilePath; - sourceMapGenerator.fillSourceMapPackageInfo(node.fileName, mixedInfo.sourceMapJson); + const key = sourceMapGenerator.isNewSourceMaps() ? moduleId! : relativeSourceFilePath; + sourceMapGenerator.fillSourceMapPackageInfo(moduleId!, mixedInfo.sourceMapJson); sourceMapGenerator.updateSourceMap(key, mixedInfo.sourceMapJson); sourceMaps = sourceMapGenerator.getSourceMaps(); } else { @@ -69,7 +69,13 @@ export async function writeFileSyncByNode(node: ts.SourceFile, projectConfig: Ob } if (projectConfig.compileHar || (!isDebug(projectConfig) && isAotMode(projectConfig))) { const eventWriteObfuscatedSourceCode = createAndStartEvent(eventWriteFileSyncByNode, 'write obfuscated source code'); - await writeObfuscatedSourceCode(mixedInfo.content, temporaryFile, logger, projectConfig, relativeSourceFilePath, sourceMaps, node.fileName); + await writeObfuscatedSourceCode({ + content: mixedInfo.content, + buildFilePath: temporaryFile, + relativeSourceFilePath: relativeSourceFilePath, + originSourceFilePath: node.fileName, + rollupModuleId: moduleId ? moduleId : undefined + }, logger, projectConfig, sourceMaps); stopEvent(eventWriteObfuscatedSourceCode); return; } diff --git a/compiler/test/ark_compiler_ut/ark_utils.test.ts b/compiler/test/ark_compiler_ut/ark_utils.test.ts index 15fd57094..8e7862913 100644 --- a/compiler/test/ark_compiler_ut/ark_utils.test.ts +++ b/compiler/test/ark_compiler_ut/ark_utils.test.ts @@ -309,7 +309,9 @@ mocha.describe('test ark_utils file api', function () { const stub = sinon.stub(logger, 'error'); const red: string = '\x1B[31m'; try { - await writeArkguardObfuscatedSourceCode(undefined, '', logger, this.rollup.share.projectConfig, '', {}, ''); + await writeArkguardObfuscatedSourceCode( + {content: undefined, buildFilePath: '', relativeSourceFilePath: '', originSourceFilePath: ''}, + logger, this.rollup.share.projectConfig, {}); } catch (e) { } expect(stub.calledWith(red, -- Gitee From 1d8ef488761d93672c8e837c48de283babbdb2db Mon Sep 17 00:00:00 2001 From: zhangchen Date: Wed, 22 May 2024 17:24:20 +0800 Subject: [PATCH 2/4] Add the judgment condition for har files record Signed-off-by: zhangchen Change-Id: I0175b3c4975d8bf649a3708a20f728b7b25b8669 --- compiler/src/ark_utils.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/compiler/src/ark_utils.ts b/compiler/src/ark_utils.ts index 87173207b..256712afb 100644 --- a/compiler/src/ark_utils.ts +++ b/compiler/src/ark_utils.ts @@ -493,16 +493,18 @@ export async function writeObfuscatedSourceCode(moduleInfo: ModuleInfo, logger: return; } - moduleInfo.originSourceFilePath = toUnixPath(moduleInfo.originSourceFilePath); - let genFileInHar: GeneratedFileInHar = harFilesRecord.get(moduleInfo.originSourceFilePath); + if (moduleInfo.originSourceFilePath) { + const originSourceFilePath = toUnixPath(moduleInfo.originSourceFilePath); + let genFileInHar: GeneratedFileInHar = harFilesRecord.get(originSourceFilePath); - if (!genFileInHar) { - genFileInHar = { sourcePath: moduleInfo.originSourceFilePath }; - } - if (!genFileInHar.sourceCachePath) { - genFileInHar.sourceCachePath = toUnixPath(moduleInfo.buildFilePath); + if (!genFileInHar) { + genFileInHar = { sourcePath: originSourceFilePath }; + } + if (!genFileInHar.sourceCachePath) { + genFileInHar.sourceCachePath = toUnixPath(moduleInfo.buildFilePath); + } + harFilesRecord.set(originSourceFilePath, genFileInHar); } - harFilesRecord.set(moduleInfo.originSourceFilePath, genFileInHar); fs.writeFileSync(moduleInfo.buildFilePath, moduleInfo.content); } -- Gitee From ce9984295601ca7594a3d7f453ca1ba03d301ee3 Mon Sep 17 00:00:00 2001 From: hufeng Date: Fri, 24 May 2024 17:50:10 +0800 Subject: [PATCH 3/4] Fix write js file with obfuscation Signed-off-by: hufeng Change-Id: Ifd1355638b0fd1076f297c90030247a880dcd3d0 --- compiler/src/fast_build/ark_compiler/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/fast_build/ark_compiler/utils.ts b/compiler/src/fast_build/ark_compiler/utils.ts index 061384e7e..4e4e9e0c6 100644 --- a/compiler/src/fast_build/ark_compiler/utils.ts +++ b/compiler/src/fast_build/ark_compiler/utils.ts @@ -164,7 +164,7 @@ async function writeFileContent(sourceFilePath: string, filePath: string, conten const relativeSourceFilePath: string = toUnixPath(sourceFilePath).replace(toUnixPath(projectConfig.projectRootPath) + '/', ''); await writeObfuscatedSourceCode({content: content, buildFilePath: filePath, - relativeSourceFilePath: relativeSourceFilePath, originSourceFilePath: sourceFilePath}, + relativeSourceFilePath: relativeSourceFilePath, originSourceFilePath: sourceFilePath, rollupModuleId: sourceFilePath}, logger, projectConfig, SourceMapGenerator.getInstance().getSourceMaps()); return; } -- Gitee From 90203835aa2d4c4fd9e628a32e7354981ea60fb1 Mon Sep 17 00:00:00 2001 From: hufeng Date: Sat, 25 May 2024 18:58:17 +0800 Subject: [PATCH 4/4] Correct spelling error Signed-off-by: hufeng Change-Id: Ib238afef965a9726877f7356bb3c425ebd93797f --- compiler/src/ark_utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/src/ark_utils.ts b/compiler/src/ark_utils.ts index 256712afb..b763c7d00 100644 --- a/compiler/src/ark_utils.ts +++ b/compiler/src/ark_utils.ts @@ -281,7 +281,7 @@ export function getBuildModeInLowerCase(projectConfig: Object): string { } /** - * This Api only used by wepack compiling process - js-loader + * This Api only used by webpack compiling process - js-loader * @param sourcePath The path in build cache dir * @param sourceCode The intermediate js source code */ @@ -689,7 +689,7 @@ export function getPackageInfo(configFile: string): Array { } /** - * This Api only used by wepack compiling process - result_process + * This Api only used by webpack compiling process - result_process * @param sourcePath The path in build cache dir * @param sourceContent The intermediate js source code */ -- Gitee