From 38d5648e391057949ca2894a00778cfbf1e18bcb Mon Sep 17 00:00:00 2001 From: songqi Date: Wed, 29 Mar 2023 14:11:06 +0800 Subject: [PATCH] Enable es2abc recognize extension automatically Issue: I6PPZM Tests: test262/parser/compiler/tsc/build_target/base64/hotfix/typeExtractor Signed-off-by: songqi Change-Id: I6e32920ccd7ab81a0b070b9c9e281b13b4b07411 --- es2panda/aot/options.cpp | 74 +++++++++++++++++-------- es2panda/aot/options.h | 8 +-- es2panda/compiler/core/compileQueue.cpp | 2 +- es2panda/compiler/core/compilerImpl.cpp | 7 +-- es2panda/es2panda.h | 6 +- es2panda/test/runner.py | 22 ++++---- 6 files changed, 69 insertions(+), 50 deletions(-) diff --git a/es2panda/aot/options.cpp b/es2panda/aot/options.cpp index e04d5e53bb..ec8d03070c 100644 --- a/es2panda/aot/options.cpp +++ b/es2panda/aot/options.cpp @@ -15,23 +15,27 @@ #include "options.h" -#include "mergeProgram.h" -#include "os/file.h" -#include -#include +#include +#include +#include +#include + #if defined(PANDA_TARGET_WINDOWS) #include #else #include #endif -#include -#include -#include +#include "os/file.h" + +#include "mergeProgram.h" +#include "util/helpers.h" +#include "utils/pandargs.h" namespace panda::es2panda::aot { constexpr char PROCESS_AS_LIST_MARK = '@'; const std::string LIST_ITEM_SEPERATOR = ";"; +const std::set VALID_EXTENSIONS = { "js", "ts", "as" }; template T RemoveExtension(T const &filename) @@ -40,6 +44,34 @@ T RemoveExtension(T const &filename) return P > 0 && P != T::npos ? filename.substr(0, P) : filename; } +static es2panda::ScriptExtension GetScriptExtensionFromStr(const std::string &extension) +{ + if (extension == "js") { + return es2panda::ScriptExtension::JS; + } else if (extension == "ts") { + return es2panda::ScriptExtension::TS; + } else if (extension == "as") { + return es2panda::ScriptExtension::AS; + } else { + return es2panda::ScriptExtension::JS; + } +} + +static es2panda::ScriptExtension GetScriptExtension(const std::string &filename, const std::string &inputExtension) +{ + std::string fileExtension = ""; + std::string::size_type pos(filename.find_last_of('.')); + if (pos > 0 && pos != std::string::npos) { + fileExtension = filename.substr(pos + 1); + } + + if (VALID_EXTENSIONS.find(fileExtension) != VALID_EXTENSIONS.end()) { + return GetScriptExtensionFromStr(fileExtension); + } + + return GetScriptExtensionFromStr(inputExtension); +} + static std::vector GetStringItems(std::string &input, const std::string &delimiter) { std::vector items; @@ -59,7 +91,7 @@ static std::vector GetStringItems(std::string &input, const std::st } // Options -bool Options::CollectInputFilesFromFileList(const std::string &input) +bool Options::CollectInputFilesFromFileList(const std::string &input, const std::string &inputExtension) { std::ifstream ifs; std::string line; @@ -90,7 +122,7 @@ bool Options::CollectInputFilesFromFileList(const std::string &input) scriptKind = parser::ScriptKind::MODULE; } - es2panda::SourceFile src(fileName, recordName, scriptKind); + es2panda::SourceFile src(fileName, recordName, scriptKind, GetScriptExtension(fileName, inputExtension)); src.sourcefile = itemList[3]; if (compilerOptions_.mergeAbc) { src.pkgName = itemList[4]; @@ -110,8 +142,9 @@ bool Options::CollectInputFilesFromFileDirectory(const std::string &input, const if (!proto::MergeProgram::GetProtoFiles(input, extension, files)) { return false; } - for (auto &f : files) { - es2panda::SourceFile src(f, util::Helpers::BaseName(f), scriptKind_); + + for (const auto &f : files) { + es2panda::SourceFile src(f, util::Helpers::BaseName(f), scriptKind_, GetScriptExtensionFromStr(extension)); sourceFiles_.push_back(src); } @@ -309,18 +342,11 @@ bool Options::Parse(int argc, const char **argv) compilerOptions_.typeDtsBuiltin << std::endl; } }; + parseTypeExtractor(); // Type Extractor is only enabled for TypeScript std::string extension = inputExtension.GetValue(); if (!extension.empty()) { - if (extension == "js") { - extension_ = es2panda::ScriptExtension::JS; - } else if (extension == "ts") { - extension_ = es2panda::ScriptExtension::TS; - // Type Extractor is only enabled for TypeScript - parseTypeExtractor(); - } else if (extension == "as") { - extension_ = es2panda::ScriptExtension::AS; - } else { + if (VALID_EXTENSIONS.find(extension) == VALID_EXTENSIONS.end()) { errorMsg_ = "Invalid extension (available options: js, ts, as)"; return false; } @@ -361,11 +387,11 @@ bool Options::Parse(int argc, const char **argv) auto fpath = inputAbs.Value(); if (isInputFileList) { - CollectInputFilesFromFileList(fpath); + CollectInputFilesFromFileList(fpath, extension); } else if (panda::os::file::File::IsDirectory(fpath)) { CollectInputFilesFromFileDirectory(fpath, extension); } else { - es2panda::SourceFile src(sourceFile_, recordName_, scriptKind_); + es2panda::SourceFile src(sourceFile_, recordName_, scriptKind_, GetScriptExtension(sourceFile_, extension)); sourceFiles_.push_back(src); } } else if (!base64InputIsEmpty) { @@ -376,7 +402,8 @@ bool Options::Parse(int argc, const char **argv) return false; } - es2panda::SourceFile src("", recordName_, es2panda::parser::ScriptKind::SCRIPT); + es2panda::SourceFile src("", recordName_, es2panda::parser::ScriptKind::SCRIPT, + GetScriptExtensionFromStr(extension)); src.source = base64Input_; sourceFiles_.push_back(src); } @@ -413,7 +440,6 @@ bool Options::Parse(int argc, const char **argv) compilerOptions_.dumpLiteralBuffer = opDumpLiteralBuffer.GetValue(); compilerOptions_.isDebuggerEvaluateExpressionMode = debuggerEvaluateExpression.GetValue(); - compilerOptions_.extension = extension_; compilerOptions_.functionThreadCount = functionThreadCount_; compilerOptions_.fileThreadCount = fileThreadCount_; compilerOptions_.output = compilerOutput_; diff --git a/es2panda/aot/options.h b/es2panda/aot/options.h index 04bd5a8c7c..00a2b7cf99 100644 --- a/es2panda/aot/options.h +++ b/es2panda/aot/options.h @@ -60,11 +60,6 @@ public: bool Parse(int argc, const char **argv); - es2panda::ScriptExtension Extension() const - { - return extension_; - } - const es2panda::CompilerOptions &CompilerOptions() const { return compilerOptions_; @@ -132,12 +127,11 @@ public: return outputFiles_; } - bool CollectInputFilesFromFileList(const std::string &input); + bool CollectInputFilesFromFileList(const std::string &input, const std::string &inputExtension); bool CollectInputFilesFromFileDirectory(const std::string &input, const std::string &extension); void ParseCacheFileOption(const std::string &cacheInput); private: - es2panda::ScriptExtension extension_ {es2panda::ScriptExtension::JS}; es2panda::CompilerOptions compilerOptions_ {}; es2panda::parser::ScriptKind scriptKind_ {es2panda::parser::ScriptKind::SCRIPT}; OptionFlags options_ {OptionFlags::DEFAULT}; diff --git a/es2panda/compiler/core/compileQueue.cpp b/es2panda/compiler/core/compileQueue.cpp index 4b952cf876..48083b4c09 100644 --- a/es2panda/compiler/core/compileQueue.cpp +++ b/es2panda/compiler/core/compileQueue.cpp @@ -95,7 +95,7 @@ void CompileFileJob::Run() } } - es2panda::Compiler compiler(options_->extension, options_->functionThreadCount); + es2panda::Compiler compiler(src_->scriptExtension, options_->functionThreadCount); auto *prog = compiler.CompileFile(*options_, src_, symbolTable_); if (prog == nullptr) { return; diff --git a/es2panda/compiler/core/compilerImpl.cpp b/es2panda/compiler/core/compilerImpl.cpp index 066a98b4d0..9d840c2be8 100644 --- a/es2panda/compiler/core/compilerImpl.cpp +++ b/es2panda/compiler/core/compilerImpl.cpp @@ -39,15 +39,14 @@ CompilerImpl::~CompilerImpl() panda::pandasm::Program *CompilerImpl::Compile(parser::Program *program, const es2panda::CompilerOptions &options, const std::string &debugInfoSourceFile, const std::string &pkgName) { + bool isTypeExtractorEnabled = ((program->Extension() == ScriptExtension::TS) && options.typeExtractor); CompilerContext context(program->Binder(), options.isDebug, options.isDebuggerEvaluateExpressionMode, - options.mergeAbc, options.typeExtractor, false, debugInfoSourceFile, pkgName, + options.mergeAbc, isTypeExtractorEnabled, false, debugInfoSourceFile, pkgName, program->RecordName(), hotfixHelper_); ArenaAllocator localAllocator(SpaceType::SPACE_TYPE_COMPILER, nullptr, true); - if (options.typeExtractor) { - ASSERT(program->Extension() == ScriptExtension::TS); - + if (isTypeExtractorEnabled) { auto rootNode = context.Binder()->TopScope()->Node()->AsBlockStatement(); extractor_ = std::make_unique(rootNode, program->IsDtsFile(), options.typeDtsBuiltin, &localAllocator, &context); diff --git a/es2panda/es2panda.h b/es2panda/es2panda.h index 8c932c504e..28e8fbae68 100644 --- a/es2panda/es2panda.h +++ b/es2panda/es2panda.h @@ -47,8 +47,8 @@ enum class ScriptExtension { }; struct SourceFile { - SourceFile(std::string fn, std::string rn, parser::ScriptKind sk) - : fileName(fn), recordName(rn), scriptKind(sk) + SourceFile(std::string fn, std::string rn, parser::ScriptKind sk, ScriptExtension se) + : fileName(fn), recordName(rn), scriptKind(sk), scriptExtension(se) { } @@ -56,6 +56,7 @@ struct SourceFile { std::string recordName {}; std::string_view source {}; parser::ScriptKind scriptKind {}; + ScriptExtension scriptExtension {}; std::string sourcefile {}; std::string pkgName {}; uint32_t hash {0}; @@ -82,7 +83,6 @@ struct CompilerOptions { bool mergeAbc {false}; bool typeExtractor {false}; bool typeDtsBuiltin {false}; - ScriptExtension extension {}; int fileThreadCount {0}; int functionThreadCount {0}; int optLevel {0}; diff --git a/es2panda/test/runner.py b/es2panda/test/runner.py index 65128dcfe9..46cc5e739d 100755 --- a/es2panda/test/runner.py +++ b/es2panda/test/runner.py @@ -398,7 +398,7 @@ class TSCTest(Test): return test_options def run(self, runner): - cmd = runner.cmd_prefix + [runner.es2panda, '--parse-only', '--extension=ts'] + cmd = runner.cmd_prefix + [runner.es2panda, '--parse-only'] cmd.extend(self.flags) if "module" in self.options: cmd.append('--module') @@ -1264,7 +1264,7 @@ class TypeExtractorTest(Test): def run(self, runner): test_abc_name = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") cmd = runner.cmd_prefix + [runner.es2panda, - '--extension=ts', '--module', '--dump-literal-buffer', '--opt-level=2', '--type-extractor'] + '--module', '--dump-literal-buffer', '--opt-level=2', '--type-extractor'] if self.is_dts_test: cmd.append("--type-dts-builtin") cmd.extend(self.flags) @@ -1313,7 +1313,7 @@ class TypeExtractorWithAssertTest(Test): def run(self, runner): test_abc_name = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") cmd = runner.cmd_prefix + [runner.es2panda, - '--extension=ts', '--module', '--merge-abc', '--opt-level=2', '--type-extractor'] + '--module', '--merge-abc', '--opt-level=2', '--type-extractor'] cmd.extend(self.flags) cmd.extend(["--output=" + test_abc_name]) cmd.append(self.path) @@ -1369,18 +1369,18 @@ def main(): runner.add_directory("parser/concurrent", "js", ["--module", "--dump-ast"]) runner.add_directory("parser/js", "js", ["--parse-only", "--dump-ast"]) runner.add_directory("parser/ts", "ts", - ["--parse-only", "--module", "--extension=ts", "--dump-ast"]) + ["--parse-only", "--module", "--dump-ast"]) runner.add_directory("parser/ts/type_checker", "ts", - ["--parse-only", "--enable-type-check", "--module", "--extension=ts", "--dump-ast"]) + ["--parse-only", "--enable-type-check", "--module", "--dump-ast"]) runner.add_directory("parser/ts/cases/declaration", "d.ts", - ["--parse-only", "--module", "--extension=ts", "--dump-ast"], TSDeclarationTest) + ["--parse-only", "--module", "--dump-ast"], TSDeclarationTest) runner.add_directory("parser/commonjs", "js", ["--commonjs", "--parse-only", "--dump-ast"]) runners.append(runner) transformer_runner = TransformerRunner(args) transformer_runner.add_directory("parser/ts/transformed_cases", "ts", - ["--parse-only", "--module", "--extension=ts", "--dump-transformed-ast", + ["--parse-only", "--module", "--dump-transformed-ast", "--check-transformed-ast-structure"]) runners.append(transformer_runner) @@ -1394,10 +1394,10 @@ def main(): if args.compiler: runner = CompilerRunner(args) runner.add_directory("compiler/js", "js", []) - runner.add_directory("compiler/ts/cases", "ts", ["--extension=ts"]) - runner.add_directory("compiler/ts/projects", "ts", ["--module", "--extension=ts"]) - runner.add_directory("compiler/ts/projects", "ts", ["--module", "--extension=ts", "--merge-abc"]) - runner.add_directory("compiler/dts", "d.ts", ["--module", "--extension=ts", "--opt-level=0"]) + runner.add_directory("compiler/ts/cases", "ts", []) + runner.add_directory("compiler/ts/projects", "ts", ["--module"]) + runner.add_directory("compiler/ts/projects", "ts", ["--module", "--merge-abc"]) + runner.add_directory("compiler/dts", "d.ts", ["--module", "--opt-level=0"]) runner.add_directory("compiler/commonjs", "js", ["--commonjs"]) runners.append(runner) -- Gitee