diff --git a/src/MapleFE/ast2cpp/runtime/src/ts2cpp.cpp b/src/MapleFE/ast2cpp/runtime/src/ts2cpp.cpp index 7c9ae38db3c05dbddd3cf27326bf8c690b7c63a7..4820cf09291685f64ec666948430893c35d36fbb 100644 --- a/src/MapleFE/ast2cpp/runtime/src/ts2cpp.cpp +++ b/src/MapleFE/ast2cpp/runtime/src/ts2cpp.cpp @@ -133,7 +133,7 @@ void GenerateDOTGraph( std::vector&obj, std::vector&name) std::cout << names[i] << " -> " << names[j] << " [label=\"__proto__\", color=red, fontcolor=red];" << std::endl; } } - std::cout << "}" << std::endl; + std::cout << "} // digraph JS" << g << std::endl; } } diff --git a/src/MapleFE/astopt/include/ast_adj.h b/src/MapleFE/astopt/include/ast_adj.h index cc85af13414c6e23b87d37071d19b4dd731566f0..025ae5d8c1565e975d849cc3405fd6c11cec11cc 100644 --- a/src/MapleFE/astopt/include/ast_adj.h +++ b/src/MapleFE/astopt/include/ast_adj.h @@ -62,6 +62,7 @@ class AdjustASTVisitor : public AstVisitor { std::unordered_map mRenameMap; void CheckAndRenameCppKeywords(TreeNode *node); + void AssignPseudoName(TreeNode *node); DeclNode *VisitDeclNode(DeclNode *node); ImportNode *VisitImportNode(ImportNode *node); diff --git a/src/MapleFE/astopt/src/ast_adj.cpp b/src/MapleFE/astopt/src/ast_adj.cpp index ec48977fe27f7506076cfb0a5a7cd132187443d7..634a282752128bd69938cf7846d40a71a8b80fe2 100644 --- a/src/MapleFE/astopt/src/ast_adj.cpp +++ b/src/MapleFE/astopt/src/ast_adj.cpp @@ -52,6 +52,7 @@ IdentifierNode *AdjustASTVisitor::VisitIdentifierNode(IdentifierNode *node) { ClassNode *AdjustASTVisitor::VisitClassNode(ClassNode *node) { (void) AstVisitor::VisitClassNode(node); CheckAndRenameCppKeywords(node); + AssignPseudoName(node); // skip getting canonical type if not only fields if (node->GetMethodsNum() || node->GetSuperClassesNum() || node->GetSuperInterfacesNum() || node->GetSuperClassesNum() || node->GetTypeParamsNum()) { @@ -517,4 +518,14 @@ void AdjustASTVisitor::CheckAndRenameCppKeywords(TreeNode *node) { } } +void AdjustASTVisitor::AssignPseudoName(TreeNode *node) { + if (!mHandler->IsTS() || node->GetStrIdx() != 0) { + return; + } + static int pseudo = 0; + // Set a pseudo name + unsigned newidx = gStringPool.GetStrIdx("__Pseudo_" + std::to_string(++pseudo)); + node->SetStrIdx(newidx); +} + } diff --git a/src/MapleFE/autogen/src/token_gen.cpp b/src/MapleFE/autogen/src/token_gen.cpp index 725192414d5cbdd64d937aa63a9a9cfb504affbc..59bb94456c7b59e7b4bf2ca2c0de2bb4b37850c4 100644 --- a/src/MapleFE/autogen/src/token_gen.cpp +++ b/src/MapleFE/autogen/src/token_gen.cpp @@ -155,7 +155,7 @@ void TokenGen::GenCppFile() { std::list::iterator oit = gTokenTable.mOperators->begin(); for (; oit != gTokenTable.mOperators->end(); oit++, overall_index++) { - std::string output = " {.mTkType = TT_OP, {.mOprId = "; + std::string output = " {.mTkType = TT_OP, .mLineNum = 0, .mColNum = 0, .mLineBegin = false, .mLineEnd = false, {.mOprId = "; Operator opr = *oit; std::string opr_name = "OPR_"; opr_name += FindOperatorName(opr.mID); @@ -186,7 +186,7 @@ void TokenGen::GenCppFile() { std::list::iterator sit = gTokenTable.mSeparators->begin(); for (; sit != gTokenTable.mSeparators->end(); sit++, overall_index++) { - std::string output = " {.mTkType = TT_SP, {.mSepId = "; + std::string output = " {.mTkType = TT_SP, .mLineNum = 0, .mColNum = 0, .mLineBegin = false, .mLineEnd = false, {.mSepId = "; Separator sep = *sit; std::string sep_name = "SEP_"; sep_name += FindSeparatorName(sep.mID); @@ -217,7 +217,7 @@ void TokenGen::GenCppFile() { unsigned kw_size = gTokenTable.mKeywords.size(); for (unsigned index = 0; index < kw_size; index++, overall_index++) { - std::string output = " {.mTkType = TT_KW, {.mName = "; + std::string output = " {.mTkType = TT_KW, .mLineNum = 0, .mColNum = 0, .mLineBegin = false, .mLineEnd = false, {.mName = "; std::string keyword = gTokenTable.mKeywords[index]; output += "\""; output += keyword; @@ -247,7 +247,7 @@ void TokenGen::GenCppFile() { } // Write the comment token - std::string output = " {.mTkType = TT_CM, {.mName = NULL}, .mAltTokens = NULL}"; + std::string output = " {.mTkType = TT_CM, .mLineNum = 0, .mColNum = 0, .mLineBegin = false, .mLineEnd = false, {.mName = NULL}, .mAltTokens = NULL}"; mCppFile.WriteOneLine(output.c_str(), output.size()); mCppFile.WriteOneLine("};", 2); mCppFile.WriteOneLine("}", 1); diff --git a/src/MapleFE/docs/utils/proto.js b/src/MapleFE/docs/utils/proto.js index 9d25dde1ef9a8002204025194fb08ba7814b190c..80bf386e32c9f037f970adf255ac12c5802c7751 100644 --- a/src/MapleFE/docs/utils/proto.js +++ b/src/MapleFE/docs/utils/proto.js @@ -10,52 +10,87 @@ class MyCar extends Car { } let car = new Car("A car"); -let mycar = new MyCar("My car"); +let myCar = new MyCar("My car"); let arr = [1, 2, 3] function* generator() { yield 1; } +const gpt = generator.prototype.__proto__; -// Dump graphs with edges for prototype, __proto__ and constructor properties of each objects -for(let g = 0; g < 4; ++g) { - let objs = [Function, Object], names = ["Function", "Object"], gen = undefined; - if (g < 2) { - objs.unshift ( arr, Array, Vehicle, Car, MyCar, car, mycar); - names.unshift("arr", "Array", "Vehicle", "Car", "MyCar", "car", "mycar"); - } else { - gen = generator.prototype.__proto__; - objs.unshift ( generator, generator(), generator.constructor, gen, gen.__proto__); - names.unshift("generator", "generator_instance", "GeneratorFunction", "GeneratorPrototype", "IteratorPrototype"); +function makeClosure(a) { + return function (b) { + return a + b; } - console.log("digraph JS" + g + " {\nranksep=0.6;\nnodesep=0.6;\n" + (g % 2 == 0 ? "newrank=true;\n" : "")); - let num = objs.length; - let k = num, suffix = "Prototype"; - // Add prototype objects and edges for them - for(let i = 0; i < num; ++i) { - console.log(names[i].includes(suffix) ? names[i] + "[shape=box];" : ""); - if(typeof objs[i].prototype !== "undefined") { - objs[k] = objs[i].prototype; - let special = names[i] === "GeneratorFunction" && objs[k].prototype === gen; - names[k] = special ? "Generator" : names[i] + suffix; - console.log(special ? "GeneratorPrototype -> " + names[k] + " [label=\"prototype\", color=blue, fontcolor=blue, dir=back];" : ""); - console.log(g % 2 == 0 ? "subgraph cluster_" + names[i] + " {\nrank=same;\ncolor=white;\n" + names[i] + ";\n" - + names[k] + "[shape=box];\n }" : names[k] + "[shape=box];"); - console.log(names[i] + " -> " + names[k] + " [label=\"prototype\", color=blue, fontcolor=blue];"); - k++; +} +const closure = makeClosure(1); + +let myMap = new Map(); +let myMapIterator = myMap[Symbol.iterator](); +let MapIteratorPrototype = Object.getPrototypeOf(new Map()[Symbol.iterator]()); + +async function asyncFunction() {} +async function* asyncGenerator() {} +const agpt = asyncGenerator.prototype.__proto__; + +// All data for generating graphs +let graphData = { + Classes : ["Array", "arr", "myCar", "car"], + Generator: ["generator", [generator(), "generator_instance"], [gpt, "GeneratorPrototype"], + [gpt.__proto__, "IteratorPrototype"], [generator.__proto__, "Generator"]], + Builtins : ["Symbol", "Math", "JSON", "Promise"], + Closure : ["makeClosure", "closure"], + Iterators: ["myMap", "myMapIterator", "MapIteratorPrototype", [gpt.__proto__, "IteratorPrototype"]], + Async : ["asyncFunction", "asyncGenerator", [asyncGenerator(), "asyncGenerator_instance"], [agpt, "AsyncGeneratorPrototype"], + [agpt.__proto__, "AsyncIteratorPrototype"], [asyncGenerator.__proto__, "AsyncGenerator"]], +}; + +generateGraph(graphData); + +function generateGraph(data) { + // Gather all reachable objects from their prototype, __proto__ and constructor properties + function insert(g, depth, ...args) { + for (let arg of args) { + let [o, name] = typeof arg === "string" ? [eval(arg), arg] : arg; + if (typeof o !== "undefined" && o !== null) + if (!g.has(o)) { + g.set(o, name !== null || typeof o !== "function" ? name : o.toString().split(" ")[1].replace(/[^a-zA-Z0-9+]/g, "")); + insert(g, depth + 1, [o.prototype, g.get(o) === null ? null : g.get(o) + "Prototype"], [o.__proto__, null], [o.constructor, null]); + } else if (name !== null) + g.set(o, name); + } + if (depth === 0) { + let visited = new Set(); + for (let [index, [key, val]] of Array.from(g).entries()) { + val = val === null || val === "" ? "Object_" + index : val.replace(/[^A-Za-z0-9]+/g, "_"); + if (visited.has(val)) val += "__" + index; + visited.add(val); + g.set(key, val); + } } } - // Add edges for __proto__ and constructor properties of each objects - num = objs.length; - for(let i = 0; i < num; ++i) { - console.log("\n/* Object.getOwnPropertyNames(" + names[i] + "):\n", Object.getOwnPropertyNames(objs[i]), "\n" - + names[i] + ".toString(): " + (names[i] === "Generator" ? "-" : objs[i].toString().replace(/\s+/g, " ")) + "\n*/"); - for(let j = 0; j < num; ++j) { - // Edges for constructor properties in the second graph only - if(g % 2 == 1 && objs[i].constructor === objs[j]) - console.log(names[i] + " -> " + names[j] + " [label=\"ctor\", color=darkgreen, fontcolor=darkgreen];"); - // Edges for __proto__ properties - if(objs[i].__proto__ === objs[j]) - console.log(names[i] + " -> " + names[j] + " [label=\"__proto__\", color=red, fontcolor=red];"); + + // Dump graphs with edges for prototype, __proto__ and constructor properties of each object + let nodejs = (typeof process !== 'undefined') && (process.release.name === 'node') + for (let prop in data) { + let graph = new Map(); + insert(graph, 0, "Function", "Object", ...data[prop]); + for (let ctor of ["", "_with_ctors"]) { + console.log("digraph JS_" + prop + ctor + " {\nrankdir = TB;\nranksep=0.6;\nnodesep=0.6;\n" + (ctor != "" ? "" : "newrank=true;")); + for (let [index, [key, val]] of Array.from(graph).entries()) { + let func = typeof key === "function"; + // Add comments with detailed information of keys + if (nodejs) + console.log("\n/* key =", key, "\nObject.getOwnPropertyNames(" + val + "):\n", Object.getOwnPropertyNames(key), + "\n" + val + ".toString(): " + (func ? key.toString().replace(/\s+/g, " ") : "-") + "\n*/"); + console.log(val + " [label=\"" + val + " " + (index < 4 ? 3 - index : index) + "\", shape=" + + (val.includes("Prototype") ? "box" : "oval") + (func ? ", style=filled" : "") + "];"); + // Add edges for prototype, constructor and __proto__ properties of objects + for (let [f, c] of [["prototype", "blue"], ["constructor", "darkgreen"], ["__proto__", "red"]]) + if (typeof key[f] !== "undefined" && key[f] !== null && graph.has(key[f]) && (ctor != "" || f !== "constructor")) + console.log((ctor != "" || f !== "prototype" ? "" : "subgraph cluster_" + val + " {\nrank=same;\ncolor=white;\n" + + val + ";\n" + graph.get(key.prototype) + " [shape=box];\n}\n") + val + " -> " + graph.get(key[f]) + + " [label=\"" + (f === "constructor" ? "ctor" : f) + "\", color=" + c + ", fontcolor=" + c + "];"); + } + console.log("} // digraph JS_" + prop + ctor); } } - console.log("}"); -} +} // generateGraph diff --git a/src/MapleFE/docs/utils/proto2.ts b/src/MapleFE/docs/utils/proto2.ts deleted file mode 100644 index d7d0c406520b346a16a93236e401f4debff57ace..0000000000000000000000000000000000000000 --- a/src/MapleFE/docs/utils/proto2.ts +++ /dev/null @@ -1,54 +0,0 @@ -interface IPrototype { prototype: any; } -type TPrototype = IPrototype & Function; -interface IProto { __proto__: any; } -type TProto = IProto & Object; - -// Command line to get graphs: tsc -t es6 proto2.ts; nodejs proto2.js | ./viewdot.sh -class Vehicle { -} -class Car implements Vehicle { - name: string; - constructor (name) { this.name = name; } -} -class MyCar extends Car { - constructor (name) { super(name); } -} - -let car = new Car("A car"); -let mycar = new MyCar("My car"); -let arr = [] - -// Dump graphs with edges for prototype, __proto__ and constructor properties of each objects -for(let g = 0; g < 2; ++g) { - let objs = [ arr, Array, Vehicle, Car, MyCar, car, mycar, Function, Object, ]; - let names = [ "arr", "Array", "Vehicle", "Car", "MyCar", "car", "mycar", "Function", "Object", ]; - console.log("digraph JS" + g + " {\nranksep=0.6;\nnodesep=0.6;\n" + (g == 0 ? "newrank=true;\n" : "")); - let num = objs.length; - let k = num; - // Add prototype objects and edges for them - for(let i = 0; i < num; ++i) { - let x = typeof (objs[i] as unknown as TPrototype).prototype; - if(x === "function" || x === "object") { - objs[k] = (objs[i] as unknown as TPrototype).prototype; - names[k] = names[i] + "_prototype"; - console.log(g == 0 ? "subgraph cluster_" + names[i] + " {\nrank=same;\ncolor=white;\n" - + names[i] + ";\n" + names[k] + "[label=\"" + names[i] + ".prototype\", shape=box];\n }" - : names[k] + "[label=\"" + names[i] + ".prototype\", shape=box];"); - console.log(names[i] + " -> " + names[k] + " [label=\"prototype\", color=blue, fontcolor=blue];"); - k++; - } - } - // Add edges for __proto__ and constructor properties of each objects - num = objs.length; - for(let i = 0; i < num; ++i) { - for(let j = 0; j < num; ++j) { - // Edges for constructor properties in the second graph only - if(g > 0 && objs[i].constructor === objs[j]) - console.log(names[i] + " -> " + names[j] + " [label=\"ctor\", color=darkgreen, fontcolor=darkgreen];"); - // Edges for __proto__ properties - if((objs[i] as unknown as TProto).__proto__ === objs[j]) - console.log(names[i] + " -> " + names[j] + " [label=\"__proto__\", color=red, fontcolor=red];"); - } - } - console.log("}"); -} diff --git a/src/MapleFE/docs/utils/viewdot.sh b/src/MapleFE/docs/utils/viewdot.sh index ab1c2f0a41b14b203649ba2e388e5a424127a203..f9252d7c0ae07e7fb98cfdfa058bcfa6bcd6a583 100755 --- a/src/MapleFE/docs/utils/viewdot.sh +++ b/src/MapleFE/docs/utils/viewdot.sh @@ -1,17 +1,19 @@ #!/bin/bash if [ $# -lt 1 ]; then out=$(cat) - ts=/tmp/viewdot-$$ - grep -n -e "digraph [^{]* {" -e "^}" <<< "$out" | grep -A1 "digraph [^{]* {" | + tmpdir=$(mktemp -dt viewdot-XXXXXX) + trap "rm -rf $tmpdir" SIGINT SIGQUIT SIGKILL + grep -n -e "digraph [^{]* {" -e "^} // digraph JS" <<< "$out" | grep -A1 "digraph [^{]* {" | grep -v ^-- | sed 'N;s/\n/ /' | sed -e 's/:.*digraph [^{]* { */,/' -e 's/:.*/p/g' | { while read cmd; do - idx=$((idx+1)) - sed -n $cmd <<< "$out" > "$ts"-$idx.dot - dot -Tpng -o "$ts"-$idx.png "$ts"-$idx.dot - env LC_ALL=C viewnior $VIEWOP "$ts"-$idx.png & + name=$(sed -n $cmd <<< "$out" | head -1 | sed 's/.*digraph \([^{]*\) {.*/\1/') + echo $$-$name + sed -n $cmd <<< "$out" > "$tmpdir"/$$-$name.dot + dot -Tpng -o "$tmpdir"/$$-$name.png "$tmpdir"/$$-$name.dot + env LC_ALL=C viewnior "$tmpdir"/$$-$name.png & done wait - rm -f "$ts"-[0-9]*.png "$ts"-[0-9]*.dot; } + rm -rf "$tmpdir"; } exit fi for f; do diff --git a/src/MapleFE/shared/include/lexer.h b/src/MapleFE/shared/include/lexer.h index 52a66ff83cbe1bfe8e7bc795c83e0aec5e712dbb..863e630e958decb28724a7a653cfd3960e9dad2e 100644 --- a/src/MapleFE/shared/include/lexer.h +++ b/src/MapleFE/shared/include/lexer.h @@ -128,9 +128,6 @@ public: // For most languages, this does nothing. TS/JS are doing something. virtual bool CharIsSeparator(const char c) {return false;} - // For most languages, this does nothing. TS/JS are doing something. - virtual void AppendSemiColon() {} - virtual TempLitData* GetTempLit() {return NULL;} virtual bool FindNextTLFormat(unsigned start, std::string& s, unsigned& end) {return false;} virtual bool FindNextTLPlaceHolder(unsigned start, std::string& s, unsigned& end) {return false;} diff --git a/src/MapleFE/shared/include/token.h b/src/MapleFE/shared/include/token.h index 2d650fab76cd1c8301a0563058c246ff827c74ef..243c09a1ddbf5afe8d427e53b0dd524f979776d1 100644 --- a/src/MapleFE/shared/include/token.h +++ b/src/MapleFE/shared/include/token.h @@ -112,6 +112,12 @@ struct RegExprData { struct Token { TK_Type mTkType; + + unsigned mLineNum; // line num + unsigned mColNum; // column num + bool mLineBegin; // first token of line? + bool mLineEnd; // last token of line? + union { const char *mName; // Identifier, Keyword. In the gStringPool LitData mLitData; diff --git a/src/MapleFE/shared/src/lexer.cpp b/src/MapleFE/shared/src/lexer.cpp index 39d870627355ab2df7a84b1aac17ac1e8cb7b7c5..f9c750fe967b20725f729cc61de7a7dba2fdc7a4 100644 --- a/src/MapleFE/shared/src/lexer.cpp +++ b/src/MapleFE/shared/src/lexer.cpp @@ -72,9 +72,6 @@ int Lexer::ReadALine() { curidx += 3; } - if (current_line_size > 0 && _linenum == _total_linenum) - AppendSemiColon(); - return current_line_size; } @@ -329,9 +326,12 @@ Token* Lexer::LexToken(void) { // Read a token until end of line. // Return NULL if no token read. Token* Lexer::LexTokenNoNewLine(void) { + unsigned old_curidx = curidx; bool is_comment = GetComment(); if (is_comment) { Token *t = FindCommentToken(); + t->mLineNum = _linenum; + t->mColNum = old_curidx; if (mTrace) t->Dump(); return t; @@ -340,6 +340,8 @@ Token* Lexer::LexTokenNoNewLine(void) { OprId opr = GetOperator(); if (opr != OPR_NA) { Token *t = FindOperatorToken(opr); + t->mLineNum = _linenum; + t->mColNum = old_curidx; if (mTrace) t->Dump(); return t; @@ -347,7 +349,6 @@ Token* Lexer::LexTokenNoNewLine(void) { // There is a corner case: .2 // The dot is lexed as separator, and 2 is an integer. But actually it's a decimal. - unsigned old_curidx = curidx; SepId sep = GetSeparator(); unsigned new_curidx = curidx; @@ -360,6 +361,8 @@ Token* Lexer::LexTokenNoNewLine(void) { if (ld.mType != LT_NA) { MASSERT(ld.mType == LT_FPLiteral || ld.mType == LT_DoubleLiteral); Token *t = (Token*)mTokenPool.NewToken(sizeof(Token)); + t->mLineNum = _linenum; + t->mColNum = old_curidx; t->SetLiteral(ld); if (mTrace) t->Dump(); @@ -370,6 +373,8 @@ Token* Lexer::LexTokenNoNewLine(void) { } Token *t = FindSeparatorToken(sep); + t->mLineNum = _linenum; + t->mColNum = old_curidx; if (mTrace) t->Dump(); return t; @@ -378,6 +383,8 @@ Token* Lexer::LexTokenNoNewLine(void) { const char *keyword = GetKeyword(); if (keyword != NULL) { Token *t = FindKeywordToken(keyword); + t->mLineNum = _linenum; + t->mColNum = old_curidx; if (mTrace) t->Dump(); return t; @@ -386,6 +393,8 @@ Token* Lexer::LexTokenNoNewLine(void) { LitData ld = GetLiteral(); if (ld.mType != LT_NA) { Token *t = (Token*)mTokenPool.NewToken(sizeof(Token)); + t->mLineNum = _linenum; + t->mColNum = old_curidx; t->SetLiteral(ld); if (mTrace) t->Dump(); @@ -396,6 +405,8 @@ Token* Lexer::LexTokenNoNewLine(void) { if (identifier != NULL) { Token *t = (Token*)mTokenPool.NewToken(sizeof(Token)); t->SetIdentifier(identifier); + t->mLineNum = _linenum; + t->mColNum = old_curidx; if (mTrace) t->Dump(); return t; @@ -404,6 +415,8 @@ Token* Lexer::LexTokenNoNewLine(void) { TempLitData* tldata = GetTempLit(); if (tldata != NULL) { Token *t = (Token*)mTokenPool.NewToken(sizeof(Token)); + t->mLineNum = _linenum; + t->mColNum = old_curidx; t->SetTempLit(tldata); if (mTrace) t->Dump(); diff --git a/src/MapleFE/shared/src/token.cpp b/src/MapleFE/shared/src/token.cpp index 66e4a85ece245a199fdaf67cef0a66be60c7a383..1c23c988d742fec91a7863429d2de2d350b3b671 100644 --- a/src/MapleFE/shared/src/token.cpp +++ b/src/MapleFE/shared/src/token.cpp @@ -30,7 +30,7 @@ const char* SeparatorTokenGetName(SepId id) { void SeparatorTokenDump(SepId id) { const char *name = SeparatorTokenGetName(id); - DUMP1("Separator Token: ", name); + DUMP1_NORETURN("Separator Token: ", name); return; } @@ -46,34 +46,34 @@ const char* OperatorTokenGetName(OprId id) { void OperatorTokenDump(OprId id) { const char *name = OperatorTokenGetName(id); - DUMP1("Operator Token: ", name); + DUMP1_NORETURN("Operator Token: ", name); return; } void LiteralTokenDump(LitData data) { switch (data.mType) { case LT_IntegerLiteral: - DUMP1("Integer Literal Token:", data.mData.mInt); + DUMP1_NORETURN("Integer Literal Token:", data.mData.mInt); break; case LT_FPLiteral: - DUMP1("Floating Literal Token:", data.mData.mFloat); + DUMP1_NORETURN("Floating Literal Token:", data.mData.mFloat); break; case LT_DoubleLiteral: - DUMP1("Double Literal Token:", data.mData.mDouble); + DUMP1_NORETURN("Double Literal Token:", data.mData.mDouble); break; case LT_BooleanLiteral: - DUMP1("Boolean Literal Token:", data.mData.mBool); + DUMP1_NORETURN("Boolean Literal Token:", data.mData.mBool); break; case LT_CharacterLiteral: { Char the_char = data.mData.mChar; if (the_char.mIsUnicode) - DUMP1("Char Literal Token(Unicode):", the_char.mData.mUniValue); + DUMP1_NORETURN("Char Literal Token(Unicode):", the_char.mData.mUniValue); else - DUMP1("Char Literal Token:", the_char.mData.mChar); + DUMP1_NORETURN("Char Literal Token:", the_char.mData.mChar); break; } case LT_StringLiteral: - DUMP1("String Literal Token:", gStringPool.GetStringFromStrIdx(data.mData.mStrIdx)); + DUMP1_NORETURN("String Literal Token:", gStringPool.GetStringFromStrIdx(data.mData.mStrIdx)); break; case LT_NullLiteral: DUMP0("Null Literal Token:"); @@ -105,10 +105,10 @@ void Token::Dump() { OperatorTokenDump(mData.mOprId); break; case TT_ID: - DUMP1("Identifier Token: ", mData.mName); + DUMP1_NORETURN("Identifier Token: ", mData.mName); break; case TT_KW: - DUMP1("Keyword Token: ", mData.mName); + DUMP1_NORETURN("Keyword Token: ", mData.mName); break; case TT_CM: DUMP0("Comment Token: "); @@ -117,9 +117,9 @@ void Token::Dump() { DUMP0("TemplateLiteral Token: "); break; case TT_RE: - DUMP1("RegExpr Token: ", mData.mRegExprData.mExpr); + DUMP1_NORETURN("RegExpr Token: ", mData.mRegExprData.mExpr); if (mData.mRegExprData.mFlags) - DUMP1(" : ", mData.mRegExprData.mFlags); + DUMP1_NORETURN(" : ", mData.mRegExprData.mFlags); break; case TT_LT: LiteralTokenDump(mData.mLitData); @@ -127,6 +127,14 @@ void Token::Dump() { default: break; } + + DUMP1_NORETURN(" line: ", mLineNum); + DUMP1_NORETURN(" col: ", mColNum); + if (mLineBegin) + DUMP0_NORETURN(" line-first "); + if (mLineEnd) + DUMP0_NORETURN(" line-last "); + DUMP_RETURN(); } } diff --git a/src/MapleFE/shared/src/tokenpool.cpp b/src/MapleFE/shared/src/tokenpool.cpp index 263e76a03ccf5e313500e6e5c9ca953c041fc817..89dfde73e94772499496049360b155e912cb9b3a 100644 --- a/src/MapleFE/shared/src/tokenpool.cpp +++ b/src/MapleFE/shared/src/tokenpool.cpp @@ -23,8 +23,15 @@ char* TokenPool::NewToken(unsigned size) { char *addr = mMemPool.Alloc(size); MASSERT(addr && "MemPool failed to alloc a token."); Token *token = (Token*)addr; + token->mAltTokens = NULL; + token->mLineNum = 0; + token->mColNum = 0; + token->mLineBegin = false; + token->mLineEnd = false; + mTokens.PushBack((Token*)addr); return addr; } + } diff --git a/src/MapleFE/typescript/include/lang_spec.h b/src/MapleFE/typescript/include/lang_spec.h index b58ce7ec9d543ad62de8d4b71486edab555a741e..aa302bfdef26782ef530ba7c93afa3acebbf424b 100644 --- a/src/MapleFE/typescript/include/lang_spec.h +++ b/src/MapleFE/typescript/include/lang_spec.h @@ -44,7 +44,6 @@ extern LitData ProcessLiteral(LitId type, const char *str); class TypescriptLexer : public Lexer { public: bool CharIsSeparator(const char); - void AppendSemiColon(); TempLitData* GetTempLit(); bool FindNextTLFormat(unsigned start, std::string& s, unsigned& end); diff --git a/src/MapleFE/typescript/src/lang_spec.cpp b/src/MapleFE/typescript/src/lang_spec.cpp index a9c107db5a9c1d27644d113f29d95ffd2dfbda68..d73777758c4ee570eca1029cd66e696c861aaec5 100644 --- a/src/MapleFE/typescript/src/lang_spec.cpp +++ b/src/MapleFE/typescript/src/lang_spec.cpp @@ -177,12 +177,6 @@ bool TypescriptLexer::CharIsSeparator(const char c) { return false; } -// Add a ';' to the end of _line. -void TypescriptLexer::AppendSemiColon() { - //line[current_line_size] = ';'; - //current_line_size++; -} - // NOTE: right now we rely on 'tsc' to assure the input is legal, // so I'll make many things easier and will skip many lexical // checks. Just make it easy for now.