From f89bd7d3e078ffe3ce8442e648b28ff000bd9e02 Mon Sep 17 00:00:00 2001 From: Matthew Brennan Jones Date: Wed, 30 Apr 2014 12:37:02 -0700 Subject: [PATCH 1/6] Updated a few remaining stdx references to std. --- README.md | 2 +- build.sh | 8 ++++---- dub.json | 2 +- std/lexer.d | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1c1ac31..d1cfbaa 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,6 @@ but not yet implemented. # Useful code The source code for DScanner has a complete lexer, parser, and abstact syntax -tree library for D code under the stdx/d/ directory. It is intended that these +tree library for D code under the std/d/ directory. It is intended that these modules eventually end up in Phobos, so feel free to use them for your own D tools. diff --git a/build.sh b/build.sh index 1a4c7de..2b3cd32 100755 --- a/build.sh +++ b/build.sh @@ -24,8 +24,8 @@ dmd\ # astprinter.d\ # formatter.d\ # outliner.d\ -# stdx/*.d\ -# stdx/d/*.d\ +# std/*.d\ +# std/d/*.d\ # analysis/*.d\ # -O3 -frelease -fno-bounds-check\ # -odscanner\ @@ -39,8 +39,8 @@ dmd\ # astprinter.d\ # formatter.d\ # outliner.d\ -# stdx/*.d\ -# stdx/d/*.d\ +# std/*.d\ +# std/d/*.d\ # analysis/*.d\ # -O3 -release\ # -oq -of=dscanner\ diff --git a/dub.json b/dub.json index 086d89e..c67d6fb 100644 --- a/dub.json +++ b/dub.json @@ -9,7 +9,7 @@ { "name": "library", "targetType": "library", - "sourcePaths": ["stdx"], + "sourcePaths": ["std"], }, { "name": "dscanner", diff --git a/std/lexer.d b/std/lexer.d index 1072e45..84fd384 100644 --- a/std/lexer.d +++ b/std/lexer.d @@ -33,7 +33,7 @@ * ) * Examples: * $(UL - * $(LI A _lexer for D is available $(LINK2 https://github.com/Hackerpilot/Dscanner/blob/master/stdx/d/lexer.d, here).) + * $(LI A _lexer for D is available $(LINK2 https://github.com/Hackerpilot/Dscanner/blob/master/std/d/lexer.d, here).) * $(LI A _lexer for Lua is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/lualexer.d, here).) * $(LI A _lexer for JSON is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/jsonlexer.d, here).) * ) From 8a8ddf45ebcd94f2062a0c13c0fe318b113a0a6f Mon Sep 17 00:00:00 2001 From: Matthew Brennan Jones Date: Wed, 30 Apr 2014 13:01:42 -0700 Subject: [PATCH 2/6] Fixed small typos. --- README.md | 2 +- std/d/parser.d | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d1cfbaa..f2b72a1 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ The following examples assume that we are analyzing a simple file called hellowo ### Token Count The "--tokenCount" or "-t" option prints the number of tokens in the given file - $ dscanner --tokencount helloworld.d + $ dscanner --tokenCount helloworld.d 20 ### Import Listing diff --git a/std/d/parser.d b/std/d/parser.d index ec60cf2..e380ead 100644 --- a/std/d/parser.d +++ b/std/d/parser.d @@ -2281,7 +2281,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c; { if (!canBeRange) { - error(`Cannot have more than one foreach varible for a foreach range statement`); + error(`Cannot have more than one foreach variable for a foreach range statement`); return null; } advance(); From 86243af5c612a9fbadf5dc5d90305bce1178f13c Mon Sep 17 00:00:00 2001 From: Matthew Brennan Jones Date: Wed, 30 Apr 2014 13:14:11 -0700 Subject: [PATCH 3/6] Fixed issues with using ulong instead of size_t that would break only on 32bit. --- std/allocator.d | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/std/allocator.d b/std/allocator.d index b8ae114..f9c5796 100644 --- a/std/allocator.d +++ b/std/allocator.d @@ -2860,7 +2860,7 @@ private struct BasicRegion(uint minAlign = platformAlignment) static if (minAlign > 1) { auto newStore = cast(void*) roundUpToMultipleOf( - cast(ulong) store.ptr, + cast(size_t) store.ptr, alignment); assert(newStore <= store.ptr + store.length); _current = newStore; @@ -2904,7 +2904,7 @@ private struct BasicRegion(uint minAlign = platformAlignment) // Just bump the pointer to the next good allocation auto save = _current; _current = cast(void*) roundUpToMultipleOf( - cast(ulong) _current, a); + cast(size_t) _current, a); if (auto b = allocate(bytes)) return b; // Failed, rollback _current = save; @@ -3064,7 +3064,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment) { assert(!_crt); _crt = cast(void*) roundUpToMultipleOf( - cast(ulong) _store.ptr, alignment); + cast(size_t) _store.ptr, alignment); _end = _store.ptr + _store.length; } @@ -3109,7 +3109,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment) // Just bump the pointer to the next good allocation auto save = _crt; _crt = cast(void*) roundUpToMultipleOf( - cast(ulong) _crt, a); + cast(size_t) _crt, a); if (auto b = allocate(bytes)) return b; // Failed, rollback _crt = save; From 3798387c19ddf10b57502f1ba1f507815863e1c6 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Thu, 1 May 2014 15:22:13 -0700 Subject: [PATCH 4/6] Added destructors, constructors, and invariants to ctags output --- ctags.d | 25 +++++++++++++++++++ std/d/ast.d | 10 +++++--- std/d/parser.d | 67 +++++++++++++++++++++++++++----------------------- 3 files changed, 68 insertions(+), 34 deletions(-) diff --git a/ctags.d b/ctags.d index 330f5aa..97e99ea 100644 --- a/ctags.d +++ b/ctags.d @@ -90,6 +90,26 @@ class CTagsPrinter : ASTVisitor context = c; } + override void visit(const Constructor dec) + { + tagLines ~= "this\t%s\t%d;\"\tf\tarity:%d%s\n".format(fileName, + dec.line, dec.parameters.parameters.length, context); + auto c = context; + context = "\tfunction: this"; + dec.accept(this); + context = c; + } + + override void visit(const Destructor dec) + { + tagLines ~= "~this\t%s\t%d;\"\tf%s\n".format(fileName, dec.line, + context); + auto c = context; + context = "\tfunction: this"; + dec.accept(this); + context = c; + } + override void visit(const EnumDeclaration dec) { if (dec.name == tok!"") @@ -136,6 +156,11 @@ class CTagsPrinter : ASTVisitor dec.accept(this); } + override void visit(const Invariant dec) + { + tagLines ~= "invariant\t%s\t%d;\"\tv%s\n".format(fileName, dec.line, context); + } + alias visit = ASTVisitor.visit; string fileName; diff --git a/std/d/ast.d b/std/d/ast.d index 02c2134..25ac3dc 100644 --- a/std/d/ast.d +++ b/std/d/ast.d @@ -1239,7 +1239,9 @@ public: } /** */ MemberFunctionAttribute[] memberFunctionAttributes; /** */ FunctionBody functionBody; - /** */ size_t location; + /** */ size_t line; + /** */ size_t column; + /** */ size_t index; /** */ string comment; mixin OpEquals; } @@ -1780,6 +1782,8 @@ public: } /** */ BlockStatement blockStatement; /** */ string comment; + size_t line; + size_t index; mixin OpEquals; } @@ -1873,12 +1877,12 @@ final class LinkageAttribute : ASTNode public: override void accept(ASTVisitor visitor) const { - version (DIP61) mixin (visitIfNotNull!(identifier, identifierChain)); + version (DIP61) mixin (visitIfNotNull!(identifier, identifierChain)); else mixin (visitIfNotNull!(identifier)); } /** */ Token identifier; /** */ bool hasPlusPlus; - version (DIP61) /** */ IdentifierChain identifierChain; + version (DIP61) /** */ IdentifierChain identifierChain; mixin OpEquals; } diff --git a/std/d/parser.d b/std/d/parser.d index 952d20e..6169229 100644 --- a/std/d/parser.d +++ b/std/d/parser.d @@ -113,7 +113,7 @@ class Parser node.linkageAttribute = parseLinkageAttribute(); } warn("Prefer the new \"'alias' identifier '=' type ';'\" syntax" - ~ " to the old \"'alias' type identifier ';'\" syntax"); + ~ " to the old \"'alias' type identifier ';'\" syntax"); if ((node.type = parseType()) is null) return null; auto ident = expect(tok!"identifier"); if (ident is null) @@ -1976,20 +1976,23 @@ class ClassFour(A, B) if (someTest()) : Super {}}c; node.comment = comment; comment = null; if (expect(tok!"~") is null) return null; + node.index = current.index; + node.line = current.line; + node.column = current.column; if (expect(tok!"this") is null) return null; if (expect(tok!"(") is null) return null; if (expect(tok!")") is null) return null; if (currentIs(tok!";")) advance(); else - { - MemberFunctionAttribute[] memberFunctionAttributes; - while(moreTokens() && currentIsMemberFunctionAttribute()) - memberFunctionAttributes ~= parseMemberFunctionAttribute(); - node.memberFunctionAttributes = ownArray(memberFunctionAttributes); + { + MemberFunctionAttribute[] memberFunctionAttributes; + while(moreTokens() && currentIsMemberFunctionAttribute()) + memberFunctionAttributes ~= parseMemberFunctionAttribute(); + node.memberFunctionAttributes = ownArray(memberFunctionAttributes); - node.functionBody = parseFunctionBody(); - } + node.functionBody = parseFunctionBody(); + } return node; } @@ -3185,6 +3188,8 @@ interface "Four" Invariant parseInvariant() { auto node = allocate!Invariant; + node.index = current.index; + node.line = current.line; if (expect(tok!"invariant") is null) return null; if (currentIs(tok!"(")) { @@ -3414,11 +3419,11 @@ invariant() foo(); { advance(); node.hasPlusPlus = true; - version(DIP61) if (currentIs(tok!",")) - { - advance(); - node.identifierChain = parseIdentifierChain(); - } + version(DIP61) if (currentIs(tok!",")) + { + advance(); + node.identifierChain = parseIdentifierChain(); + } } expect(tok!")"); return node; @@ -4545,18 +4550,18 @@ q{(int a, ...) if (!currentIs(tok!"]")) { node.lower = parseAssignExpression(); - if (node.lower is null) - { - error("assignExpression expected"); - return null; - } + if (node.lower is null) + { + error("assignExpression expected"); + return null; + } expect(tok!".."); node.upper = parseAssignExpression(); - if (node.upper is null) - { - error("assignExpression expected"); - return null; - } + if (node.upper is null) + { + error("assignExpression expected"); + return null; + } } if (expect(tok!"]") is null) return null; return node; @@ -4810,13 +4815,13 @@ q{(int a, ...) mixin(traceEnterAndExit!(__FUNCTION__)); auto node = allocate!StructInitializer; expect(tok!"{"); - if (currentIs(tok!"}")) - advance(); - else - { - node.structMemberInitializers = parseStructMemberInitializers(); - expect(tok!"}"); - } + if (currentIs(tok!"}")) + advance(); + else + { + node.structMemberInitializers = parseStructMemberInitializers(); + expect(tok!"}"); + } return node; } @@ -6349,7 +6354,7 @@ protected: case tok!"abstract": case tok!"pure": case tok!"nothrow": - return true; + return true; mixin(BASIC_TYPE_CASES); return !peekIs(tok!"."); case tok!"case": From 258aa066f7c370ac71ccb8f7e9f6e491eac2d11e Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Tue, 6 May 2014 16:10:16 -0700 Subject: [PATCH 5/6] Fix #172 --- astprinter.d | 3 ++- std/d/ast.d | 3 ++- std/d/lexer.d | 23 +++++++++++++++++++++++ std/d/parser.d | 2 +- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/astprinter.d b/astprinter.d index 6a7523f..f78a97b 100644 --- a/astprinter.d +++ b/astprinter.d @@ -1314,8 +1314,9 @@ class XMLPrinter : ASTVisitor case tok!"stringLiteral": tagName = "stringLiteral"; break; case tok!"dstringLiteral": tagName = "dstringLiteral"; break; case tok!"wstringLiteral": tagName = "wstringLiteral"; break; + case tok!"scriptLine": tagName = "scriptLine"; break; case tok!"$": output.writeln(""); return; - default: output.writeln("<", str(token.type), "/>"); return; + default: tagName = "token"; break; } output.writeln("<", tagName, ">", xmlEscape(token.text), ""); } diff --git a/std/d/ast.d b/std/d/ast.d index 25ac3dc..04a5c71 100644 --- a/std/d/ast.d +++ b/std/d/ast.d @@ -1956,8 +1956,9 @@ final class Module : ASTNode public: override void accept(ASTVisitor visitor) const { - mixin (visitIfNotNull!(moduleDeclaration, declarations)); + mixin (visitIfNotNull!(scriptLine, moduleDeclaration, declarations)); } + /** */ Token scriptLine; /** */ ModuleDeclaration moduleDeclaration; /** */ Declaration[] declarations; mixin OpEquals; diff --git a/std/d/lexer.d b/std/d/lexer.d index 52381e2..abdc99e 100644 --- a/std/d/lexer.d +++ b/std/d/lexer.d @@ -119,6 +119,18 @@ public enum WhitespaceBehavior : ubyte /// Whitespace is treated as a token include } + +/** + * Configure special token handling behavior + */ +public enum SpecialTokenBehavior : ubyte +{ + /// Special tokens are skipped + skip, + /// Special tokens are treated as a token + include +} + /** * Configure comment handling behavior */ @@ -136,6 +148,7 @@ public struct LexerConfig StringBehavior stringBehavior; WhitespaceBehavior whitespaceBehavior; CommentBehavior commentBehavior; + SpecialTokenBehavior specialTokenBehavior; } public bool isBasicType(IdType type) nothrow pure @safe @@ -434,6 +447,7 @@ public struct DLexer } do _popFront(); while (front == tok!"comment"); if (front == tok!"whitespace") goto case tok!"whitespace"; + if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence"; } break; case tok!"whitespace": @@ -441,6 +455,15 @@ public struct DLexer { do _popFront(); while (front == tok!"whitespace"); if (front == tok!"comment") goto case tok!"comment"; + if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence"; + } + break; + case tok!"specialTokenSequence": + if (config.specialTokenBehavior == SpecialTokenBehavior.skip) + { + do _popFront(); while (front == tok!"specialTokenSequence"); + if (front == tok!"comment") goto case tok!"comment"; + if (front == tok!"whitespace") goto case tok!"whitespace"; } break; default: diff --git a/std/d/parser.d b/std/d/parser.d index 1f4ee0b..f6e84e6 100644 --- a/std/d/parser.d +++ b/std/d/parser.d @@ -3561,7 +3561,7 @@ invariant() foo(); mixin(traceEnterAndExit!(__FUNCTION__)); Module m = allocate!Module; if (currentIs(tok!"scriptLine")) - advance(); + m.scriptLine = advance(); if (currentIs(tok!"module")) m.moduleDeclaration = parseModuleDeclaration(); while (moreTokens()) From 3081fa86c2c865b6dc6e60c2680407332d32de9e Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Tue, 6 May 2014 16:41:19 -0700 Subject: [PATCH 6/6] Fix highlighting of special token sequences and script lines --- highlighter.d | 2 ++ main.d | 1 + std/d/lexer.d | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/highlighter.d b/highlighter.d index 3be4c0c..cf48e27 100644 --- a/highlighter.d +++ b/highlighter.d @@ -49,6 +49,8 @@ html { background-color: #fdf6e3; color: #002b36; } writeSpan("num", t.text); else if (isOperator(t.type)) writeSpan("op", str(t.type)); + else if (t.type == tok!"specialTokenSequence" || t.type == tok!"scriptLine") + writeSpan("cons", t.text.replace("<", "<")); else { version(Windows) diff --git a/main.d b/main.d index 89a822d..cb011f8 100644 --- a/main.d +++ b/main.d @@ -101,6 +101,7 @@ int main(string[] args) config.whitespaceBehavior = WhitespaceBehavior.include; config.stringBehavior = StringBehavior.source; config.commentBehavior = CommentBehavior.include; + config.specialTokenBehavior = SpecialTokenBehavior.include; auto tokens = byToken(bytes, config, cache); if (highlight) { diff --git a/std/d/lexer.d b/std/d/lexer.d index abdc99e..03095b7 100644 --- a/std/d/lexer.d +++ b/std/d/lexer.d @@ -41,7 +41,7 @@ private enum dynamicTokens = [ "whitespace", "doubleLiteral", "floatLiteral", "idoubleLiteral", "ifloatLiteral", "intLiteral", "longLiteral", "realLiteral", "irealLiteral", "uintLiteral", "ulongLiteral", "characterLiteral", - "dstringLiteral", "stringLiteral", "wstringLiteral", "scriptLine" + "dstringLiteral", "stringLiteral", "wstringLiteral" ]; private enum pseudoTokenHandlers = [