diff --git a/README.md b/README.md index 1c1ac31..f2b72a1 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ The following examples assume that we are analyzing a simple file called hellowo ### Token Count The "--tokenCount" or "-t" option prints the number of tokens in the given file - $ dscanner --tokencount helloworld.d + $ dscanner --tokenCount helloworld.d 20 ### Import Listing @@ -198,6 +198,6 @@ but not yet implemented. # Useful code The source code for DScanner has a complete lexer, parser, and abstact syntax -tree library for D code under the stdx/d/ directory. It is intended that these +tree library for D code under the std/d/ directory. It is intended that these modules eventually end up in Phobos, so feel free to use them for your own D tools. diff --git a/astprinter.d b/astprinter.d index 6a7523f..f78a97b 100644 --- a/astprinter.d +++ b/astprinter.d @@ -1314,8 +1314,9 @@ class XMLPrinter : ASTVisitor case tok!"stringLiteral": tagName = "stringLiteral"; break; case tok!"dstringLiteral": tagName = "dstringLiteral"; break; case tok!"wstringLiteral": tagName = "wstringLiteral"; break; + case tok!"scriptLine": tagName = "scriptLine"; break; case tok!"$": output.writeln(""); return; - default: output.writeln("<", str(token.type), "/>"); return; + default: tagName = "token"; break; } output.writeln("<", tagName, ">", xmlEscape(token.text), ""); } diff --git a/build.sh b/build.sh index 1a4c7de..2b3cd32 100755 --- a/build.sh +++ b/build.sh @@ -24,8 +24,8 @@ dmd\ # astprinter.d\ # formatter.d\ # outliner.d\ -# stdx/*.d\ -# stdx/d/*.d\ +# std/*.d\ +# std/d/*.d\ # analysis/*.d\ # -O3 -frelease -fno-bounds-check\ # -odscanner\ @@ -39,8 +39,8 @@ dmd\ # astprinter.d\ # formatter.d\ # outliner.d\ -# stdx/*.d\ -# stdx/d/*.d\ +# std/*.d\ +# std/d/*.d\ # analysis/*.d\ # -O3 -release\ # -oq -of=dscanner\ diff --git a/ctags.d b/ctags.d index 330f5aa..97e99ea 100644 --- a/ctags.d +++ b/ctags.d @@ -90,6 +90,26 @@ class CTagsPrinter : ASTVisitor context = c; } + override void visit(const Constructor dec) + { + tagLines ~= "this\t%s\t%d;\"\tf\tarity:%d%s\n".format(fileName, + dec.line, dec.parameters.parameters.length, context); + auto c = context; + context = "\tfunction: this"; + dec.accept(this); + context = c; + } + + override void visit(const Destructor dec) + { + tagLines ~= "~this\t%s\t%d;\"\tf%s\n".format(fileName, dec.line, + context); + auto c = context; + context = "\tfunction: this"; + dec.accept(this); + context = c; + } + override void visit(const EnumDeclaration dec) { if (dec.name == tok!"") @@ -136,6 +156,11 @@ class CTagsPrinter : ASTVisitor dec.accept(this); } + override void visit(const Invariant dec) + { + tagLines ~= "invariant\t%s\t%d;\"\tv%s\n".format(fileName, dec.line, context); + } + alias visit = ASTVisitor.visit; string fileName; diff --git a/dub.json b/dub.json index 086d89e..c67d6fb 100644 --- a/dub.json +++ b/dub.json @@ -9,7 +9,7 @@ { "name": "library", "targetType": "library", - "sourcePaths": ["stdx"], + "sourcePaths": ["std"], }, { "name": "dscanner", diff --git a/highlighter.d b/highlighter.d index 3be4c0c..cf48e27 100644 --- a/highlighter.d +++ b/highlighter.d @@ -49,6 +49,8 @@ html { background-color: #fdf6e3; color: #002b36; } writeSpan("num", t.text); else if (isOperator(t.type)) writeSpan("op", str(t.type)); + else if (t.type == tok!"specialTokenSequence" || t.type == tok!"scriptLine") + writeSpan("cons", t.text.replace("<", "<")); else { version(Windows) diff --git a/main.d b/main.d index 89a822d..cb011f8 100644 --- a/main.d +++ b/main.d @@ -101,6 +101,7 @@ int main(string[] args) config.whitespaceBehavior = WhitespaceBehavior.include; config.stringBehavior = StringBehavior.source; config.commentBehavior = CommentBehavior.include; + config.specialTokenBehavior = SpecialTokenBehavior.include; auto tokens = byToken(bytes, config, cache); if (highlight) { diff --git a/std/allocator.d b/std/allocator.d index b8ae114..f9c5796 100644 --- a/std/allocator.d +++ b/std/allocator.d @@ -2860,7 +2860,7 @@ private struct BasicRegion(uint minAlign = platformAlignment) static if (minAlign > 1) { auto newStore = cast(void*) roundUpToMultipleOf( - cast(ulong) store.ptr, + cast(size_t) store.ptr, alignment); assert(newStore <= store.ptr + store.length); _current = newStore; @@ -2904,7 +2904,7 @@ private struct BasicRegion(uint minAlign = platformAlignment) // Just bump the pointer to the next good allocation auto save = _current; _current = cast(void*) roundUpToMultipleOf( - cast(ulong) _current, a); + cast(size_t) _current, a); if (auto b = allocate(bytes)) return b; // Failed, rollback _current = save; @@ -3064,7 +3064,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment) { assert(!_crt); _crt = cast(void*) roundUpToMultipleOf( - cast(ulong) _store.ptr, alignment); + cast(size_t) _store.ptr, alignment); _end = _store.ptr + _store.length; } @@ -3109,7 +3109,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment) // Just bump the pointer to the next good allocation auto save = _crt; _crt = cast(void*) roundUpToMultipleOf( - cast(ulong) _crt, a); + cast(size_t) _crt, a); if (auto b = allocate(bytes)) return b; // Failed, rollback _crt = save; diff --git a/std/d/ast.d b/std/d/ast.d index 0a760f3..04a5c71 100644 --- a/std/d/ast.d +++ b/std/d/ast.d @@ -1239,7 +1239,9 @@ public: } /** */ MemberFunctionAttribute[] memberFunctionAttributes; /** */ FunctionBody functionBody; - /** */ size_t location; + /** */ size_t line; + /** */ size_t column; + /** */ size_t index; /** */ string comment; mixin OpEquals; } @@ -1780,6 +1782,8 @@ public: } /** */ BlockStatement blockStatement; /** */ string comment; + size_t line; + size_t index; mixin OpEquals; } @@ -1952,8 +1956,9 @@ final class Module : ASTNode public: override void accept(ASTVisitor visitor) const { - mixin (visitIfNotNull!(moduleDeclaration, declarations)); + mixin (visitIfNotNull!(scriptLine, moduleDeclaration, declarations)); } + /** */ Token scriptLine; /** */ ModuleDeclaration moduleDeclaration; /** */ Declaration[] declarations; mixin OpEquals; diff --git a/std/d/lexer.d b/std/d/lexer.d index 52381e2..03095b7 100644 --- a/std/d/lexer.d +++ b/std/d/lexer.d @@ -41,7 +41,7 @@ private enum dynamicTokens = [ "whitespace", "doubleLiteral", "floatLiteral", "idoubleLiteral", "ifloatLiteral", "intLiteral", "longLiteral", "realLiteral", "irealLiteral", "uintLiteral", "ulongLiteral", "characterLiteral", - "dstringLiteral", "stringLiteral", "wstringLiteral", "scriptLine" + "dstringLiteral", "stringLiteral", "wstringLiteral" ]; private enum pseudoTokenHandlers = [ @@ -119,6 +119,18 @@ public enum WhitespaceBehavior : ubyte /// Whitespace is treated as a token include } + +/** + * Configure special token handling behavior + */ +public enum SpecialTokenBehavior : ubyte +{ + /// Special tokens are skipped + skip, + /// Special tokens are treated as a token + include +} + /** * Configure comment handling behavior */ @@ -136,6 +148,7 @@ public struct LexerConfig StringBehavior stringBehavior; WhitespaceBehavior whitespaceBehavior; CommentBehavior commentBehavior; + SpecialTokenBehavior specialTokenBehavior; } public bool isBasicType(IdType type) nothrow pure @safe @@ -434,6 +447,7 @@ public struct DLexer } do _popFront(); while (front == tok!"comment"); if (front == tok!"whitespace") goto case tok!"whitespace"; + if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence"; } break; case tok!"whitespace": @@ -441,6 +455,15 @@ public struct DLexer { do _popFront(); while (front == tok!"whitespace"); if (front == tok!"comment") goto case tok!"comment"; + if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence"; + } + break; + case tok!"specialTokenSequence": + if (config.specialTokenBehavior == SpecialTokenBehavior.skip) + { + do _popFront(); while (front == tok!"specialTokenSequence"); + if (front == tok!"comment") goto case tok!"comment"; + if (front == tok!"whitespace") goto case tok!"whitespace"; } break; default: diff --git a/std/d/parser.d b/std/d/parser.d index ec60cf2..f6e84e6 100644 --- a/std/d/parser.d +++ b/std/d/parser.d @@ -1976,6 +1976,9 @@ class ClassFour(A, B) if (someTest()) : Super {}}c; node.comment = comment; comment = null; if (expect(tok!"~") is null) return null; + node.index = current.index; + node.line = current.line; + node.column = current.column; if (expect(tok!"this") is null) return null; if (expect(tok!"(") is null) return null; if (expect(tok!")") is null) return null; @@ -2281,7 +2284,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c; { if (!canBeRange) { - error(`Cannot have more than one foreach varible for a foreach range statement`); + error(`Cannot have more than one foreach variable for a foreach range statement`); return null; } advance(); @@ -3184,6 +3187,8 @@ interface "Four" Invariant parseInvariant() { auto node = allocate!Invariant; + node.index = current.index; + node.line = current.line; if (expect(tok!"invariant") is null) return null; if (currentIs(tok!"(")) { @@ -3556,7 +3561,7 @@ invariant() foo(); mixin(traceEnterAndExit!(__FUNCTION__)); Module m = allocate!Module; if (currentIs(tok!"scriptLine")) - advance(); + m.scriptLine = advance(); if (currentIs(tok!"module")) m.moduleDeclaration = parseModuleDeclaration(); while (moreTokens()) diff --git a/std/lexer.d b/std/lexer.d index 9994954..7d39043 100644 --- a/std/lexer.d +++ b/std/lexer.d @@ -33,7 +33,7 @@ * ) * Examples: * $(UL - * $(LI A _lexer for D is available $(LINK2 https://github.com/Hackerpilot/Dscanner/blob/master/stdx/d/lexer.d, here).) + * $(LI A _lexer for D is available $(LINK2 https://github.com/Hackerpilot/Dscanner/blob/master/std/d/lexer.d, here).) * $(LI A _lexer for Lua is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/lualexer.d, here).) * $(LI A _lexer for JSON is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/jsonlexer.d, here).) * )