diff --git a/astprinter.d b/astprinter.d index 6a7523f..f78a97b 100644 --- a/astprinter.d +++ b/astprinter.d @@ -1314,8 +1314,9 @@ class XMLPrinter : ASTVisitor case tok!"stringLiteral": tagName = "stringLiteral"; break; case tok!"dstringLiteral": tagName = "dstringLiteral"; break; case tok!"wstringLiteral": tagName = "wstringLiteral"; break; + case tok!"scriptLine": tagName = "scriptLine"; break; case tok!"$": output.writeln(""); return; - default: output.writeln("<", str(token.type), "/>"); return; + default: tagName = "token"; break; } output.writeln("<", tagName, ">", xmlEscape(token.text), ""); } diff --git a/std/d/ast.d b/std/d/ast.d index 25ac3dc..04a5c71 100644 --- a/std/d/ast.d +++ b/std/d/ast.d @@ -1956,8 +1956,9 @@ final class Module : ASTNode public: override void accept(ASTVisitor visitor) const { - mixin (visitIfNotNull!(moduleDeclaration, declarations)); + mixin (visitIfNotNull!(scriptLine, moduleDeclaration, declarations)); } + /** */ Token scriptLine; /** */ ModuleDeclaration moduleDeclaration; /** */ Declaration[] declarations; mixin OpEquals; diff --git a/std/d/lexer.d b/std/d/lexer.d index 52381e2..abdc99e 100644 --- a/std/d/lexer.d +++ b/std/d/lexer.d @@ -119,6 +119,18 @@ public enum WhitespaceBehavior : ubyte /// Whitespace is treated as a token include } + +/** + * Configure special token handling behavior + */ +public enum SpecialTokenBehavior : ubyte +{ + /// Special tokens are skipped + skip, + /// Special tokens are treated as a token + include +} + /** * Configure comment handling behavior */ @@ -136,6 +148,7 @@ public struct LexerConfig StringBehavior stringBehavior; WhitespaceBehavior whitespaceBehavior; CommentBehavior commentBehavior; + SpecialTokenBehavior specialTokenBehavior; } public bool isBasicType(IdType type) nothrow pure @safe @@ -434,6 +447,7 @@ public struct DLexer } do _popFront(); while (front == tok!"comment"); if (front == tok!"whitespace") goto case tok!"whitespace"; + if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence"; } break; case tok!"whitespace": @@ -441,6 +455,15 @@ public struct DLexer { do _popFront(); while (front == tok!"whitespace"); if (front == tok!"comment") goto case tok!"comment"; + if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence"; + } + break; + case tok!"specialTokenSequence": + if (config.specialTokenBehavior == SpecialTokenBehavior.skip) + { + do _popFront(); while (front == tok!"specialTokenSequence"); + if (front == tok!"comment") goto case tok!"comment"; + if (front == tok!"whitespace") goto case tok!"whitespace"; } break; default: diff --git a/std/d/parser.d b/std/d/parser.d index 1f4ee0b..f6e84e6 100644 --- a/std/d/parser.d +++ b/std/d/parser.d @@ -3561,7 +3561,7 @@ invariant() foo(); mixin(traceEnterAndExit!(__FUNCTION__)); Module m = allocate!Module; if (currentIs(tok!"scriptLine")) - advance(); + m.scriptLine = advance(); if (currentIs(tok!"module")) m.moduleDeclaration = parseModuleDeclaration(); while (moreTokens())