forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix #172

This commit is contained in:
Hackerpilot 2014-05-06 16:10:16 -07:00
parent 867f8b426c
commit 258aa066f7
4 changed files with 28 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
astprinter.d
View File

@ -1314,8 +1314,9 @@ class XMLPrinter : ASTVisitor
case tok!"stringLiteral": tagName = "stringLiteral"; break; case tok!"stringLiteral": tagName = "stringLiteral"; break;
case tok!"dstringLiteral": tagName = "dstringLiteral"; break; case tok!"dstringLiteral": tagName = "dstringLiteral"; break;
case tok!"wstringLiteral": tagName = "wstringLiteral"; break; case tok!"wstringLiteral": tagName = "wstringLiteral"; break;
case tok!"scriptLine": tagName = "scriptLine"; break;
case tok!"$": output.writeln("<dollar/>"); return; case tok!"$": output.writeln("<dollar/>"); return;
default: output.writeln("<", str(token.type), "/>"); return; default: tagName = "token"; break;
} }
output.writeln("<", tagName, ">", xmlEscape(token.text), "</", tagName, ">"); output.writeln("<", tagName, ">", xmlEscape(token.text), "</", tagName, ">");
} }

3
std/d/ast.d
View File

@ -1956,8 +1956,9 @@ final class Module : ASTNode
public: public:
override void accept(ASTVisitor visitor) const override void accept(ASTVisitor visitor) const
{ {
mixin (visitIfNotNull!(moduleDeclaration, declarations)); mixin (visitIfNotNull!(scriptLine, moduleDeclaration, declarations));
} }
/** */ Token scriptLine;
/** */ ModuleDeclaration moduleDeclaration; /** */ ModuleDeclaration moduleDeclaration;
/** */ Declaration[] declarations; /** */ Declaration[] declarations;
mixin OpEquals; mixin OpEquals;

23
std/d/lexer.d
View File

@ -119,6 +119,18 @@ public enum WhitespaceBehavior : ubyte
/// Whitespace is treated as a token /// Whitespace is treated as a token
include include
} }
/**
* Configure special token handling behavior
*/
public enum SpecialTokenBehavior : ubyte
{
/// Special tokens are skipped
skip,
/// Special tokens are treated as a token
include
}
/** /**
* Configure comment handling behavior * Configure comment handling behavior
*/ */
@ -136,6 +148,7 @@ public struct LexerConfig
StringBehavior stringBehavior; StringBehavior stringBehavior;
WhitespaceBehavior whitespaceBehavior; WhitespaceBehavior whitespaceBehavior;
CommentBehavior commentBehavior; CommentBehavior commentBehavior;
SpecialTokenBehavior specialTokenBehavior;
} }
public bool isBasicType(IdType type) nothrow pure @safe public bool isBasicType(IdType type) nothrow pure @safe
@ -434,6 +447,7 @@ public struct DLexer
} }
do _popFront(); while (front == tok!"comment"); do _popFront(); while (front == tok!"comment");
if (front == tok!"whitespace") goto case tok!"whitespace"; if (front == tok!"whitespace") goto case tok!"whitespace";
if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence";
} }
break; break;
case tok!"whitespace": case tok!"whitespace":
@ -441,6 +455,15 @@ public struct DLexer
{ {
do _popFront(); while (front == tok!"whitespace"); do _popFront(); while (front == tok!"whitespace");
if (front == tok!"comment") goto case tok!"comment"; if (front == tok!"comment") goto case tok!"comment";
if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence";
}
break;
case tok!"specialTokenSequence":
if (config.specialTokenBehavior == SpecialTokenBehavior.skip)
{
do _popFront(); while (front == tok!"specialTokenSequence");
if (front == tok!"comment") goto case tok!"comment";
if (front == tok!"whitespace") goto case tok!"whitespace";
} }
break; break;
default: default:

2
std/d/parser.d
View File

@ -3561,7 +3561,7 @@ invariant() foo();
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
Module m = allocate!Module; Module m = allocate!Module;
if (currentIs(tok!"scriptLine")) if (currentIs(tok!"scriptLine"))
advance(); m.scriptLine = advance();
if (currentIs(tok!"module")) if (currentIs(tok!"module"))
m.moduleDeclaration = parseModuleDeclaration(); m.moduleDeclaration = parseModuleDeclaration();
while (moreTokens()) while (moreTokens())