forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added ddoc comment handling

This commit is contained in:
Hackerpilot 2014-01-13 20:51:42 -08:00
parent 22ca0cd284
commit 0e3adafd48
6 changed files with 195 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
astprinter.d
View File

@ -411,6 +411,7 @@ class XMLPrinter : ASTVisitor
override void visit(EnumDeclaration enumDec) override void visit(EnumDeclaration enumDec)
{ {
output.writeln("<enumDeclaration line=\"", enumDec.name.line, "\">"); output.writeln("<enumDeclaration line=\"", enumDec.name.line, "\">");
writeDdoc(enumDec.comment);
if (enumDec.name.type == tok!"identifier") if (enumDec.name.type == tok!"identifier")
output.writeln("<name>", enumDec.name.text, "</name>"); output.writeln("<name>", enumDec.name.text, "</name>");
enumDec.accept(this); enumDec.accept(this);
@ -420,6 +421,7 @@ class XMLPrinter : ASTVisitor
override void visit(EnumMember enumMem) override void visit(EnumMember enumMem)
{ {
output.writeln("<enumMember line=\"", enumMem.name.line, "\">"); output.writeln("<enumMember line=\"", enumMem.name.line, "\">");
writeDdoc(enumMem.comment);
enumMem.accept(this); enumMem.accept(this);
output.writeln("</enumMember>"); output.writeln("</enumMember>");
} }
@ -553,6 +555,7 @@ class XMLPrinter : ASTVisitor
{ {
output.writeln("<functionDeclaration line=\"", functionDec.name.line, "\">"); output.writeln("<functionDeclaration line=\"", functionDec.name.line, "\">");
output.writeln("<name>", functionDec.name.text, "</name>"); output.writeln("<name>", functionDec.name.text, "</name>");
writeDdoc(functionDec.comment);
if (functionDec.hasAuto) if (functionDec.hasAuto)
output.writeln("<auto/>"); output.writeln("<auto/>");
if (functionDec.hasRef) if (functionDec.hasRef)
@ -726,6 +729,7 @@ class XMLPrinter : ASTVisitor
{ {
output.writeln("<interfaceDeclaration line=\"", interfaceDec.name.line, "\">"); output.writeln("<interfaceDeclaration line=\"", interfaceDec.name.line, "\">");
output.writeln("<name>", interfaceDec.name.text, "</name>"); output.writeln("<name>", interfaceDec.name.text, "</name>");
writeDdoc(interfaceDec.comment);
interfaceDec.accept(this); interfaceDec.accept(this);
output.writeln("</interfaceDeclaration>"); output.writeln("</interfaceDeclaration>");
} }
@ -733,6 +737,7 @@ class XMLPrinter : ASTVisitor
override void visit(Invariant invariant_) override void visit(Invariant invariant_)
{ {
output.writeln("<invariant>"); output.writeln("<invariant>");
writeDdoc(invariant_.comment);
invariant_.accept(this); invariant_.accept(this);
output.writeln("</invariant>"); output.writeln("</invariant>");
} }
@ -1193,7 +1198,7 @@ class XMLPrinter : ASTVisitor
output.writeln("</templateDeclaration>"); output.writeln("</templateDeclaration>");
return; return;
} }
writeDdoc(templateDeclaration.comment);
output.writeln("<templateDeclaration line=\"", output.writeln("<templateDeclaration line=\"",
templateDeclaration.name.line, "\">"); templateDeclaration.name.line, "\">");
output.writeln("<name>", templateDeclaration.name.text, "</name>"); output.writeln("<name>", templateDeclaration.name.text, "</name>");
@ -1432,7 +1437,10 @@ class XMLPrinter : ASTVisitor
override void visit(VariableDeclaration variableDeclaration) override void visit(VariableDeclaration variableDeclaration)
{ {
mixin (tagAndAccept!"variableDeclaration"); output.writeln("<variableDeclaration>");
writeDdoc(variableDeclaration.comment);
variableDeclaration.accept(this);
output.writeln("</variableDeclaration>");
} }
override void visit(Vector vector) override void visit(Vector vector)
@ -1477,10 +1485,16 @@ class XMLPrinter : ASTVisitor
alias ASTVisitor.visit visit; alias ASTVisitor.visit visit;
private string xmlEscape(string s) private static string xmlEscape(string s)
{ {
return s.translate(['<' : "&lt;", '>' : "&gt;", '&' : "&amp;"]); return s.translate(['<' : "&lt;", '>' : "&gt;", '&' : "&amp;"]);
} }
private void writeDdoc(string comment)
{
if (comment is null) return;
output.writeln("<ddoc>", xmlEscape(comment), "</ddoc>");
}
File output; File output;
} }

21
main.d
View File

@ -96,7 +96,11 @@ int main(string[] args)
{ {
bool usingStdin = args.length == 1; bool usingStdin = args.length == 1;
ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]); ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
auto tokens = byToken!(ubyte[], false, false)(bytes); LexerConfig config;
config.whitespaceBehavior = WhitespaceBehavior.include;
config.stringBehavior = StringBehavior.source;
config.commentBehavior = CommentBehavior.include;
auto tokens = byToken(bytes, config);
highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]); highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]);
return 0; return 0;
} }
@ -104,11 +108,16 @@ int main(string[] args)
{ {
bool usingStdin = args.length == 1; bool usingStdin = args.length == 1;
ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]); ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
auto tokens = byToken!(ubyte[], false, false)(bytes); LexerConfig config;
config.whitespaceBehavior = WhitespaceBehavior.skip;
config.stringBehavior = StringBehavior.source;
config.commentBehavior = CommentBehavior.attach;
auto tokens = byToken(bytes, config);
foreach (ref token; tokens) foreach (ref token; tokens)
{ {
writeln("«", token.text is null ? str(token.type) : token.text, writeln("«", token.text is null ? str(token.type) : token.text,
" ", token.index, " ", token.line, " ", token.column, "»"); " ", token.index, " ", token.line, " ", token.column, " ",
token.comment, "»");
} }
} }
else if (ctags) else if (ctags)
@ -126,7 +135,11 @@ int main(string[] args)
{ {
if (usingStdin) if (usingStdin)
{ {
auto tokens = byToken!(ubyte[], false, false)(readStdin()); LexerConfig config;
config.whitespaceBehavior = WhitespaceBehavior.include;
config.stringBehavior = StringBehavior.source;
config.commentBehavior = CommentBehavior.include;
auto tokens = byToken(readStdin(), config);
if (tokenCount) if (tokenCount)
printTokenCount(stdout, "stdin", tokens); printTokenCount(stdout, "stdin", tokens);
else else

16
stdx/d/ast.d
View File

@ -290,6 +290,7 @@ public:
/** */ Type type; /** */ Type type;
/** */ Token name; /** */ Token name;
/** */ AliasInitializer[] initializers; /** */ AliasInitializer[] initializers;
/** */ string comment;
} }
/// ///
@ -808,6 +809,7 @@ public:
/** */ Constraint constraint; /** */ Constraint constraint;
/** */ BaseClassList baseClassList; /** */ BaseClassList baseClassList;
/** */ StructBody structBody; /** */ StructBody structBody;
/** */ string comment;
} }
/// ///
@ -891,6 +893,7 @@ public:
/** */ MemberFunctionAttribute[] memberFunctionAttributes; /** */ MemberFunctionAttribute[] memberFunctionAttributes;
/** */ TemplateParameters templateParameters; /** */ TemplateParameters templateParameters;
/** */ size_t location; /** */ size_t location;
/** */ string comment;
} }
/// ///
@ -1067,6 +1070,7 @@ public:
} }
/** */ FunctionBody functionBody; /** */ FunctionBody functionBody;
/** */ size_t location; /** */ size_t location;
/** */ string comment;
} }
/// ///
@ -1113,6 +1117,7 @@ public:
/** */ Token name; /** */ Token name;
/** */ Type type; /** */ Type type;
/** */ EnumBody enumBody; /** */ EnumBody enumBody;
/** */ string comment;
} }
/// ///
@ -1126,6 +1131,7 @@ public:
/** */ Token name; /** */ Token name;
/** */ Type type; /** */ Type type;
/** */ AssignExpression assignExpression; /** */ AssignExpression assignExpression;
/** */ string comment;
} }
/// ///
@ -1325,6 +1331,7 @@ public:
/** */ Constraint constraint; /** */ Constraint constraint;
/** */ FunctionBody functionBody; /** */ FunctionBody functionBody;
/** */ MemberFunctionAttribute[] memberFunctionAttributes; /** */ MemberFunctionAttribute[] memberFunctionAttributes;
/** */ string comment;
} }
/// ///
@ -1549,6 +1556,7 @@ public:
/** */ Constraint constraint; /** */ Constraint constraint;
/** */ BaseClassList baseClassList; /** */ BaseClassList baseClassList;
/** */ StructBody structBody; /** */ StructBody structBody;
/** */ string comment;
} }
/// ///
@ -1560,6 +1568,7 @@ public:
mixin (visitIfNotNull!(blockStatement)); mixin (visitIfNotNull!(blockStatement));
} }
/** */ BlockStatement blockStatement; /** */ BlockStatement blockStatement;
/** */ string comment;
} }
/// ///
@ -2067,6 +2076,7 @@ public:
} }
/** */ FunctionBody functionBody; /** */ FunctionBody functionBody;
/** */ size_t location; /** */ size_t location;
/** */ string comment;
} }
/// ///
@ -2079,6 +2089,7 @@ public:
} }
/** */ FunctionBody functionBody; /** */ FunctionBody functionBody;
/** */ size_t location; /** */ size_t location;
/** */ string comment;
} }
/// ///
@ -2236,6 +2247,7 @@ public:
/** */ TemplateParameters templateParameters; /** */ TemplateParameters templateParameters;
/** */ Constraint constraint; /** */ Constraint constraint;
/** */ StructBody structBody; /** */ StructBody structBody;
/** */ string comment;
} }
/// ///
@ -2376,6 +2388,7 @@ public:
/** */ Constraint constraint; /** */ Constraint constraint;
/** */ Declaration[] declarations; /** */ Declaration[] declarations;
/** */ EponymousTemplateDeclaration eponymousTemplateDeclaration; /** */ EponymousTemplateDeclaration eponymousTemplateDeclaration;
/** */ string comment;
} }
/// ///
@ -2694,6 +2707,7 @@ public:
/** */ TemplateParameters templateParameters; /** */ TemplateParameters templateParameters;
/** */ Constraint constraint; /** */ Constraint constraint;
/** */ StructBody structBody; /** */ StructBody structBody;
/** */ string comment;
} }
/// ///
@ -2705,6 +2719,7 @@ public:
mixin (visitIfNotNull!(blockStatement)); mixin (visitIfNotNull!(blockStatement));
} }
/** */ BlockStatement blockStatement; /** */ BlockStatement blockStatement;
/** */ string comment;
} }
/// ///
@ -2719,6 +2734,7 @@ public:
/** */ Declarator[] declarators; /** */ Declarator[] declarators;
/** */ StorageClass storageClass; /** */ StorageClass storageClass;
/** */ AutoDeclaration autoDeclaration; /** */ AutoDeclaration autoDeclaration;
/** */ string comment;
} }
/// ///

128
stdx/d/lexer.d
View File

@ -50,31 +50,69 @@ private enum dynamicTokens = [
]; ];
public alias TokenIdType!(staticTokens, dynamicTokens, possibleDefaultTokens) IdType; public alias TokenIdType!(staticTokens, dynamicTokens, possibleDefaultTokens) IdType;
public alias TokenStringRepresentation!(IdType, staticTokens, dynamicTokens, possibleDefaultTokens) str; public alias tokenStringRepresentation!(IdType, staticTokens, dynamicTokens, possibleDefaultTokens) str;
public template tok(string token) public template tok(string token)
{ {
alias TokenId!(IdType, staticTokens, dynamicTokens, possibleDefaultTokens, token) tok; alias TokenId!(IdType, staticTokens, dynamicTokens, possibleDefaultTokens, token) tok;
} }
public alias stdx.lexer.TokenStructure!(IdType) Token; enum extraFields = q{
string comment;
};
public alias stdx.lexer.TokenStructure!(IdType, extraFields) Token;
pure nothrow bool isNotComment(const Token t) { return t.type != tok!"comment"; } /**
pure nothrow bool isNotWhitespace(const Token t) { return t.type != tok!"whitespace"; } * Configure string lexing behavior
pure nothrow bool isNotEither(const Token t) { return t.type != tok!"whitespace" && t.type != tok!"comment"; } */
public enum StringBehavior : ubyte
public auto byToken(R, bool skipComments = true, bool skipWhitespace = true)(R range)
{ {
auto tokens = DLexer!(R)(range); /// Do not include quote characters, process escape sequences
static if (skipComments) compiler = 0b0000_0000,
{ /// Opening quotes, closing quotes, and string suffixes are included in the
static if (skipWhitespace) /// string token
return filter!isNotEither(tokens); includeQuoteChars = 0b0000_0001,
else /// String escape sequences are not replaced
return filter!isNotComment(tokens); notEscaped = 0b0000_0010,
} /// Not modified at all. Useful for formatters or highlighters
else static if (skipWhitespace) source = includeQuoteChars | notEscaped
return filter!isNotWhitespace(tokens); }
else
return tokens; /**
* Configure whitespace handling behavior
*/
public enum WhitespaceBehavior : ubyte
{
/// Whitespace is skipped
skip,
/// Whitespace is treated as a token
include
}
/**
* Configure comment handling behavior
*/
public enum CommentBehavior : ubyte
{
/// Comments are attached to the non-whitespace token that follows them
attach,
/// Comments are tokens, and can be returned by calls to the token range's front()
include
}
public struct LexerConfig
{
StringBehavior stringBehavior;
WhitespaceBehavior whitespaceBehavior;
CommentBehavior commentBehavior;
}
public auto byToken(R)(R range)
{
LexerConfig config;
return byToken(range, config);
}
public auto byToken(R)(R range, const LexerConfig config)
{
return DLexer!(R)(range, config);
} }
unittest unittest
@ -371,12 +409,49 @@ public struct DLexer(R)
private alias typeof(range).Mark Mark; private alias typeof(range).Mark Mark;
this(R range) this(R range, const LexerConfig config)
{ {
this.range = LexerRange!(typeof(buffer(range)))(buffer(range)); this.range = LexerRange!(typeof(buffer(range)))(buffer(range));
this.config = config;
popFront(); popFront();
} }
private static bool isDocComment(string comment) pure nothrow @safe
{
return comment.length >= 3 && (comment[0 .. 3] == "///"
|| comment[0 .. 3] == "/**" || comment[0 .. 3] == "/++");
}
public void popFront()
{
_popFront();
string comment = null;
switch (_front.type)
{
case tok!"comment":
if (config.commentBehavior == CommentBehavior.attach)
{
import std.string;
if (isDocComment(front.text))
comment = comment == null ? front.text : format("%s\n%s", comment, front.text);
do _popFront(); while (front == tok!"comment");
if (front == tok!"whitespace") goto case tok!"whitespace";
}
break;
case tok!"whitespace":
if (config.whitespaceBehavior == WhitespaceBehavior.skip)
{
do _popFront(); while (front == tok!"whitespace");
if (front == tok!"comment") goto case tok!"comment";
}
break;
default:
break;
}
_front.comment = comment;
}
bool isWhitespace() pure /*const*/ nothrow bool isWhitespace() pure /*const*/ nothrow
{ {
switch (range.front) switch (range.front)
@ -597,7 +672,7 @@ public struct DLexer(R)
mixin (tokenStart); mixin (tokenStart);
return lexDecimal(mark, line, column, index); return lexDecimal(mark, line, column, index);
} }
Token lexDecimal(Mark mark, size_t line, size_t column, size_t index) pure nothrow Token lexDecimal(Mark mark, size_t line, size_t column, size_t index) pure nothrow
{ {
bool foundDot = range.front == '.'; bool foundDot = range.front == '.';
@ -728,6 +803,7 @@ public struct DLexer(R)
} }
if (!range.empty && range.front == 'i') if (!range.empty && range.front == 'i')
{ {
warning("Complex number literals are deprecated");
range.popFront(); range.popFront();
if (type == tok!"floatLiteral") if (type == tok!"floatLiteral")
type = tok!"ifloatLiteral"; type = tok!"ifloatLiteral";
@ -783,7 +859,7 @@ public struct DLexer(R)
return Token(tok!"scriptLine", cast(string) range.slice(mark), return Token(tok!"scriptLine", cast(string) range.slice(mark),
line, column, index); line, column, index);
} }
Token lexSpecialTokenSequence() pure Token lexSpecialTokenSequence() pure
{ {
mixin (tokenStart); mixin (tokenStart);
@ -1329,12 +1405,14 @@ public struct DLexer(R)
size_t line = range.line; size_t line = range.line;
auto mark = range.mark(); auto mark = range.mark();
}; };
void error(...) pure { void error(...) pure {
} }
void warning(...) pure { void warning(...) pure {
} }
const LexerConfig config;
} }

45
stdx/d/parser.d
View File

@ -127,7 +127,7 @@ alias core.sys.posix.stdio.fileno fileno;
node.type = parseType(); node.type = parseType();
return node; return node;
} }
unittest unittest
{ {
auto sourceCode = q{a = abcde!def}; auto sourceCode = q{a = abcde!def};
@ -156,7 +156,7 @@ alias core.sys.posix.stdio.fileno fileno;
if (expect(tok!";") is null) return null; if (expect(tok!";") is null) return null;
return node; return node;
} }
unittest unittest
{ {
auto sourceCode = q{alias oneTwoThree this;}; auto sourceCode = q{alias oneTwoThree this;};
@ -188,7 +188,7 @@ alias core.sys.posix.stdio.fileno fileno;
} }
return node; return node;
} }
unittest unittest
{ {
auto sourceCode = q{align(42) align}; auto sourceCode = q{align(42) align};
@ -1170,6 +1170,8 @@ incorrect;
auto ident = expect(tok!"identifier"); auto ident = expect(tok!"identifier");
if (ident is null) return null; if (ident is null) return null;
node.name = *ident; node.name = *ident;
node.comment = comment;
comment = null;
if (currentIs(tok!"(")) if (currentIs(tok!"("))
{ {
node.templateParameters = parseTemplateParameters(); node.templateParameters = parseTemplateParameters();
@ -1409,6 +1411,8 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
Constructor node = new Constructor; Constructor node = new Constructor;
node.comment = comment;
comment = null;
auto t = expect(tok!"this"); auto t = expect(tok!"this");
if (t is null) return null; if (t is null) return null;
node.location = t.index; node.location = t.index;
@ -1557,7 +1561,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new Declaration; auto node = new Declaration;
comment = current.comment;
do do
{ {
if (!isAttribute()) if (!isAttribute())
@ -1904,6 +1908,8 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new Destructor; auto node = new Destructor;
node.comment = comment;
comment = null;
if (expect(tok!"~") is null) return null; if (expect(tok!"~") is null) return null;
if (expect(tok!"this") is null) return null; if (expect(tok!"this") is null) return null;
if (expect(tok!"(") is null) return null; if (expect(tok!"(") is null) return null;
@ -2005,6 +2011,8 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
node.name = advance(); node.name = advance();
else else
node.name.line = tokens[index - 1].line; // preserve line number if anonymous node.name.line = tokens[index - 1].line; // preserve line number if anonymous
node.comment = comment;
comment = null;
if (currentIs(tok!":")) if (currentIs(tok!":"))
{ {
advance(); advance();
@ -2026,6 +2034,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new EnumMember; auto node = new EnumMember;
node.comment = current.comment;
if (currentIs(tok!"identifier")) if (currentIs(tok!"identifier"))
{ {
if (peekIsOneOf(tok!",", tok!"}")) if (peekIsOneOf(tok!",", tok!"}"))
@ -2425,6 +2434,8 @@ body {} // six
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new FunctionDeclaration; auto node = new FunctionDeclaration;
node.comment = comment;
comment = null;
if (isAuto) if (isAuto)
goto functionName; goto functionName;
@ -2994,6 +3005,8 @@ import core.stdc.stdio, std.string : KeepTerminator;
auto ident = expect(tok!"identifier"); auto ident = expect(tok!"identifier");
if (ident is null) return null; if (ident is null) return null;
node.name = *ident; node.name = *ident;
node.comment = comment;
comment = null;
if (currentIs(tok!"(")) if (currentIs(tok!"("))
{ {
node.templateParameters = parseTemplateParameters(); node.templateParameters = parseTemplateParameters();
@ -3129,7 +3142,7 @@ invariant() foo();
if (expect(tok!")") is null) return null; if (expect(tok!")") is null) return null;
return node; return node;
} }
unittest unittest
{ {
auto sourceCode = q{is ( x : uybte)}c; auto sourceCode = q{is ( x : uybte)}c;
@ -3410,7 +3423,7 @@ invariant() foo();
node.symbol = parseSymbol(); node.symbol = parseSymbol();
return node; return node;
} }
unittest unittest
{ {
} }
@ -4585,6 +4598,8 @@ q{(int a, ...)
{ {
node.name = advance(); node.name = advance();
} }
node.comment = comment;
comment = null;
if (currentIs(tok!"(")) if (currentIs(tok!"("))
{ {
@ -4856,6 +4871,8 @@ q{(int a, ...)
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new TemplateDeclaration; auto node = new TemplateDeclaration;
node.comment = comment;
comment = null;
if (currentIs(tok!"enum")) if (currentIs(tok!"enum"))
{ {
node.eponymousTemplateDeclaration = parseEponymousTemplateDeclaration(); node.eponymousTemplateDeclaration = parseEponymousTemplateDeclaration();
@ -5784,7 +5801,7 @@ q{doStuff(5)}c;
Unittest parseUnittest() Unittest parseUnittest()
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
mixin (simpleParse!(Unittest, tok!"unittest", "blockStatement|parseBlockStatement")); mixin (simpleParse!(Unittest, tok!"unittest", "blockStatement|parseBlockStatement", true));
} }
/** /**
@ -6490,9 +6507,16 @@ protected:
template simpleParse(NodeType, parts ...) template simpleParse(NodeType, parts ...)
{ {
enum simpleParse = "auto node = new " ~ NodeType.stringof ~ ";\n" static if (__traits(hasMember, NodeType, "comment"))
~ simpleParseItems!(parts) enum simpleParse = "auto node = new " ~ NodeType.stringof ~ ";\n"
~ "\nreturn node;\n"; ~ "node.comment = comment;\n"
~ "comment = null;\n"
~ simpleParseItems!(parts)
~ "\nreturn node;\n";
else
enum simpleParse = "auto node = new " ~ NodeType.stringof ~ ";\n"
~ simpleParseItems!(parts)
~ "\nreturn node;\n";
} }
template simpleParseItems(items ...) template simpleParseItems(items ...)
@ -6609,4 +6633,5 @@ protected:
int suppressMessages; int suppressMessages;
size_t index; size_t index;
int _traceDepth; int _traceDepth;
string comment;
} }

13
stdx/lexer.d
View File

@ -33,7 +33,7 @@ template TokenIdType(alias staticTokens, alias dynamicTokens,
static assert (false); static assert (false);
} }
string TokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens, alias possibleDefaultTokens)(IdType type) @property string tokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens, alias possibleDefaultTokens)(IdType type) @property
{ {
if (type == 0) if (type == 0)
return "!ERROR!"; return "!ERROR!";
@ -90,7 +90,7 @@ template TokenId(IdType, alias staticTokens, alias dynamicTokens,
} }
} }
struct TokenStructure(IDType) struct TokenStructure(IDType, string extraFields = "")
{ {
bool opEquals(IDType type) const pure nothrow @safe bool opEquals(IDType type) const pure nothrow @safe
{ {
@ -116,6 +116,7 @@ struct TokenStructure(IDType)
size_t column; size_t column;
size_t index; size_t index;
IDType type; IDType type;
mixin (extraFields);
} }
mixin template Lexer(R, IDType, Token, alias defaultTokenFunction, mixin template Lexer(R, IDType, Token, alias defaultTokenFunction,
@ -210,7 +211,7 @@ mixin template Lexer(R, IDType, Token, alias defaultTokenFunction,
return _front; return _front;
} }
void popFront() pure void _popFront() pure
{ {
_front = advance(); _front = advance();
} }
@ -277,20 +278,20 @@ struct LexerRange(BufferType) if (isBuffer!BufferType)
column = 1; column = 1;
line = 1; line = 1;
} }
void popFront() pure void popFront() pure
{ {
index++; index++;
column++; column++;
range.popFront(); range.popFront();
} }
void incrementLine() pure nothrow void incrementLine() pure nothrow
{ {
column = 1; column = 1;
line++; line++;
} }
BufferType range; BufferType range;
alias range this; alias range this;
size_t index; size_t index;