forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

More work on the lexer transition

This commit is contained in:
Hackerpilot 2013-12-23 00:53:46 -08:00
parent aedf58eb81
commit d5a3918dd0
3 changed files with 86 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
stdx/d/lexer.d
View File

@ -51,7 +51,7 @@ private enum dynamicTokens = [
];
public alias TokenIdType!(staticTokens, dynamicTokens, possibleDefaultTokens) IdType;
public alias TokenStringRepresentation!(IdType, staticTokens, possibleDefaultTokens) str;
public alias TokenStringRepresentation!(IdType, staticTokens, dynamicTokens, possibleDefaultTokens) str;
public template tok(string token)
{
alias TokenId!(IdType, staticTokens, dynamicTokens, possibleDefaultTokens, token) tok;
@ -60,19 +60,19 @@ public alias stdx.lexer.TokenStructure!(IdType) Token;
public auto byToken(R, bool skipComments = true, bool skipWhitespace = true)(R range)
{
pure nothrow bool isComment(const Token t) { return t.type == tok!"comment"; }
pure nothrow bool isWhitespace(const Token t) { return t.type == tok!"whitespace"; }
pure nothrow bool isEither(const Token t) { return t.type == tok!"whitespace" || t.type == tok!"comment"; }
pure nothrow bool isNotComment(const Token t) { return t.type != tok!"comment"; }
pure nothrow bool isNotWhitespace(const Token t) { return t.type != tok!"whitespace"; }
pure nothrow bool isNotEither(const Token t) { return t.type != tok!"whitespace" && t.type != tok!"comment"; }
static if (skipComments)
{
static if (skipWhitespace)
return DLexer!(R)(range).filter!isEither;
return DLexer!(R)(range).filter!isNotEither;
else
return DLexer!(R)(range).filter.isComment;
return DLexer!(R)(range).filter!isNotComment;
}
else static if (skipWhitespace)
return DLexer!(R)(range).filter!isWhitespace;
return DLexer!(R)(range).filter!isNotWhitespace;
else
return DLexer!(R)(range);
}
@ -322,10 +322,10 @@ public bool isStringLiteral(IdType type) pure nothrow @safe
public struct DLexer(R)
{
import std.conv;
mixin Lexer!(R, IdType, Token, isSeparating, lexIdentifier, staticTokens, dynamicTokens,
pseudoTokens, possibleDefaultTokens);
this(R range)
{
registerPostProcess!"\""(&lexStringLiteral!RangeType);
@ -395,7 +395,7 @@ public struct DLexer(R)
return;
case '\n':
range.popFront();
range.incrementLine();
range.incrementLine();
return;
case 0xe2:
if (range.canPeek(2) && range.peek() == 0x80
@ -770,7 +770,7 @@ public struct DLexer(R)
return Token(type, cast(string) range.getMarked(), range.line, range.column,
range.index);
}
Token lexSlashSlashComment(LR)(ref LR range)
{
range.mark();
@ -786,7 +786,7 @@ public struct DLexer(R)
return Token(type, cast(string) range.getMarked(), range.line, range.column,
range.index);
}
Token lexSlashPlusComment(LR)(ref LR range)
{
range.mark();
@ -1024,7 +1024,7 @@ public struct DLexer(R)
range.mark();
range.popFront();
range.popFront();
loop: while (true)
{
if (range.empty)
@ -1037,7 +1037,7 @@ public struct DLexer(R)
else switch (range.front)
{
case '0': .. case '9':
case 'F': .. case 'F':
case 'A': .. case 'F':
case 'a': .. case 'f':
range.popFront();
break;
@ -1049,7 +1049,7 @@ public struct DLexer(R)
return Token();
}
}
IdType type = tok!"stringLiteral";
lexStringSuffix(range, type);
return Token(type, cast(string) range.getMarked(), range.line, range.column,
@ -1105,7 +1105,7 @@ public struct DLexer(R)
range.popFront();
break;
default:
writeln("Error: at least 4 hex digits expected.");
writeln("Error: at least 4 hex digits expected.");
return false;
}
}

149
stdx/d/parser.d
View File

@ -198,6 +198,7 @@ alias core.sys.posix.stdio.fileno fileno;
if (expect(tok!"alias") is null) return null;
auto ident = expect(tok!"identifier");
if (ident is null) return null;
node.identifier = *ident;
if (expect(tok!"this") is null) return null;
if (expect(tok!";") is null) return null;
return node;
@ -689,11 +690,8 @@ alias core.sys.posix.stdio.fileno fileno;
AssocArrayLiteral parseAssocArrayLiteral()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new AssocArrayLiteral;
if (expect(tok!"[") is null) return null;
node.keyValuePairs = parseKeyValuePairs();
if (expect(tok!"]") is null) return null;
return node;
mixin (simpleParse!(AssocArrayLiteral, tok!"[",
"keyValuePairs|parseKeyValuePairs", tok!"]"));
}
/**
@ -853,10 +851,8 @@ alias core.sys.posix.stdio.fileno fileno;
BodyStatement parseBodyStatement()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new BodyStatement;
expect(tok!"body");
node.blockStatement = parseBlockStatement();
return node;
mixin (simpleParse!(BodyStatement, tok!"body",
"blockStatement|parseBlockStatement"));
}
/**
@ -4318,15 +4314,8 @@ q{(int a, ...)
SharedStaticConstructor parseSharedStaticConstructor()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new SharedStaticConstructor;
if (expect(tok!"shared") is null) return null;
if (expect(tok!"static") is null) return null;
if (expect(tok!"this") is null) return null;
if (expect(tok!"(") is null) return null;
if (expect(tok!")") is null) return null;
node.functionBody = parseFunctionBody();
if (node.functionBody is null) return null;
return node;
mixin (simpleParse!(SharedStaticConstructor, tok!"shared", tok!"static",
tok!"this", tok!"(", tok!")", "functionBody|parseFunctionBody"));
}
/**
@ -4339,15 +4328,9 @@ q{(int a, ...)
SharedStaticDestructor parseSharedStaticDestructor()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new SharedStaticDestructor;
expect(tok!"shared");
expect(tok!"static");
expect(tok!"~");
expect(tok!"this");
expect(tok!"(");
expect(tok!")");
node.functionBody = parseFunctionBody();
return node;
mixin (simpleParse!(SharedStaticDestructor, tok!"shared", tok!"static",
tok!"~", tok!"this", tok!"(", tok!")",
"functionBody|parseFunctionBody"));
}
/**
@ -4455,10 +4438,8 @@ q{(int a, ...)
StaticAssertDeclaration parseStaticAssertDeclaration()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new StaticAssertDeclaration;
node.staticAssertStatement = parseStaticAssertStatement();
if (node.staticAssertStatement is null) return null;
return node;
mixin (simpleParse!(StaticAssertDeclaration,
"staticAssertStatement|parseStaticAssertStatement"));
}
@ -4472,12 +4453,8 @@ q{(int a, ...)
StaticAssertStatement parseStaticAssertStatement()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new StaticAssertStatement;
if (expect(tok!"static") is null) return null;
node.assertExpression = parseAssertExpression();
if (node.assertExpression is null) return null;
if (expect(tok!";") is null) return null;
return node;
mixin (simpleParse!(StaticAssertStatement,
tok!"static", "assertExpression|parseAssertExpression", tok!";"));
}
/**
@ -4490,13 +4467,8 @@ q{(int a, ...)
StaticConstructor parseStaticConstructor()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new StaticConstructor;
expect(tok!"static");
expect(tok!"this");
expect(tok!"(");
expect(tok!")");
node.functionBody = parseFunctionBody();
return node;
mixin (simpleParse!(StaticConstructor, tok!"static", tok!"this",
tok!"(", tok!")", "functionBody|parseFunctionBody"));
}
/**
@ -4509,14 +4481,8 @@ q{(int a, ...)
StaticDestructor parseStaticDestructor()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new StaticDestructor;
expect(tok!"static");
expect(tok!"~");
expect(tok!"this");
expect(tok!"(");
expect(tok!")");
node.functionBody = parseFunctionBody();
return node;
mixin (simpleParse!(StaticDestructor, tok!"static", tok!"~", tok!"this",
tok!"(", tok!")", "functionBody|parseFunctionBody"));
}
/**
@ -4529,13 +4495,8 @@ q{(int a, ...)
StaticIfCondition parseStaticIfCondition()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new StaticIfCondition;
expect(tok!"static");
expect(tok!"if");
expect(tok!"(");
node.assignExpression = parseAssignExpression();
expect(tok!")");
return node;
mixin (simpleParse!(StaticIfCondition, tok!"static", tok!"if", tok!"(",
"assignExpression|parseAssignExpression", tok!")"));
}
/**
@ -5740,7 +5701,7 @@ q{(int a, ...)
goto case tok!"(";
else
break loop;
case tok!"(":
auto newUnary = new UnaryExpression();
newUnary.functionCallExpression = parseFunctionCallExpression(node);
@ -5837,10 +5798,7 @@ q{doStuff(5)}c;
Unittest parseUnittest()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new Unittest;
expect(tok!"unittest");
node.blockStatement = parseBlockStatement();
return node;
mixin (simpleParse!(Unittest, tok!"unittest", "blockStatement|parseBlockStatement"));
}
/**
@ -5887,12 +5845,7 @@ q{doStuff(5)}c;
Vector parseVector()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new Vector;
if (expect(tok!"__vector") is null) return null;
if (expect(tok!"(") is null) return null;
if ((node.type = parseType()) is null) return null;
if (expect(tok!")") is null) return null;
return node;
mixin (simpleParse!(Vector, tok!"__vector", tok!"(", "type|parseType", tok!")"));
}
/**
@ -5906,8 +5859,7 @@ q{doStuff(5)}c;
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new VersionCondition;
if (expect(tok!"version") is null) return null;
if (expect(tok!"(") is null) return null;
mixin (expectSequence!(tok!"version", tok!"("));
if (currentIsOneOf(tok!"intLiteral", tok!"identifier",
tok!"unittest", tok!"assert"))
{
@ -5933,8 +5885,7 @@ q{doStuff(5)}c;
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new VersionSpecification;
expect(tok!"version");
expect(tok!"=");
mixin (expectSequence!(tok!"version", tok!"="));
if (!currentIsOneOf(tok!"identifier", tok!"intLiteral"))
{
error("Identifier or integer literal expected");
@ -5980,14 +5931,9 @@ q{doStuff(5)}c;
WithStatement parseWithStatement()
{
mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new WithStatement;
if (expect(tok!"with") is null) return null;
if (expect(tok!"(") is null) return null;
if ((node.expression = parseExpression()) is null) return null;
if (expect(tok!")") is null) return null;
node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault();
if (node.statementNoCaseNoDefault is null) return null;
return node;
mixin (simpleParse!(WithStatement, tok!"with", tok!"(",
"expression|parseExpression", tok!")",
"statementNoCaseNoDefault|parseStatementNoCaseNoDefault"));
}
/**
@ -6557,6 +6503,42 @@ protected:
return p;
}
template simpleParse(NodeType, parts ...)
{
enum simpleParse = "auto node = new " ~ NodeType.stringof ~ ";\n"
~ simpleParseItems!(parts)
~ "\nreturn node;\n";
}
template simpleParseItems(items ...)
{
static if (items.length > 1)
enum simpleParseItems = simpleParseItem!(items[0]) ~ "\n"
~ simpleParseItems!(items[1 .. $]);
else static if (items.length == 1)
enum simpleParseItems = simpleParseItem!(items[0]);
else
enum simpleParseItems = "";
}
template simpleParseItem(alias item)
{
static if (is (typeof(item) == string))
enum simpleParseItem = "if ((node." ~ item[0 .. item.countUntil("|")]
~ " = " ~ item[item.countUntil("|") + 1 .. $] ~ "()) is null) return null;";
else
enum simpleParseItem = "if (expect(" ~ item.stringof ~ ") is null) return null;";
}
template expectSequence(sequence ...)
{
static if (sequence.length == 1)
enum expectSequence = "if (expect(" ~ sequence[0].stringof ~ ") is null) return null;";
else
enum expectSequence = "if (expect(" ~ sequence[0].stringof ~ ") is null) return null;\n"
~ expectSequence!(sequence[1..$]);
}
template traceEnterAndExit(string fun)
{
enum traceEnterAndExit = `version (std_parser_verbose) trace("`
@ -6619,6 +6601,9 @@ protected:
case tok!"irealLiteral":
case tok!"uintLiteral":
case tok!"ulongLiteral":
case tok!"stringLiteral":
case tok!"wstringLiteral":
case tok!"dstringLiteral":
};
enum string SPECIAL_CASES = q{
case tok!"__DATE__":

4
stdx/lexer.d
View File

@ -30,7 +30,7 @@ template TokenIdType(alias staticTokens, alias dynamicTokens,
static assert (false);
}
string TokenStringRepresentation(IdType, alias staticTokens, alias possibleDefaultTokens)(IdType type) @property
string TokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens, alias possibleDefaultTokens)(IdType type) @property
{
if (type == 0)
return "!ERROR!";
@ -38,6 +38,8 @@ string TokenStringRepresentation(IdType, alias staticTokens, alias possibleDefau
return staticTokens[type - 1];
else if (type < staticTokens.length + possibleDefaultTokens.length)
return possibleDefaultTokens[type - staticTokens.length];
else if (type < staticTokens.length + possibleDefaultTokens.length + dynamicTokens.length)
return dynamicTokens[type - staticTokens.length - possibleDefaultTokens.length];
else
return null;
}