More work on the lexer transition

stdx/d/lexer.d

@@ -51,7 +51,7 @@ private enum dynamicTokens = [
 ];
 
 public alias TokenIdType!(staticTokens, dynamicTokens, possibleDefaultTokens) IdType;
-public alias TokenStringRepresentation!(IdType, staticTokens, possibleDefaultTokens) str;
+public alias TokenStringRepresentation!(IdType, staticTokens, dynamicTokens, possibleDefaultTokens) str;
 public template tok(string token)
 {
   alias TokenId!(IdType, staticTokens, dynamicTokens, possibleDefaultTokens, token) tok;
@@ -60,19 +60,19 @@ public alias stdx.lexer.TokenStructure!(IdType) Token;
 
 public auto byToken(R, bool skipComments = true, bool skipWhitespace = true)(R range)
 {
-	pure nothrow bool isComment(const Token t) { return t.type == tok!"comment"; }
+	pure nothrow bool isNotComment(const Token t) { return t.type != tok!"comment"; }
-	pure nothrow bool isWhitespace(const Token t) { return t.type == tok!"whitespace"; }
+	pure nothrow bool isNotWhitespace(const Token t) { return t.type != tok!"whitespace"; }
-	pure nothrow bool isEither(const Token t) { return t.type == tok!"whitespace" || t.type == tok!"comment"; }
+	pure nothrow bool isNotEither(const Token t) { return t.type != tok!"whitespace" && t.type != tok!"comment"; }
 
 	static if (skipComments)
 	{
 		static if (skipWhitespace)
-			return DLexer!(R)(range).filter!isEither;
+			return DLexer!(R)(range).filter!isNotEither;
 		else
-			return DLexer!(R)(range).filter.isComment;
+			return DLexer!(R)(range).filter!isNotComment;
 	}
 	else static if (skipWhitespace)
-		return DLexer!(R)(range).filter!isWhitespace;
+		return DLexer!(R)(range).filter!isNotWhitespace;
 	else
 		return DLexer!(R)(range);
 }
@@ -1037,7 +1037,7 @@ public struct DLexer(R)
 			else switch (range.front)
 			{
 			case '0': .. case '9':
-			case 'F': .. case 'F':
+			case 'A': .. case 'F':
 			case 'a': .. case 'f':
 				range.popFront();
 				break;

stdx/d/parser.d

@@ -198,6 +198,7 @@ alias core.sys.posix.stdio.fileno fileno;
         if (expect(tok!"alias") is null) return null;
         auto ident = expect(tok!"identifier");
         if (ident is null) return null;
+        node.identifier = *ident;
         if (expect(tok!"this") is null) return null;
         if (expect(tok!";") is null) return null;
         return node;
@@ -689,11 +690,8 @@ alias core.sys.posix.stdio.fileno fileno;
     AssocArrayLiteral parseAssocArrayLiteral()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new AssocArrayLiteral;
+        mixin (simpleParse!(AssocArrayLiteral, tok!"[",
-        if (expect(tok!"[") is null) return null;
+            "keyValuePairs|parseKeyValuePairs", tok!"]"));
-        node.keyValuePairs = parseKeyValuePairs();
-        if (expect(tok!"]") is null) return null;
-        return node;
     }
 
     /**
@@ -853,10 +851,8 @@ alias core.sys.posix.stdio.fileno fileno;
     BodyStatement parseBodyStatement()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new BodyStatement;
+        mixin (simpleParse!(BodyStatement, tok!"body",
-        expect(tok!"body");
+            "blockStatement|parseBlockStatement"));
-        node.blockStatement = parseBlockStatement();
-        return node;
     }
 
     /**
@@ -4318,15 +4314,8 @@ q{(int a, ...)
     SharedStaticConstructor parseSharedStaticConstructor()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new SharedStaticConstructor;
+        mixin (simpleParse!(SharedStaticConstructor, tok!"shared", tok!"static",
-        if (expect(tok!"shared") is null) return null;
+            tok!"this", tok!"(", tok!")", "functionBody|parseFunctionBody"));
-        if (expect(tok!"static") is null) return null;
-        if (expect(tok!"this") is null) return null;
-        if (expect(tok!"(") is null) return null;
-        if (expect(tok!")") is null) return null;
-        node.functionBody = parseFunctionBody();
-        if (node.functionBody is null) return null;
-        return node;
     }
 
     /**
@@ -4339,15 +4328,9 @@ q{(int a, ...)
     SharedStaticDestructor parseSharedStaticDestructor()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new SharedStaticDestructor;
+        mixin (simpleParse!(SharedStaticDestructor, tok!"shared", tok!"static",
-        expect(tok!"shared");
+            tok!"~", tok!"this", tok!"(", tok!")",
-        expect(tok!"static");
+            "functionBody|parseFunctionBody"));
-        expect(tok!"~");
-        expect(tok!"this");
-        expect(tok!"(");
-        expect(tok!")");
-        node.functionBody = parseFunctionBody();
-        return node;
     }
 
     /**
@@ -4455,10 +4438,8 @@ q{(int a, ...)
     StaticAssertDeclaration parseStaticAssertDeclaration()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new StaticAssertDeclaration;
+        mixin (simpleParse!(StaticAssertDeclaration,
-        node.staticAssertStatement = parseStaticAssertStatement();
+            "staticAssertStatement|parseStaticAssertStatement"));
-        if (node.staticAssertStatement is null) return null;
-        return node;
     }
 
 
@@ -4472,12 +4453,8 @@ q{(int a, ...)
     StaticAssertStatement parseStaticAssertStatement()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new StaticAssertStatement;
+        mixin (simpleParse!(StaticAssertStatement,
-        if (expect(tok!"static") is null) return null;
+            tok!"static", "assertExpression|parseAssertExpression", tok!";"));
-        node.assertExpression = parseAssertExpression();
-        if (node.assertExpression is null) return null;
-        if (expect(tok!";") is null) return null;
-        return node;
     }
 
     /**
@@ -4490,13 +4467,8 @@ q{(int a, ...)
     StaticConstructor parseStaticConstructor()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new StaticConstructor;
+        mixin (simpleParse!(StaticConstructor, tok!"static", tok!"this",
-        expect(tok!"static");
+            tok!"(", tok!")", "functionBody|parseFunctionBody"));
-        expect(tok!"this");
-        expect(tok!"(");
-        expect(tok!")");
-        node.functionBody = parseFunctionBody();
-        return node;
     }
 
     /**
@@ -4509,14 +4481,8 @@ q{(int a, ...)
     StaticDestructor parseStaticDestructor()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new StaticDestructor;
+        mixin (simpleParse!(StaticDestructor, tok!"static", tok!"~", tok!"this",
-        expect(tok!"static");
+            tok!"(", tok!")", "functionBody|parseFunctionBody"));
-        expect(tok!"~");
-        expect(tok!"this");
-        expect(tok!"(");
-        expect(tok!")");
-        node.functionBody = parseFunctionBody();
-        return node;
     }
 
     /**
@@ -4529,13 +4495,8 @@ q{(int a, ...)
     StaticIfCondition parseStaticIfCondition()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new StaticIfCondition;
+        mixin (simpleParse!(StaticIfCondition, tok!"static", tok!"if", tok!"(",
-        expect(tok!"static");
+            "assignExpression|parseAssignExpression", tok!")"));
-        expect(tok!"if");
-        expect(tok!"(");
-        node.assignExpression = parseAssignExpression();
-        expect(tok!")");
-        return node;
     }
 
     /**
@@ -5837,10 +5798,7 @@ q{doStuff(5)}c;
     Unittest parseUnittest()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new Unittest;
+        mixin (simpleParse!(Unittest, tok!"unittest", "blockStatement|parseBlockStatement"));
-        expect(tok!"unittest");
-        node.blockStatement = parseBlockStatement();
-        return node;
     }
 
     /**
@@ -5887,12 +5845,7 @@ q{doStuff(5)}c;
     Vector parseVector()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new Vector;
+        mixin (simpleParse!(Vector, tok!"__vector", tok!"(", "type|parseType", tok!")"));
-        if (expect(tok!"__vector") is null) return null;
-        if (expect(tok!"(") is null) return null;
-        if ((node.type = parseType()) is null) return null;
-        if (expect(tok!")") is null) return null;
-        return node;
     }
 
     /**
@@ -5906,8 +5859,7 @@ q{doStuff(5)}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new VersionCondition;
-        if (expect(tok!"version") is null) return null;
+        mixin (expectSequence!(tok!"version", tok!"("));
-        if (expect(tok!"(") is null) return null;
         if (currentIsOneOf(tok!"intLiteral", tok!"identifier",
             tok!"unittest", tok!"assert"))
         {
@@ -5933,8 +5885,7 @@ q{doStuff(5)}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new VersionSpecification;
-        expect(tok!"version");
+        mixin (expectSequence!(tok!"version", tok!"="));
-        expect(tok!"=");
         if (!currentIsOneOf(tok!"identifier", tok!"intLiteral"))
         {
             error("Identifier or integer literal expected");
@@ -5980,14 +5931,9 @@ q{doStuff(5)}c;
     WithStatement parseWithStatement()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new WithStatement;
+        mixin (simpleParse!(WithStatement, tok!"with", tok!"(",
-        if (expect(tok!"with") is null) return null;
+            "expression|parseExpression", tok!")",
-        if (expect(tok!"(") is null) return null;
+            "statementNoCaseNoDefault|parseStatementNoCaseNoDefault"));
-        if ((node.expression = parseExpression()) is null) return null;
-        if (expect(tok!")") is null) return null;
-        node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault();
-        if (node.statementNoCaseNoDefault is null) return null;
-        return node;
     }
 
     /**
@@ -6557,6 +6503,42 @@ protected:
         return p;
     }
 
+    template simpleParse(NodeType, parts ...)
+    {
+        enum simpleParse = "auto node = new " ~ NodeType.stringof ~ ";\n"
+            ~ simpleParseItems!(parts)
+            ~ "\nreturn node;\n";
+    }
+
+    template simpleParseItems(items ...)
+    {
+        static if (items.length > 1)
+            enum simpleParseItems = simpleParseItem!(items[0]) ~ "\n"
+                ~ simpleParseItems!(items[1 .. $]);
+        else static if (items.length == 1)
+            enum simpleParseItems = simpleParseItem!(items[0]);
+        else
+            enum simpleParseItems = "";
+    }
+
+    template simpleParseItem(alias item)
+    {
+        static if (is (typeof(item) == string))
+            enum simpleParseItem = "if ((node." ~ item[0 .. item.countUntil("|")]
+                ~ " = " ~ item[item.countUntil("|") + 1 .. $] ~ "()) is null) return null;";
+        else
+            enum simpleParseItem = "if (expect(" ~ item.stringof ~ ") is null) return null;";
+    }
+
+    template expectSequence(sequence ...)
+    {
+        static if (sequence.length == 1)
+            enum expectSequence = "if (expect(" ~ sequence[0].stringof ~ ") is null) return null;";
+        else
+            enum expectSequence = "if (expect(" ~ sequence[0].stringof ~ ") is null) return null;\n"
+                ~ expectSequence!(sequence[1..$]);
+    }
+
     template traceEnterAndExit(string fun)
     {
         enum traceEnterAndExit = `version (std_parser_verbose) trace("`
@@ -6619,6 +6601,9 @@ protected:
         case tok!"irealLiteral":
         case tok!"uintLiteral":
         case tok!"ulongLiteral":
+        case tok!"stringLiteral":
+        case tok!"wstringLiteral":
+        case tok!"dstringLiteral":
     };
     enum string SPECIAL_CASES = q{
         case tok!"__DATE__":

stdx/lexer.d

@@ -30,7 +30,7 @@ template TokenIdType(alias staticTokens, alias dynamicTokens,
 		static assert (false);
 }
 
-string TokenStringRepresentation(IdType, alias staticTokens, alias possibleDefaultTokens)(IdType type) @property
+string TokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens, alias possibleDefaultTokens)(IdType type) @property
 {
 	if (type == 0)
 		return "!ERROR!";
@@ -38,6 +38,8 @@ string TokenStringRepresentation(IdType, alias staticTokens, alias possibleDefau
 		return staticTokens[type - 1];
 	else if (type < staticTokens.length + possibleDefaultTokens.length)
 		return possibleDefaultTokens[type - staticTokens.length];
+    else if (type < staticTokens.length + possibleDefaultTokens.length + dynamicTokens.length)
+		return dynamicTokens[type - staticTokens.length - possibleDefaultTokens.length];
 	else
 		return null;
 }