diff --git a/.gitmodules b/.gitmodules
old mode 100755
new mode 100644
index e69de29..c34a4b6
--- a/.gitmodules
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "datapicked"]
+	path = datapicked
+	url = ./datapicked/
diff --git a/astprinter.d b/astprinter.d
index 3d8f7e9..57ba4e0 100644
--- a/astprinter.d
+++ b/astprinter.d
@@ -21,7 +21,7 @@ class XMLPrinter : ASTVisitor
 {
 	override void visit(AddExpression addExpression)
 	{
-		output.writeln("<addExpression operator=\"", getTokenValue(addExpression.operator) ,"\">");
+		output.writeln("<addExpression operator=\"", str(addExpression.operator) ,"\">");
 		output.writeln("<left>");
 		addExpression.left.accept(this);
 		output.writeln("</left>");
@@ -51,7 +51,7 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(AlignAttribute alignAttribute)
 	{
-		output.writeln("<alignAttribute align=\"", alignAttribute.intLiteral.value, "\">");
+		output.writeln("<alignAttribute align=\"", alignAttribute.intLiteral.text, "\">");
 	}
 
 	override void visit(AndAndExpression andAndExpression)
@@ -130,7 +130,7 @@ class XMLPrinter : ASTVisitor
 			output.writeln("<assignExpression>");
 		else
 			output.writeln("<assignExpression operator=\"",
-				getTokenValue(assignExpression.operator), "\">");
+				str(assignExpression.operator), "\">");
 		assignExpression.accept(this);
 		output.writeln("</assignExpression>");
 	}
@@ -143,20 +143,20 @@ class XMLPrinter : ASTVisitor
 	override void visit(AtAttribute atAttribute)
 	{
 		output.writeln("<atAttribute>");
-		if (atAttribute.identifier.type == TokenType.invalid)
+		if (atAttribute.identifier.type == tok!"")
 			atAttribute.accept(this);
 		else
-			output.writeln("<identifier>", atAttribute.identifier.value, "</identifier>");
+			output.writeln("<identifier>", atAttribute.identifier.text, "</identifier>");
 		output.writeln("</atAttribute>");
 	}
 
 	override void visit(Attribute attribute)
 	{
 		output.writeln("<attribute>");
-		if (attribute.attribute == TokenType.invalid)
+		if (attribute.attribute == tok!"")
 			attribute.accept(this);
 		else
-			output.writeln(getTokenValue(attribute.attribute));
+			output.writeln(str(attribute.attribute));
 		output.writeln("</attribute>");
 	}
 
@@ -173,7 +173,7 @@ class XMLPrinter : ASTVisitor
 		{
 			output.writeln("<item>");
 			output.writeln("<name line=\"", autoDec.identifiers[i].line, "\">",
-				autoDec.identifiers[i].value, "</name>");
+				autoDec.identifiers[i].text, "</name>");
 			visit(autoDec.initializers[i]);
 			output.writeln("</item>");
 		}
@@ -196,10 +196,10 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(BreakStatement breakStatement)
 	{
-		if (breakStatement.label.type == TokenType.invalid)
+		if (breakStatement.label.type == tok!"")
 			output.writeln("<breakStatement/>");
 		else
-			output.writeln("<breakStatement label=\"", breakStatement.label.value, "\"/>");
+			output.writeln("<breakStatement label=\"", breakStatement.label.text, "\"/>");
 	}
 
 	override void visit(BaseClass baseClass)
@@ -256,7 +256,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(ClassDeclaration classDec)
 	{
 		output.writeln("<classDeclaration line=\"", classDec.name.line, "\">");
-		output.writeln("<name>", classDec.name.value, "</name>");
+		output.writeln("<name>", classDec.name.text, "</name>");
 		classDec.accept(this);
 		output.writeln("</classDeclaration>");
 	}
@@ -318,29 +318,29 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(ContinueStatement continueStatement)
 	{
-		if (continueStatement.label.type == TokenType.invalid)
+		if (continueStatement.label.type == tok!"")
 			output.writeln("<continueStatement/>");
 		else
 			output.writeln("<continueStatement label=\"",
-				continueStatement.label.value, "\"/>");
+				continueStatement.label.text, "\"/>");
 	}
 
 	override void visit(DebugCondition debugCondition)
 	{
-		if (debugCondition.identifierOrInteger.type == TokenType.invalid)
+		if (debugCondition.identifierOrInteger.type == tok!"")
 			output.writeln("<debugCondition/>");
 		else
 			output.writeln("<debugCondition condition=\"",
-				debugCondition.identifierOrInteger.value, "\"/>");
+				debugCondition.identifierOrInteger.text, "\"/>");
 	}
 
 	override void visit(DebugSpecification debugSpecification)
 	{
-		if (debugSpecification.identifierOrInteger.type == TokenType.invalid)
+		if (debugSpecification.identifierOrInteger.type == tok!"")
 			output.writeln("<debugSpecification/>");
 		else
 			output.writeln("<debugSpecification condition=\"",
-				debugSpecification.identifierOrInteger.value, "\"/>");
+				debugSpecification.identifierOrInteger.text, "\"/>");
 	}
 
 	override void visit(Declaration declaration)
@@ -361,7 +361,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(Declarator declarator)
 	{
 		output.writeln("<declarator line=\"", declarator.name.line, "\">");
-		output.writeln("<name>", declarator.name.value, "</name>");
+		output.writeln("<name>", declarator.name.text, "</name>");
 		declarator.accept(this);
 		output.writeln("</declarator>");
 	}
@@ -411,8 +411,9 @@ class XMLPrinter : ASTVisitor
 	override void visit(EnumDeclaration enumDec)
 	{
 		output.writeln("<enumDeclaration line=\"", enumDec.name.line, "\">");
-		if (enumDec.name.type == TokenType.identifier)
-			output.writeln("<name>", enumDec.name.value, "</name>");
+        writeDdoc(enumDec.comment);
+		if (enumDec.name.type == tok!"identifier")
+			output.writeln("<name>", enumDec.name.text, "</name>");
 		enumDec.accept(this);
 		output.writeln("</enumDeclaration>");
 	}
@@ -420,13 +421,14 @@ class XMLPrinter : ASTVisitor
 	override void visit(EnumMember enumMem)
 	{
 		output.writeln("<enumMember line=\"", enumMem.name.line, "\">");
+        writeDdoc(enumMem.comment);
 		enumMem.accept(this);
 		output.writeln("</enumMember>");
 	}
 
 	override void visit(EqualExpression equalExpression)
 	{
-		output.writeln("<enumMember operator=\"", getTokenValue(equalExpression.operator), "\">");
+		output.writeln("<enumMember operator=\"", str(equalExpression.operator), "\">");
 		output.writeln("<left>");
 		visit(equalExpression.left);
 		output.writeln("</left>");
@@ -469,9 +471,9 @@ class XMLPrinter : ASTVisitor
 		output.writeln("<forStatement>");
 		if (forStatement.declarationOrStatement !is null)
 		{
-			output.writeln("<initialize>");
-			visit(forStatement.declarationOrStatement);
-			output.writeln("</initialize>");
+			output.writeln("<initialization>");
+			visit(forStatement.initialization);
+			output.writeln("</initialization>");
 		}
 		if (forStatement.test !is null)
 		{
@@ -491,7 +493,7 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(ForeachStatement foreachStatement)
 	{
-		output.writeln("<foreachStatement type=\"", getTokenValue(
+		output.writeln("<foreachStatement type=\"", str(
 			foreachStatement.type), "\">");
 		if (foreachStatement.foreachType !is null)
 			visit(foreachStatement.foreachType);
@@ -515,7 +517,7 @@ class XMLPrinter : ASTVisitor
 		output.writeln("<foreachType>");
 		foreach (constructor; foreachType.typeConstructors)
 		{
-			output.writeln("<typeConstructor>", getTokenValue(constructor), "</typeConstructor>");
+			output.writeln("<typeConstructor>", str(constructor), "</typeConstructor>");
 		}
 		if (foreachType.type !is null)
 			visit(foreachType.type);
@@ -552,7 +554,8 @@ class XMLPrinter : ASTVisitor
 	override void visit(FunctionDeclaration functionDec)
 	{
 		output.writeln("<functionDeclaration line=\"", functionDec.name.line, "\">");
-		output.writeln("<name>", functionDec.name.value, "</name>");
+		output.writeln("<name>", functionDec.name.text, "</name>");
+        writeDdoc(functionDec.comment);
 		if (functionDec.hasAuto)
 			output.writeln("<auto/>");
 		if (functionDec.hasRef)
@@ -564,17 +567,17 @@ class XMLPrinter : ASTVisitor
 	override void visit(FunctionLiteralExpression functionLiteralExpression)
 	{
 		output.writeln("<functionLiteralExpression type=\"",
-			getTokenValue(functionLiteralExpression.functionOrDelegate), "\">");
+			str(functionLiteralExpression.functionOrDelegate), "\">");
 		functionLiteralExpression.accept(this);
 		output.writeln("</functionLiteralExpression>");
 	}
 
 	override void visit(GotoStatement gotoStatement)
 	{
-		if (gotoStatement.label.type == TokenType.default_)
+		if (gotoStatement.label.type == tok!"default")
 			output.writeln("<gotoStatement default=\"true\"/>");
-		else if (gotoStatement.label.type == TokenType.identifier)
-			output.writeln("<gotoStatement label=\"", gotoStatement.label.value, "\"/>");
+		else if (gotoStatement.label.type == tok!"identifier")
+			output.writeln("<gotoStatement label=\"", gotoStatement.label.text, "\"/>");
 		else
 		{
 			output.writeln("<gotoStatement>");
@@ -625,7 +628,7 @@ class XMLPrinter : ASTVisitor
 		output.writeln("<ifStatement>");
 
 		output.writeln("<condition>");
-		if (ifStatement.identifier.type != TokenType.invalid)
+		if (ifStatement.identifier.type != tok!"")
 		{
 			if (ifStatement.type is null)
 				output.writeln("<auto/>");
@@ -651,11 +654,11 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(ImportBind importBind)
 	{
-		if (importBind.right.type == TokenType.invalid)
-			output.writeln("<importBind symbol=\"", importBind.left.value, "\">");
+		if (importBind.right.type == tok!"")
+			output.writeln("<importBind symbol=\"", importBind.left.text, "\">");
 		else
-			output.writeln("<importBind symbol=\"", importBind.right.value,
-				"\" rename=\"", importBind.left.value, "\">");
+			output.writeln("<importBind symbol=\"", importBind.right.text,
+				"\" rename=\"", importBind.left.text, "\">");
 	}
 
 	override void visit(ImportBindings importBindings)
@@ -725,7 +728,8 @@ class XMLPrinter : ASTVisitor
 	override void visit(InterfaceDeclaration interfaceDec)
 	{
 		output.writeln("<interfaceDeclaration line=\"", interfaceDec.name.line, "\">");
-		output.writeln("<name>", interfaceDec.name.value, "</name>");
+		output.writeln("<name>", interfaceDec.name.text, "</name>");
+        writeDdoc(interfaceDec.comment);
 		interfaceDec.accept(this);
 		output.writeln("</interfaceDeclaration>");
 	}
@@ -733,6 +737,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(Invariant invariant_)
 	{
 		output.writeln("<invariant>");
+        writeDdoc(invariant_.comment);
 		invariant_.accept(this);
 		output.writeln("</invariant>");
 	}
@@ -741,11 +746,11 @@ class XMLPrinter : ASTVisitor
 	{
 		output.writeln("<isExpression>");
 		visit(isExpression.type);
-		if (isExpression.identifier.type != TokenType.invalid)
+		if (isExpression.identifier.type != tok!"")
 			visit(isExpression.identifier);
 		if (isExpression.typeSpecialization !is null)
 		{
-			if (isExpression.equalsOrColon == TokenType.colon)
+			if (isExpression.equalsOrColon == tok!":")
 				output.writeln("<colon/>");
 			else
 				output.writeln("<equals/>");
@@ -776,7 +781,7 @@ class XMLPrinter : ASTVisitor
 	override void visit (LabeledStatement labeledStatement)
 	{
 		output.writeln("<labeledStatement label=\"",
-			labeledStatement.identifier.value ,"\">");
+			labeledStatement.identifier.text ,"\">");
 		visit(labeledStatement.declarationOrStatement);
 		output.writeln("</labeledStatement>");
 	}
@@ -784,9 +789,9 @@ class XMLPrinter : ASTVisitor
 	override void visit(LambdaExpression lambdaExpression)
 	{
 		output.writeln("<lambdaExpression>");
-		if (lambdaExpression.functionType == TokenType.function_)
+		if (lambdaExpression.functionType == tok!"function")
 			output.writeln("<function/>");
-		if (lambdaExpression.functionType == TokenType.delegate_)
+		if (lambdaExpression.functionType == tok!"delegate")
 			output.writeln("<delegate/>");
 		lambdaExpression.accept(this);
 		output.writeln("</lambdaExpression>");
@@ -803,14 +808,14 @@ class XMLPrinter : ASTVisitor
 			output.writeln("<linkageAttribute linkage=\"c++\"/>");
 		else
 			output.writeln("<linkageAttribute linkage=\"",
-				linkageAttribute.identifier.value, "\"/>");
+				linkageAttribute.identifier.text, "\"/>");
 	}
 
 	override void visit(MemberFunctionAttribute memberFunctionAttribute)
 	{
 		output.writeln("<memberFunctionAttribute>");
 		if (memberFunctionAttribute.atAttribute is null)
-			output.writeln(getTokenValue(memberFunctionAttribute.tokenType));
+			output.writeln(str(memberFunctionAttribute.tokenType));
 		else
 			memberFunctionAttribute.accept(this);
 		output.writeln("</memberFunctionAttribute>");
@@ -851,7 +856,7 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(MulExpression mulExpression)
 	{
-		output.writeln("<mulExpression operator=\"", getTokenValue(mulExpression.operator) ,"\">");
+		output.writeln("<mulExpression operator=\"", str(mulExpression.operator) ,"\">");
 		output.writeln("<left>");
 		mulExpression.left.accept(this);
 		output.writeln("</left>");
@@ -902,11 +907,11 @@ class XMLPrinter : ASTVisitor
 	override void visit(Parameter param)
 	{
 		output.writeln("<parameter>");
-		if (param.name.type == TokenType.identifier)
-			output.writeln("<name>", param.name.value, "</name>");
+		if (param.name.type == tok!"identifier")
+			output.writeln("<name>", param.name.text, "</name>");
 		foreach (attribute; param.parameterAttributes)
 		{
-			output.writeln("<parameterAttribute>", getTokenValue(attribute), "</parameterAttribute>");
+			output.writeln("<parameterAttribute>", str(attribute), "</parameterAttribute>");
 		}
 		param.accept(this);
 		if (param.vararg)
@@ -927,7 +932,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(PostIncDecExpression postIncDecExpression)
 	{
 		output.writeln("<postIncDecExpression operator=\"",
-			getTokenValue(postIncDecExpression.operator), "\">");
+			str(postIncDecExpression.operator), "\">");
 		postIncDecExpression.accept(this);
 		output.writeln("</postIncDecExpression>");
 	}
@@ -960,7 +965,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(PreIncDecExpression preIncDecExpression)
 	{
 		output.writeln("<preIncDecExpression operator=\"",
-			getTokenValue(preIncDecExpression.operator), "\">");
+			str(preIncDecExpression.operator), "\">");
 		preIncDecExpression.accept(this);
 		output.writeln("</preIncDecExpression>");
 	}
@@ -975,7 +980,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(RelExpression relExpression)
 	{
 		output.writeln("<relExpression operator=\"",
-			xmlEscape(getTokenValue(relExpression.operator)), "\">");
+			xmlEscape(str(relExpression.operator)), "\">");
 		output.writeln("<left>");
 		visit(relExpression.left);
 		output.writeln("</left>");
@@ -1015,7 +1020,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(ShiftExpression shiftExpression)
 	{
 		output.writeln("<shiftExpression operator=\"",
-			xmlEscape(getTokenValue(shiftExpression.operator)), "\">");
+			xmlEscape(str(shiftExpression.operator)), "\">");
 		output.writeln("<left>");
 		visit(shiftExpression.left);
 		output.writeln("</left>");
@@ -1027,10 +1032,10 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(SingleImport singleImport)
 	{
-		if (singleImport.rename.type == TokenType.invalid)
+		if (singleImport.rename.type == tok!"")
 			output.writeln("<singleImport>");
 		else
-			output.writeln("<singleImport rename=\"", singleImport.rename.value, "\">");
+			output.writeln("<singleImport rename=\"", singleImport.rename.text, "\">");
 		visit(singleImport.identifierChain);
 		output.writeln("</singleImport>");
 	}
@@ -1095,7 +1100,7 @@ class XMLPrinter : ASTVisitor
 	override void visit(StructDeclaration structDec)
 	{
 		output.writeln("<structDeclaration line=\"", structDec.name.line, "\">");
-		output.writeln("<name>", structDec.name.value, "</name>");
+		output.writeln("<name>", structDec.name.text, "</name>");
 		structDec.accept(this);
 		output.writeln("</structDeclaration>");
 	}
@@ -1194,10 +1199,10 @@ class XMLPrinter : ASTVisitor
 			output.writeln("</templateDeclaration>");
 			return;
 		}
-
+        writeDdoc(templateDeclaration.comment);
 		output.writeln("<templateDeclaration line=\"",
 			templateDeclaration.name.line, "\">");
-		output.writeln("<name>", templateDeclaration.name.value, "</name>");
+		output.writeln("<name>", templateDeclaration.name.text, "</name>");
 		visit(templateDeclaration.templateParameters);
 		if (templateDeclaration.constraint !is null)
 			visit(templateDeclaration.constraint);
@@ -1276,28 +1281,28 @@ class XMLPrinter : ASTVisitor
 	override void visit(Token token)
 	{
 		string tagName;
-		with (TokenType) switch (token.type)
+		switch (token.type)
 		{
-		case invalid: return;
-		case identifier: tagName = "identifier"; break;
-		case doubleLiteral: tagName = "doubleLiteral"; break;
-		case idoubleLiteral: tagName = "idoubleLiteral"; break;
-		case floatLiteral: tagName = "floatLiteral"; break;
-		case ifloatLiteral: tagName = "ifloatLiteral"; break;
-		case intLiteral: tagName = "intLiteral"; break;
-		case uintLiteral: tagName = "uintLiteral"; break;
-		case longLiteral: tagName = "longLiteral"; break;
-		case ulongLiteral: tagName = "ulongLiteral"; break;
-		case realLiteral: tagName = "realLiteral"; break;
-		case irealLiteral: tagName = "irealLiteral"; break;
-		case characterLiteral: tagName = "characterLiteral"; break;
-		case stringLiteral: tagName = "stringLiteral"; break;
-		case dstringLiteral: tagName = "dstringLiteral"; break;
-		case wstringLiteral: tagName = "wstringLiteral"; break;
-		case dollar: output.writeln("<dollar/>"); return;
-		default: output.writeln("<", getTokenValue(token.type), "/>"); return;
+		case tok!"": return;
+		case tok!"identifier": tagName = "identifier"; break;
+		case tok!"doubleLiteral": tagName = "doubleLiteral"; break;
+		case tok!"idoubleLiteral": tagName = "idoubleLiteral"; break;
+		case tok!"floatLiteral": tagName = "floatLiteral"; break;
+		case tok!"ifloatLiteral": tagName = "ifloatLiteral"; break;
+		case tok!"intLiteral": tagName = "intLiteral"; break;
+		case tok!"uintLiteral": tagName = "uintLiteral"; break;
+		case tok!"longLiteral": tagName = "longLiteral"; break;
+		case tok!"ulongLiteral": tagName = "ulongLiteral"; break;
+		case tok!"realLiteral": tagName = "realLiteral"; break;
+		case tok!"irealLiteral": tagName = "irealLiteral"; break;
+		case tok!"characterLiteral": tagName = "characterLiteral"; break;
+		case tok!"stringLiteral": tagName = "stringLiteral"; break;
+		case tok!"dstringLiteral": tagName = "dstringLiteral"; break;
+		case tok!"wstringLiteral": tagName = "wstringLiteral"; break;
+		case tok!"$": output.writeln("<dollar/>"); return;
+		default: output.writeln("<", str(token.type), "/>"); return;
 		}
-		output.writeln("<", tagName, ">", xmlEscape(token.value), "</", tagName, ">");
+		output.writeln("<", tagName, ">", xmlEscape(token.text), "</", tagName, ">");
 	}
 
 	override void visit(TraitsExpression traitsExpression)
@@ -1323,8 +1328,8 @@ class XMLPrinter : ASTVisitor
 	override void visit(Type2 type2)
 	{
 		output.writeln("<type2>");
-		if (type2.builtinType != TokenType.invalid)
-			output.writeln(getTokenValue(type2.builtinType));
+		if (type2.builtinType != tok!"")
+			output.writeln(str(type2.builtinType));
 		else
 			type2.accept(this);
 		output.writeln("</type2>");
@@ -1393,16 +1398,16 @@ class XMLPrinter : ASTVisitor
 	override void visit(UnaryExpression unaryExpression)
 	{
 		output.writeln("<unaryExpression>");
-		if (unaryExpression.prefix != TokenType.invalid)
+		if (unaryExpression.prefix != tok!"")
 		{
-			output.writeln("<prefix>", xmlEscape(unaryExpression.prefix.value),
+			output.writeln("<prefix>", xmlEscape(unaryExpression.prefix.text),
 				"</prefix>");
 			visit(unaryExpression.unaryExpression);
 		}
-		if (unaryExpression.suffix != TokenType.invalid)
+		if (unaryExpression.suffix != tok!"")
 		{
 			visit(unaryExpression.unaryExpression);
-			output.writeln("<suffix>", unaryExpression.suffix.value,
+			output.writeln("<suffix>", unaryExpression.suffix.text,
 				"</suffix>");
 		}
 		else
@@ -1413,8 +1418,8 @@ class XMLPrinter : ASTVisitor
 	override void visit(UnionDeclaration unionDeclaration)
 	{
 		output.writeln("<unionDeclaration line=\"", unionDeclaration.name.line, "\">");
-		if (unionDeclaration.name != TokenType.invalid)
-			output.writeln("<name>", unionDeclaration.name.value, "</name>");
+		if (unionDeclaration.name != tok!"")
+			output.writeln("<name>", unionDeclaration.name.text, "</name>");
 		if (unionDeclaration.templateParameters !is null)
 			visit(unionDeclaration.templateParameters);
 		if (unionDeclaration.constraint !is null)
@@ -1433,7 +1438,10 @@ class XMLPrinter : ASTVisitor
 
 	override void visit(VariableDeclaration variableDeclaration)
 	{
-		mixin (tagAndAccept!"variableDeclaration");
+        output.writeln("<variableDeclaration>");
+        writeDdoc(variableDeclaration.comment);
+        variableDeclaration.accept(this);
+        output.writeln("</variableDeclaration>");
 	}
 
 	override void visit(Vector vector)
@@ -1478,10 +1486,16 @@ class XMLPrinter : ASTVisitor
 
 	alias ASTVisitor.visit visit;
 
-	private string xmlEscape(string s)
+	private static string xmlEscape(string s)
 	{
 		return s.translate(['<' : "&lt;", '>' : "&gt;", '&' : "&amp;"]);
 	}
 
+    private void writeDdoc(string comment)
+    {
+        if (comment is null) return;
+        output.writeln("<ddoc>", xmlEscape(comment), "</ddoc>");
+    }
+
 	File output;
 }
diff --git a/build.bat b/build.bat
index d3cc301..7eb3101 100644
--- a/build.bat
+++ b/build.bat
@@ -1 +1 @@
-dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d outliner.d formatter.d stdx/d/lexer.d stdx/d/parser.d stdx/d/entities.d stdx/d/ast.d -wi -ofdscanner
+dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d outliner.d formatter.d stdx/d/lexer.d stdx/d/parser.d stdx/d/entities.d stdx/d/ast.d stdx/lexer.d -wi -ofdscanner
diff --git a/build.sh b/build.sh
index 225ef97..be987c0 100755
--- a/build.sh
+++ b/build.sh
@@ -1,5 +1,19 @@
 #dmd *.d stdx/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner-dmd
-dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/d/*.d -g -m64 -wi -ofdscanner
-#ldc2 -O3 *.d stdx/d/*.d -of=dscanner-ldc -release -m64
+dmd\
+	main.d\
+	stats.d\
+	imports.d\
+	highlighter.d\
+	ctags.d\
+	astprinter.d\
+	formatter.d\
+	outliner.d\
+	style.d\
+	stdx/*.d\
+	stdx/d/*.d\
+	datapicked/dpick/buffer/*.d\
+	-Idatapicked\
+	-g -m64 -wi -ofdscanner
+#ldc2 main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/*.d stdx/d/*.d -of=dscanner-ldc -m64 -oq
 #ldc2 *.d stdx/d/*.d -of=dscanner -unittest -m64 -g
 #/opt/gdc/bin/gdc -O3 -odscanner-gdc -fno-bounds-check -frelease -m64 *.d stdx/d/*.d
diff --git a/ctags.d b/ctags.d
index d29515d..7000833 100644
--- a/ctags.d
+++ b/ctags.d
@@ -14,7 +14,7 @@ import std.stdio;
 import std.array;
 import std.conv;
 
-void doNothing(string, int, int, string) {}
+void doNothing(string, size_t, size_t, string) {}
 
 void printCtags(File output, string[] fileNames)
 {
@@ -24,9 +24,8 @@ void printCtags(File output, string[] fileNames)
 		File f = File(fileName);
 		auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
 		f.rawRead(bytes);
-		LexerConfig config;
-		auto tokens = byToken(bytes, config);
-		Module m = parseModule(tokens.array(), fileName, &doNothing);
+		auto tokens = byToken(bytes);
+		Module m = parseModule(tokens.array, fileName, &doNothing);
 		auto printer = new CTagsPrinter;
 		printer.fileName = fileName;
 		printer.visit(m);
@@ -41,88 +40,85 @@ void printCtags(File output, string[] fileNames)
 
 class CTagsPrinter : ASTVisitor
 {
-
-	alias ASTVisitor.visit visit;
-
 	override void visit(ClassDeclaration dec)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\tc%s\n".format(dec.name.value, fileName, dec.name.line, context);
+		tagLines ~= "%s\t%s\t%d;\"\tc%s\n".format(dec.name.text, fileName, dec.name.line, context);
 		auto c = context;
-		context = "\tclass:" ~ dec.name.value;
+		context = "\tclass:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(StructDeclaration dec)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\ts%s\n".format(dec.name.value, fileName, dec.name.line, context);
+		tagLines ~= "%s\t%s\t%d;\"\ts%s\n".format(dec.name.text, fileName, dec.name.line, context);
 		auto c = context;
-		context = "\tstruct:" ~ dec.name.value;
+		context = "\tstruct:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(InterfaceDeclaration dec)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\ti%s\n".format(dec.name.value, fileName, dec.name.line, context);
+		tagLines ~= "%s\t%s\t%d;\"\ti%s\n".format(dec.name.text, fileName, dec.name.line, context);
 		auto c = context;
-		context = "\tclass:" ~ dec.name.value;
+		context = "\tclass:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(TemplateDeclaration dec)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.value, fileName, dec.name.line, context);
+		tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.text, fileName, dec.name.line, context);
 		auto c = context;
-		context = "\ttemplate:" ~ dec.name.value;
+		context = "\ttemplate:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(FunctionDeclaration dec)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d%s\n".format(dec.name.value, fileName,
+		tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d%s\n".format(dec.name.text, fileName,
 			dec.name.line, dec.parameters.parameters.length, context);
 		auto c = context;
-		context = "\tfunction:" ~ dec.name.value;
+		context = "\tfunction:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(EnumDeclaration dec)
 	{
-		if (dec.name == TokenType.invalid)
+		if (dec.name == tok!"")
 		{
 			dec.accept(this);
 			return;
 		}
-		tagLines ~= "%s\t%s\t%d;\"\tg%s\n".format(dec.name.value, fileName,
+		tagLines ~= "%s\t%s\t%d;\"\tg%s\n".format(dec.name.text, fileName,
 			dec.name.line, context);
 		auto c = context;
-		context = "\tenum:" ~ dec.name.value;
+		context = "\tenum:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(UnionDeclaration dec)
 	{
-		if (dec.name == TokenType.invalid)
+		if (dec.name == tok!"")
 		{
 			dec.accept(this);
 			return;
 		}
-		tagLines ~= "%s\t%s\t%d;\"\tu%s\n".format(dec.name.value, fileName,
+		tagLines ~= "%s\t%s\t%d;\"\tu%s\n".format(dec.name.text, fileName,
 			dec.name.line, context);
 		auto c = context;
-		context = "\tunion:" ~ dec.name.value;
+		context = "\tunion:" ~ dec.name.text;
 		dec.accept(this);
 		context = c;
 	}
 
 	override void visit(EnumMember mem)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\te%s\n".format(mem.name.value, fileName,
+		tagLines ~= "%s\t%s\t%d;\"\te%s\n".format(mem.name.text, fileName,
 			mem.name.line, context);
 	}
 
@@ -130,11 +126,13 @@ class CTagsPrinter : ASTVisitor
 	{
 		foreach (d; dec.declarators)
 		{
-			tagLines ~= "%s\t%s\t%d;\"\tv%s\n".format(d.name.value, fileName,
+			tagLines ~= "%s\t%s\t%d;\"\tv%s\n".format(d.name.text, fileName,
 				d.name.line, context);
 		}
 		dec.accept(this);
 	}
+	
+	alias ASTVisitor.visit visit;
 
 	string fileName;
 	string[] tagLines;
diff --git a/datapicked b/datapicked
new file mode 160000
index 0000000..f63a843
--- /dev/null
+++ b/datapicked
@@ -0,0 +1 @@
+Subproject commit f63a843e9c0ce8db7fd897684fe323697255d87d
diff --git a/dscanner.exe b/dscanner.exe
new file mode 100644
index 0000000..df8ccab
Binary files /dev/null and b/dscanner.exe differ
diff --git a/dscanner.obj b/dscanner.obj
new file mode 100644
index 0000000..aeaff27
Binary files /dev/null and b/dscanner.obj differ
diff --git a/formatter.d b/formatter.d
index ff1224e..b98ac75 100644
--- a/formatter.d
+++ b/formatter.d
@@ -70,7 +70,7 @@ class Formatter(Sink)
 		if (addExpression.right is null)
 			return;
 		sink.put(" ");
-		sink.put(getTokenValue(addExpression.operator));
+		sink.put(str(addExpression.operator));
 		sink.put(" ");
 		format(addExpression.right);
 	}
@@ -333,8 +333,8 @@ class Formatter(Sink)
 	void format(const GotoStatement gotoStatement)
 	{
 		sink.put("goto ");
-		if (gotoStatement.label != TokenType.invalid)
-			sink.put(gotoStatement.label.value);
+		if (gotoStatement.label != tok!"")
+			sink.put(gotoStatement.label.text);
 		else
 			format(gotoStatement.expression);
 		sink.put(";");
@@ -348,7 +348,7 @@ class Formatter(Sink)
 			if (!first)
 				sink.put(".");
 			first = false;
-			sink.put(ident.value);
+			sink.put(ident.text);
 		}
 	}
 
@@ -372,7 +372,7 @@ class Formatter(Sink)
 		if (identifierOrTemplateInstance.templateInstance !is null)
 			format(identifierOrTemplateInstance.templateInstance);
 		else
-			sink.put(identifierOrTemplateInstance.identifier.value);
+			sink.put(identifierOrTemplateInstance.identifier.text);
 
 	}
 
@@ -488,14 +488,14 @@ class Formatter(Sink)
 	{
 		foreach (attribute; parameter.parameterAttributes)
 		{
-			sink.put(getTokenValue(attribute));
+			sink.put(str(attribute));
 		}
 		if (parameter.type !is null)
 			format(parameter.type);
-		if (parameter.name.type != TokenType.invalid)
+		if (parameter.name.type != tok!"")
 		{
 			sink.put(" ");
-			sink.put(parameter.name.value);
+			sink.put(parameter.name.text);
 		}
 		if (parameter.vararg)
 			sink.put(" ...");
@@ -607,7 +607,7 @@ class Formatter(Sink)
 
 	void format(const Symbol symbol)
 	{
-        if (symbol.dot != TokenType.invalid)
+        if (symbol.dot != tok!"")
             sink.put(".");
 		format(symbol.identifierOrTemplateChain);
 	}
@@ -671,7 +671,7 @@ class Formatter(Sink)
 
 	void format(const Token token)
 	{
-		sink.put(token.value);
+		sink.put(token.text);
 	}
 
 	void format(const TraitsExpression traitsExpression)
@@ -688,7 +688,7 @@ class Formatter(Sink)
             if (first)
                 sink.put(" ");
             first = false;
-            sink.put(getTokenValue(constructor));
+            sink.put(str(constructor));
         }
         if (type.typeConstructors.length > 0)
             sink.put(" ");
@@ -711,16 +711,16 @@ class Formatter(Sink)
             format(type2.typeofExpression);
 			return;
         }
-        else if (type2.typeConstructor != TokenType.invalid)
+        else if (type2.typeConstructor != tok!"")
         {
-			sink.put(getTokenValue(type2.typeConstructor));
+			sink.put(str(type2.typeConstructor));
 			sink.put("(");
 			format(type2.type);
 			sink.put(")");
 			return;
         }
         else
-            sink.put(getTokenValue(type2.builtinType));
+            sink.put(str(type2.builtinType));
 	}
 
 	void format(const TypeSpecialization typeSpecialization)
diff --git a/highlighter.d b/highlighter.d
index 843a2bc..3b24e2f 100644
--- a/highlighter.d
+++ b/highlighter.d
@@ -11,7 +11,7 @@ import std.array;
 import stdx.d.lexer;
 
 // http://ethanschoonover.com/solarized
-void highlight(R)(TokenRange!R tokens, string fileName)
+void highlight(R)(ref R tokens, string fileName)
 {
 	stdout.writeln(q"[
 <!DOCTYPE html>
@@ -33,30 +33,33 @@ html  { background-color: #fdf6e3; color: #002b36; }
 </style>
 <pre>]");
 
-	foreach (Token t; tokens)
+	while (!tokens.empty)
 	{
+		auto t = tokens.front;
+		tokens.popFront();
 		if (isBasicType(t.type))
-			writeSpan("type", t.value);
+			writeSpan("type", str(t.type));
 		else if (isKeyword(t.type))
-			writeSpan("kwrd", t.value);
-		else if (t.type == TokenType.comment)
-			writeSpan("com", t.value);
-		else if (isStringLiteral(t.type) || t.type == TokenType.characterLiteral)
-			writeSpan("str", t.value);
+			writeSpan("kwrd", str(t.type));
+		else if (t.type == tok!"comment")
+			writeSpan("com", t.text);
+		else if (isStringLiteral(t.type) || t.type == tok!"characterLiteral")
+			writeSpan("str", t.text);
 		else if (isNumberLiteral(t.type))
-			writeSpan("num", t.value);
+			writeSpan("num", t.text);
 		else if (isOperator(t.type))
-			writeSpan("op", t.value);
+			writeSpan("op", str(t.type));
 		else
 		{
 			version(Windows)
 			{
-				// Stupid Windows automatically does a LF → CRLF, so CRLF → CRCRLF, which is obviously wrong.
+				// Stupid Windows automatically does a LF → CRLF, so
+				// CRLF → CRCRLF, which is obviously wrong.
 				// Strip out the CR characters here to avoid this.
-				stdout.write(t.value.replace("<", "&lt;").replace("\r", ""));
+				stdout.write(t.text.replace("<", "&lt;").replace("\r", ""));
 			}
 			else
-				stdout.write(t.value.replace("<", "&lt;"));
+				stdout.write(t.text.replace("<", "&lt;"));
 		}
 
 	}
@@ -70,13 +73,3 @@ void writeSpan(string cssClass, string value)
 	else
 		stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`);
 }
-
-/+void main(string[] args)
-{
-	LexerConfig config;
-	config.tokenStyle = TokenStyle.source;
-	config.iterStyle = IterationStyle.everything;
-	config.fileName = args[1];
-	auto f = File(args[1]);
-	(cast(ubyte[]) f.byLine(KeepTerminator.yes).join()).byToken(config).highlight();
-}+/
diff --git a/imports.d b/imports.d
index 6fef4aa..772a297 100644
--- a/imports.d
+++ b/imports.d
@@ -26,7 +26,7 @@ class ImportPrinter : ASTVisitor
 		{
 			if (!first)
 				write(".");
-			write(ident.value);
+			write(ident.text);
 			first = false;
 		}
 	}
diff --git a/main.d b/main.d
index c692c93..a999515 100644
--- a/main.d
+++ b/main.d
@@ -17,6 +17,7 @@ import std.stdio;
 import std.range;
 import stdx.d.lexer;
 import stdx.d.parser;
+import dpick.buffer.buffer;
 
 import highlighter;
 import stats;
@@ -24,6 +25,7 @@ import ctags;
 import astprinter;
 import imports;
 import outliner;
+import style;
 
 int main(string[] args)
 {
@@ -39,6 +41,8 @@ int main(string[] args)
 	bool imports;
 	bool muffin;
 	bool outline;
+	bool tokenDump;
+	bool styleCheck;
 
 	try
 	{
@@ -46,6 +50,7 @@ int main(string[] args)
 			"ctags|c", &ctags, "recursive|r|R", &recursive, "help|h", &help,
 			"tokenCount|t", &tokenCount, "syntaxCheck|s", &syntaxCheck,
 			"ast|xml", &ast, "imports|i", &imports, "outline|o", &outline,
+			"tokenDump", &tokenDump, "styleCheck", &styleCheck,
 			"muffinButton", &muffin);
 	}
 	catch (Exception e)
@@ -75,7 +80,7 @@ int main(string[] args)
 	}
 
 	auto optionCount = count!"a"([sloc, highlight, ctags, tokenCount,
-		syntaxCheck, ast, imports, outline]);
+		syntaxCheck, ast, imports, outline, tokenDump, styleCheck]);
 	if (optionCount > 1)
 	{
 		stderr.writeln("Too many options specified");
@@ -89,27 +94,51 @@ int main(string[] args)
 
 	if (highlight)
 	{
-		LexerConfig config;
-		config.iterStyle = IterationStyle.everything;
-		config.tokenStyle = TokenStyle.source;
 		bool usingStdin = args.length == 1;
 		ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
-		highlighter.highlight(byToken(bytes, config),
-			args.length == 1 ? "stdin" : args[1]);
+        LexerConfig config;
+        config.whitespaceBehavior = WhitespaceBehavior.include;
+        config.stringBehavior = StringBehavior.source;
+        config.commentBehavior = CommentBehavior.include;
+		auto tokens = byToken(bytes, config);
+		highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]);
 		return 0;
 	}
+	else if (tokenDump)
+	{
+		bool usingStdin = args.length == 1;
+		ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
+		LexerConfig config;
+        config.whitespaceBehavior = WhitespaceBehavior.skip;
+        config.stringBehavior = StringBehavior.source;
+        config.commentBehavior = CommentBehavior.attach;
+		auto tokens = byToken(bytes, config);
+		foreach (ref token; tokens)
+		{
+			writeln("«", token.text is null ? str(token.type) : token.text,
+				" ", token.index, " ", token.line, " ", token.column, " ",
+                token.comment, "»");
+		}
+	}
 	else if (ctags)
 	{
 		stdout.printCtags(expandArgs(args, recursive));
 	}
+	else if (styleCheck)
+	{
+		stdout.styleCheck(expandArgs(args, recursive));
+	}
 	else
 	{
-		LexerConfig config;
 		bool usingStdin = args.length == 1;
 		if (sloc || tokenCount)
 		{
 			if (usingStdin)
 			{
+                LexerConfig config;
+                config.whitespaceBehavior = WhitespaceBehavior.include;
+                config.stringBehavior = StringBehavior.source;
+                config.commentBehavior = CommentBehavior.include;
 				auto tokens = byToken(readStdin(), config);
 				if (tokenCount)
 					printTokenCount(stdout, "stdin", tokens);
@@ -121,7 +150,7 @@ int main(string[] args)
 				ulong count;
 				foreach (f; expandArgs(args, recursive))
 				{
-					auto tokens = byToken(readFile(f), config);
+					auto tokens = byToken!(ubyte[])(readFile(f));
 					if (tokenCount)
 						count += printTokenCount(stdout, f, tokens);
 					else
@@ -132,48 +161,28 @@ int main(string[] args)
 		}
 		else if (syntaxCheck)
 		{
-			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
-				config);
-			if (usingStdin)
-				config.fileName = "stdin";
-			else
-				config.fileName = args[1];
-			parseModule(tokens.array(), config.fileName);
+			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
+			parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
 		}
 		else if (imports)
 		{
-			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
-				config);
-			if (usingStdin)
-				config.fileName = "stdin";
-			else
-				config.fileName = args[1];
-			auto mod = parseModule(tokens.array(), config.fileName);
+			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
+			auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
 			auto visitor = new ImportPrinter;
 			visitor.visit(mod);
 		}
 		else if (ast)
 		{
-			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
-				config);
-			if (usingStdin)
-				config.fileName = "stdin";
-			else
-				config.fileName = args[1];
-			auto mod = parseModule(tokens.array(), config.fileName);
+			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
+			auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
 			auto printer = new XMLPrinter;
 			printer.output = stdout;
 			printer.visit(mod);
 		}
 		else if (outline)
 		{
-			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
-				config);
-			if (usingStdin)
-				config.fileName = "stdin";
-			else
-				config.fileName = args[1];
-			auto mod = parseModule(tokens.array(), config.fileName);
+			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
+			auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
 			auto outliner = new Outliner(stdout);
 			outliner.visit(mod);
 		}
@@ -245,7 +254,7 @@ options:
         Prints the number of logical lines of code in the given
         source files. If no files are specified, input is read from stdin.
 
-    --tokenCount | t [sourceFiles]
+    --tokenCount | -t [sourceFiles]
         Prints the number of tokens in the given source files. If no files are
         specified, input is read from stdin.
 
@@ -262,6 +271,10 @@ options:
         syntax errors to stdout. One error or warning is printed per line.
         If no files are specified, input is read from stdin.
 
+    --styleCheck [sourceFiles]
+        Lexes and parses sourceFiles, printing the line and column number of any
+        style guideline violations to stdout.
+
     --ctags | -c sourceFile
         Generates ctags information from the given source code file. Note that
         ctags information requires a filename, so stdin cannot be used in place
diff --git a/outliner.d b/outliner.d
index 46647a1..9f3db55 100644
--- a/outliner.d
+++ b/outliner.d
@@ -21,7 +21,7 @@ class Outliner : ASTVisitor
 	override void visit(ClassDeclaration classDec)
 	{
 		printIndentation();
-		output.writeln("class ", classDec.name.value, " : ", classDec.name.line);
+		output.writeln("class ", classDec.name.text, " : ", classDec.name.line);
 		indent();
 		classDec.accept(this);
 		outdent();
@@ -31,7 +31,7 @@ class Outliner : ASTVisitor
 	override void visit(EnumDeclaration enumDec)
 	{
 		printIndentation();
-		output.writeln("enum ", enumDec.name.value, " : ", enumDec.name.line);
+		output.writeln("enum ", enumDec.name.text, " : ", enumDec.name.line);
 		indent();
 		enumDec.accept(this);
 		outdent();
@@ -41,7 +41,7 @@ class Outliner : ASTVisitor
 	override void visit(EnumMember enumMem)
 	{
 		printIndentation();
-		output.writeln(enumMem.name.value, " : ", enumMem.name.line);
+		output.writeln(enumMem.name.text, " : ", enumMem.name.line);
 		finish();
 	}
 
@@ -57,7 +57,7 @@ class Outliner : ASTVisitor
 		if (functionDec.returnType !is null)
 			f.format(functionDec.returnType);
 		app.put(" ");
-		app.put(functionDec.name.value);
+		app.put(functionDec.name.text);
 		f.format(functionDec.parameters);
 		app.put(" : ");
 		app.put(to!string(functionDec.name.line));
@@ -68,7 +68,7 @@ class Outliner : ASTVisitor
 	override void visit(InterfaceDeclaration interfaceDec)
 	{
 		printIndentation();
-		output.writeln("interface ", interfaceDec.name.value, " : ",
+		output.writeln("interface ", interfaceDec.name.text, " : ",
 			interfaceDec.name.line);
 		indent();
 		interfaceDec.accept(this);
@@ -79,7 +79,7 @@ class Outliner : ASTVisitor
 	override void visit(StructDeclaration structDec)
 	{
 		printIndentation();
-		output.writeln("struct ", structDec.name.value, " : ",
+		output.writeln("struct ", structDec.name.text, " : ",
 			structDec.name.line);
 		indent();
 		structDec.accept(this);
@@ -90,7 +90,7 @@ class Outliner : ASTVisitor
 	override void visit(TemplateDeclaration templateDeclaration)
 	{
 		printIndentation();
-		output.writeln("template", templateDeclaration.name.value, " : ",
+		output.writeln("template", templateDeclaration.name.text, " : ",
 			templateDeclaration.name.line);
 		finish();
 	}
@@ -105,7 +105,7 @@ class Outliner : ASTVisitor
 	override void visit(UnionDeclaration unionDeclaration)
 	{
 		printIndentation();
-		output.writeln("union ", unionDeclaration.name.value, " : ",
+		output.writeln("union ", unionDeclaration.name.text, " : ",
 			unionDeclaration.name.line);
 		indent();
 		unionDeclaration.accept(this);
@@ -125,7 +125,7 @@ class Outliner : ASTVisitor
 				f.format(variableDeclaration.type);
 			}
 			app.put(" ");
-			app.put(d.name.value);
+			app.put(d.name.text);
 			app.put(" : ");
 			app.put(to!string(d.name.line));
 			output.writeln(app.data);
diff --git a/stats.d b/stats.d
index 78dfd16..ee55ccb 100644
--- a/stats.d
+++ b/stats.d
@@ -9,21 +9,21 @@ import std.stdio;
 import std.algorithm;
 import stdx.d.lexer;
 
-pure nothrow bool isLineOfCode(TokenType t)
+pure nothrow bool isLineOfCode(IdType t)
 {
-	with (TokenType) switch(t)
+	switch(t)
 	{
-	case semicolon:
-	case while_:
-	case if_:
-	case do_:
-	case else_:
-	case switch_:
-	case for_:
-	case foreach_:
-	case foreach_reverse_:
-	case default_:
-	case case_:
+	case tok!";":
+	case tok!"while":
+	case tok!"if":
+	case tok!"do":
+	case tok!"else":
+	case tok!"switch":
+	case tok!"for":
+	case tok!"foreach":
+	case tok!"foreach_reverse":
+	case tok!"default":
+	case tok!"case":
 		return true;
 	default:
 		return false;
diff --git a/stdx/d/ast.d b/stdx/d/ast.d
index 1437808..dd9f3e3 100644
--- a/stdx/d/ast.d
+++ b/stdx/d/ast.d
@@ -275,7 +275,7 @@ public:
     {
         mixin (visitIfNotNull!(left, right));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -290,6 +290,7 @@ public:
     /** */ Type type;
     /** */ Token name;
     /** */ AliasInitializer[] initializers;
+    /** */ string comment;
 }
 
 ///
@@ -410,7 +411,7 @@ class AsmAddExp : ExpressionNode
 {
 public:
     mixin (DEFAULT_ACCEPT);
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -483,7 +484,7 @@ class AsmMulExp : ExpressionNode
 {
 public:
     mixin (DEFAULT_ACCEPT);
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 
 }
@@ -583,7 +584,7 @@ public:
     }
     /** */ ExpressionNode ternaryExpression;
     /** */ ExpressionNode assignExpression;
-    /** */ TokenType operator;
+    /** */ IdType operator;
 }
 
 ///
@@ -623,7 +624,7 @@ public:
     /** */ AlignAttribute alignAttribute;
     /** */ PragmaExpression pragmaExpression;
     /** */ StorageClass storageClass;
-    /** */ TokenType attribute;
+    /** */ IdType attribute;
 }
 
 ///
@@ -808,6 +809,7 @@ public:
     /** */ Constraint constraint;
     /** */ BaseClassList baseClassList;
     /** */ StructBody structBody;
+    /** */ string comment;
 }
 
 ///
@@ -891,6 +893,7 @@ public:
     /** */ MemberFunctionAttribute[] memberFunctionAttributes;
     /** */ TemplateParameters templateParameters;
     /** */ size_t location;
+    /** */ string comment;
 }
 
 ///
@@ -943,7 +946,7 @@ public:
             destructor, staticConstructor, staticDestructor,
             sharedStaticDestructor, sharedStaticConstructor,
             conditionalDeclaration, pragmaDeclaration, versionSpecification,
-            declarations));
+			invariant_, postblit, declarations));
     }
 
     /** */ Attribute[] attributes;
@@ -1067,6 +1070,7 @@ public:
     }
     /** */ FunctionBody functionBody;
     /** */ size_t location;
+    /** */ string comment;
 }
 
 ///
@@ -1113,6 +1117,7 @@ public:
     /** */ Token name;
     /** */ Type type;
     /** */ EnumBody enumBody;
+    /** */ string comment;
 }
 
 ///
@@ -1126,6 +1131,7 @@ public:
     /** */ Token name;
     /** */ Type type;
     /** */ AssignExpression assignExpression;
+    /** */ string comment;
 }
 
 ///
@@ -1149,7 +1155,7 @@ public:
     {
         mixin (visitIfNotNull!(left, right));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -1222,7 +1228,7 @@ public:
         mixin (visitIfNotNull!(foreachType, foreachTypeList, low, high,
             declarationOrStatement));
     }
-    /** */ TokenType type;
+    /** */ IdType type;
     /** */ ForeachTypeList foreachTypeList;
 	/** */ ForeachType foreachType;
     /** */ Expression low;
@@ -1239,7 +1245,7 @@ public:
     {
         mixin (visitIfNotNull!(type, identifier));
     }
-    /** */ TokenType[] typeConstructors;
+    /** */ IdType[] typeConstructors;
     /** */ Type type;
     /** */ Token identifier;
 }
@@ -1325,6 +1331,7 @@ public:
     /** */ Constraint constraint;
     /** */ FunctionBody functionBody;
     /** */ MemberFunctionAttribute[] memberFunctionAttributes;
+    /** */ string comment;
 }
 
 ///
@@ -1336,7 +1343,7 @@ public:
         mixin (visitIfNotNull!(type, parameters, functionAttributes,
             functionBody));
     }
-    /** */ TokenType functionOrDelegate;
+    /** */ IdType functionOrDelegate;
     /** */ Type type;
     /** */ Parameters parameters;
     /** */ FunctionAttribute[] functionAttributes;
@@ -1549,6 +1556,7 @@ public:
     /** */ Constraint constraint;
     /** */ BaseClassList baseClassList;
     /** */ StructBody structBody;
+    /** */ string comment;
 }
 
 ///
@@ -1560,6 +1568,7 @@ public:
         mixin (visitIfNotNull!(blockStatement));
     }
     /** */ BlockStatement blockStatement;
+    /** */ string comment;
 }
 
 ///
@@ -1575,7 +1584,7 @@ public:
     /** */ Token identifier;
     /** */ TypeSpecialization typeSpecialization;
     /** */ TemplateParameterList templateParameterList;
-    /** */ TokenType equalsOrColon;
+    /** */ IdType equalsOrColon;
 }
 
 ///
@@ -1622,7 +1631,7 @@ public:
         mixin (visitIfNotNull!(identifier, parameters, functionAttributes,
             assignExpression));
     }
-    /** */ TokenType functionType;
+    /** */ IdType functionType;
     /** */ Token identifier;
     /** */ Parameters parameters;
     /** */ FunctionAttribute[] functionAttributes;
@@ -1660,7 +1669,7 @@ public:
     {
         mixin (visitIfNotNull!(atAttribute));
     }
-    /** */ TokenType tokenType;
+    /** */ IdType tokenType;
     /** */ AtAttribute atAttribute;
 }
 
@@ -1743,7 +1752,7 @@ public:
     {
         mixin (visitIfNotNull!(left, right));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -1893,7 +1902,7 @@ public:
         mixin (visitIfNotNull!(type, name, default_));
     }
 
-    /** */ TokenType[] parameterAttributes;
+    /** */ IdType[] parameterAttributes;
     /** */ Type type;
     /** */ Token name;
     /** */ bool vararg;
@@ -1932,7 +1941,7 @@ public:
     {
         mixin (visitIfNotNull!(unaryExpression));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     /** */ UnaryExpression unaryExpression;
 }
 
@@ -1978,7 +1987,7 @@ public:
     {
         mixin (visitIfNotNull!(unaryExpression));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     /** */ UnaryExpression unaryExpression;
 }
 
@@ -2030,7 +2039,7 @@ public:
     {
         mixin (visitIfNotNull!(left, right));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -2067,6 +2076,7 @@ public:
     }
     /** */ FunctionBody functionBody;
     /** */ size_t location;
+    /** */ string comment;
 }
 
 ///
@@ -2079,6 +2089,7 @@ public:
     }
     /** */ FunctionBody functionBody;
     /** */ size_t location;
+    /** */ string comment;
 }
 
 ///
@@ -2089,7 +2100,7 @@ public:
     {
         mixin (visitIfNotNull!(left, right));
     }
-    /** */ TokenType operator;
+    /** */ IdType operator;
     mixin BinaryExpressionBody;
 }
 
@@ -2236,6 +2247,7 @@ public:
     /** */ TemplateParameters templateParameters;
     /** */ Constraint constraint;
     /** */ StructBody structBody;
+    /** */ string comment;
 }
 
 ///
@@ -2376,6 +2388,7 @@ public:
     /** */ Constraint constraint;
     /** */ Declaration[] declarations;
     /** */ EponymousTemplateDeclaration eponymousTemplateDeclaration;
+    /** */ string comment;
 }
 
 ///
@@ -2573,7 +2586,7 @@ public:
         mixin (visitIfNotNull!(type2, typeSuffixes));
     }
 
-    /** */ TokenType[] typeConstructors;
+    /** */ IdType[] typeConstructors;
     /** */ TypeSuffix[] typeSuffixes;
     /** */ Type2 type2;
 }
@@ -2588,11 +2601,11 @@ public:
             identifierOrTemplateChain, type));
     }
 
-    /** */ TokenType builtinType;
+    /** */ IdType builtinType;
     /** */ Symbol symbol;
     /** */ TypeofExpression typeofExpression;
     /** */ IdentifierOrTemplateChain identifierOrTemplateChain;
-    /** */ TokenType typeConstructor;
+    /** */ IdType typeConstructor;
     /** */ Type type;
 }
 
@@ -2694,6 +2707,7 @@ public:
     /** */ TemplateParameters templateParameters;
     /** */ Constraint constraint;
     /** */ StructBody structBody;
+    /** */ string comment;
 }
 
 ///
@@ -2705,6 +2719,7 @@ public:
         mixin (visitIfNotNull!(blockStatement));
     }
     /** */ BlockStatement blockStatement;
+    /** */ string comment;
 }
 
 ///
@@ -2719,6 +2734,7 @@ public:
     /** */ Declarator[] declarators;
     /** */ StorageClass storageClass;
     /** */ AutoDeclaration autoDeclaration;
+    /** */ string comment;
 }
 
 ///
diff --git a/stdx/d/entities.d b/stdx/d/entities.d
index dd2cd9e..7b967d7 100644
--- a/stdx/d/entities.d
+++ b/stdx/d/entities.d
@@ -6,7 +6,7 @@
  * Copyright: Brian Schott 2013
  * License: <a href="http://www.boost.org/LICENSE_1_0.txt">Boost License 1.0</a>.
  * Authors: Brian Schott
- * Source: $(PHOBOSSRC std/d/_lexer.d)
+ * Source: $(PHOBOSSRC std/d/_entities.d)
  */
 
 module stdx.d.entities;
@@ -18,7 +18,7 @@ module stdx.d.entities;
 struct HtmlEntity
 {
     string name, value;
-} 
+}
 
 immutable HtmlEntity[] characterEntities = [
         HtmlEntity("AElig", "\u00C6"),
diff --git a/stdx/d/lexer.d b/stdx/d/lexer.d
index 9b90c7a..76e27b3 100644
--- a/stdx/d/lexer.d
+++ b/stdx/d/lexer.d
@@ -1,3618 +1,1418 @@
-// Written in the D programming language
-
-/**
- * This module contains a range-based _lexer for the D programming language.
- *
- * For performance reasons the _lexer contained in this module operates only on
- * UTF-8 encoded source code. If the use of other encodings is
- * desired, the source code must be converted to UTF-8 before passing it to this
- * _lexer.
- *
- * To use the _lexer, create a $(LREF LexerConfig) struct. The
- * $(LREF LexerConfig) contains fields for configuring the behavior of the
- * lexer.
- * ---
- * LexerConfig config;
- * config.iterStyle = IterationStyle.everything;
- * config.tokenStyle = TokenStyle.source;
- * config.versionNumber = 2064;
- * config.vendorString = "Lexer Example";
- * ---
- * Once you have configured the _lexer, call $(LREF byToken)$(LPAREN)$(RPAREN)
- * on your source code, passing in the configuration.
- * ---
- * // UTF-8 encoded source code
- * auto source = "import std.stdio;"c;
- * auto tokens = byToken(source, config);
- * // or auto tokens = source.byToken(config);
- * ---
- * The result of $(LREF byToken)$(LPAREN)$(RPAREN) is a forward range of tokens
- * that can be easily used with the algorithms from std.algorithm or iterated
- * over with $(D_KEYWORD foreach).
- * ---
- * assert (tokens.front.type == TokenType.import_);
- * assert (tokens.front.value == "import");
- * assert (tokens.front.line == 1);
- * assert (tokens.front.startIndex == 0);
- * ---
- *
- * Examples:
- *
- * Generate HTML markup of D code.
- * ---
- * module highlighter;
- *
- * import std.stdio;
- * import std.array;
- * import std.d.lexer;
- *
- * void writeSpan(string cssClass, string value)
- * {
- *     stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`);
- * }
- *
- * // http://ethanschoonover.com/solarized
- * void highlight(R)(R tokens)
- * {
- *     stdout.writeln(q"[<!DOCTYPE html>
- * <html>
- * <head>
- * <meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
- * </head>
- * <body>
- * <style type="text/css">
- * html  { background-color: #fdf6e3; color: #002b36; }
- * .kwrd { color: #b58900; font-weight: bold;  }
- * .com  { color: #93a1a1; font-style: italic; }
- * .num  { color: #dc322f; font-weigth: bold;  }
- * .str  { color: #2aa198; font-style: italic; }
- * .op   { color: #586e75; font-weight: bold;  }
- * .type { color: #268bd2; font-weight: bold;  }
- * .cons { color: #859900; font-weight: bold;  }
- * </style>
- * <pre>]");
- *
- *     foreach (Token t; tokens)
- *     {
- *         if (isBuiltType(t.type))
- *             writeSpan("type", t.value);
- *         else if (isKeyword(t.type))
- *             writeSpan("kwrd", t.value);
- *         else if (t.type == TokenType.comment)
- *             writeSpan("com", t.value);
- *         else if (isStringLiteral(t.type))
- *             writeSpan("str", t.value);
- *         else if (isNumberLiteral(t.type))
- *             writeSpan("num", t.value);
- *         else if (isOperator(t.type))
- *             writeSpan("op", t.value);
- *         else
- *             stdout.write(t.value.replace("<", "&lt;"));
- *     }
- *     stdout.writeln("</pre>\n</body></html>");
- * }
- *
- * void main(string[] args)
- * {
- *     // Create the configuration
- *     LexerConfig config;
- *     // Specify that we want tokens to appear exactly as they did in the source
- *     config.tokenStyle = TokenStyle.source;
- *     // Include whitespace, comments, etc.
- *     config.iterStyle = IterationStyle.everything;
- *     // Tell the lexer to use the name of the file being read when generating
- *     // error messages.
- *     config.fileName = args[1];
- *     // Open the file (error checking ommitted for brevity)
- *     auto f = File(args[1]);
- *     // Read the lines of the file, and combine them. Then create the token
- *     // range, which is then passed on to highlight.
- *     (cast(ubyte[]) f.byLine(KeepTerminator.yes).join()).byToken(config).highlight();
- * }
- * ---
- *
- * Copyright: Brian Schott 2013
- * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0)
- * Authors: Brian Schott, Dmitry Olshansky
- * Source: $(PHOBOSSRC std/d/_lexer.d)
- */
-
 module stdx.d.lexer;
 
+import std.typecons;
+import std.typetuple;
+import std.array;
 import std.algorithm;
-import std.ascii;
-import std.conv;
-import std.datetime;
-import stdx.d.entities;
-import std.exception;
 import std.range;
-import std.regex;
-import std.string;
-import std.traits;
-import std.utf;
-version (unittest) import std.stdio;
+import stdx.lexer;
 
+private enum staticTokens = [
+	",", ".", "..", "...", "/", "/=", "!", "!<", "!<=", "!<>", "!<>=", "!=",
+	"!>", "!>=", "$", "%", "%=", "&", "&&", "&=", "(", ")", "*", "*=", "+", "++",
+	"+=", "-", "--", "-=", ":", ";", "<", "<<", "<<=", "<=", "<>", "<>=", "=",
+	"==", "=>", ">", ">=", ">>", ">>=", ">>>", ">>>=", "?", "@", "[", "]", "^",
+	"^=", "^^", "^^=", "{", "|", "|=", "||", "}", "~", "~="
+];
 
-public:
+private enum pseudoTokens = [
+	"\"", "`", "//", "/*", "/+", ".", "'", "0", "1", "2", "3", "4", "5", "6",
+	"7", "8", "9", "q\"", "q{", "r\"", "x\"", " ", "\t", "\r", "\n", "#!",
+	"#line", "\u2028", "\u2029"
+];
+
+private enum possibleDefaultTokens = [
+	"abstract", "alias", "align", "asm", "assert", "auto", "body", "bool",
+	"break", "byte", "case", "cast", "catch", "cdouble", "cent", "cfloat",
+	"char", "class", "const", "continue", "creal", "dchar", "debug", "default",
+	"delegate", "delete", "deprecated", "do", "double", "else", "enum",
+	"export", "extern", "false", "final", "finally", "float", "for", "foreach",
+	"foreach_reverse", "function", "goto", "idouble", "if", "ifloat",
+	"immutable", "import", "in", "inout", "int", "interface", "invariant",
+	"ireal", "is", "lazy", "long", "macro", "mixin", "module", "new", "nothrow",
+	"null", "out", "override", "package", "pragma", "private", "protected",
+	"public", "pure", "real", "ref", "return", "scope", "shared", "short",
+	"static", "struct", "super", "switch", "synchronized", "template", "this",
+	"throw", "true", "try", "typedef", "typeid", "typeof", "ubyte", "ucent",
+	"uint", "ulong", "union", "unittest", "ushort", "version", "void",
+	"volatile", "wchar", "while", "with", "__DATE__", "__EOF__", "__FILE__",
+	"__FUNCTION__", "__gshared", "__LINE__", "__MODULE__", "__parameters",
+	"__PRETTY_FUNCTION__", "__TIME__", "__TIMESTAMP__", "__traits", "__vector",
+	"__VENDOR__", "__VERSION__"
+];
+
+private enum dynamicTokens = [
+	"specialTokenSequence", "comment", "identifier", "scriptLine",
+	"whitespace", "doubleLiteral", "floatLiteral", "idoubleLiteral",
+	"ifloatLiteral", "intLiteral", "longLiteral", "realLiteral",
+	"irealLiteral", "uintLiteral", "ulongLiteral", "characterLiteral",
+	"dstringLiteral", "stringLiteral", "wstringLiteral", "scriptLine"
+];
+
+public alias TokenIdType!(staticTokens, dynamicTokens, possibleDefaultTokens) IdType;
+public alias tokenStringRepresentation!(IdType, staticTokens, dynamicTokens, possibleDefaultTokens) str;
+public template tok(string token)
+{
+  alias TokenId!(IdType, staticTokens, dynamicTokens, possibleDefaultTokens, token) tok;
+}
+enum extraFields = q{
+    string comment;
+};
+public alias stdx.lexer.TokenStructure!(IdType, extraFields) Token;
 
 /**
- * Represents a D token
+ * Configure string lexing behavior
  */
-struct Token
+public enum StringBehavior : ubyte
 {
-    /**
-     * The characters that comprise the token.
-     */
-    string value;
-
-    /**
-     * The index of the start of the token in the original source.
-     * $(LPAREN)measured in UTF-8 code units$(RPAREN)
-     */
-    size_t startIndex;
-
-    /**
-     * The number of the line the token is on.
-     */
-    uint line;
-
-    /**
-     * The column number of the start of the token in the original source.
-     * $(LPAREN)measured in ASCII characters or UTF-8 code units$(RPAREN)
-     */
-    ushort column;
-
-    /**
-     * The token type.
-     */
-    TokenType type;
-
-    /**
-     * Check to see if the token is of the same type and has the same string
-     * representation as the given token.
-     */
-    bool opEquals(ref const(Token) other) const nothrow pure
-    {
-        return other.type == type && other.value == value;
-    }
-
-    ///
-    unittest
-    {
-        Token a;
-        a.type = TokenType.intLiteral;
-        a.value = "1";
-        Token b;
-        b.type = TokenType.intLiteral;
-        b.value = "1";
-        assert (a == b);
-        b.value = "2";
-        assert (a != b);
-    }
-
-    /**
-     * Checks to see if the token's string representation is equal to the given
-     * string.
-     */
-    bool opEquals(string value) const nothrow pure
-    {
-        return this.value == value;
-    }
-
-    ///
-    unittest
-    {
-        Token t;
-        t.value = "abcde";
-        assert (t == "abcde");
-    }
-
-    /**
-     * Checks to see if the token is of the given type.
-     */
-    bool opEquals(TokenType type) const nothrow pure
-    {
-        return this.type == type;
-    }
-
-    ///
-    unittest
-    {
-        Token t;
-        t.type = TokenType.class_;
-        assert (t == TokenType.class_);
-    }
-
-    /**
-     * Comparison operator orders tokens by start index.
-     */
-    int opCmp(ref const(Token) other) const nothrow pure
-    {
-        if (startIndex < other.startIndex) return -1;
-        if (startIndex > other.startIndex) return 1;
-        return 0;
-    }
-
-    ///
-    unittest
-    {
-        Token a;
-        a.startIndex = 10;
-        Token b;
-        b.startIndex = 20;
-        assert (a < b);
-    }
-
-    /**
-     * Comparison operator overload for checking if the token's start index is
-     * before, after, or the same as the given index.
-     */
-    int opCmp(size_t index) const nothrow pure
-    {
-        if (startIndex < index) return -1;
-        if (startIndex > index) return 1;
-        return 0;
-    }
-
-    ///
-    unittest
-    {
-        import std.array;
-        import std.range;
-        auto source = cast(ubyte[]) "a b c"c;
-        LexerConfig c;
-        auto tokens = source.byToken(c).array();
-        assert (tokens.length == 3);
-        //assert (tokens.assumeSorted().lowerBound(1)[$ - 1] == "b");
-    }
+    /// Do not include quote characters, process escape sequences
+    compiler = 0b0000_0000,
+    /// Opening quotes, closing quotes, and string suffixes are included in the
+    /// string token
+    includeQuoteChars = 0b0000_0001,
+    /// String escape sequences are not replaced
+    notEscaped = 0b0000_0010,
+    /// Not modified at all. Useful for formatters or highlighters
+    source = includeQuoteChars | notEscaped
 }
 
 /**
- * Configure the behavior of the $(LREF byToken)$(LPAREN)$(RPAREN) function.
- * These flags may be combined using a bitwise or.
+ * Configure whitespace handling behavior
  */
-enum IterationStyle : ushort
+public enum WhitespaceBehavior : ubyte
 {
-    /// Only include code, not whitespace or comments
-    codeOnly = 0,
-    /// Includes comments
-    includeComments = 0b0001,
-    /// Includes whitespace
-    includeWhitespace = 0b0010,
-    /// Include $(LINK2 http://dlang.org/lex.html#specialtokens, special tokens)
-    includeSpecialTokens = 0b0100,
-    /// Do not stop iteration on reaching the $(D_KEYWORD ___EOF__) token
-    ignoreEOF = 0b1000,
-    /// Include _everything
-    everything = includeComments | includeWhitespace | ignoreEOF
+    /// Whitespace is skipped
+    skip,
+    /// Whitespace is treated as a token
+    include
 }
-
 /**
- * Configuration of the token lexing style. These flags may be combined with a
- * bitwise or.
+ * Configure comment handling behavior
  */
-enum TokenStyle : ushort
+public enum CommentBehavior : ubyte
 {
-    /**
-     * Escape sequences will be replaced with their equivalent characters,
-     * enclosing quote characters will not be included. Special tokens such as
-     * $(D_KEYWORD ___VENDOR__) will be replaced with their equivalent strings.
-     * Useful for creating a compiler or interpreter.
-     */
-    default_ = 0b0000,
-
-    /**
-     * Escape sequences will not be processed. An escaped quote character will
-     * not terminate string lexing, but it will not be replaced with the quote
-     * character in the token.
-     */
-    notEscaped = 0b0001,
-
-    /**
-     * Strings will include their opening and closing quote characters as well
-     * as any prefixes or suffixes $(LPAREN)e.g.: $(D_STRING "abcde"w) will
-     * include the $(D_STRING 'w') character as well as the opening and closing
-     * quotes$(RPAREN)
-     */
-    includeQuotes = 0b0010,
-
-    /**
-     * Do not replace the value field of the special tokens such as
-     * $(D_KEYWORD ___DATE__) with their string equivalents.
-     */
-    doNotReplaceSpecial = 0b0100,
-
-    /**
-     * Strings will be read exactly as they appeared in the source, including
-     * their opening and closing quote characters. Useful for syntax
-     * highlighting.
-     */
-    source = notEscaped | includeQuotes | doNotReplaceSpecial
+    /// Comments are attached to the non-whitespace token that follows them
+    attach,
+    /// Comments are tokens, and can be returned by calls to the token range's front()
+    include
 }
 
-/**
- * Lexer configuration
- */
-struct LexerConfig
+public struct LexerConfig
 {
-    /**
-     * Configure the lexer's iteration style.
-     * See_Also: $(LREF IterationStyle)
-     */
-    IterationStyle iterStyle = IterationStyle.codeOnly;
-
-    /**
-     * Configure the style of the tokens produced by the lexer.
-     * See_Also: $(LREF TokenStyle)
-     */
-    TokenStyle tokenStyle = tokenStyle.default_;
-
-    /**
-     * Replacement for the $(D_KEYWORD ___VERSION__) token. Defaults to 100.
-     */
-    uint versionNumber = 100;
-
-    /**
-     * Replacement for the $(D_KEYWORD ___VENDOR__) token. Defaults to $(D_STRING "std.d.lexer")
-     */
-    string vendorString = "std.d.lexer";
-
-    /**
-     * Name used when creating error messages that are sent to errorFunc. This
-     * is needed because the lexer operates on any forward range of ASCII
-     * characters or UTF-8 code units and does not know what to call its input
-     * source. Defaults to the empty string.
-     */
-    string fileName = "";
-
-    /**
-     * The starting line and column numbers for the lexer. These can be set when
-     * partially lexing D code to provide correct token locations and better
-     * error messages. These should be left to their default values of 1 when
-     * lexing entire files. Line and column numbers are 1-indexed in this lexer
-     * because this produces more useful error messages. The start index is
-     * zero-indexed, as it is more useful to machines than users.
-     */
-    uint startLine = 1;
-
-    /**
-     * ditto
-     */
-    ushort startColumn = 1;
-
-    /**
-     * ditto
-     */
-    size_t startIndex = 0;
-
-    /**
-     * This function is called when an error is encountered during lexing.
-     * If this field is not set, the lexer will throw an exception including the
-	 * line, column, and error message.
-     *
-     * $(BOOKTABLE Error Function Parameters:,
-     * $(TR $(TD string) $(TD File name))
-     * $(TR $(TD size_t) $(TD Code unit index))
-     * $(TR $(TD uint) $(TD Line number))
-     * $(TR $(TD ushort) $(TD Column number))
-     * $(TR $(TD string) $(TD Error message))
-     * )
-     */
-    void delegate(string, size_t, uint, ushort, string) errorFunc;
+    StringBehavior stringBehavior;
+    WhitespaceBehavior whitespaceBehavior;
+    CommentBehavior commentBehavior;
 }
 
-/**
- * Iterate over the given range of characters by D tokens.
- *
- * The lexing process is able to handle a forward range of code units by using
- * an internal circular buffer to provide efficient extracting of the token
- * values from the input. It is more efficient, however, to provide a range
- * that supports random accessing and slicing. If the input range supports
- * slicing, the caching layer aliases itself away and the lexing process
- * is much more efficient.
- *
- * Params:
- *     range = the range of characters to lex
- *     config = the lexer configuration
- *     bufferSize = initial size of internal circular buffer
- * Returns:
- *     a $(LREF TokenRange) that iterates over the given range
- */
-auto byToken(R)(R range, LexerConfig config, size_t bufferSize = 4*1024)
-    if (isForwardRange!(R) && !isRandomAccessRange!(R)
-        && is(ElementType!R : const(ubyte)))
+public auto byToken(R)(R range)
 {
-    // 4K of circular buffer by default
-    auto r = TokenRange!(typeof(lexerSource(range)))
-        (lexerSource(range, bufferSize), config);
-    r.popFront();
-    return r;
-}
-
-///ditto
-auto byToken(R)(R range, LexerConfig config)
-    if (isRandomAccessRange!(R) && is(ElementType!R : const(ubyte)))
-{
-    auto r = TokenRange!(typeof(lexerSource(range)))
-        (lexerSource(range), config);
-    r.popFront();
-    return r;
-}
-
-/**
- * Range of tokens. Use $(LREF byToken)$(LPAREN)$(RPAREN) to instantiate.
- */
-struct TokenRange(LexSrc)
-    //if ( is(LexSrc : LexSource!(U...), U...)) //check for LexSource
-{
-    /**
-     * Returns: true if the range is empty
-     */
-    bool empty() const @property
-    {
-        return _empty;
-    }
-
-    /**
-     * Returns: the current token
-     */
-    ref const(Token) front() const @property
-    {
-        assert(!empty, "trying to get front of an empty token range");
-        return current;
-    }
-
-    /**
-     * Returns: the current token and then removes it from the range
-     */
-    Token moveFront()
-    {
-        auto r = move(current);
-        popFront();
-        return r;
-    }
-
-    /**
-     * Removes the current token from the range
-     */
-    void popFront()
-    {
-        advance();
-    }
-
-private:
-
-    /*
-     * Advances the range to the next token
-     */
-    void advance()
-    {
-L_advance:
-        if (src.empty)
-        {
-            _empty = true;
-            return;
-        }
-        src.mark(); // mark a start of a lexing "frame"
-        current.line = lineNumber;
-        current.startIndex = src.index;
-        current.column = column;
-        current.value = null;
-        switch (src.front)
-        {
-        // handle sentinels for end of input
-        case 0:
-        case 0x1a:
-            // TODO: check config flags, it's cheap
-            // since this branch at most is taken once per file
-            _empty = true;
-            return;
-        mixin(generateCaseTrie(
-            "=",               "TokenType.assign",
-            "@",               "TokenType.at",
-            "&",               "TokenType.amp",
-            "&=",              "TokenType.bitAndAssign",
-            "|",               "TokenType.bitOr",
-            "|=",              "TokenType.bitOrAssign",
-            "~=",              "TokenType.catAssign",
-            ":",               "TokenType.colon",
-            ",",               "TokenType.comma",
-            "--",              "TokenType.decrement",
-            "$",               "TokenType.dollar",
-            "==",              "TokenType.equal",
-            "=>",              "TokenType.goesTo",
-            ">",               "TokenType.greater",
-            ">=",              "TokenType.greaterEqual",
-            "++",              "TokenType.increment",
-            "{",               "TokenType.lBrace",
-            "[",               "TokenType.lBracket",
-            "<",               "TokenType.less",
-            "<=",              "TokenType.lessEqual",
-            "<>=",             "TokenType.lessEqualGreater",
-            "<>",              "TokenType.lessOrGreater",
-            "&&",              "TokenType.logicAnd",
-            "||",              "TokenType.logicOr",
-            "(",               "TokenType.lParen",
-            "-",               "TokenType.minus",
-            "-=",              "TokenType.minusAssign",
-            "%",               "TokenType.mod",
-            "%=",              "TokenType.modAssign",
-            "*=",              "TokenType.mulAssign",
-            "!",               "TokenType.not",
-            "!=",              "TokenType.notEqual",
-            "!>",              "TokenType.notGreater",
-            "!>=",             "TokenType.notGreaterEqual",
-            "!<",              "TokenType.notLess",
-            "!<=",             "TokenType.notLessEqual",
-            "!<>",             "TokenType.notLessEqualGreater",
-            "+",               "TokenType.plus",
-            "+=",              "TokenType.plusAssign",
-            "^^",              "TokenType.pow",
-            "^^=",             "TokenType.powAssign",
-            "}",               "TokenType.rBrace",
-            "]",               "TokenType.rBracket",
-            ")",               "TokenType.rParen",
-            ";",               "TokenType.semicolon",
-            "<<",              "TokenType.shiftLeft",
-            "<<=",             "TokenType.shiftLeftAssign",
-            ">>",              "TokenType.shiftRight",
-            ">>=",             "TokenType.shiftRightAssign",
-            "*",               "TokenType.star",
-            "?",               "TokenType.ternary",
-            "~",               "TokenType.tilde",
-            "!<>=",            "TokenType.unordered",
-            ">>>",             "TokenType.unsignedShiftRight",
-            ">>>=",            "TokenType.unsignedShiftRightAssign",
-            "^",               "TokenType.xor",
-            "^=",              "TokenType.xorAssign"
-        ));
-        case '/':
-            nextCharNonLF();
-            if (isEoF())
-            {
-                current.type = TokenType.div;
-                current.value = "/";
-                return;
-            }
-            switch (src.front)
-            {
-            case '/':
-            case '*':
-            case '+':
-                if (config.iterStyle & IterationStyle.includeComments)
-                    return lexComment!true();
-                lexComment!false();
-                goto L_advance; // tail-recursion
-
-            case '=':
-                current.type = TokenType.divAssign;
-                current.value = "/=";
-                src.popFront();
-                return;
-            default:
-                current.type = TokenType.div;
-                current.value = "/";
-                return;
-            }
-        case '.':
-            if (!src.canPeek())
-            {
-                current.type = TokenType.dot;
-                current.value = tokenValue!(TokenType.dot);
-                nextCharNonLF();
-                return;
-            }
-            switch (src.peek())
-            {
-            case '0': .. case '9':
-                lexNumber();
-                return;
-            case '.':
-                nextCharNonLF();
-                nextCharNonLF();
-                current.type = TokenType.dotdot;
-                if (src.front == '.')
-                {
-                    current.type = TokenType.vararg;
-                    nextCharNonLF();
-                    current.value = tokenValue!(TokenType.vararg);
-                }
-                else
-                    current.value = tokenValue!(TokenType.dotdot);
-                return;
-            default:
-                nextCharNonLF();
-                current.type = TokenType.dot;
-                current.value = tokenValue!(TokenType.dot);
-                return;
-            }
-        case '0': .. case '9':
-            lexNumber();
-            return;
-        case '\'':
-            lexCharacterLiteral();
-            return;
-        case '"':
-        case '`':
-            lexString();
-            return;
-        case 'q':
-            nextCharNonLF();
-            if (isEoF())
-                goto default;
-            switch (src.front)
-            {
-            case '{':
-                lexTokenString();
-                return;
-            case '"':
-                lexDelimitedString();
-                return;
-            default:
-                break;
-            }
-            goto default;
-        case 'r':
-            nextCharNonLF();
-            if (isEoF())
-                goto default;
-            else if (src.front == '"')
-            {
-                lexString();
-                return;
-            }
-            else
-                goto default;
-        case 'x':
-            nextCharNonLF();
-            if (isEoF())
-                goto default;
-            else if (src.front == '"')
-            {
-                lexHexString();
-                return;
-            }
-            else
-                goto default;
-        case '#':
-            lexSpecialTokenSequence();
-            if(config.iterStyle & IterationStyle.includeSpecialTokens)
-                return;
-            goto L_advance; // tail-recursion
-        // "short" ASCII whites
-        case 0x20:
-        case 0x09: .. case 0x0d:
-             if (config.iterStyle & IterationStyle.includeWhitespace)
-                return lexWhitespace!true();
-             lexWhitespace!false();
-             goto L_advance; // tail-recursion
-        default:
-            if ((src.front & 0x80) && isLongWhite())
-            {
-                if (config.iterStyle & IterationStyle.includeWhitespace)
-                    return lexWhitespace!true();
-                lexWhitespace!false();
-                goto L_advance; // tail-recursion
-            }
-            for(;;)
-            {
-                if(isSeparating())
-                    break;
-                nextCharNonLF();
-                if(isEoF())
-                    break;
-            }
-
-            current.type = lookupTokenType(src.slice);
-            current.value = getTokenValue(current.type);
-            if (current.value is null)
-                setTokenValue();
-            if (!(config.iterStyle & IterationStyle.ignoreEOF) && current.type == TokenType.specialEof)
-            {
-                _empty = true;
-                return;
-            }
-
-            if (config.tokenStyle & TokenStyle.doNotReplaceSpecial)
-                return;
-            expandSpecialToken();
-        }
-    }
-
-    // TODO: LexSource could be improved for forward ranges
-    // to avoid buffering at all (by disabling it for a moment)
-    // so keep the 'keep' parameter here and elsewhere
-    void lexWhitespace(bool keep)()
-    {
-        current.type = TokenType.whitespace;
-        do
-        {
-            nextChar();
-        } while (!isEoF() && isWhite());
-        static if (keep) setTokenValue();
-    }
-
-    void lexComment(bool keep)()
-    in
-    {
-        assert (src.front == '/' || src.front == '*' || src.front == '+');
-    }
-    body
-    {
-        current.type = TokenType.comment;
-        switch(src.front)
-        {
-        case '/':
-            while (!isEoF() && !isNewline(src.front))
-            {
-                nextCharNonLF();
-            }
-            break;
-        case '*':
-            while (!isEoF())
-            {
-                if (src.front == '*')
-                {
-                    static if (keep) nextCharNonLF();
-                    else src.popFront();
-                    if (src.front == '/')
-                    {
-                        nextCharNonLF();
-                        break;
-                    }
-                }
-                else
-                    nextChar();
-            }
-            break;
-        case '+':
-            int depth = 1;
-            while (depth > 0 && !isEoF())
-            {
-                if (src.front == '+')
-                {
-                    nextCharNonLF();
-                    if (src.front == '/')
-                    {
-                        nextCharNonLF();
-                        --depth;
-                    }
-                }
-                else if (src.front == '/')
-                {
-                    nextCharNonLF();
-                    if (src.front == '+')
-                    {
-                        nextCharNonLF();
-                        ++depth;
-                    }
-                }
-                else
-                    nextChar();
-            }
-            break;
-        default:
-            assert(false);
-        }
-        static if (keep)
-        setTokenValue();
-    }
-
-    void lexHexString()
-    in
-    {
-        assert (src.front == '"');
-    }
-    body
-    {
-        current.type = TokenType.stringLiteral;
-        nextChar();
-        while (true)
-        {
-            if (isEoF())
-            {
-                errorMessage("Unterminated hex string literal");
-                return;
-            }
-            else if (isHexDigit(src.front))
-            {
-                nextCharNonLF();
-            }
-            else if (isWhite() && (config.tokenStyle & TokenStyle.notEscaped))
-            {
-                nextChar();
-            }
-            else if (src.front == '"')
-            {
-                nextCharNonLF();
-                break;
-            }
-            else
-            {
-                errorMessage(format("Invalid character '%s' in hex string literal",
-                    cast(char) src.front));
-                return;
-            }
-        }
-        bool hasSuffix = lexStringSuffix();
-        if (config.tokenStyle & TokenStyle.notEscaped)
-        {
-            if (config.tokenStyle & TokenStyle.includeQuotes)
-                setTokenValue();
-            else
-                setTokenValue(2, hasSuffix ? -2 : -1);
-        }
-        else
-        {
-            // TODO: appender is an allocation happy fat pig
-            // remove it later
-            auto a = appender!(char[])();
-            foreach (b; std.range.chunks(src.slice[2 .. $ - 1], 2))
-            {
-                auto s = cast(char[])b;
-                ubyte ch = cast(ubyte)parse!uint(s, 16);
-                a.put(ch);
-            }
-            // can safely assume ownership of data
-            current.value = cast(string)a.data;
-        }
-    }
-
-    void lexNumber()
-    in
-    {
-        assert(isDigit(src.front) || src.front == '.');
-    }
-    body
-    {
-        if (src.front != '0')
-        {
-            lexDecimal();
-            return;
-        }
-        else
-        {
-            switch (src.peek())
-            {
-            case 'x':
-            case 'X':
-                nextCharNonLF();
-                nextCharNonLF();
-                lexHex();
-                break;
-            case 'b':
-            case 'B':
-                nextCharNonLF();
-                nextCharNonLF();
-                lexBinary();
-                break;
-            default:
-                lexDecimal();
-                break;
-            }
-        }
-    }
-
-    void lexFloatSuffix()
-    {
-        switch (src.front)
-        {
-        case 'L':
-            nextCharNonLF();
-            current.type = TokenType.doubleLiteral;
-            break;
-        case 'f':
-        case 'F':
-            nextCharNonLF();
-            current.type = TokenType.floatLiteral;
-            break;
-        default:
-            break;
-        }
-        if (!isEoF() && src.front == 'i')
-        {
-            nextCharNonLF();
-            if (current.type == TokenType.floatLiteral)
-                current.type = TokenType.ifloatLiteral;
-            else
-                current.type = TokenType.idoubleLiteral;
-        }
-    }
-
-    void lexIntSuffix()
-    {
-        bool foundU;
-        bool foundL;
-        while (!isEoF())
-        {
-            switch (src.front)
-            {
-            case 'u':
-            case 'U':
-                if (foundU)
-                    return;
-                switch (current.type)
-                {
-                case TokenType.intLiteral:
-                    current.type = TokenType.uintLiteral;
-                    nextCharNonLF();
-                    break;
-                case TokenType.longLiteral:
-                    current.type = TokenType.ulongLiteral;
-                    nextCharNonLF();
-                    break;
-                default:
-                    assert (false);
-                }
-                foundU = true;
-                break;
-            case 'L':
-                if (foundL)
-                    return;
-                switch (current.type)
-                {
-                case TokenType.intLiteral:
-                    current.type = TokenType.longLiteral;
-                    nextCharNonLF();
-                    break;
-                case TokenType.uintLiteral:
-                    current.type = TokenType.ulongLiteral;
-                    nextCharNonLF();
-                    break;
-                default:
-                    assert (false);
-                }
-                foundL = true;
-                break;
-            default:
-                return;
-            }
-        }
-    }
-
-    void lexExponent()
-    in
-    {
-        assert (src.front == 'e' || src.front == 'E' || src.front == 'p'
-            || src.front == 'P');
-    }
-    body
-    {
-        nextCharNonLF();
-        bool foundSign = false;
-        bool foundDigit = false;
-        while (!isEoF())
-        {
-            switch (src.front)
-            {
-            case '-':
-            case '+':
-                if (foundSign)
-                {
-                    if (!foundDigit)
-                    errorMessage("Expected an exponent");
-                    return;
-                }
-                foundSign = true;
-                nextCharNonLF();
-                break;
-            case '0': .. case '9':
-            case '_':
-                foundDigit = true;
-                nextCharNonLF();
-                break;
-            case 'L':
-            case 'f':
-            case 'F':
-            case 'i':
-                lexFloatSuffix();
-                return;
-            default:
-                if (!foundDigit)
-                    errorMessage("Expected an exponent");
-                return;
-            }
-        }
-    }
-
-    void lexDecimal()
-    in
-    {
-        assert (isDigit(src.front) || src.front == '.');
-    }
-    body
-    {
-        bool foundDot = src.front == '.';
-        if (foundDot)
-            nextCharNonLF();
-        current.type = TokenType.intLiteral;
-        decimalLoop: while (!isEoF())
-        {
-            switch (src.front)
-            {
-            case '0': .. case '9':
-            case '_':
-                nextCharNonLF();
-                break;
-            case 'u':
-            case 'U':
-                if (!foundDot)
-                    lexIntSuffix();
-                break decimalLoop;
-            case 'i':
-                lexFloatSuffix();
-                break decimalLoop;
-            case 'L':
-                if (foundDot)
-                    lexFloatSuffix();
-                else
-                    lexIntSuffix();
-                break decimalLoop;
-            case 'f':
-            case 'F':
-                lexFloatSuffix();
-                break decimalLoop;
-            case 'e':
-            case 'E':
-                lexExponent();
-                break decimalLoop;
-            case '.':
-                if (foundDot)
-                    break decimalLoop;
-                if (src.canPeek() && src.peek() == '.')
-                    break decimalLoop;
-                else
-                {
-                    // The following bit of silliness tries to tell the
-                    // difference between "int dot identifier" and
-                    // "double identifier".
-                    if (src.canPeek())
-                    {
-                        switch (src.peek())
-                        {
-                        case '0': .. case '9':
-                            goto doubleLiteral;
-                        default:
-                            break decimalLoop;
-                        }
-                    }
-                    else
-                    {
-                    doubleLiteral:
-                        nextCharNonLF();
-                        foundDot = true;
-                        current.type = TokenType.doubleLiteral;
-                    }
-                }
-                break;
-            default:
-                break decimalLoop;
-            }
-        }
-        setTokenValue();
-    }
-
-    void lexBinary()
-    {
-        current.type = TokenType.intLiteral;
-        binaryLoop: while (!isEoF())
-        {
-            switch (src.front)
-            {
-            case '0':
-            case '1':
-            case '_':
-                nextCharNonLF();
-                break;
-            case 'u':
-            case 'U':
-            case 'L':
-                lexIntSuffix();
-                break binaryLoop;
-            default:
-                break binaryLoop;
-            }
-        }
-        setTokenValue();
-    }
-
-    void lexHex()
-    {
-        current.type = TokenType.intLiteral;
-        bool foundDot;
-        hexLoop: while (!isEoF())
-        {
-            switch (src.front)
-            {
-            case 'a': .. case 'f':
-            case 'A': .. case 'F':
-            case '0': .. case '9':
-            case '_':
-                nextCharNonLF();
-                break;
-            case 'u':
-            case 'U':
-                lexIntSuffix();
-                break hexLoop;
-            case 'i':
-                if (foundDot)
-                    lexFloatSuffix();
-                break hexLoop;
-            case 'L':
-                if (foundDot)
-                {
-                    lexFloatSuffix();
-                    break hexLoop;
-                }
-                else
-                {
-                    lexIntSuffix();
-                    break hexLoop;
-                }
-            case 'p':
-            case 'P':
-                lexExponent();
-                break hexLoop;
-            case '.':
-                if (foundDot)
-                    break hexLoop;
-                if (src.canPeek() && src.peek() == '.')
-                    break hexLoop;
-                nextCharNonLF();
-                foundDot = true;
-                current.type = TokenType.doubleLiteral;
-                break;
-            default:
-                break hexLoop;
-            }
-        }
-        setTokenValue();
-    }
-
-    bool lexStringSuffix()
-    {
-        current.type = TokenType.stringLiteral;
-        bool foundSuffix = false;
-        if (!isEoF())
-        {
-            switch (src.front)
-            {
-            case 'w':
-                current.type = TokenType.wstringLiteral;
-                goto case 'c';
-            case 'd':
-                current.type = TokenType.dstringLiteral;
-                goto case 'c';
-            case 'c':
-                foundSuffix = true;
-                nextCharNonLF();
-                break;
-            default:
-                break;
-            }
-        }
-        return foundSuffix;
-    }
-
-    void lexCharacterLiteral()
-    in
-    {
-        assert (src.front == '\'');
-    }
-    body
-    {
-        current.type = TokenType.characterLiteral;
-        nextChar();
-        if (isEoF())
-        {
-            errorMessage("Unterminated character literal");
-            return;
-        }
-        switch (src.front)
-        {
-        case '\'':
-            break;
-        case '\\':
-            if (config.tokenStyle & TokenStyle.notEscaped)
-                skipEscapeSequence();
-            else
-            {
-                // the only special path
-                // 40 bytes is enough for 2 quotes
-                // and the longest character entity
-                ubyte[40] utf8;
-                size_t len;
-                if (config.tokenStyle & TokenStyle.includeQuotes)
-                {
-                    utf8[0] = '\'';
-                    len = decodeEscapeSequence(utf8[1..$]);
-                    utf8[len++] = '\'';
-                }
-                else
-                    len = decodeEscapeSequence(utf8[]);
-                if (src.front != '\'')
-                {
-                    errorMessage("Expected \"'\" to end character literal");
-                }
-                // skip over last "'"
-                nextChar();
-                setTokenValue(utf8[0..len]);
-            return;
-            }
-            break;
-        default:
-            if (src.front & 0x80)
-            {
-                while (src.front & 0x80)
-                    nextChar();
-                break;
-            }
-            else
-            {
-                nextChar();
-                break;
-            }
-        }
-        if (src.front != '\'')
-            errorMessage("Expected \"'\" to end character literal");
-        nextChar();
-        if (config.tokenStyle & TokenStyle.includeQuotes)
-            setTokenValue();
-        else
-            setTokenValue(1, -1);
-    }
-
-    void lexString()
-    in
-    {
-        //assert (src.front == '"');
-    }
-    body
-    {
-        current.type = TokenType.stringLiteral;
-        bool longWysiwg = src.slice.length > 0 && src.slice[0] == 'r'; // 2 chars : r"
-        bool isWysiwyg = src.front == '`';
-        // in case we need to unescape string
-        Appender!(ubyte[]) unescaped;
-        auto quote = src.front;
-        nextChar();
-        while (true)
-        {
-            if (isEoF())
-            {
-                errorMessage("Unterminated string literal");
-                return;
-            }
-            else if (src.front == '\\')
-            {
-                if (isWysiwyg || longWysiwg)
-                    nextChar();
-                else if(config.tokenStyle & TokenStyle.notEscaped)
-                {
-                    skipEscapeSequence();
-                }
-                else
-                {
-                    if(unescaped == Appender!(ubyte[]).init)
-                        unescaped = appender!(ubyte[])();
-                    unescaped.put(src.slice());
-                    decodeEscapeSequence(unescaped);
-                    src.mark(); //start next slice after escape sequence
-                }
-            }
-            else if (src.front == quote)
-            {
-                nextCharNonLF();
-                break;
-            }
-            else
-                nextChar();
-        }
-        lexStringSuffix();
-        // helper to handle quotes
-        void setData(R)(R range)
-        {
-            if (config.tokenStyle & TokenStyle.includeQuotes)
-                setTokenValue(range);
-            else if (longWysiwg)
-                setTokenValue(range[2..$-1]);
-            else
-                setTokenValue(range[1..$-1]);
-        }
-        import std.stdio;
-        if(unescaped != Appender!(ubyte[]).init)
-        {
-            //stuff in the last slice and use buffered data
-            unescaped.put(src.slice);
-            setData(unescaped.data);
-        }
-        else
-        {
-            setData(src.slice); //slice directly
-        }
-    }
-
-    void lexDelimitedString()
-    in
-    {
-        assert(src.front == '"');
-    }
-    body
-    {
-        current.type = TokenType.stringLiteral;
-
-        nextChar();
-
-        bool heredoc;
-        ubyte open;
-        ubyte close;
-
-        switch (src.front)
-        {
-        case '[': open = '['; close = ']'; break;
-        case '{': open = '{'; close = '}'; break;
-        case '(': open = '('; close = ')'; break;
-        case '<': open = '<'; close = '>'; break;
-        default: heredoc = true; break;
-        }
-        if (heredoc)
-            lexHeredocString();
-        else
-            lexNormalDelimitedString(open, close);
-    }
-
-    void lexNormalDelimitedString(ubyte open, ubyte close)
-    in
-    {
-        assert(src.slice[0 .. 2] == `q"`);
-    }
-    body
-    {
-        current.type = TokenType.stringLiteral;
-        int depth = 1;
-        nextChar();
-        while (true)
-        {
-            if (isEoF())
-            {
-                errorMessage("Unterminated string literal");
-                break;
-            }
-            if (src.front == open)
-            {
-                nextChar();
-                ++depth;
-            }
-            else if (src.front == close)
-            {
-                nextChar();
-                --depth;
-                if (depth <= 0)
-                {
-                    auto r = src.save(); //TODO: allocates for Fwd range
-                    if (r.front == '"')
-                    {
-                        nextChar();
-                        break;
-                    }
-                    else
-                    {
-                        errorMessage("Expected \" after balanced "
-                            ~ cast(char) close ~ " but found "
-                            ~ cast(char) r.front ~ " instead.");
-                        break;
-                    }
-                }
-            }
-            else
-                nextChar();
-        }
-        if (config.tokenStyle & TokenStyle.includeQuotes)
-            setTokenValue();
-        else
-            setTokenValue(3, -2);
-    }
-
-    void lexHeredocString()
-    in
-    {
-        assert (src.slice.equal("q\""));
-    }
-    body
-    {
-        typeof(src.slice) ident;
-        uint newlineBytes;
-        while (true)
-        {
-            if (isEoF())
-            {
-                errorMessage("Unterminated string literal");
-                return;
-            }
-            else if (isNewline(src.front))
-            {
-                ident = src.slice[2..$];
-                nextChar();
-                newlineBytes = cast(uint) (src.slice.length - 2 - ident.length);
-                break;
-            }
-            else if (isSeparating())
-            {
-                nextChar();
-                ident = src.slice[2..$];
-                nextChar();
-                newlineBytes = 0;
-                break;
-            }
-            else
-            {
-                nextChar();
-            }
-        }
-        while (true)
-        {
-            if (isEoF())
-            {
-                errorMessage("Unterminated string literal");
-                break;
-            }
-            else if (src.slice.length > ident.length
-                && src.slice[$-ident.length .. $].equal(ident))
-            {
-                if (src.front == '"')
-                {
-                    nextChar();
-                    lexStringSuffix();
-                    break;
-                }
-                else
-                {
-                    errorMessage("Unterminated string literal: " ~ cast(string) src.slice);
-                    break;
-                }
-            }
-            else
-                nextChar();
-        }
-
-        bool hasSuffix = lexStringSuffix();
-
-        if (config.tokenStyle & TokenStyle.includeQuotes)
-            setTokenValue();
-        else
-        {
-            setTokenValue(cast(int) (2 + newlineBytes + ident.length),
-                cast(int) (-(ident.length + (hasSuffix ? 2 : 1))));
-        }
-    }
-
-    void lexTokenString()
-    in
-    {
-        assert (src.front == '{');
-    }
-    body
-    {
-        current.type = TokenType.stringLiteral;
-        nextChar();
-        auto app = appender!(ubyte[])();
-        if (config.tokenStyle & TokenStyle.includeQuotes)
-        {
-            app.put('q');
-            app.put('{');
-        }
-        LexerConfig c = config;
-        scope (exit) config = c;
-        config.iterStyle = IterationStyle.everything;
-        config.tokenStyle = TokenStyle.source;
-        int depth = 1;
-
-        while (!isEoF())
-        {
-            advance();
-            if (current.type == TokenType.lBrace)
-                ++depth;
-            else if (current.type == TokenType.rBrace)
-            {
-                --depth;
-                if (depth <= 0)
-                    break;
-            }
-            app.put(representation(current.value));
-        }
-        config = c;
-        if (config.tokenStyle & TokenStyle.includeQuotes)
-        {
-            app.put('}');
-        }
-        if (src.empty)
-            current.type = TokenType.stringLiteral;
-        else
-        {
-            switch (src.front)
-            {
-                case 'd':
-                    if (config.tokenStyle & TokenStyle.includeQuotes)
-                        app.put('d');
-                    current.type = TokenType.dstringLiteral;
-                    src.popFront();
-                    break;
-                case 'w':
-                    if (config.tokenStyle & TokenStyle.includeQuotes)
-                        app.put('w');
-                    current.type = TokenType.wstringLiteral;
-                    src.popFront();
-                    break;
-                case 'c':
-                    if (config.tokenStyle & TokenStyle.includeQuotes)
-                        app.put('c');
-                    src.popFront();
-                    goto default;
-                default:
-                    current.type = TokenType.stringLiteral;
-                    break;
-            }
-        }
-        current.value = cast(string) app.data;
-    }
-
-    void lexSpecialTokenSequence()
-    in
-    {
-        assert (src.front == '#');
-    }
-    body
-    {
-        nextChar();
-        auto r = src.save();
-        auto app = appender!(ubyte[])();
-        app.put('#');
-        while (true)
-        {
-            if (r.isRangeEoF())
-            {
-                errorMessage("Found EOF when interpreting special token sequence");
-                return;
-            }
-            else if (isNewline(r.front))
-                break;
-            else
-            {
-                app.put(r.front);
-                r.popFront();
-            }
-        }
-        auto m = match((cast(char[]) app.data),
-            `#line\s+(?P<line>\d+)\s*(?P<filespec>".+")*?`);
-        if (m)
-        {
-            current.type = TokenType.specialTokenSequence;
-            current.value = (cast(char[]) app.data).idup;
-            column += app.data.length;
-            foreach (i; 0 .. app.data.length)
-                src.popFront();
-            auto c = m.captures;
-            if (c["filespec"])
-                config.fileName = c["filespec"].idup;
-            auto l = c["line"];
-            lineNumber = parse!uint(l);
-        }
-        else
-        {
-            current.type = TokenType.hash;
-            current.value = tokenValue!(TokenType.hash);
-        }
-    }
-
-//=====================================================================
-//          Helpers for lexXYZ functions
-//=====================================================================
-    void skipEscapeSequence()
-    {
-        // no decoding, just minor sanity checks
-        nextChar();
-        switch (src.front)
-        {
-        case '\'':
-        case '"':
-        case '?':
-        case '\\':
-        case 'a':
-        case 'b':
-        case 'f':
-        case 'n':
-        case 'r':
-        case 't':
-        case 'v':
-        case 0x0a:
-        case 0x00:
-            nextChar();
-            return;
-        case '0': .. case '7':
-            foreach(i; 0 .. 3)
-            {
-                nextChar();
-                if (src.front < '0' || src.front > '7') return;
-            }
-            return;
-        case 'x':
-            nextChar();
-            foreach(i; 0 .. 2)
-            {
-                if (!isHexDigit(src.front))
-                {
-                    errorMessage("Expected hex digit");
-                    return;
-                }
-                nextChar();
-            }
-            return;
-        case 'u':
-        case 'U':
-            uint digits = src.front == 'u' ? 4 : 8;
-            nextChar();
-            foreach (i; 0 .. digits)
-            {
-                if (!isHexDigit(src.front))
-                {
-                    errorMessage("Expected hex digit instead of %s".format(
-                        cast(char) src.front));
-                    return;
-                }
-                nextChar();
-            }
-            return;
-        case '&':
-            while (!isEoF())
-            {
-                nextChar();
-                if (src.front == ';')
-                    break;
-            }
-            return;
-        default:
-            errorMessage("Invalid escape sequence");
-            return;
-        }
-    }
-
-    size_t decodeEscapeSequence(OutputRange)(OutputRange dest)
-    in
-    {
-        assert (src.front == '\\');
-    }
-    body
-    {
-        size_t reencodeNumeric(ubyte[] src, int radix, OutputRange dest)
-        {
-            char[] chunk = cast(char[])src;
-            char[4] utfBuf;
-            uint codepoint = parse!uint(chunk, radix);
-            size_t len;
-            try
-                 len = encode(utfBuf, codepoint);
-            catch (UTFException ex)
-            {
-                errorMessage(ex.msg);
-                return 0;
-            }
-            dest.put(cast(ubyte[]) utfBuf[0..len]);
-            return len;
-        }
-
-        ubyte[40] buffer;
-        src.popFront();
-        switch (src.front)
-            {
-        case '\'':
-        case '"':
-        case '?':
-        case '\\':
-            buffer[0] = src.front;
-            src.popFront();
-            return 1;
-        case 'a':  dest.put('\a'); src.popFront(); return 1;
-        case 'b':  dest.put('\b'); src.popFront(); return 1;
-        case 'f':  dest.put('\f'); src.popFront(); return 1;
-        case 'n':  dest.put('\n'); src.popFront(); return 1;
-        case 'r':  dest.put('\r'); src.popFront(); return 1;
-        case 't':  dest.put('\t'); src.popFront(); return 1;
-        case 'v':  dest.put('\v'); src.popFront(); return 1;
-        case 0x0a: dest.put(cast(ubyte)0x0a); src.popFront(); return 1;
-        case 0x00: dest.put(cast(ubyte)0x00); src.popFront(); return 1;
-        case '0': .. case '7':
-            size_t idx = 0;
-            while(idx < 3 && !isEoF())
-            {
-                buffer[idx++] = src.front;
-                src.popFront();
-                if (src.front < '0' || src.front > '7') break;
-            }
-            return reencodeNumeric(buffer[0..idx], 8, dest);
-        case 'x':
-            src.popFront();
-            foreach(i; 0 .. 2)
-            {
-                if (!isHexDigit(src.front))
-                {
-                    errorMessage("Expected hex digit");
-                    return 1;
-                }
-                buffer[i] = src.front;
-                src.popFront();
-            }
-            return reencodeNumeric(buffer[0..2], 16, dest);
-        case 'u':
-        case 'U':
-            uint digitCount = src.front == 'u' ? 4 : 8;
-            src.popFront();
-            foreach (i; 0 .. digitCount)
-            {
-                if (!isHexDigit(src.front))
-                {
-                    errorMessage("Expected hex digit");
-                    return 1;
-                }
-                buffer[i] = src.front;
-                src.popFront();
-            }
-            return reencodeNumeric(buffer[0..digitCount], 16, dest);
-        case '&':
-            src.popFront();
-            size_t idx = 0;
-            while (!isEoF())
-            {
-                if (std.ascii.isAlpha(src.front))
-                {
-                    buffer[idx++] = src.front;
-                    if(idx == buffer.length) // way over maximum length
-                        errorMessage("Invalid character entity");
-                    src.popFront();
-                }
-                else if (src.front == ';')
-                {
-                    src.popFront();
-                    break;
-                }
-                else
-                {
-                    errorMessage("Invalid character entity");
-                    return idx;
-                }
-            }
-            //TODO: avoid looking up as UTF string, use raw bytes
-            string chunk = cast(string)buffer[0..idx];
-            auto names = assumeSorted(map!"a.name"(characterEntities));
-            auto place = names.lowerBound(chunk).length;
-            if (place == names.length || names[place] != chunk)
-            {
-                errorMessage("Invalid character entity \"&%s;\""
-                    .format(cast(string) chunk));
-                return 1;
-            }
-            auto entity = characterEntities[place].value;
-            dest.put(cast(ubyte[]) entity);
-            return entity.length;
-        default:
-            errorMessage("Invalid escape sequence");
-            return 1;
-        }
-    }
-
-    // advances underlying mark-slice range and counts lines, cols
-    void nextChar()
-    {
-        bool foundNewline;
-        if (src.front == '\r')
-        {
-            src.popFront();
-            foundNewline = true;
-        }
-        if (!src.empty)
-        {
-            if (src.front == '\n')
-            {
-                src.popFront();
-                foundNewline = true;
-            }
-            else
-            {
-                src.popFront();
-            }
-        }
-        if (foundNewline)
-        {
-            ++lineNumber;
-            column = 1;
-        }
-        else
-            ++column;
-
-    }
-
-    //same but don't bother for LF sequences
-    void nextCharNonLF()
-    {
-        src.popFront();
-        ++column;
-    }
-
-    void setTokenValue()()
-    {
-        current.value = cache.get(src.slice);
-    }
-
-    void setTokenValue()(int startOffset, int endOffset)
-    in
-    {
-        assert(startOffset >= 0);
-        assert(endOffset <= 0);
-    }
-    body
-    {
-        auto piece = src.slice;
-        // avoid unsigned arithmetic as endOffset is negative
-        int end = cast(int)piece.length + endOffset;
-        current.value = cache.get(src.slice[startOffset .. end]);
-    }
-
-    void setTokenValue(R)(R range)
-        if(isRandomAccessRange!R && is(ElementType!R : const(ubyte)))
-    {
-        current.value = cache.get(range);
-    }
-
-    bool isEoF() const
-    {
-        return src.empty || src.front == 0 || src.front == 0x1a;
-    }
-
-    bool isSeparating()
-    {
-        auto ch = src.front;
-        if (ch <= 0x2f) return true;
-        if (ch >= ':' && ch <= '@') return true;
-        if (ch >= '[' && ch <= '^') return true;
-        if (ch >= '{' && ch <= '~') return true;
-        if (ch == '`') return true;
-        if ((ch & 0x80) && isLongWhite()) return true;
-        return false;
-    }
-
-    bool isWhite()
-    {
-        auto c = src.front;
-        if (c & 0x80) // multi-byte utf-8
-        {
-            return isLongWhite();
-        }
-        else
-            return c == 0x20 || (c >= 0x09 && c <= 0x0d);
-    }
-
-    bool isLongWhite()
-    {
-        assert(src.front & 0x80); // only non-ascii
-        //TODO: here and elsewhere we'd better have
-        // some kind of lookahead in LexSource instead of .save
-        auto r = src.save();
-        if (r.front != 0xe2)
-            return false;
-        else
-            r.popFront();
-        if (r.empty || r.front != 0x80)
-            return false;
-        else
-            r.popFront();
-        if (r.empty || (r.front != 0xa8 && r.front != 0xa9))
-            return false;
-        return true;
-    }
-
-    void expandSpecialToken()
-    {
-        switch (current.type)
-        {
-        case TokenType.specialDate:
-            current.type = TokenType.stringLiteral;
-            auto time = Clock.currTime();
-            current.value = format("%s %02d %04d", time.month, time.day, time.year);
-            return;
-        case TokenType.specialTime:
-            auto time = Clock.currTime();
-            current.type = TokenType.stringLiteral;
-            current.value = (cast(TimeOfDay)(time)).toISOExtString();
-            return;
-        case TokenType.specialTimestamp:
-            auto time = Clock.currTime();
-            auto dt = cast(DateTime) time;
-            current.type = TokenType.stringLiteral;
-            current.value = format("%s %s %02d %02d:%02d:%02d %04d",
-                dt.dayOfWeek, dt.month, dt.day, dt.hour, dt.minute,
-                dt.second, dt.year);
-            return;
-        case TokenType.specialVendor:
-            current.type = TokenType.stringLiteral;
-            current.value = config.vendorString;
-            return;
-        case TokenType.specialVersion:
-            current.type = TokenType.stringLiteral;
-            current.value = format("%d", config.versionNumber);
-            return;
-        case TokenType.specialLine:
-            current.type = TokenType.intLiteral;
-            current.value = format("%d", current.line);
-            return;
-        case TokenType.specialFile:
-            current.type = TokenType.stringLiteral;
-            current.value = config.fileName;
-            return;
-        default:
-            return;
-        }
-    }
-
-    void errorMessage(string s)
-    {
-        import std.string: format;
-        if (config.errorFunc !is null)
-            config.errorFunc(config.fileName, current.startIndex,
-                current.line, current.column, s);
-        else
-            throw new Exception(format("%s(%d:%d): %s",
-                config.fileName, current.line, current.column, s));
-    }
-
-    this(LexSrc lex, LexerConfig cfg)
-    {
-        src = move(lex); // lex is r-value
-        lineNumber = cfg.startLine;
-        column = cfg.startColumn;
-        //src._index = cfg.startIndex;
-        _empty = false;
-        config = move(cfg); // ditto with cfg
-        cache = StringCache(initialTableSize);
-    }
-
-    enum initialTableSize = 2048;
-    Token current;
-    uint lineNumber;
-    ushort column;
-    LexSrc src;
-    bool _empty;
     LexerConfig config;
-    StringCache cache;
+    return byToken(range, config);
 }
 
-/**
- * Returns: true if the token is an operator
- */
-pure nothrow bool isOperator(const TokenType t)
+public auto byToken(R)(R range, const LexerConfig config)
 {
-    return t >= TokenType.assign && t <= TokenType.xorAssign;
-}
-
-/**
- * ditto
- */
-pure nothrow bool isOperator(ref const Token t)
-{
-    return isOperator(t.type);
-}
-
-/**
- * Returns: true if the token is a keyword
- */
-pure nothrow bool isKeyword(const TokenType t)
-{
-    return t >= TokenType.bool_ && t <= TokenType.with_;
-}
-
-/**
- * ditto
- */
-pure nothrow bool isKeyword(ref const Token t)
-{
-    return isKeyword(t.type);
-}
-
-/**
- * Returns: true if the token is a built-in type
- */
-pure nothrow bool isBasicType(const TokenType t)
-{
-    return t >= TokenType.bool_ && t <= TokenType.wchar_;
-}
-
-/**
- * ditto
- */
-pure nothrow bool isBasicType(ref const Token t)
-{
-    return isBasicType(t.type);
-}
-
-/**
- * Returns: true if the token is an attribute
- */
-pure nothrow bool isAttribute(const TokenType t)
-{
-    return t >= TokenType.align_ && t <= TokenType.static_;
-}
-
-/**
- * ditto
- */
-pure nothrow bool isAttribute(ref const Token t)
-{
-    return isAttribute(t.type);
-}
-
-/**
- * Returns: true if the token is a protection attribute
- */
-pure nothrow bool isProtection(const TokenType t)
-{
-    return t >= TokenType.export_ && t <= TokenType.public_;
-}
-
-/**
- * ditto
- */
-pure nothrow bool isProtection(ref const Token t)
-{
-    return isProtection(t.type);
-}
-
-/**
- * Returns: true if the token is a compile-time constant such as ___DATE__
- */
-pure nothrow bool isConstant(const TokenType t)
-{
-    return t >= TokenType.specialDate && t <= TokenType.traits;
-}
-
-/**
- * ditto
- */
-pure nothrow bool isConstant(ref const Token t)
-{
-    return isConstant(t.type);
-}
-
-/**
- * Returns: true if the token is a string or number literal
- */
-pure nothrow bool isLiteral(const TokenType t)
-{
-    return t >= TokenType.doubleLiteral && t <= TokenType.wstringLiteral;
-}
-
-/**
- * ditto
- */
-pure nothrow bool isLiteral(ref const Token t)
-{
-    return isLiteral(t.type);
-}
-
-/**
- * Returns: true if the token is a number literal
- */
-pure nothrow bool isNumberLiteral(const TokenType t)
-{
-    return t >= TokenType.doubleLiteral && t <= TokenType.ulongLiteral;
-}
-
-/**
- * ditto
- */
-pure nothrow bool isNumberLiteral(ref const Token t)
-{
-    return isNumberLiteral(t.type);
-}
-
-/**
- * Returns: true if the token is a string literal
- */
-pure nothrow bool isStringLiteral(const TokenType t)
-{
-    return t >= TokenType.dstringLiteral && t <= TokenType.wstringLiteral;
-}
-
-/**
- * ditto
- */
-pure nothrow bool isStringLiteral(ref const Token t)
-{
-    return isStringLiteral(t.type);
-}
-
-/**
- * Returns: true if the token is whitespace, a comment, a special token
- *     sequence, or an identifier
- */
-pure nothrow bool isMisc(const TokenType t)
-{
-    return t >= TokenType.comment && t <= TokenType.specialTokenSequence;
-}
-
-/**
- * ditto
- */
-pure nothrow bool isMisc(ref const Token t)
-{
-    return isMisc(t.type);
-}
-
-/**
- * Listing of all the tokens in the D language.
- */
-enum TokenType: ushort
-{
-    invalid, /// Not a valid token
-    assign, /// =
-    at, /// @
-    amp, /// &
-    bitAndAssign, /// &=
-    bitOr, /// |
-    bitOrAssign, /// |=
-    catAssign, /// ~=
-    colon, /// :
-    comma, /// ,
-    decrement, /// --
-    div, /// /
-    divAssign, /// /=
-    dollar, /// $
-    dot, /// .
-    equal, /// ==
-    goesTo, /// =>
-    greater, /// >
-    greaterEqual, /// >=
-    hash, /// #
-    increment, /// ++
-    lBrace, /// {
-    lBracket, /// [
-    less, /// <
-    lessEqual, /// <=
-    lessEqualGreater, /// <>=
-    lessOrGreater, /// <>
-    logicAnd, /// &&
-    logicOr, /// ||
-    lParen, /// $(LPAREN)
-    minus, /// -
-    minusAssign, /// -=
-    mod, /// %
-    modAssign, /// %=
-    mulAssign, /// *=
-    not, /// !
-    notEqual, /// !=
-    notGreater, /// !>
-    notGreaterEqual, /// !>=
-    notLess, /// !<
-    notLessEqual, /// !<=
-    notLessEqualGreater, /// !<>
-    plus, /// +
-    plusAssign, /// +=
-    pow, /// ^^
-    powAssign, /// ^^=
-    rBrace, /// }
-    rBracket, /// ]
-    rParen, /// $(RPAREN)
-    semicolon, /// ;
-    shiftLeft, /// <<
-    shiftLeftAssign, /// <<=
-    shiftRight, /// >>
-    shiftRightAssign, /// >>=
-    dotdot, /// ..
-    star, /// *
-    ternary, /// ?
-    tilde, /// ~
-    unordered, /// !<>=
-    unsignedShiftRight, /// >>>
-    unsignedShiftRightAssign, /// >>>=
-    vararg, /// ...
-    xor, /// ^
-    xorAssign, /// ^=
-
-    bool_, /// $(D_KEYWORD bool)
-    byte_, /// $(D_KEYWORD byte)
-    cdouble_, /// $(D_KEYWORD cdouble)
-    cent_, /// $(D_KEYWORD cent)
-    cfloat_, /// $(D_KEYWORD cfloat)
-    char_, /// $(D_KEYWORD char)
-    creal_, /// $(D_KEYWORD creal)
-    dchar_, /// $(D_KEYWORD dchar)
-    double_, /// $(D_KEYWORD double)
-    float_, /// $(D_KEYWORD float)
-    idouble_, /// $(D_KEYWORD idouble)
-    ifloat_, /// $(D_KEYWORD ifloat)
-    int_, /// $(D_KEYWORD int)
-    ireal_, /// $(D_KEYWORD ireal)
-    long_, /// $(D_KEYWORD long)
-    real_, /// $(D_KEYWORD real)
-    short_, /// $(D_KEYWORD short)
-    ubyte_, /// $(D_KEYWORD ubyte)
-    ucent_, /// $(D_KEYWORD ucent)
-    uint_, /// $(D_KEYWORD uint)
-    ulong_, /// $(D_KEYWORD ulong)
-    ushort_, /// $(D_KEYWORD ushort)
-    void_, /// $(D_KEYWORD void)
-    wchar_, /// $(D_KEYWORD wchar)
-
-    align_, /// $(D_KEYWORD align)
-    deprecated_, /// $(D_KEYWORD deprecated)
-    extern_, /// $(D_KEYWORD extern)
-    pragma_, /// $(D_KEYWORD pragma)
-    export_, /// $(D_KEYWORD export)
-    package_, /// $(D_KEYWORD package)
-    private_, /// $(D_KEYWORD private)
-    protected_, /// $(D_KEYWORD protected)
-    public_, /// $(D_KEYWORD public)
-    abstract_, /// $(D_KEYWORD abstract)
-    auto_, /// $(D_KEYWORD auto)
-    const_, /// $(D_KEYWORD const)
-    final_, /// $(D_KEYWORD final)
-    gshared, /// $(D_KEYWORD __gshared)
-    immutable_, /// $(D_KEYWORD immutable)
-    inout_, /// $(D_KEYWORD inout)
-    scope_, /// $(D_KEYWORD scope)
-    shared_, /// $(D_KEYWORD shared)
-    static_, /// $(D_KEYWORD static)
-
-    synchronized_, /// $(D_KEYWORD synchronized)
-    alias_, /// $(D_KEYWORD alias)
-    asm_, /// $(D_KEYWORD asm)
-    assert_, /// $(D_KEYWORD assert)
-    body_, /// $(D_KEYWORD body)
-    break_, /// $(D_KEYWORD break)
-    case_, /// $(D_KEYWORD case)
-    cast_, /// $(D_KEYWORD cast)
-    catch_, /// $(D_KEYWORD catch)
-    class_, /// $(D_KEYWORD class)
-    continue_, /// $(D_KEYWORD continue)
-    debug_, /// $(D_KEYWORD debug)
-    default_, /// $(D_KEYWORD default)
-    delegate_, /// $(D_KEYWORD delegate)
-    function_, /// $(D_KEYWORD function)
-    delete_, /// $(D_KEYWORD delete)
-    do_, /// $(D_KEYWORD do)
-    else_, /// $(D_KEYWORD else)
-    enum_, /// $(D_KEYWORD enum)
-    false_, /// $(D_KEYWORD false)
-    finally_, /// $(D_KEYWORD finally)
-    foreach_, /// $(D_KEYWORD foreach)
-    foreach_reverse_, /// $(D_KEYWORD foreach_reverse)
-    for_, /// $(D_KEYWORD for)
-    goto_, /// $(D_KEYWORD goto)
-    if_, /// $(D_KEYWORD if)
-    import_, /// $(D_KEYWORD import)
-    in_, /// $(D_KEYWORD in)
-    interface_, /// $(D_KEYWORD interface)
-    invariant_, /// $(D_KEYWORD invariant)
-    is_, /// $(D_KEYWORD is)
-    lazy_, /// $(D_KEYWORD lazy)
-    macro_, /// $(D_KEYWORD macro)
-    mixin_, /// $(D_KEYWORD mixin)
-    module_, /// $(D_KEYWORD module)
-    new_, /// $(D_KEYWORD new)
-    nothrow_, /// $(D_KEYWORD nothrow)
-    null_, /// $(D_KEYWORD null)
-    out_, /// $(D_KEYWORD out)
-    override_, /// $(D_KEYWORD override)
-    pure_, /// $(D_KEYWORD pure)
-    ref_, /// $(D_KEYWORD ref)
-    return_, /// $(D_KEYWORD return)
-    struct_, /// $(D_KEYWORD struct)
-    super_, /// $(D_KEYWORD super)
-    switch_, /// $(D_KEYWORD switch)
-    template_, /// $(D_KEYWORD template)
-    this_, /// $(D_KEYWORD this)
-    throw_, /// $(D_KEYWORD throw)
-    true_, /// $(D_KEYWORD true)
-    try_, /// $(D_KEYWORD try)
-    typedef_, /// $(D_KEYWORD typedef)
-    typeid_, /// $(D_KEYWORD typeid)
-    typeof_, /// $(D_KEYWORD typeof)
-    union_, /// $(D_KEYWORD union)
-    unittest_, /// $(D_KEYWORD unittest)
-    version_, /// $(D_KEYWORD version)
-    volatile_, /// $(D_KEYWORD volatile)
-    while_, /// $(D_KEYWORD while)
-    with_, /// $(D_KEYWORD with)
-
-    specialDate, /// $(D_KEYWORD ___DATE__)
-    specialEof, /// $(D_KEYWORD ___EOF__)
-    specialTime, /// $(D_KEYWORD ___TIME__)
-    specialTimestamp, /// $(D_KEYWORD ___TIMESTAMP__)
-    specialVendor, /// $(D_KEYWORD ___VENDOR__)
-    specialVersion, /// $(D_KEYWORD ___VERSION__)
-    specialFile, /// $(D_KEYWORD ___FILE__)
-    specialLine, /// $(D_KEYWORD ___LINE__)
-    specialModule, /// $(D_KEYWORD ___MODULE__)
-    specialFunction, /// $(D_KEYWORD ___FUNCTION__)
-    specialPrettyFunction, /// $(D_KEYWORD ___PRETTY_FUNCTION__)
-    specialTokenSequence, /// #line 10 "file.d"
-
-    comment, /// $(D_COMMENT /** comment */) or $(D_COMMENT // comment) or $(D_COMMENT ///comment)
-    identifier, /// anything else
-    scriptLine, /// Line at the beginning of source file that starts from #!
-    traits, /// $(D_KEYWORD ___traits)
-    parameters, /// $(D_KEYWORD ___parameters)
-    vector, /// $(D_KEYWORD ___vector)
-    whitespace, /// whitespace
-    doubleLiteral, /// 123.456
-    floatLiteral, /// 123.456f or 0x123_45p-3
-    idoubleLiteral, /// 123.456i
-    ifloatLiteral, /// 123.456fi
-    intLiteral, /// 123 or 0b1101010101
-    longLiteral, /// 123L
-    realLiteral, /// 123.456L
-    irealLiteral, /// 123.456Li
-    uintLiteral, /// 123u
-    ulongLiteral, /// 123uL
-    characterLiteral, /// 'a'
-    dstringLiteral, /// $(D_STRING "32-bit string"d)
-    stringLiteral, /// $(D_STRING "an 8-bit string")
-    wstringLiteral, /// $(D_STRING "16-bit string"w)
-}
-
-/**
- * Look up a token's string representation by its type.
- * Params:
- *     type = the token type
- * Returns: a string representing the token, or null for token types such as
- *     identifier or integer literal whose string representations vary
- */
-pure string getTokenValue(const TokenType type)
-{
-    return tokenValues[type];
-}
-
-///
-unittest
-{
-    // The class token always has one value
-    assert (getTokenValue(TokenType.class_) == "class");
-    // Identifiers do not
-    assert (getTokenValue(TokenType.identifier) is null);
-}
-
-// Implementation details follow
-private:
-
-// For now a private helper that is tailored to the way lexer works
-// hides away forwardness of range by buffering
-// random-access version is a straightforward thin wrapping
-// ATM it is byte-oriented
-private struct LexSource(R)
-    if(isForwardRange!R && !isRandomAccessRange!R)
-    {
-    bool empty() const { return _empty; }
-
-    auto ref front() const
-    {
-        return accum[accumIdx];
-    }
-
-    auto ref peek() const
-    in
-    {
-        assert (accumIdx + 1 < accum.length);
-    }
-    body
-    {
-        return accum[accumIdx + 1];
-    }
-
-    void popFront()
-    {
-        ++_index;
-        range.popFront();
-        // if that was last byte
-        // just advance so that open-righted slice just works
-        accumIdx =  (accumIdx+1) & mask;
-        if(range.empty)
-        {
-            _empty = true;
-            return;
-        }
-        if(accumIdx == savedAccumIdx)
-        {
-            // and move stuff around
-            auto oldLen = accum.length;
-            auto toCopy = oldLen - accumIdx;
-            accum.length *= 2; // keep pow of 2
-            // copy starting with last item
-            copy(retro(accum[accumIdx..oldLen]),
-                retro(accum[$-toCopy..$]));
-            savedAccumIdx = accum.length - toCopy;
-        }
-        accum[accumIdx] = range.front;
-    }
-
-    auto save()
-    {
-        typeof(this) copy = this;
-        copy.range = range.save;
-        // sadly need to dup circular buffer, as it overwrites items
-        copy.accum = copy.accum.dup;
-        return copy;
-    }
-
-    // mark a position to slice from later on
-    size_t mark()
-    {
-        savedAccumIdx = accumIdx;
-        return accumIdx;
-    }
-
-    // slice to current position from previously marked position
-    auto slice() @property
-    {
-        // it's an open right range as usual
-        return CircularRange(accum, savedAccumIdx, accumIdx);
-    }
-
-    size_t index() const @property
-    {
-        return _index;
-    }
-
-private:
-    this(R src, size_t bufferSize)
-    {
-        range = src;
-        assert(bufferSize > 0);
-        assert((bufferSize & (bufferSize-1)) == 0); //is power of 2
-        accum = new ubyte[bufferSize];
-        if(range.empty)
-            _empty = true;
-        else
-            accum[accumIdx] = range.front; // load front
-    }
-
-    // a true RA-range of ubyte
-    struct CircularRange
-    {
-        this(ubyte[] buf, size_t s, size_t e)
-        {
-            assert((buffer.length & (buffer.length-1)) == 0);
-            buffer = buf;
-            start = s;
-            end = e;
-        }
-        //Forward range primitives
-        @property bool empty() const { return start == end; }
-        @property auto ref front() const { return buffer[start]; }
-        void popFront() { start = (start + 1) & mask; }
-        @property auto save() { return this; }
-
-        //Backwards is a bit slower, but should be rarely used (if at all)
-        @property ref back(){ return buffer[(end-1) & mask]; }
-        void popBack() { end  = (end - 1) & mask; }
-
-        // RA range primitives
-        ref opIndex(size_t idx){ return buffer[(start+idx) & mask]; }
-        @property size_t length()
-        {
-            return end < start ? end + buffer.length -start : end - start;
-        }
-        alias length opDollar;
-
-        auto opSlice(size_t newStart, size_t newEnd)
-        {
-            size_t maskedStart = (start+newStart) & mask;
-            size_t maskedEnd = (start+newEnd) & mask;
-            return typeof(this)(buffer, maskedStart, maskedEnd);
-        }
-        // @@@bug fwd-ref in ldc0.10 (if placed above previous one)
-        auto opSlice(){ return opSlice(0, length); }
-    private:
-        @property auto mask(){ return buffer.length-1; }
-        size_t start, end;
-        ubyte[] buffer;
-    }
-
-    @property auto mask(){ return accum.length-1; }
-
-    R range;
-    bool _empty;
-    ubyte[] accum; // accumulator buffer for non-RA ranges
-    size_t savedAccumIdx;
-    size_t accumIdx; // current index in accumulator
-    size_t _index; // index of current element in original range
-}
-
-// TODO: make sure it's RandomAccess later
-/*static assert(isRandomAccessRange!(
-    LexSource!(typeof(filter!"true"(cast(ubyte[])null)))
-    .CircularRange)
-);*/
-
-//trivial pass-through for RA ranges
-private struct LexSource(R)
-    if(isRandomAccessRange!R)
-{
-    bool empty() const @property { return cur >= range.length; }
-    bool canPeek() const { return cur + 1 < range.length; }
-    auto ref front() const @property { return range[cur]; }
-    void popFront(){ cur++; }
-
-    auto ref peek() const
-    in
-    {
-        assert (canPeek());
-    }
-    body
-    {
-        return range[cur + 1];
-    }
-
-    auto save()
-    {
-        typeof(this) copy = this;
-        copy.range = range.save;
-        return copy;
-    }
-
-    auto mark()
-    {
-        saved = cur;
-    }
-
-    // use the underlying range slicing capability
-    auto slice() @property
-    {
-        return range[saved..cur];
-    }
-
-    size_t index() const @property
-    {
-        return cur;
-    }
-
-private:
-    this(R src)
-    {
-        range = src;
-    }
-    size_t cur, saved;
-    R range;
-}
-
-auto lexerSource(Range)(Range range, size_t bufSize=8)
-    if(isForwardRange!Range && !isRandomAccessRange!Range
-    && is(ElementType!Range : const(ubyte)))
-{
-    return LexSource!(Range)(range, bufSize);
-}
-
-auto lexerSource(Range)(Range range)
-    if(isRandomAccessRange!Range
-    && is(ElementType!Range : const(ubyte)))
-{
-    return LexSource!(Range)(range);
-}
-
-unittest
-{
-    // test the basic functionality of a "mark-slice" range
-    import std.string, std.stdio;
-
-    static void test_hello(T)(T lexs)
-    {
-        assert(lexs.front == 'H');
-        lexs.popFront();
-        assert(lexs.front == 'e');
-        foreach(i; 0..2)
-        {
-            auto saved = lexs.save;
-            lexs.mark();
-            assert(lexs.slice.equal(""));
-            lexs.popFront();
-            assert(lexs.slice.equal("e"), text(cast(char)lexs.front));
-            lexs.popFrontN(4);
-            auto bytes = lexs.slice.map!"cast(char)a".array();
-            assert(bytes.equal("ello,"), bytes.to!string);
-            lexs.mark();
-            assert(lexs.slice.equal(""));
-            assert(lexs.front == 'w');
-            lexs.popFrontN(6);
-            assert(lexs.empty);
-            auto s = lexs.slice();
-            auto msg = s.save.map!"cast(char)a".array;
-            assert(s[].equal("world!"), msg);
-            assert(s[2..$-1].equal("rld"), msg);
-            assert(s[0] == 'w' && s[$-1] == '!');
-            s.popFront();
-            assert(s.front == 'o' && s.back == '!');
-            s.popBack();
-            assert(s.front == 'o' && s.back == 'd');
-            //restore and repeat again
-            lexs = saved;
-        }
-    }
-
-    static void test_empty(T)(T lexs)
-    {
-        assert(lexs.empty);
-        lexs.mark();
-        assert(lexs.slice().equal(""));
-    }
-
-    auto fwdLex = lexerSource(
-        "Hello, world!"
-        .representation
-        .filter!"a != ' '", 16 // and the one that is more then enough
-    );
-    test_hello(fwdLex);
-    fwdLex = lexerSource(
-        "Hello, world!"
-        .representation
-        .filter!"a != ' '", 1 // try the smallest initial buffer
-    );
-    test_hello(fwdLex);
-    fwdLex = lexerSource("".representation.filter!"a != ' '");
-    auto raLex = lexerSource("".representation);
-    test_empty(raLex);
-    test_empty(fwdLex);
-    raLex = lexerSource("Hello,world!".representation);
-    test_hello(raLex);
-}
-
-// uses auto-detection for pure, safe nothrow
-bool isRangeEoF(R)(ref R range)
-{
-    return range.empty || range.front == 0 || range.front == 0x1a;
-}
-
-// Lookup table for token string representations
-immutable(string[TokenType.max + 1]) tokenValues = [
-    null,
-    "=",
-    "@",
-    "&",
-    "&=",
-    "|",
-    "|=",
-    "~=",
-    ":",
-    ",",
-    "--",
-    "/",
-    "/=",
-    "$",
-    ".",
-    "==",
-    "=>",
-    ">",
-    ">=",
-    "#",
-    "++",
-    "{",
-    "[",
-    "<",
-    "<=",
-    "<>=",
-    "<>",
-    "&&",
-    "||",
-    "(",
-    "-",
-    "-=",
-    "%",
-    "%=",
-    "*=",
-    "!",
-    "!=",
-    "!>",
-    "!>=",
-    "!<",
-    "!<=",
-    "!<>",
-    "+",
-    "+=",
-    "^^",
-    "^^=",
-    "}",
-    "]",
-    ")",
-    ";",
-    "<<",
-    "<<=",
-    ">>",
-    ">>=",
-    "..",
-    "*",
-    "?",
-    "~",
-    "!<>=",
-    ">>>",
-    ">>>=",
-    "...",
-    "^",
-    "^=",
-    "bool",
-    "byte",
-    "cdouble",
-    "cent",
-    "cfloat",
-    "char",
-    "creal",
-    "dchar",
-    "double",
-    "float",
-    "idouble",
-    "ifloat",
-    "int",
-    "ireal",
-    "long",
-    "real",
-    "short",
-    "ubyte",
-    "ucent",
-    "uint",
-    "ulong",
-    "ushort",
-    "void",
-    "wchar",
-    "align",
-    "deprecated",
-    "extern",
-    "pragma",
-    "export",
-    "package",
-    "private",
-    "protected",
-    "public",
-    "abstract",
-    "auto",
-    "const",
-    "final",
-    "__gshared",
-    "immutable",
-    "inout",
-    "scope",
-    "shared",
-    "static",
-    "synchronized",
-    "alias",
-    "asm",
-    "assert",
-    "body",
-    "break",
-    "case",
-    "cast",
-    "catch",
-    "class",
-    "continue",
-    "debug",
-    "default",
-    "delegate",
-    "function",
-    "delete",
-    "do",
-    "else",
-    "enum",
-    "false",
-    "finally",
-    "foreach",
-    "foreach_reverse",
-    "for",
-    "goto",
-    "if",
-    "import",
-    "in",
-    "interface",
-    "invariant",
-    "is",
-    "lazy",
-    "macro",
-    "mixin",
-    "module",
-    "new",
-    "nothrow",
-    "null",
-    "out",
-    "override",
-    "pure",
-    "ref",
-    "return",
-    "struct",
-    "super",
-    "switch",
-    "template",
-    "this",
-    "throw",
-    "true",
-    "try",
-    "typedef",
-    "typeid",
-    "typeof",
-    "union",
-    "unittest",
-    "version",
-    "volatile",
-    "while",
-    "with",
-    "__DATE__",
-    "__EOF__",
-    "__TIME__",
-    "__TIMESTAMP__",
-    "__VENDOR__",
-    "__VERSION__",
-    "__FILE__",
-    "__LINE__",
-    "__MODULE__",
-    "__FUNCTION__",
-    "__PRETTY_FUNCTION__",
-    null,
-    null,
-    null,
-    null,
-    "__traits",
-    "__parameters",
-    "__vector",
-    null,
-    null,
-    null,
-    null,
-    null,
-    null,
-    null,
-    null,
-    null,
-    null,
-    null,
-    null,
-    null,
-    null,
-    null,
-];
-
-template tokenValue(TokenType val)
-{
-    enum tokenValue = getTokenValue(val);
-}
-
-private pure bool isNewline(ubyte ch)
-{
-    return ch == '\n' || ch == '\r';
-}
-
-pure TokenType lookupTokenType(R)(R input)
-{
-    switch(input.length)
-    {
-    case 2:
-        switch (input[0])
-        {
-        case 'd': if (input[1] == 'o') return TokenType.do_; else break;
-        case 'i':
-            if (input[1] == 'f') return TokenType.if_;
-            else if (input[1] == 'n') return TokenType.in_;
-            else if (input[1] == 's') return TokenType.is_;
-            else break;
-        default: break;
-        }
-        break;
-    case 3:
-        switch (input[0])
-        {
-        case 'a': if (input[1..$].equal("sm")) return TokenType.asm_; else break;
-        case 'f': if (input[1..$].equal("or")) return TokenType.for_; else break;
-        case 'i': if (input[1..$].equal("nt")) return TokenType.int_; else break;
-        case 'n': if (input[1..$].equal("ew")) return TokenType.new_; else break;
-        case 'o': if (input[1..$].equal("ut")) return TokenType.out_; else break;
-        case 'r': if (input[1..$].equal("ef")) return TokenType.ref_; else break;
-        case 't': if (input[1..$].equal("ry")) return TokenType.try_; else break;
-        default: break;
-        }
-        break;
-    case 4:
-        switch (input[0])
-        {
-        case 'a': if (input[1..$].equal("uto")) return TokenType.auto_; else break;
-        case 'b': if (input[1..$].equal("ody")) return TokenType.body_;
-            else if (input[1..$].equal("ool")) return TokenType.bool_;
-            else if (input[1..$].equal("yte")) return TokenType.byte_;
-            else break;
-        case 'c': if (input[1..$].equal("ase")) return TokenType.case_;
-            else if (input[1..$].equal("ast")) return TokenType.cast_;
-            else if (input[1..$].equal("ent")) return TokenType.cent_;
-            else if (input[1..$].equal("har")) return TokenType.char_;
-            else break;
-        case 'e': if (input[1..$].equal("lse")) return TokenType.else_;
-            else if (input[1..$].equal("num")) return TokenType.enum_;
-            else break;
-        case 'g': if (input[1..$].equal("oto")) return TokenType.goto_; else break;
-        case 'l': if (input[1..$].equal("azy")) return TokenType.lazy_;
-            else if (input[1..$].equal("ong")) return TokenType.long_;
-            else break;
-        case 'n': if (input[1..$].equal("ull")) return TokenType.null_; else break;
-        case 'p': if (input[1..$].equal("ure")) return TokenType.pure_; else break;
-        case 'r': if (input[1..$].equal("eal")) return TokenType.real_; else break;
-        case 't': if (input[1..$].equal("his")) return TokenType.this_;
-            else if (input[1..$].equal("rue")) return TokenType.true_;
-            else break;
-        case 'u': if (input[1..$].equal("int")) return TokenType.uint_; else break;
-        case 'v': if (input[1..$].equal("oid")) return TokenType.void_; else break;
-        case 'w': if (input[1..$].equal("ith")) return TokenType.with_; else break;
-        default: break;
-        }
-        break;
-    case 5:
-        switch (input[0])
-        {
-        case 'a': if (input[1..$].equal("lias")) return TokenType.alias_;
-            else if (input[1..$].equal("lign")) return TokenType.align_; else break;
-        case 'b': if (input[1..$].equal("reak")) return TokenType.break_; else break;
-        case 'c': if (input[1..$].equal("atch")) return TokenType.catch_;
-            else if (input[1..$].equal("lass")) return TokenType.class_;
-            else if (input[1..$].equal("onst")) return TokenType.const_;
-            else if (input[1..$].equal("real")) return TokenType.creal_;
-            else break;
-        case 'd': if (input[1..$].equal("char")) return TokenType.dchar_;
-            else if (input[1..$].equal("ebug")) return TokenType.debug_; else break;
-        case 'f': if (input[1..$].equal("alse")) return TokenType.false_;
-            else if (input[1..$].equal("inal")) return TokenType.final_;
-            else if (input[1..$].equal("loat")) return TokenType.float_;
-            else break;
-        case 'i': if (input[1..$].equal("nout")) return TokenType.inout_;
-            else if (input[1..$].equal("real")) return TokenType.ireal_; else break;
-        case 'm': if (input[1..$].equal("acro")) return TokenType.macro_;
-            else if (input[1..$].equal("ixin")) return TokenType.mixin_; else break;
-        case 's': if (input[1..$].equal("cope")) return TokenType.scope_;
-            else if (input[1..$].equal("hort")) return TokenType.short_;
-            else if (input[1..$].equal("uper")) return TokenType.super_; else break;
-        case 't': if (input[1..$].equal("hrow")) return TokenType.throw_; else break;
-        case 'u': if (input[1..$].equal("byte")) return TokenType.ubyte_;
-            else if (input[1..$].equal("cent")) return TokenType.ucent_;
-            else if (input[1..$].equal("long")) return TokenType.ulong_;
-            else if (input[1..$].equal("nion")) return TokenType.union_;
-            else break;
-        case 'w': if (input[1..$].equal("char")) return TokenType.wchar_;
-            else if (input[1..$].equal("hile")) return TokenType.while_;
-            else break;
-        default: break;
-        }
-        break;
-    case 6:
-        switch (input[0])
-        {
-        case 'a': if (input[1..$].equal("ssert")) return TokenType.assert_; else break;
-        case 'c': if (input[1..$].equal("float")) return TokenType.cfloat_; else break;
-        case 'd': if (input[1..$].equal("elete")) return TokenType.delete_;
-            else if (input[1..$].equal("ouble")) return TokenType.double_; else break;
-        case 'e': if (input[1..$].equal("xport")) return TokenType.export_;
-            else if (input[1..$].equal("xtern")) return TokenType.extern_; else break;
-        case 'i': if (input[1..$].equal("float")) return TokenType.ifloat_;
-            else if (input[1..$].equal("mport")) return TokenType.import_; else break;
-        case 'm': if (input[1..$].equal("odule")) return TokenType.module_; else break;
-        case 'p': if (input[1..$].equal("ragma")) return TokenType.pragma_;
-            else if (input[1..$].equal("ublic")) return TokenType.public_; else break;
-        case 'r': if (input[1..$].equal("eturn")) return TokenType.return_; else break;
-        case 's': if (input[1..$].equal("hared")) return TokenType.shared_;
-            else if (input[1..$].equal("tatic")) return TokenType.static_;
-            else if (input[1..$].equal("truct")) return TokenType.struct_;
-            else if (input[1..$].equal("witch")) return TokenType.switch_; else break;
-        case 't': if (input[1..$].equal("ypeid")) return TokenType.typeid_;
-            else if (input[1..$].equal("ypeof")) return TokenType.typeof_; else break;
-        case 'u': if (input[1..$].equal("short")) return TokenType.ushort_; else break;
-        default: break;
-        }
-        break;
-    case 7:
-        switch (input[0])
-        {
-        case '_': if (input[1..$].equal("_EOF__")) return TokenType.specialEof; else break;
-        case 'c': if (input[1..$].equal("double")) return TokenType.cdouble_; else break;
-        case 'd': if (input[1..$].equal("efault")) return TokenType.default_; else break;
-        case 'f': if (input[1..$].equal("inally")) return TokenType.finally_;
-            else if (input[1..$].equal("oreach")) return TokenType.foreach_; else break;
-        case 'i': if (input[1..$].equal("double")) return TokenType.idouble_; else break;
-        case 'n': if (input[1..$].equal("othrow")) return TokenType.nothrow_; else break;
-        case 'p': if (input[1..$].equal("ackage")) return TokenType.package_;
-            else if (input[1..$].equal("rivate")) return TokenType.private_; else break;
-        case 't': if (input[1..$].equal("ypedef")) return TokenType.typedef_; else break;
-        case 'v': if (input[1..$].equal("ersion")) return TokenType.version_; else break;
-        default: break;
-        }
-        break;
-    case 8:
-        switch (input[0])
-        {
-        case '_': if (input[1..$].equal("_DATE__")) return TokenType.specialDate;
-            else if (input[1..$].equal("_FILE__")) return TokenType.specialFile;
-            else if (input[1..$].equal("_LINE__")) return TokenType.specialLine;
-            else if (input[1..$].equal("_vector")) return TokenType.vector;
-            else if (input[1..$].equal("_TIME__")) return TokenType.specialTime;
-            else if (input[1..$].equal("_traits")) return TokenType.traits; else break;
-        case 'a': if (input[1..$].equal("bstract")) return TokenType.abstract_; else break;
-        case 'c': if (input[1..$].equal("ontinue")) return TokenType.continue_; else break;
-        case 'd': if (input[1..$].equal("elegate")) return TokenType.delegate_; else break;
-        case 'f': if (input[1..$].equal("unction")) return TokenType.function_; else break;
-        case 'o': if (input[1..$].equal("verride")) return TokenType.override_; else break;
-        case 't': if (input[1..$].equal("emplate")) return TokenType.template_; else break;
-        case 'u': if (input[1..$].equal("nittest")) return TokenType.unittest_; else break;
-        case 'v': if (input[1..$].equal("olatile")) return TokenType.volatile_; else break;
-        default: break;
-        }
-        break;
-    case 9:
-        switch (input[0])
-        {
-        case '_': if (input[1..$].equal("_gshared")) return TokenType.gshared; else break;
-        case 'i': if (input[1..$].equal("mmutable")) return TokenType.immutable_;
-            else if (input[1..$].equal("nterface")) return TokenType.interface_;
-            else if (input[1..$].equal("nvariant")) return TokenType.invariant_; else break;
-        case 'p': if (input[1..$].equal("rotected")) return TokenType.protected_; else break;
-        default: break;
-        }
-        break;
-    case 10:
-        switch (input[0])
-        {
-        case 'd': if (input[1..$].equal("eprecated")) return TokenType.deprecated_; else break;
-        case '_':
-            if (input[1..$].equal("_VENDOR__")) return TokenType.specialVendor;
-            else if (input[1..$].equal("_MODULE__")) return TokenType.specialModule; else break;
-        default: break;
-        }
-        break;
-    case 11:
-        if (input.equal("__VERSION__")) return TokenType.specialVersion;
-        break;
-    case 12:
-        switch (input[0])
-        {
-        case 's': if (input[1..$].equal("ynchronized")) return TokenType.synchronized_; else break;
-        case '_': if (input[1..$].equal("_FUNCTION__")) return TokenType.specialFunction;
-            else if (input[1..$].equal("_parameters")) return TokenType.parameters; else break;
-        default: break;
-        }
-        break;
-    case 13:
-        if (input.equal("__TIMESTAMP__")) return TokenType.specialTimestamp;
-        break;
-    case 15:
-        if (input.equal("foreach_reverse")) return TokenType.foreach_reverse_;
-        break;
-    case 19:
-        if (input.equal("__PRETTY_FUNCTION__")) return TokenType.specialPrettyFunction;
-        break;
-    default: break;
-    }
-    return TokenType.identifier;
-}
-
-class Trie(K, V) if (isInputRange!K): TrieNode!(K, V)
-{
-    /**
-    * Adds the given value to the trie with the given key
-    */
-    void add(K key, V value) pure
-    {
-        TrieNode!(K,V) current = this;
-        foreach(keyPart; key)
-        {
-            if ((keyPart in current.children) is null)
-            {
-                auto node = new TrieNode!(K, V);
-                current.children[keyPart] = node;
-                current = node;
-            }
-            else
-                current = current.children[keyPart];
-        }
-        current.value = value;
-    }
-}
-
-class TrieNode(K, V) if (isInputRange!K)
-{
-    V value;
-    TrieNode!(K,V)[ElementType!K] children;
-}
-
-string printCaseStatements(K, V)(TrieNode!(K,V) node, string indentString)
-{
-    string caseStatement = "";
-    foreach(dchar k, TrieNode!(K,V) v; node.children)
-    {
-        caseStatement ~= indentString;
-        caseStatement ~= "case '";
-        caseStatement ~= k;
-        caseStatement ~= "':\n";
-        caseStatement ~= indentString;
-        caseStatement ~= "\tnextCharNonLF();\n";
-        if (v.children.length > 0)
-        {
-            caseStatement ~= indentString;
-            caseStatement ~= "\tif (isEoF())\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\t{\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\t\tcurrent.value = tokenValue!("~node.children[k].value~");\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\t\tcurrent.type = " ~ node.children[k].value;
-            caseStatement ~= ";\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\t\treturn;\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\t}\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\tswitch (src.front)\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\t{\n";
-            caseStatement ~= printCaseStatements(v, indentString ~ "\t");
-            caseStatement ~= indentString;
-            caseStatement ~= "\tdefault:\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\t\tcurrent.type = ";
-            caseStatement ~= v.value;
-            caseStatement ~= ";\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\t\tcurrent.value = tokenValue!("~v.value~");\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\t\treturn;\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\t}\n";
-        }
-        else
-        {
-            caseStatement ~= indentString;
-            caseStatement ~= "\tcurrent.type = ";
-            caseStatement ~= v.value;
-            caseStatement ~= ";\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\tcurrent.value = tokenValue!("~v.value~");\n";
-            caseStatement ~= indentString;
-            caseStatement ~= "\treturn;\n";
-        }
-    }
-    return caseStatement;
-}
-
-string generateCaseTrie(string[] args ...)
-{
-    auto t = new Trie!(string, string);
-    for(int i = 0; i < args.length; i+=2)
-    {
-        t.add(args[i], args[i+1]);
-    }
-    return printCaseStatements(t, "");
-}
-
-struct StringCache
-{
-    this(size_t startSize)
-    {
-        assert((startSize & (startSize-1)) == 0);
-        index = new Slot*[startSize];
-    }
-
-    string get(R)(R range) if (isRandomAccessRange!R
-        && is(Unqual!(ElementType!R) : const(ubyte)))
-    {
-        uint h = hash(range);
-        uint bucket = h & (index.length-1);
-        Slot *s = index[bucket];
-        if (s == null)
-        {
-            string str = putIntoCache(range);
-            index[bucket] = allocateSlot(str, h);
-            uniqueSlots++;
-            return str;
-        }
-        while (true)
-        {
-            if(s.hash == h && s.value.equal(range))
-                return s.value;
-            if(s.next == null) break;
-            s = s.next;
-        }
-        string str = putIntoCache(range);
-        s.next = allocateSlot(str, h);
-        uniqueSlots++;
-        // had at least 1 item in this bucket
-        // and inserted another one - check load factor
-        if (uniqueSlots * loadDenom > index.length * loadQuot)
-            rehash();
-        return str;
-    }
-
-private:
-
-    static uint hash(R)(R data)
-    {
-        uint hash = 0;
-        foreach (b; data)
-        {
-            hash ^= sbox[b];
-            hash *= 3;
-        }
-        return hash;
-    }
-
-    struct Slot
-    {
-        string value;
-        Slot* next;
-        uint hash;
-    }
-
-    void printLoadFactor()
-    {
-        size_t cnt = 0, maxChain = 0;
-        foreach(Slot* s; index)
-        {
-            size_t chain = 0;
-            for(Slot* p = s; p; p = p.next)
-            {
-                chain++;
-            }
-            maxChain = max(chain, maxChain);
-            cnt += chain;
-        }
-        import std.stdio;
-        assert(cnt == uniqueSlots);
-        writefln("Load factor: %.3f; max bucket %d",
-            cast(double)cnt/index.length,
-                maxChain);
-    }
-
-    void rehash()
-    {
-        //writefln("BEFORE (size = %d):", index.length);
-        //printLoadFactor();
-        size_t oldLen = index.length;
-        index.length *= 2;
-        for (size_t i = 0; i < oldLen; i++)
-        {
-            Slot* cur = index[i], prev;
-            while(cur)
-            {
-                //has extra bit set - move it out
-                if(cur.hash & oldLen)
-                {
-                    if(prev == null)
-                    {
-                        Slot* r = cur;
-                        index[i] = cur.next;
-                        cur = cur.next;
-                        insertIntoBucket(r, i + oldLen);
-                    }
-                    else
-                    {
-                        Slot* r = removeLink(cur, prev);
-                        insertIntoBucket(r, i + oldLen);
-                    }
-                }
-                else
-                {
-                    prev = cur;
-                    cur = cur.next;
-                }
-            }
-        }
-        //writefln("AFTER (size = %d):", index.length);
-        //printLoadFactor();
-    }
-
-    static Slot* removeLink(ref Slot* cur, Slot* prev)
-    {
-        prev.next = cur.next;
-        Slot* r = cur;
-        cur = cur.next;
-        return r;
-    }
-
-    //insert at front of bucket
-    void insertIntoBucket(Slot* what, size_t bucket)
-    {
-        what.next = null;
-        Slot* p = index[bucket];
-        what.next = p;
-        index[bucket] = what;
-    }
-
-    Slot* allocateSlot(string val, uint hash)
-    {
-        auto slice = allocateInCache(Slot.sizeof);
-        auto newSlot = cast(Slot*)slice.ptr;
-        *newSlot = Slot(val, null, hash);
-        return newSlot;
-    }
-
-    Slot*[] index;
-    size_t uniqueSlots;
-    enum loadQuot = 2, loadDenom = 3;
-
-    // leave some slack for allocators/GC meta-data
-    enum chunkSize = 16*1024 - size_t.sizeof*8;
-    ubyte*[] chunkS;
-    size_t next = chunkSize;
-    //TODO: add aligned variant that allocates at word boundary
-    ubyte[] allocateInCache(size_t size)
-    {
-        import core.memory;
-        if(next + size > chunkSize)
-        {
-            // avoid huge allocations
-            if(size> chunkSize/4)
-            {
-                ubyte* p = cast(ubyte*)GC.malloc(size,
-                    GC.BlkAttr.NO_SCAN);
-                return p[0..size];
-            }
-            chunkS ~= cast(ubyte*)GC.malloc(chunkSize,
-                GC.BlkAttr.NO_SCAN);
-            next = 0;
-        }
-        auto slice = chunkS[$-1][next..next+size];
-        next += size;
-        return slice;
-    }
-
-    string putIntoCache(R)(R data)
-    {
-        auto slice = allocateInCache(data.length);
-        slice[] = data[];
-        return cast(string)slice;
-    }
-
-}
-
-immutable uint[] sbox = [
-    0xF53E1837, 0x5F14C86B, 0x9EE3964C, 0xFA796D53,
-    0x32223FC3, 0x4D82BC98, 0xA0C7FA62, 0x63E2C982,
-    0x24994A5B, 0x1ECE7BEE, 0x292B38EF, 0xD5CD4E56,
-    0x514F4303, 0x7BE12B83, 0x7192F195, 0x82DC7300,
-    0x084380B4, 0x480B55D3, 0x5F430471, 0x13F75991,
-    0x3F9CF22C, 0x2FE0907A, 0xFD8E1E69, 0x7B1D5DE8,
-    0xD575A85C, 0xAD01C50A, 0x7EE00737, 0x3CE981E8,
-    0x0E447EFA, 0x23089DD6, 0xB59F149F, 0x13600EC7,
-    0xE802C8E6, 0x670921E4, 0x7207EFF0, 0xE74761B0,
-    0x69035234, 0xBFA40F19, 0xF63651A0, 0x29E64C26,
-    0x1F98CCA7, 0xD957007E, 0xE71DDC75, 0x3E729595,
-    0x7580B7CC, 0xD7FAF60B, 0x92484323, 0xA44113EB,
-    0xE4CBDE08, 0x346827C9, 0x3CF32AFA, 0x0B29BCF1,
-    0x6E29F7DF, 0xB01E71CB, 0x3BFBC0D1, 0x62EDC5B8,
-    0xB7DE789A, 0xA4748EC9, 0xE17A4C4F, 0x67E5BD03,
-    0xF3B33D1A, 0x97D8D3E9, 0x09121BC0, 0x347B2D2C,
-    0x79A1913C, 0x504172DE, 0x7F1F8483, 0x13AC3CF6,
-    0x7A2094DB, 0xC778FA12, 0xADF7469F, 0x21786B7B,
-    0x71A445D0, 0xA8896C1B, 0x656F62FB, 0x83A059B3,
-    0x972DFE6E, 0x4122000C, 0x97D9DA19, 0x17D5947B,
-    0xB1AFFD0C, 0x6EF83B97, 0xAF7F780B, 0x4613138A,
-    0x7C3E73A6, 0xCF15E03D, 0x41576322, 0x672DF292,
-    0xB658588D, 0x33EBEFA9, 0x938CBF06, 0x06B67381,
-    0x07F192C6, 0x2BDA5855, 0x348EE0E8, 0x19DBB6E3,
-    0x3222184B, 0xB69D5DBA, 0x7E760B88, 0xAF4D8154,
-    0x007A51AD, 0x35112500, 0xC9CD2D7D, 0x4F4FB761,
-    0x694772E3, 0x694C8351, 0x4A7E3AF5, 0x67D65CE1,
-    0x9287DE92, 0x2518DB3C, 0x8CB4EC06, 0xD154D38F,
-    0xE19A26BB, 0x295EE439, 0xC50A1104, 0x2153C6A7,
-    0x82366656, 0x0713BC2F, 0x6462215A, 0x21D9BFCE,
-    0xBA8EACE6, 0xAE2DF4C1, 0x2A8D5E80, 0x3F7E52D1,
-    0x29359399, 0xFEA1D19C, 0x18879313, 0x455AFA81,
-    0xFADFE838, 0x62609838, 0xD1028839, 0x0736E92F,
-    0x3BCA22A3, 0x1485B08A, 0x2DA7900B, 0x852C156D,
-    0xE8F24803, 0x00078472, 0x13F0D332, 0x2ACFD0CF,
-    0x5F747F5C, 0x87BB1E2F, 0xA7EFCB63, 0x23F432F0,
-    0xE6CE7C5C, 0x1F954EF6, 0xB609C91B, 0x3B4571BF,
-    0xEED17DC0, 0xE556CDA0, 0xA7846A8D, 0xFF105F94,
-    0x52B7CCDE, 0x0E33E801, 0x664455EA, 0xF2C70414,
-    0x73E7B486, 0x8F830661, 0x8B59E826, 0xBB8AEDCA,
-    0xF3D70AB9, 0xD739F2B9, 0x4A04C34A, 0x88D0F089,
-    0xE02191A2, 0xD89D9C78, 0x192C2749, 0xFC43A78F,
-    0x0AAC88CB, 0x9438D42D, 0x9E280F7A, 0x36063802,
-    0x38E8D018, 0x1C42A9CB, 0x92AAFF6C, 0xA24820C5,
-    0x007F077F, 0xCE5BC543, 0x69668D58, 0x10D6FF74,
-    0xBE00F621, 0x21300BBE, 0x2E9E8F46, 0x5ACEA629,
-    0xFA1F86C7, 0x52F206B8, 0x3EDF1A75, 0x6DA8D843,
-    0xCF719928, 0x73E3891F, 0xB4B95DD6, 0xB2A42D27,
-    0xEDA20BBF, 0x1A58DBDF, 0xA449AD03, 0x6DDEF22B,
-    0x900531E6, 0x3D3BFF35, 0x5B24ABA2, 0x472B3E4C,
-    0x387F2D75, 0x4D8DBA36, 0x71CB5641, 0xE3473F3F,
-    0xF6CD4B7F, 0xBF7D1428, 0x344B64D0, 0xC5CDFCB6,
-    0xFE2E0182, 0x2C37A673, 0xDE4EB7A3, 0x63FDC933,
-    0x01DC4063, 0x611F3571, 0xD167BFAF, 0x4496596F,
-    0x3DEE0689, 0xD8704910, 0x7052A114, 0x068C9EC5,
-    0x75D0E766, 0x4D54CC20, 0xB44ECDE2, 0x4ABC653E,
-    0x2C550A21, 0x1A52C0DB, 0xCFED03D0, 0x119BAFE2,
-    0x876A6133, 0xBC232088, 0x435BA1B2, 0xAE99BBFA,
-    0xBB4F08E4, 0xA62B5F49, 0x1DA4B695, 0x336B84DE,
-    0xDC813D31, 0x00C134FB, 0x397A98E6, 0x151F0E64,
-    0xD9EB3E69, 0xD3C7DF60, 0xD2F2C336, 0x2DDD067B,
-    0xBD122835, 0xB0B3BD3A, 0xB0D54E46, 0x8641F1E4,
-    0xA0B38F96, 0x51D39199, 0x37A6AD75, 0xDF84EE41,
-    0x3C034CBA, 0xACDA62FC, 0x11923B8B, 0x45EF170A,
-];
-
-unittest
-{
-    LexerConfig cfg;
-    auto tkr = "void main(){ }".representation.byToken(cfg);
-    assert(tkr.map!"a.value".equal(["void", "main", "(", ")", "{", "}"]));
-    tkr = "1234 54.23232".representation.byToken(cfg);
-    assert(tkr.equal(["1234", "54.23232"]));
-    auto str =  r"0 0. .0 1 0x3 0b102 007";
-    cfg.iterStyle = IterationStyle.everything;
-    tkr = str.representation.byToken(cfg);
-    assert(tkr.map!"a.value".equal(["0", " ", "0.", " ",
-        ".0", " ", "1", " ", "0x3", " ", "0b10",
-        "2", " ", "007"]
-    ), text(tkr.map!"a.value"));
+	return DLexer!(R)(range, config);
 }
 
 unittest
 {
     import std.stdio;
-    auto source = cast(ubyte[]) (
-        " bool byte cdouble cent cfloat char creal dchar double float function"
-        ~ " idouble ifloat int ireal long real short ubyte ucent uint ulong"
-        ~ " ushort void wchar align deprecated extern pragma export package private"
-        ~ " protected public abstract auto const final __gshared immutable inout"
-        ~ " scope shared static synchronized alias asm assert body break case"
-        ~ " cast catch class continue debug default delegate delete do else"
-        ~ " enum false finally foreach foreach_reverse for goto if import in"
-        ~ " interface invariant is lazy macro mixin module new nothrow null"
-        ~ " out override pure ref return struct super switch template this"
-        ~ " throw true try typedef typeid typeof union unittest version volatile"
-        ~ " while with __traits __parameters __vector __VENDOR__ __MODULE__"
-        ~ " __VERSION__ __TIMESTAMP__ __PRETTY_FUNCTION__");
-    auto expected = ["bool", "byte", "cdouble",
-        "cent", "cfloat", "char", "creal",
-        "dchar", "double", "float", "function",
-        "idouble", "ifloat", "int", "ireal", "long",
-        "real", "short", "ubyte", "ucent", "uint",
-        "ulong", "ushort", "void", "wchar", "align",
-        "deprecated", "extern", "pragma", "export",
-        "package", "private", "protected", "public",
-        "abstract", "auto", "const", "final", "__gshared",
-        "immutable", "inout", "scope", "shared",
-        "static", "synchronized", "alias", "asm", "assert",
-        "body", "break", "case", "cast", "catch",
-        "class", "continue", "debug", "default", "delegate",
-        "delete", "do", "else", "enum", "false",
-        "finally", "foreach", "foreach_reverse", "for",
-        "goto", "if", "import", "in", "interface",
-        "invariant", "is", "lazy","macro", "mixin",
-        "module", "new", "nothrow", "null", "out",
-        "override", "pure", "ref", "return", "struct",
-        "super", "switch", "template", "this", "throw",
-        "true", "try", "typedef", "typeid", "typeof",
-        "union", "unittest", "version", "volatile",
-        "while", "with", "__traits", "__parameters", "__vector",
-        "__VENDOR__", "__MODULE__", "__VERSION__", "__TIMESTAMP__",
-        "__PRETTY_FUNCTION__"];
-    LexerConfig config;
-    config.tokenStyle = TokenStyle.doNotReplaceSpecial;
-    auto tokens = byToken(source, config);
-//    writeln(tokens.map!"a.value"().array());
-    assert (equal(map!"a.value"(tokens), expected));
+    auto source = cast(ubyte[]) q{ import std.stdio;}c;
+    auto tokens = byToken(source);
+    assert (tokens.map!"a.type"().equal([tok!"import", tok!"identifier", tok!".",
+        tok!"identifier", tok!";"]));
 }
 
-unittest
+public bool isBasicType(IdType type) nothrow pure @safe
 {
-    auto source = cast(ubyte[]) ("=@& &=| |=~=:,--/ /=$.===>> >=++{[< <=<>=<>&&||(- -=%%=*=!!=!>!>=!<!<=!<>+ +=^^^^=}]);<< <<=>> >>=..*?~!<>=>>>>>>=...^ ^=");
-    auto expected = ["=",  "@", "&", "&=", "|", "|=", "~=",
-        ":", ",", "--", "/", "/=", "$", ".", "==",
-        "=>", ">", ">=", "++", "{", "[", "<",
-        "<=", "<>=", "<>", "&&", "||", "(", "-", "-=", "%",
-        "%=", "*=", "!", "!=", "!>", "!>=", "!<",
-        "!<=", "!<>", "+", "+=", "^^", "^^=",
-        "}", "]", ")", ";", "<<", "<<=", ">>",
-        ">>=", "..", "*", "?", "~", "!<>=",
-        ">>>", ">>>=", "...", "^", "^="];
-    LexerConfig config;
-    auto tokens = byToken(source, config);
-    //writeln(tokens.map!"a.value"().array());
-    assert (equal(map!"a.value"(tokens), expected), map!"a.value"(tokens).text());
+	switch (type)
+	{
+	case tok!"int":
+	case tok!"uint":
+	case tok!"double":
+	case tok!"idouble":
+	case tok!"float":
+	case tok!"ifloat":
+	case tok!"short":
+	case tok!"ushort":
+	case tok!"long":
+	case tok!"ulong":
+	case tok!"char":
+	case tok!"wchar":
+	case tok!"dchar":
+	case tok!"bool":
+	case tok!"void":
+	case tok!"cent":
+	case tok!"ucent":
+	case tok!"real":
+	case tok!"ireal":
+	case tok!"byte":
+	case tok!"ubyte":
+	case tok!"cdouble":
+	case tok!"cfloat":
+	case tok!"creal":
+		return true;
+	default:
+		return false;
+	}
 }
 
-unittest
+public bool isNumberLiteral(IdType type) nothrow pure @safe
 {
-    auto source = cast(ubyte[]) (`
-        1 1.2 //comment
-        1.2f 1u 1uL 0b011 0b1uu 0b1 /+abc/+def+/+/0x11001uL
-        123e1L 123e+1f 123e-1i 15e++ 4ea 1.2u 4i 1337L 4.2L 1..2 4.3.5.8
-        0xabc 0xabcp4 0x1P-10 0x40u 0x29L 0x4Lu 0xdeadbeef
-    `);
-    auto expected = ["1", "1.2", "1.2f", "1u", "1uL", "0b011", "0b1u", "u", "0b1",
-        "0x11001uL", "123e1L", "123e+1f", "123e-1i", "15e+", "+", "4e", "a",
-        "1.2", "u", "4i", "1337L", "4.2L", "1", "..", "2", "4.3", ".5", ".8",
-        "0xabc", "0xabcp4", "0x1P-10", "0x40u", "0x29L", "0x4Lu", "0xdeadbeef"];
-    int errCount = 0;
-    void errorFunction(string file, size_t index, uint line, ushort col, string msg)
+	switch (type)
+	{
+	case tok!"doubleLiteral":
+	case tok!"floatLiteral":
+	case tok!"idoubleLiteral":
+	case tok!"ifloatLiteral":
+	case tok!"intLiteral":
+	case tok!"longLiteral":
+	case tok!"realLiteral":
+	case tok!"irealLiteral":
+	case tok!"uintLiteral":
+	case tok!"ulongLiteral":
+		return true;
+	default:
+		return false;
+	}
+}
+
+public bool isOperator(IdType type) nothrow pure @safe
+{
+	switch (type)
+	{
+	case tok!",":
+	case tok!".":
+	case tok!"..":
+	case tok!"...":
+	case tok!"/":
+	case tok!"/=":
+	case tok!"!":
+	case tok!"!<":
+	case tok!"!<=":
+	case tok!"!<>":
+	case tok!"!<>=":
+	case tok!"!=":
+	case tok!"!>":
+	case tok!"!>=":
+	case tok!"$":
+	case tok!"%":
+	case tok!"%=":
+	case tok!"&":
+	case tok!"&&":
+	case tok!"&=":
+	case tok!"(":
+	case tok!")":
+	case tok!"*":
+	case tok!"*=":
+	case tok!"+":
+	case tok!"++":
+	case tok!"+=":
+	case tok!"-":
+	case tok!"--":
+	case tok!"-=":
+	case tok!":":
+	case tok!";":
+	case tok!"<":
+	case tok!"<<":
+	case tok!"<<=":
+	case tok!"<=":
+	case tok!"<>":
+	case tok!"<>=":
+	case tok!"=":
+	case tok!"==":
+	case tok!"=>":
+	case tok!">":
+	case tok!">=":
+	case tok!">>":
+	case tok!">>=":
+	case tok!">>>":
+	case tok!">>>=":
+	case tok!"?":
+	case tok!"@":
+	case tok!"[":
+	case tok!"]":
+	case tok!"^":
+	case tok!"^=":
+	case tok!"^^":
+	case tok!"^^=":
+	case tok!"{":
+	case tok!"|":
+	case tok!"|=":
+	case tok!"||":
+	case tok!"}":
+	case tok!"~":
+	case tok!"~=":
+		return true;
+	default:
+		return false;
+	}
+}
+
+public bool isKeyword(IdType type) pure nothrow @safe
+{
+	switch (type)
+	{
+	case tok!"abstract":
+	case tok!"alias":
+	case tok!"align":
+	case tok!"asm":
+	case tok!"assert":
+	case tok!"auto":
+	case tok!"body":
+	case tok!"break":
+	case tok!"case":
+	case tok!"cast":
+	case tok!"catch":
+	case tok!"class":
+	case tok!"const":
+	case tok!"continue":
+	case tok!"debug":
+	case tok!"default":
+	case tok!"delegate":
+	case tok!"delete":
+	case tok!"deprecated":
+	case tok!"do":
+	case tok!"else":
+	case tok!"enum":
+	case tok!"export":
+	case tok!"extern":
+	case tok!"false":
+	case tok!"final":
+	case tok!"finally":
+	case tok!"for":
+	case tok!"foreach":
+	case tok!"foreach_reverse":
+	case tok!"function":
+	case tok!"goto":
+	case tok!"if":
+	case tok!"immutable":
+	case tok!"import":
+	case tok!"in":
+	case tok!"inout":
+	case tok!"interface":
+	case tok!"invariant":
+	case tok!"is":
+	case tok!"lazy":
+	case tok!"macro":
+	case tok!"mixin":
+	case tok!"module":
+	case tok!"new":
+	case tok!"nothrow":
+	case tok!"null":
+	case tok!"out":
+	case tok!"override":
+	case tok!"package":
+	case tok!"pragma":
+	case tok!"private":
+	case tok!"protected":
+	case tok!"public":
+	case tok!"pure":
+	case tok!"ref":
+	case tok!"return":
+	case tok!"scope":
+	case tok!"shared":
+	case tok!"static":
+	case tok!"struct":
+	case tok!"super":
+	case tok!"switch":
+	case tok!"synchronized":
+	case tok!"template":
+	case tok!"this":
+	case tok!"throw":
+	case tok!"true":
+	case tok!"try":
+	case tok!"typedef":
+	case tok!"typeid":
+	case tok!"typeof":
+	case tok!"union":
+	case tok!"unittest":
+	case tok!"version":
+	case tok!"volatile":
+	case tok!"while":
+	case tok!"with":
+	case tok!"__DATE__":
+	case tok!"__EOF__":
+	case tok!"__FILE__":
+	case tok!"__FUNCTION__":
+	case tok!"__gshared":
+	case tok!"__LINE__":
+	case tok!"__MODULE__":
+	case tok!"__parameters":
+	case tok!"__PRETTY_FUNCTION__":
+	case tok!"__TIME__":
+	case tok!"__TIMESTAMP__":
+	case tok!"__traits":
+	case tok!"__vector":
+	case tok!"__VENDOR__":
+	case tok!"__VERSION__":
+		return true;
+	default:
+		return false;
+	}
+}
+
+public bool isStringLiteral(IdType type) pure nothrow @safe
+{
+	switch (type)
+	{
+	case tok!"dstringLiteral":
+	case tok!"stringLiteral":
+	case tok!"wstringLiteral":
+		return true;
+	default:
+		return false;
+	}
+}
+
+public struct DLexer(R)
+{
+	import std.conv;
+	import core.vararg;
+	import dpick.buffer.buffer;
+
+	private enum pseudoTokenHandlers = [
+		"\"", "lexStringLiteral",
+		"`", "lexWysiwygString",
+		"//", "lexSlashSlashComment",
+		"/*", "lexSlashStarComment",
+		"/+", "lexSlashPlusComment",
+		".", "lexDot",
+		"'", "lexCharacterLiteral",
+		"0", "lexNumber",
+		"1", "lexDecimal",
+		"2", "lexDecimal",
+		"3", "lexDecimal",
+		"4", "lexDecimal",
+		"5", "lexDecimal",
+		"6", "lexDecimal",
+		"7", "lexDecimal",
+		"8", "lexDecimal",
+		"9", "lexDecimal",
+		"q\"", "lexDelimitedString",
+		"q{", "lexTokenString",
+		"r\"", "lexWysiwygString",
+		"x\"", "lexHexString",
+		" ", "lexWhitespace",
+		"\t", "lexWhitespace",
+		"\r", "lexWhitespace",
+		"\n", "lexWhitespace",
+		"\u2028", "lexLongNewline",
+		"\u2029", "lexLongNewline",
+		"#!", "lexScriptLine",
+		"#line", "lexSpecialTokenSequence"
+	];
+
+	mixin Lexer!(R, IdType, Token, lexIdentifier, staticTokens,
+		dynamicTokens, pseudoTokens, pseudoTokenHandlers, possibleDefaultTokens);
+
+	private alias typeof(range).Mark Mark;
+
+	this(R range, const LexerConfig config)
+	{
+		this.range = LexerRange!(typeof(buffer(range)))(buffer(range));
+        this.config = config;
+        popFront();
+	}
+
+    private static bool isDocComment(string comment) pure nothrow @safe
     {
-        ++errCount;
+        return comment.length >= 3 && (comment[0 .. 3] == "///"
+            || comment[0 .. 3] == "/**" || comment[0 .. 3] == "/++");
     }
-    LexerConfig config;
-    config.errorFunc = &errorFunction;
-    auto tokens = byToken(source, config);
-    //writeln(tokens.map!"a.value"());
-    assert (equal(map!"a.value"(tokens), expected), map!"a.value"(tokens).text());
-    assert (errCount == 2);
-}
 
-unittest
-{
-    auto source = cast(ubyte[]) ("int #line 4\n double q{abcde (a + b) == 0} '\\u0020' q\"HEREDOC\r\nabcde\r\nHEREDOC\"");
-    LexerConfig config;
-    auto tokens = byToken(source, config);
-    assert (tokens.front.line == 1);
-    assert (tokens.moveFront() == TokenType.int_);
-    assert (tokens.front.line == 4);
-    assert (isBasicType(tokens.front));
-    assert (tokens.front.value == "double");
-    tokens.popFront();
-    assert (tokens.front.value == "abcde (a + b) == 0", tokens.front.value);
-    assert (isStringLiteral(tokens.front), tokens.front.type.text());
-    tokens.popFront();
-    assert (tokens.front.value == " ");
-    assert (tokens.front.type == TokenType.characterLiteral);
-    tokens.popFront();
-    assert (tokens.front.value == "abcde\r\n", "[%s]".format(tokens.front.value));
-}
-
-unittest
-{
-    auto source = cast(ubyte[]) "q{(a & 1) == 0} q\"/foo]/\" q\"HEREDOC\r\nabcde\r\nHEREDOC\"";
-    LexerConfig config;
-    config.tokenStyle = TokenStyle.includeQuotes;
-    auto tokens = byToken(source, config);
-    assert (tokens.front.value == "q{(a & 1) == 0}", tokens.front.value);
-    tokens.popFront();
-    assert (tokens.front.value == "q\"/foo]/\"", tokens.front.value);
-    tokens.popFront();
-    assert (tokens.front.value == "q\"HEREDOC\r\nabcde\r\nHEREDOC\"", tokens.front.value);
-}
-
-unittest
-{
-    auto source = cast(ubyte[]) (`"string`);
-    int errCount = 0;
-    void errorFunction(string file, size_t index, uint line, ushort col, string msg)
+    public void popFront() pure
     {
-        ++errCount;
+        _popFront();
+        string comment = null;
+        switch (_front.type)
+        {
+            case tok!"comment":
+                if (config.commentBehavior == CommentBehavior.attach)
+                {
+                    import std.string;
+                    if (isDocComment(front.text))
+                        comment = comment == null ? front.text : format("%s\n%s", comment, front.text);
+                    do _popFront(); while (front == tok!"comment");
+                    if (front == tok!"whitespace") goto case tok!"whitespace";
+                }
+                break;
+            case tok!"whitespace":
+                if (config.whitespaceBehavior == WhitespaceBehavior.skip)
+                {
+                    do _popFront(); while (front == tok!"whitespace");
+                    if (front == tok!"comment") goto case tok!"comment";
+                }
+                break;
+            default:
+                break;
+        }
+        _front.comment = comment;
     }
-    LexerConfig config;
-    config.errorFunc = &errorFunction;
-    auto tokens = byToken(source, config);
-    assert (errCount == 1);
-}
 
-unittest
-{
-    auto source = cast(ubyte[]) ("import foo");
-    LexerConfig config;
-    auto tokens = byToken(source, config);
-    Token a = tokens.moveFront();
-    assert (a.type == TokenType.import_);
-    Token b = tokens.moveFront();
-    assert (b.type == TokenType.identifier);
-    assert (a != b);
-    assert (a != "foo");
-    assert (a < b);
-    assert (b == "foo");
-    assert (b > a);
-    assert (!(a > a));
-    assert (tokens.empty);
-}
 
-unittest
-{
-    auto source = cast(ubyte[]) ("import std.stdio; void main(){writeln(\"hello world\");}");
-    LexerConfig config;
-    auto tokens = byToken(source, config);
-    int tokenCount = 0;
-    foreach (t; tokens)
-    {
-        ++tokenCount;
-    }
-    assert (tokenCount == 16);
+	bool isWhitespace() pure /*const*/ nothrow
+	{
+		switch (range.front)
+		{
+		case ' ':
+		case '\r':
+		case '\n':
+		case '\t':
+			return true;
+		case 0xe2:
+			auto peek = range.lookahead(2);
+			return peek.length == 2
+				&& peek[0] == 0x80
+				&& (peek[1] == 0xa8 || peek[1] == 0xa9);
+		default:
+			return false;
+		}
+	}
+
+	void popFrontWhitespaceAware() pure nothrow
+	{
+		switch (range.front)
+		{
+		case '\r':
+			range.popFront();
+			if (!range.empty && range.front == '\n')
+			{
+				range.popFront();
+				range.incrementLine();
+			}
+			else
+				range.incrementLine();
+			return;
+		case '\n':
+			range.popFront();
+			range.incrementLine();
+			return;
+		case 0xe2:
+			auto lookahead = range.lookahead(3);
+			if (lookahead.length == 3 && lookahead[1] == 0x80
+				&& (lookahead[2] == 0xa8 || lookahead[2] == 0xa9))
+			{
+				range.popFront();
+				range.popFront();
+				range.popFront();
+				range.incrementLine();
+				return;
+			}
+			else
+			{
+				range.popFront();
+				return;
+			}
+		default:
+			range.popFront();
+			return;
+		}
+	}
+
+	Token lexWhitespace() pure nothrow
+	{
+		mixin (tokenStart);
+		loop: do
+		{
+			switch (range.front)
+			{
+			case '\r':
+				range.popFront();
+				if (!range.empty && range.front == '\n')
+					range.popFront();
+				range.incrementLine();
+				break;
+			case '\n':
+				range.popFront();
+				range.incrementLine();
+				break;
+			case ' ':
+			case '\t':
+				range.popFront();
+				break;
+			case 0xe2:
+				auto lookahead = range.lookahead(3);
+				if (lookahead.length != 3)
+					break loop;
+				if (lookahead[1] != 0x80)
+					break loop;
+				if (lookahead[2] == 0xa8 || lookahead[2] == 0xa9)
+				{
+					range.popFront();
+					range.popFront();
+					range.popFront();
+					range.incrementLine();
+					break;
+				}
+				break loop;
+			default:
+				break loop;
+			}
+		} while (!range.empty);
+		return Token(tok!"whitespace", cast(string) range.slice(mark), line,
+			column, index);
+	}
+
+	Token lexNumber() pure nothrow
+	{
+		mixin (tokenStart);
+		auto lookahead = range.lookahead(2);
+		if (range.front == '0' && lookahead.length == 2)
+		{
+			switch (lookahead[1])
+			{
+			case 'x':
+			case 'X':
+				range.popFront();
+				range.popFront();
+				return lexHex(mark, line, column, index);
+			case 'b':
+			case 'B':
+				range.popFront();
+				range.popFront();
+				return lexBinary(mark, line, column, index);
+			default:
+				return lexDecimal(mark, line, column, index);
+			}
+		}
+		else
+			return lexDecimal(mark, line, column, index);
+	}
+
+	Token lexHex() pure nothrow
+	{
+		mixin (tokenStart);
+		return lexHex(mark, line, column, index);
+	}
+
+	Token lexHex(Mark mark, size_t line, size_t column, size_t index) pure nothrow
+	{
+		IdType type = tok!"intLiteral";
+		bool foundDot;
+		hexLoop: while (!range.empty)
+		{
+			switch (range.front)
+			{
+			case 'a': .. case 'f':
+			case 'A': .. case 'F':
+			case '0': .. case '9':
+			case '_':
+				range.popFront();
+				break;
+			case 'u':
+			case 'U':
+				lexIntSuffix(type);
+				break hexLoop;
+			case 'i':
+				if (foundDot)
+					lexFloatSuffix(type);
+				break hexLoop;
+			case 'L':
+				if (foundDot)
+					lexFloatSuffix(type);
+				else
+					lexIntSuffix(type);
+                break hexLoop;
+			case 'p':
+			case 'P':
+				lexExponent(type);
+				break hexLoop;
+			case '.':
+				if (foundDot)
+					break hexLoop;
+				if (range.lookahead(1).length && range.lookahead(1)[0] == '.')
+					break hexLoop;
+				range.popFront();
+				foundDot = true;
+				type = tok!"doubleLiteral";
+				break;
+			default:
+				break hexLoop;
+			}
+		}
+		return Token(type, cast(string) range.slice(mark), line, column,
+			index);
+	}
+
+	Token lexBinary() pure nothrow
+	{
+		mixin (tokenStart);
+		return lexBinary(mark, line, column, index);
+	}
+
+	Token lexBinary(Mark mark, size_t line, size_t column, size_t index) pure nothrow
+	{
+		IdType type = tok!"intLiteral";
+		binaryLoop: while (!range.empty)
+		{
+			switch (range.front)
+			{
+			case '0':
+			case '1':
+			case '_':
+				range.popFront();
+				break;
+			case 'u':
+			case 'U':
+			case 'L':
+				lexIntSuffix(type);
+				break binaryLoop;
+			default:
+				break binaryLoop;
+			}
+		}
+		return Token(type, cast(string) range.slice(mark), line, column,
+			index);
+	}
+
+	Token lexDecimal()
+	{
+		mixin (tokenStart);
+		return lexDecimal(mark, line, column, index);
+	}
+
+	Token lexDecimal(Mark mark, size_t line, size_t column, size_t index) pure nothrow
+	{
+		bool foundDot = range.front == '.';
+		IdType type = tok!"intLiteral";
+		if (foundDot)
+		{
+			range.popFront();
+			type = tok!"doubleLiteral";
+		}
+
+		decimalLoop: while (!range.empty)
+		{
+			switch (range.front)
+			{
+			case '0': .. case '9':
+			case '_':
+				range.popFront();
+				break;
+			case 'u':
+			case 'U':
+				if (!foundDot)
+					lexIntSuffix(type);
+				break decimalLoop;
+			case 'i':
+				lexFloatSuffix(type);
+				break decimalLoop;
+			case 'L':
+				if (foundDot)
+					lexFloatSuffix(type);
+				else
+					lexIntSuffix(type);
+				break decimalLoop;
+			case 'f':
+			case 'F':
+				lexFloatSuffix(type);
+				break decimalLoop;
+			case 'e':
+			case 'E':
+				lexExponent(type);
+				break decimalLoop;
+			case '.':
+				if (foundDot)
+					break decimalLoop;
+				auto lookahead = range.lookahead(1);
+				if (lookahead.length == 1 && lookahead[0] == '.')
+					break decimalLoop;
+				else
+				{
+					// The following bit of silliness tries to tell the
+					// difference between "int dot identifier" and
+					// "double identifier".
+					if (lookahead.length == 1)
+					{
+						switch (lookahead[0])
+						{
+						case '0': .. case '9':
+							goto doubleLiteral;
+						default:
+							break decimalLoop;
+						}
+					}
+					else
+					{
+					doubleLiteral:
+						range.popFront();
+						foundDot = true;
+						type = tok!"doubleLiteral";
+					}
+				}
+				break;
+			default:
+				break decimalLoop;
+			}
+		}
+		return Token(type, cast(string) range.slice(mark), line, column,
+			index);
+	}
+
+	void lexIntSuffix(ref IdType type) pure nothrow @safe
+	{
+		bool secondPass;
+		if (range.front == 'u' || range.front == 'U')
+		{
+	U:
+			if (type == tok!"intLiteral")
+				type = tok!"uintLiteral";
+			else
+				type = tok!"ulongLiteral";
+			range.popFront();
+			if (secondPass)
+				return;
+			if (range.front == 'L' || range.front == 'l')
+				goto L;
+			return;
+		}
+		if (range.front == 'L' || range.front == 'l')
+		{
+	L:
+			if (type == tok!"uintLiteral")
+				type = tok!"ulongLiteral";
+			else
+				type = tok!"longLiteral";
+			range.popFront();
+			if (range.front == 'U' || range.front == 'u')
+			{
+				secondPass = true;
+				goto U;
+			}
+			return;
+		}
+	}
+
+	void lexFloatSuffix(ref IdType type) pure nothrow @safe
+	{
+		switch (range.front)
+		{
+		case 'L':
+			range.popFront();
+			type = tok!"doubleLiteral";
+			break;
+		case 'f':
+		case 'F':
+			range.popFront();
+			type = tok!"floatLiteral";
+			break;
+		default:
+			break;
+		}
+		if (!range.empty && range.front == 'i')
+		{
+            warning("Complex number literals are deprecated");
+			range.popFront();
+			if (type == tok!"floatLiteral")
+				type = tok!"ifloatLiteral";
+			else
+				type = tok!"idoubleLiteral";
+		}
+	}
+
+	void lexExponent(ref IdType type) pure nothrow @safe
+	{
+		range.popFront();
+		bool foundSign = false;
+		bool foundDigit = false;
+		while (!range.empty)
+		{
+			switch (range.front)
+			{
+			case '-':
+			case '+':
+				if (foundSign)
+				{
+					if (!foundDigit)
+					error("Expected an exponent");
+					return;
+				}
+				foundSign = true;
+				range.popFront();
+				break;
+			case '0': .. case '9':
+			case '_':
+				foundDigit = true;
+				range.popFront();
+				break;
+			case 'L':
+			case 'f':
+			case 'F':
+			case 'i':
+				lexFloatSuffix(type);
+				return;
+			default:
+				if (!foundDigit)
+					error("Expected an exponent");
+				return;
+			}
+		}
+	}
+
+	Token lexScriptLine() pure
+	{
+		mixin (tokenStart);
+		while (!range.empty && !isNewline)
+			range.popFront();
+		return Token(tok!"scriptLine", cast(string) range.slice(mark),
+			line, column, index);
+	}
+
+	Token lexSpecialTokenSequence() pure
+	{
+		mixin (tokenStart);
+		while (!range.empty && !isNewline)
+			range.popFront();
+		return Token(tok!"specialTokenSequence", cast(string) range.slice(mark),
+			line, column, index);
+	}
+
+	Token lexSlashStarComment() pure
+	{
+		mixin (tokenStart);
+		IdType type = tok!"comment";
+		range.popFront();
+		range.popFront();
+		while (!range.empty)
+		{
+			if (range.front == '*')
+			{
+				range.popFront();
+				if (!range.empty && range.front == '/')
+				{
+					range.popFront();
+					break;
+				}
+			}
+			else
+				popFrontWhitespaceAware();
+		}
+		return Token(type, cast(string) range.slice(mark), line, column,
+			index);
+	}
+
+	Token lexSlashSlashComment() pure nothrow
+	{
+		mixin (tokenStart);
+		IdType type = tok!"comment";
+		range.popFront();
+		range.popFront();
+		while (!range.empty)
+		{
+			if (range.front == '\r' || range.front == '\n')
+				break;
+			range.popFront();
+		}
+		return Token(type, cast(string) range.slice(mark), line, column,
+			index);
+	}
+
+	Token lexSlashPlusComment() pure nothrow
+	{
+		mixin (tokenStart);
+		IdType type = tok!"comment";
+		range.popFront();
+		range.popFront();
+		int depth = 1;
+		while (depth > 0 && !range.empty)
+		{
+			if (range.front == '+')
+			{
+				range.popFront();
+				if (!range.empty && range.front == '/')
+				{
+					range.popFront();
+					depth--;
+				}
+			}
+			else if (range.front == '/')
+			{
+				range.popFront();
+				if (!range.empty && range.front == '+')
+				{
+					range.popFront();
+					depth++;
+				}
+			}
+			else
+				popFrontWhitespaceAware();
+		}
+		return Token(type, cast(string) range.slice(mark), line, column,
+			index);
+	}
+
+	Token lexStringLiteral() pure nothrow
+	{
+		mixin (tokenStart);
+		range.popFront();
+		while (true)
+		{
+			if (range.empty)
+			{
+				error("Error: unterminated string literal");
+				return Token();
+			}
+			else if (range.front == '"')
+			{
+				range.popFront();
+				break;
+			}
+			else if (range.front == '\\')
+			{
+				lexEscapeSequence();
+			}
+			else
+				range.popFront();
+		}
+		IdType type = tok!"stringLiteral";
+		lexStringSuffix(type);
+		return Token(type, cast(string) range.slice(mark), line, column,
+			index);
+	}
+
+	Token lexWysiwygString() pure nothrow
+	{
+		mixin (tokenStart);
+		IdType type = tok!"stringLiteral";
+		bool backtick = range.front == '`';
+		if (backtick)
+		{
+			range.popFront();
+			while (true)
+			{
+				if (range.empty)
+				{
+					error("Error: unterminated string literal");
+					return Token(tok!"");
+				}
+				else if (range.front == '`')
+				{
+					range.popFront();
+					break;
+				}
+				else
+					popFrontWhitespaceAware();
+			}
+		}
+		else
+		{
+			range.popFront();
+			if (range.empty)
+			{
+				error("Error: unterminated string literal");
+				return Token(tok!"");
+			}
+			range.popFront();
+			while (true)
+			{
+				if (range.empty)
+				{
+					error("Error: unterminated string literal");
+					return Token(tok!"");
+				}
+				else if (range.front == '"')
+				{
+					range.popFront();
+					break;
+				}
+				else
+					popFrontWhitespaceAware();
+			}
+		}
+		lexStringSuffix(type);
+		return Token(type, cast(string) range.slice(mark), line, column,
+			index);
+	}
+
+	void lexStringSuffix(ref IdType type) pure
+	{
+		if (range.empty)
+			type = tok!"stringLiteral";
+		else
+		{
+			switch (range.front)
+			{
+			case 'w': range.popFront(); type = tok!"wstringLiteral"; break;
+			case 'd': range.popFront(); type = tok!"dstringLiteral"; break;
+			case 'c': range.popFront(); type = tok!"stringLiteral"; break;
+			default: type = tok!"stringLiteral"; break;
+			}
+		}
+	}
+
+	Token lexDelimitedString() pure nothrow
+	{
+        import std.traits;
+		mixin (tokenStart);
+		range.popFront();
+		range.popFront();
+		Unqual!(ElementEncodingType!R) open;
+		Unqual!(ElementEncodingType!R) close;
+		switch (range.front)
+		{
+		case '<':
+			open = '<';
+			close = '>';
+			range.popFront();
+			return lexNormalDelimitedString(mark, line, column, index, open, close);
+		case '{':
+			open = '{';
+			close = '}';
+			range.popFront();
+			return lexNormalDelimitedString(mark, line, column, index, open, close);
+		case '[':
+			open = '[';
+			close = ']';
+			range.popFront();
+			return lexNormalDelimitedString(mark, line, column, index, open, close);
+		case '(':
+			open = '(';
+			close = ')';
+			range.popFront();
+			return lexNormalDelimitedString(mark, line, column, index, open, close);
+		default:
+			return lexHeredocString();
+		}
+	}
+
+	Token lexNormalDelimitedString(Mark mark, size_t line, size_t column,
+		size_t index, ElementEncodingType!R open, ElementEncodingType!R close)
+		pure nothrow
+	{
+		int depth = 1;
+		while (!range.empty && depth > 0)
+		{
+			if (range.front == open)
+			{
+				depth++;
+				range.popFront();
+			}
+			else if (range.front == close)
+			{
+				depth--;
+				range.popFront();
+				if (depth <= 0)
+				{
+					if (range.front == '"')
+						range.popFront();
+					else
+					{
+						error("Error: \" expected to end delimited string literal");
+						return Token(tok!"");
+					}
+				}
+			}
+			else
+				popFrontWhitespaceAware();
+		}
+		IdType type = tok!"stringLiteral";
+		lexStringSuffix(type);
+		return Token(type, cast(string) range.slice(mark), line, column, index);
+	}
+
+	Token lexHeredocString() pure nothrow
+	{
+		assert (false, "unimplemented");
+	}
+
+	Token lexTokenString() pure
+	{
+		mixin (tokenStart);
+		assert(range.front == 'q');
+		range.popFront();
+		assert(range.front == '{');
+		range.popFront();
+		auto app = appender!string();
+		app.put("q{");
+		int depth = 1;
+
+		_front = advance();
+		while (depth > 0 && !empty)
+		{
+			auto t = front();
+			if (t.text is null)
+				app.put(str(t.type));
+			else
+				app.put(t.text);
+			if (t.type == tok!"}")
+			{
+				depth--;
+				if (depth > 0)
+				popFront();
+			}
+			else if (t.type == tok!"{")
+			{
+				depth++;
+				popFront();
+			}
+			else
+				popFront();
+		}
+		IdType type = tok!"stringLiteral";
+		lexStringSuffix(type);
+		return Token(type, app.data, line, column, index);
+	}
+
+	Token lexHexString() pure nothrow
+	{
+		mixin (tokenStart);
+		range.popFront();
+		range.popFront();
+
+		loop: while (true)
+		{
+			if (range.empty)
+			{
+				error("Error: unterminated hex string literal");
+				return Token();
+			}
+			else if (isWhitespace())
+				popFrontWhitespaceAware();
+			else switch (range.front)
+			{
+			case '0': .. case '9':
+			case 'A': .. case 'F':
+			case 'a': .. case 'f':
+				range.popFront();
+				break;
+			case '"':
+				range.popFront();
+				break loop;
+			default:
+				error("Error: invalid character in hex string");
+				return Token();
+			}
+		}
+
+		IdType type = tok!"stringLiteral";
+		lexStringSuffix(type);
+		return Token(type, cast(string) range.slice(mark), line, column,
+			index);
+	}
+
+	bool lexEscapeSequence() pure nothrow
+	{
+		range.popFront();
+		if (range.empty)
+		{
+			error("Error: non-terminated character escape sequence.");
+			return false;
+		}
+		switch (range.front)
+		{
+		case '\'':
+		case '"':
+		case '?':
+		case '\\':
+		case '0':
+		case 'a':
+		case 'b':
+		case 'f':
+		case 'n':
+		case 'r':
+		case 't':
+		case 'v':
+			range.popFront();
+			break;
+		case 'x':
+			// TODO
+			range.popFront();
+			break;
+		case '1': .. case '7':
+			for (size_t i = 0; i < 3 && !range.empty && range.front >= '0' && range.front <= '7'; i++)
+				range.popFront();
+			break;
+		case 'u':
+			range.popFront();
+			foreach (i; 0 .. 4)
+			{
+				if (range.empty)
+				{
+					error("Error: at least 4 hex digits expected.");
+					return false;
+				}
+				switch (range.front)
+				{
+				case '0': .. case '9':
+				case 'a': .. case 'f':
+				case 'A': .. case 'F':
+					range.popFront();
+					break;
+				default:
+					error("Error: at least 4 hex digits expected.");
+					return false;
+				}
+			}
+			break;
+		case 'U':
+			range.popFront();
+			foreach (i; 0 .. 8)
+			{
+				if (range.empty)
+				{
+					error("Error: at least 8 hex digits expected.");
+					return false;
+				}
+				switch (range.front)
+				{
+				case '0': .. case '9':
+				case 'a': .. case 'f':
+				case 'A': .. case 'F':
+					range.popFront();
+					break;
+				default:
+					error("Error: at least 8 hex digits expected.");
+					return false;
+				}
+			}
+			break;
+		default:
+			while (true)
+			{
+				if (range.empty)
+				{
+					error("Error: non-terminated character escape sequence.");
+					return false;
+				}
+				if (range.front == ';')
+					break;
+				else
+					range.popFront();
+			}
+		}
+		return true;
+	}
+
+	Token lexCharacterLiteral() pure nothrow
+	{
+		mixin (tokenStart);
+		range.popFront();
+		if (range.front == '\\')
+		{
+			lexEscapeSequence();
+			goto close;
+		}
+		else if (range.front == '\'')
+		{
+			range.popFront();
+			return Token(tok!"characterLiteral", cast(string) range.slice(mark),
+				line, column, index);
+		}
+		else if (range.front & 0x80)
+		{
+			while (range.front & 0x80)
+				range.popFront();
+			goto close;
+		}
+		else
+		{
+			popFrontWhitespaceAware();
+			goto close;
+		}
+	close:
+		if (range.front == '\'')
+		{
+			range.popFront();
+			return Token(tok!"characterLiteral", cast(string) range.slice(mark),
+				line, column, index);
+		}
+		else
+		{
+			error("Error: Expected ' to end character literal ", cast(char) range.front);
+			return Token();
+		}
+	}
+
+	Token lexIdentifier() pure nothrow
+	{
+		mixin (tokenStart);
+		while (!range.empty && !isSeparating(range.front))
+		{
+			range.popFront();
+		}
+		return Token(tok!"identifier", cast(string) range.slice(mark), line,
+			column, index);
+	}
+
+	Token lexDot() pure nothrow
+	{
+		mixin (tokenStart);
+		auto lookahead = range.lookahead(1);
+		if (lookahead.length == 0)
+		{
+			range.popFront();
+			return Token(tok!".", null, line, column, index);
+		}
+		switch (lookahead[0])
+		{
+		case '0': .. case '9':
+			return lexNumber();
+		case '.':
+			range.popFront();
+			range.popFront();
+			if (!range.empty && range.front == '.')
+			{
+				range.popFront();
+				return Token(tok!"...", null, line, column, index);
+			}
+			else
+				return Token(tok!"..", null, line, column, index);
+		default:
+			range.popFront();
+			return Token(tok!".", null, line, column, index);
+		}
+	}
+
+	Token lexLongNewline() pure nothrow
+	{
+		mixin (tokenStart);
+		range.popFront();
+		range.popFront();
+		range.popFront();
+		range.incrementLine();
+		return Token(tok!"whitespace", cast(string) range.slice(mark), line,
+			column, index);
+	}
+
+	bool isNewline() pure @safe
+	{
+		if (range.front == '\n') return true;
+		if (range.front == '\r') return true;
+		auto lookahead = range.lookahead(3);
+		if (lookahead.length == 0) return false;
+		if (lookahead.startsWith("\u2028") || lookahead.startsWith("\u2029"))
+			return true;
+		return false;
+	}
+
+	bool isSeparating(ElementType!R c) nothrow pure @safe
+	{
+		if (c <= 0x2f) return true;
+		if (c >= ':' && c <= '@') return true;
+		if (c >= '[' && c <= '^') return true;
+		if (c >= '{' && c <= '~') return true;
+		if (c == '`') return true;
+//		if (c & 0x80 && (range.lookahead(3).startsWith("\u2028")
+//			|| range.lookahead(3).startsWith("\u2029"))) return true;
+		return false;
+	}
+
+	enum tokenStart = q{
+		size_t index = range.index;
+		size_t column = range.column;
+		size_t line = range.line;
+		auto mark = range.mark();
+	};
+
+	void error(...) pure {
+
+	}
+
+	void warning(...) pure {
+
+	}
+
+    const LexerConfig config;
 }
diff --git a/stdx/d/parser.d b/stdx/d/parser.d
index 01f5565..c684501 100644
--- a/stdx/d/parser.d
+++ b/stdx/d/parser.d
@@ -1,62 +1,5 @@
 // Written in the D programming language
 
-/**
- * This module contains a _parser for D source code.
- *
- * Grammar:
- * The grammar format used in the documentation of this module generally follows
- * the format used by the ANTLR _parser generator.
- * $(UL
- * $(LI Tokens and rules can be grouped by parenthesis.)
- * $(LI An asterisk (*) indicates that the previous rule, token, or group
- * can repeat 0 or more times.)
- * $(LI A question mark (?) indicates that the previous rule, token, or group
- * will be present either 0 or 1 times.)
- * $(LI A plus sign (+) indicates that the previous rule, token, or group
- * repeats one or more times. (i.e. it is optional))
- * $(LI If there is more than one way to match a rule, the alternatives will be
- * separated by a pipe character (|).)
- * $(LI Rule definitions begin with the rule name followed by a colon (:). Rule
- * definitions end with a semicolon (;).)
- * )
- *
- * The grammar for D starts with the $(LINK2 #module, module) rule.
- *
- * Examples:
- * ---
- * import std.d.lexer;
- * import std.d.parser;
- * import std.d.ast;
- * import std.array;
- *
- * string sourceCode = q{
- * import std.stdio;
- *
- * void main()
- * {
- *     writeln("Hello, World.");
- * }
- * }c;
- * void main()
- * {
- *     LexerConfig config;
- *     auto tokens = byToken(cast(ubyte[]) sourceCode, config).array();
- *     Module mod = parseModule(tokens);
- *     // Use module here...
- * }
- * ---
- *
- * Copyright: Brian Schott 2013
- * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
- * Authors: Brian Schott
- * Source: $(PHOBOSSRC std/d/_parser.d)
- * Macros:
- * GRAMMAR = <pre>$0</pre>
- * RULEDEF = <a name="$0"><span style="font-weight: bold;">$0</span></a>
- * RULE = <a href="#$0"><span style="font-weight: bold;">$0</span></a>
- * LITERAL = <span style="color: green;">$0</span>
- */
-
 module stdx.d.parser;
 
 import stdx.d.lexer;
@@ -80,12 +23,12 @@ import std.string : format;
  * Returns: the parsed module
  */
 Module parseModule(const(Token)[] tokens, string fileName,
-	void function(string, int, int, string) messageFunction = null)
+    void function(string, size_t, size_t, string) messageFunction = null)
 {
     auto parser = new Parser();
     parser.fileName = fileName;
     parser.tokens = tokens;
-	parser.messageFunction = messageFunction;
+    parser.messageFunction = messageFunction;
     auto mod = parser.parseModule();
     // writefln("Parsing finished with %d errors and %d warnings.",
     //     parser.errorCount, parser.warningCount);
@@ -109,7 +52,7 @@ class Parser
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         return parseLeftAssocBinaryExpression!(AddExpression, MulExpression,
-            TokenType.plus, TokenType.minus, TokenType.tilde)();
+            tok!"+", tok!"-", tok!"~")();
     }
 
     /**
@@ -124,15 +67,15 @@ class Parser
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new AliasDeclaration;
-        if (expect(TokenType.alias_) is null) return null;
-        if (startsWith(TokenType.identifier, TokenType.assign))
+        if (expect(tok!"alias") is null) return null;
+        if (startsWith(tok!"identifier", tok!"="))
         {
             do
             {
                 auto initializer = parseAliasInitializer();
                 if (initializer is null) return null;
                 node.initializers ~= initializer;
-                if (currentIs(TokenType.comma))
+                if (currentIs(tok!","))
                     advance();
                 else
                     break;
@@ -142,12 +85,12 @@ class Parser
         else
         {
             if ((node.type = parseType()) is null) return null;
-            auto ident = expect(TokenType.identifier);
+            auto ident = expect(tok!"identifier");
             if (ident is null)
                 return null;
             node.name = *ident;
         }
-        if (expect(TokenType.semicolon) is null) return null;
+        if (expect(tok!";") is null) return null;
         return node;
     }
 
@@ -177,14 +120,24 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new AliasInitializer;
-        auto i = expect(TokenType.identifier);
+        auto i = expect(tok!"identifier");
         if (i is null) return null;
         node.name = *i;
-        if (expect(TokenType.assign) is null) return null;
+        if (expect(tok!"=") is null) return null;
         node.type = parseType();
         return node;
     }
 
+	unittest
+    {
+        auto sourceCode = q{a = abcde!def};
+        Parser p = getParserForUnittest(sourceCode, "parseAliasInitializer");
+        auto initializer = p.parseAliasInitializer();
+        assert (initializer !is null);
+        assert (p.errorCount == 0);
+        stderr.writeln("Unittest for parseAliasInitializer() passed.");
+    }
+
     /**
      * Parses an AliasThisDeclaration
      * $(GRAMMAR $(RULEDEF aliasThisDeclaration):
@@ -195,14 +148,25 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new AliasThisDeclaration;
-        if (expect(TokenType.alias_) is null) return null;
-        auto ident = expect(TokenType.identifier);
+        if (expect(tok!"alias") is null) return null;
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
-        if (expect(TokenType.this_) is null) return null;
-        if (expect(TokenType.semicolon) is null) return null;
+        node.identifier = *ident;
+        if (expect(tok!"this") is null) return null;
+        if (expect(tok!";") is null) return null;
         return node;
     }
 
+    unittest
+    {
+        auto sourceCode = q{alias oneTwoThree this;};
+        Parser p = getParserForUnittest(sourceCode, "parseAliasThisDeclaration");
+        auto aliasThis = p.parseAliasThisDeclaration();
+        assert (aliasThis !is null);
+        assert (p.errorCount == 0);
+        stderr.writeln("Unittest for parseAliasThisDeclaration() passed.");
+    }
+
     /**
      * Parses an AlignAttribute.
      * $(GRAMMAR $(RULEDEF alignAttribute):
@@ -213,18 +177,30 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new AlignAttribute;
-        expect(TokenType.align_);
-        if (currentIs(TokenType.lParen))
+        expect(tok!"align");
+        if (currentIs(tok!"("))
         {
-            if (expect(TokenType.lParen) is null) return null;
-            auto intLit = expect(TokenType.intLiteral);
+            if (expect(tok!"(") is null) return null;
+            auto intLit = expect(tok!"intLiteral");
             if (intLit is null) return null;
             node.intLiteral = *intLit;
-            if (expect(TokenType.rParen) is null) return null;
+            if (expect(tok!")") is null) return null;
         }
         return node;
     }
 
+    unittest
+    {
+        auto sourceCode = q{align(42) align};
+        Parser p = getParserForUnittest(sourceCode, "parseAlignAttribute");
+        auto attribute = p.parseAlignAttribute();
+        assert (attribute !is null);
+        attribute = p.parseAlignAttribute();
+        assert (attribute !is null);
+        assert (p.errorCount == 0);
+        stderr.writeln("Unittest for parseAlignAttribute() passed.");
+    }
+
     /**
      * Parses an AndAndExpression
      * $(GRAMMAR $(RULEDEF andAndExpression):
@@ -236,7 +212,7 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         return parseLeftAssocBinaryExpression!(AndAndExpression, OrExpression,
-            TokenType.logicAnd)();
+            tok!"&&")();
     }
 
     /**
@@ -251,7 +227,7 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         return parseLeftAssocBinaryExpression!(AndExpression, CmpExpression,
-            TokenType.amp)();
+            tok!"&")();
     }
 
     /**
@@ -278,10 +254,10 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Arguments;
-        if (expect(TokenType.lParen) is null) return null;
-        if (!currentIs(TokenType.rParen))
+        if (expect(tok!"(") is null) return null;
+        if (!currentIs(tok!")"))
             node.argumentList = parseArgumentList();
-        if (expect(TokenType.rParen) is null) return null;
+        if (expect(tok!")") is null) return null;
         return node;
     }
 
@@ -297,18 +273,18 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new ArrayInitializer;
-        if (expect(TokenType.lBracket) is null) return null;
+        if (expect(tok!"[") is null) return null;
         while (moreTokens())
         {
-            if (currentIs(TokenType.rBracket))
+            if (currentIs(tok!"]"))
                 break;
             node.arrayMemberInitializations ~= parseArrayMemberInitialization();
-            if (currentIs(TokenType.comma))
+            if (currentIs(tok!","))
                 advance();
             else
                 break;
         }
-        if (expect(TokenType.rBracket) is null) return null;
+        if (expect(tok!"]") is null) return null;
         return node;
     }
 
@@ -323,10 +299,10 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new ArrayLiteral;
-        if (expect(TokenType.lBracket) is null) return null;
-        if (!currentIs(TokenType.rBracket))
+        if (expect(tok!"[") is null) return null;
+        if (!currentIs(tok!"]"))
             node.argumentList = parseArgumentList();
-        if (expect(TokenType.rBracket) is null) return null;
+        if (expect(tok!"]") is null) return null;
         return node;
     }
 
@@ -341,17 +317,17 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new ArrayMemberInitialization;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case lBrace:
-        case lBracket:
+        case tok!"{":
+        case tok!"[":
             node.nonVoidInitializer = parseNonVoidInitializer();
             if (node.nonVoidInitializer is null) return null;
             break;
         default:
             auto assignExpression = parseAssignExpression();
             if (assignExpression is null) return null;
-            if (currentIs(colon))
+            if (currentIs(tok!":"))
             {
                 node.assignExpression = assignExpression;
                 advance();
@@ -378,7 +354,7 @@ alias core.sys.posix.stdio.fileno fileno;
     ExpressionNode parseAsmAddExp()
     {
         return parseLeftAssocBinaryExpression!(AsmAddExp, AsmMulExp,
-            TokenType.plus, TokenType.minus)();
+            tok!"+", tok!"-")();
     }
 
     /**
@@ -626,15 +602,15 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new AssertExpression;
-        expect(TokenType.assert_);
-        if (expect(TokenType.lParen) is null) return null;
+        expect(tok!"assert");
+        if (expect(tok!"(") is null) return null;
         node.assertion = parseAssignExpression();
-        if (currentIs(TokenType.comma))
+        if (currentIs(tok!","))
         {
             advance();
             node.message = parseAssignExpression();
         }
-        if (expect(TokenType.rParen) is null) return null;
+        if (expect(tok!")") is null) return null;
         return node;
     }
 
@@ -666,12 +642,12 @@ alias core.sys.posix.stdio.fileno fileno;
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new AssignExpression;
         node.ternaryExpression = parseTernaryExpression();
-        if (currentIsOneOf(TokenType.assign, TokenType.unsignedShiftRightAssign,
-            TokenType.shiftRightAssign, TokenType.shiftLeftAssign,
-            TokenType.plusAssign, TokenType.minusAssign, TokenType.mulAssign,
-            TokenType.modAssign, TokenType.bitAndAssign, TokenType.divAssign,
-            TokenType.bitOrAssign, TokenType.powAssign, TokenType.xorAssign,
-            TokenType.catAssign))
+        if (currentIsOneOf(tok!"=", tok!">>>=",
+            tok!">>=", tok!"<<=",
+            tok!"+=", tok!"-=", tok!"*=",
+            tok!"%=", tok!"&=", tok!"/=",
+            tok!"|=", tok!"^^=", tok!"^=",
+            tok!"~="))
         {
             node.operator = advance().type;
             node.assignExpression = parseAssignExpression();
@@ -689,11 +665,8 @@ alias core.sys.posix.stdio.fileno fileno;
     AssocArrayLiteral parseAssocArrayLiteral()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new AssocArrayLiteral;
-        if (expect(TokenType.lBracket) is null) return null;
-        node.keyValuePairs = parseKeyValuePairs();
-        if (expect(TokenType.rBracket) is null) return null;
-        return node;
+        mixin (simpleParse!(AssocArrayLiteral, tok!"[",
+            "keyValuePairs|parseKeyValuePairs", tok!"]"));
     }
 
     /**
@@ -707,19 +680,19 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new AtAttribute;
-        if (expect(TokenType.at) is null) return null;
-        with (TokenType) switch (current.type)
+        if (expect(tok!"@") is null) return null;
+        switch (current.type)
         {
-        case identifier:
-            if (peekIsOneOf(lParen, dot, not))
+        case tok!"identifier":
+            if (peekIsOneOf(tok!"(", tok!".", tok!"!"))
                 node.functionCallExpression = parseFunctionCallExpression();
             else
                 node.identifier = advance();
             break;
-        case lParen:
+        case tok!"(":
             advance();
             node.argumentList = parseArgumentList();
-            expect(rParen);
+            expect(tok!")");
             break;
         default:
             error(`"(", or identifier expected`);
@@ -749,23 +722,23 @@ alias core.sys.posix.stdio.fileno fileno;
         auto node = new Attribute;
         switch (current.type)
         {
-        case TokenType.extern_:
-            if (peekIs(TokenType.lParen))
+        case tok!"extern":
+            if (peekIs(tok!"("))
                 node.linkageAttribute = parseLinkageAttribute();
             else
                 goto default;
             break;
-        case TokenType.align_:
+        case tok!"align":
             node.alignAttribute = parseAlignAttribute();
             break;
-        case TokenType.pragma_:
+        case tok!"pragma":
             node.pragmaExpression = parsePragmaExpression();
             break;
-        case TokenType.private_:
-        case TokenType.package_:
-        case TokenType.protected_:
-        case TokenType.public_:
-        case TokenType.export_:
+        case tok!"private":
+        case tok!"package":
+        case tok!"protected":
+        case tok!"public":
+        case tok!"export":
             node.attribute = advance().type;
             break;
         default:
@@ -786,7 +759,7 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         auto node = new AttributeDeclaration;
         node.attribute = attribute is null ? parseAttribute() : attribute;
-        expect(TokenType.colon);
+        expect(tok!":");
         return node;
     }
 
@@ -803,19 +776,19 @@ alias core.sys.posix.stdio.fileno fileno;
         auto node = new AutoDeclaration;
         do
         {
-            auto ident = expect(TokenType.identifier);
+            auto ident = expect(tok!"identifier");
             if (ident is null) return null;
             node.identifiers ~= *ident;
-            if (expect(TokenType.assign) is null) return null;
+            if (expect(tok!"=") is null) return null;
             auto init = parseInitializer();
             if (init is null) return null;
             node.initializers ~= init;
-            if (currentIs(TokenType.comma))
+            if (currentIs(tok!","))
                 advance();
             else
                 break;
         } while (moreTokens());
-        if (expect(TokenType.semicolon) is null) return null;
+        if (expect(tok!";") is null) return null;
         return node;
     }
 
@@ -830,14 +803,14 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new BlockStatement();
-        auto openBrace = expect(TokenType.lBrace);
+        auto openBrace = expect(tok!"{");
         if (openBrace is null) return null;
-        node.startLocation = openBrace.startIndex;
-        if (!currentIs(TokenType.rBrace))
+        node.startLocation = openBrace.index;
+        if (!currentIs(tok!"}"))
             node.declarationsAndStatements = parseDeclarationsAndStatements();
-        auto closeBrace = expect(TokenType.rBrace);
+        auto closeBrace = expect(tok!"}");
         if (closeBrace !is null)
-            node.endLocation = closeBrace.startIndex;
+            node.endLocation = closeBrace.index;
         else
             node.endLocation = size_t.max;
         return node;
@@ -853,10 +826,8 @@ alias core.sys.posix.stdio.fileno fileno;
     BodyStatement parseBodyStatement()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new BodyStatement;
-        expect(TokenType.body_);
-        node.blockStatement = parseBlockStatement();
-        return node;
+        mixin (simpleParse!(BodyStatement, tok!"body",
+            "blockStatement|parseBlockStatement"));
     }
 
     /**
@@ -869,15 +840,15 @@ alias core.sys.posix.stdio.fileno fileno;
     BreakStatement parseBreakStatement()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        expect(TokenType.break_);
+        expect(tok!"break");
         auto node = new BreakStatement;
         switch (current.type)
         {
-        case TokenType.identifier:
+        case tok!"identifier":
             node.label = advance();
-            if (expect(TokenType.semicolon) is null) return null;
+            if (expect(tok!";") is null) return null;
             break;
-        case TokenType.semicolon:
+        case tok!";":
             advance();
             break;
         default:
@@ -898,10 +869,10 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new BaseClass;
-        if (currentIs(TokenType.typeof_))
+        if (currentIs(tok!"typeof"))
         {
             node.typeofExpression = parseTypeofExpression();
-            if (expect(TokenType.dot) is null) return null;
+            if (expect(tok!".") is null) return null;
         }
         node.identifierOrTemplateChain = parseIdentifierOrTemplateChain();
         return node;
@@ -948,13 +919,13 @@ alias core.sys.posix.stdio.fileno fileno;
      *    | $(LITERAL 'void')
      *    ;)
      */
-    TokenType parseBasicType()
+    IdType parseBasicType()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         if (isBasicType(current.type))
             return advance().type;
         error("Basic type expected");
-        return TokenType.invalid;
+        return tok!"";
     }
 
     /**
@@ -970,14 +941,14 @@ alias core.sys.posix.stdio.fileno fileno;
         auto node = new CaseRangeStatement;
         if (low is null)
         {
-            expect(TokenType.case_);
+            expect(tok!"case");
             node.low = parseAssignExpression();
         }
-        if (expect(TokenType.colon) is null) return null;
-        if (expect(TokenType.dotdot) is null) return null;
-        expect(TokenType.case_);
+        if (expect(tok!":") is null) return null;
+        if (expect(tok!"..") is null) return null;
+        expect(tok!"case");
         node.high = parseAssignExpression();
-        if (expect(TokenType.colon) is null) return null;
+        if (expect(tok!":") is null) return null;
         node.declarationsAndStatements = parseDeclarationsAndStatements();
         return node;
     }
@@ -995,12 +966,12 @@ alias core.sys.posix.stdio.fileno fileno;
         auto node = new CaseStatement;
         if (argumentList is null)
         {
-            expect(TokenType.case_);
+            expect(tok!"case");
             node.argumentList = parseArgumentList();
         }
         else
             node.argumentList = argumentList;
-        if (expect(TokenType.colon) is null) return null;
+        if (expect(tok!":") is null) return null;
         node.declarationsAndStatements = parseDeclarationsAndStatements();
         return node;
     }
@@ -1016,16 +987,16 @@ alias core.sys.posix.stdio.fileno fileno;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new CastExpression;
-        expect(TokenType.cast_);
-        if (expect(TokenType.lParen) is null) return null;
-        if (!currentIs(TokenType.rParen))
+        expect(tok!"cast");
+        if (expect(tok!"(") is null) return null;
+        if (!currentIs(tok!")"))
         {
             if (isCastQualifier())
                 node.castQualifier = parseCastQualifier();
             else
                 node.type = parseType();
         }
-        if (expect(TokenType.rParen) is null) return null;
+        if (expect(tok!")") is null) return null;
         node.unaryExpression = parseUnaryExpression();
         return node;
     }
@@ -1050,18 +1021,18 @@ alias core.sys.posix.stdio.fileno fileno;
         auto node = new CastQualifier;
         switch (current.type)
         {
-        case TokenType.inout_:
-        case TokenType.const_:
+        case tok!"inout":
+        case tok!"const":
             node.first = advance();
-            if (currentIs(TokenType.shared_))
+            if (currentIs(tok!"shared"))
                 node.second = advance();
             break;
-        case TokenType.shared_:
+        case tok!"shared":
             node.first = advance();
-            if (currentIsOneOf(TokenType.const_, TokenType.inout_))
+            if (currentIsOneOf(tok!"const", tok!"inout"))
                 node.second = advance();
             break;
-        case TokenType.immutable_:
+        case tok!"immutable":
             node.first = advance();
             break;
         default:
@@ -1088,48 +1059,44 @@ incorrect;
         Parser p = getParserForUnittest(sourceCode, "parseCastQualifier");
 
         CastQualifier one = p.parseCastQualifier();
-        assert (one.first == TokenType.const_);
-        assert (!one.hasSecond);
-        p.expect(TokenType.semicolon);
+        assert (one.first == tok!"const");
+        assert (one.second == tok!"");
+        p.expect(tok!";");
 
         CastQualifier two = p.parseCastQualifier();
-        assert (two.first == TokenType.const_);
-        assert (two.hasSecond);
-        assert (two.second == TokenType.shared_);
-        p.expect(TokenType.semicolon);
+        assert (two.first == tok!"const");
+        assert (two.second == tok!"shared");
+        p.expect(tok!";");
 
         CastQualifier three = p.parseCastQualifier();
-        assert (three.first == TokenType.immutable_);
-        assert (!three.hasSecond);
-        p.expect(TokenType.semicolon);
+        assert (three.first == tok!"immutable");
+        assert (three.second == tok!"");
+        p.expect(tok!";");
 
         CastQualifier four = p.parseCastQualifier();
-        assert (four.first == TokenType.inout_);
-        assert (!four.hasSecond);
-        p.expect(TokenType.semicolon);
+        assert (four.first == tok!"inout");
+        assert (four.second == tok!"");
+        p.expect(tok!";");
 
         CastQualifier five = p.parseCastQualifier();
-        assert (five.first == TokenType.inout_);
-        assert (five.hasSecond);
-        assert (five.second == TokenType.shared_);
-        p.expect(TokenType.semicolon);
+        assert (five.first == tok!"inout");
+        assert (five.second == tok!"shared");
+        p.expect(tok!";");
 
         CastQualifier six = p.parseCastQualifier();
-        assert (six.first == TokenType.shared_);
-        assert (!six.hasSecond);
-        p.expect(TokenType.semicolon);
+        assert (six.first == tok!"shared");
+        assert (six.second == tok!"");
+        p.expect(tok!";");
 
         CastQualifier seven = p.parseCastQualifier();
-        assert (seven.first == TokenType.shared_);
-        assert (seven.hasSecond);
-        assert (seven.second == TokenType.const_);
-        p.expect(TokenType.semicolon);
+        assert (seven.first == tok!"shared");
+        assert (seven.second == tok!"const");
+        p.expect(tok!";");
 
         CastQualifier eight = p.parseCastQualifier();
-        assert (eight.first == TokenType.shared_);
-        assert (eight.hasSecond);
-        assert (eight.second == TokenType.inout_);
-        p.expect(TokenType.semicolon);
+        assert (eight.first == tok!"shared");
+        assert (eight.second == tok!"inout");
+        p.expect(tok!";");
 
         CastQualifier nine = p.parseCastQualifier();
         assert (nine is null);
@@ -1149,12 +1116,12 @@ incorrect;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Catch;
-        expect(TokenType.catch_);
-        if (expect(TokenType.lParen) is null) return null;
+        expect(tok!"catch");
+        if (expect(tok!"(") is null) return null;
         node.type = parseType();
-        if (currentIs(TokenType.identifier))
+        if (currentIs(tok!"identifier"))
             node.identifier = advance();
-        if (expect(TokenType.rParen) is null) return null;
+        if (expect(tok!")") is null) return null;
         node.declarationOrStatement = parseDeclarationOrStatement();
         return node;
     }
@@ -1173,9 +1140,9 @@ incorrect;
         auto node = new Catches;
         while (moreTokens())
         {
-            if (!currentIs(TokenType.catch_))
+            if (!currentIs(tok!"catch"))
                 break;
-            if (peekIs(TokenType.lParen))
+            if (peekIs(tok!"("))
             {
                 node.catches ~= parseCatch();
             }
@@ -1199,19 +1166,21 @@ incorrect;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new ClassDeclaration;
-        expect(TokenType.class_);
-        auto ident = expect(TokenType.identifier);
+        expect(tok!"class");
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.name = *ident;
-        if (currentIs(TokenType.lParen))
+        node.comment = comment;
+        comment = null;
+        if (currentIs(tok!"("))
         {
             node.templateParameters = parseTemplateParameters();
-            if (currentIs(TokenType.if_))
+            if (currentIs(tok!"if"))
             {
                 node.constraint = parseConstraint();
             }
         }
-        if (currentIs(TokenType.colon))
+        if (currentIs(tok!":"))
         {
             advance();
             node.baseClassList = parseBaseClassList();
@@ -1231,14 +1200,14 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         Parser p = getParserForUnittest(sourceCode, "parseClassDeclaration");
 
         auto classOne = p.parseClassDeclaration();
-        assert (classOne.name == "ClassOne");
+        assert (classOne.name.text == "ClassOne");
         assert (classOne.structBody.declarations.length == 0);
         assert (classOne.baseClassList is null);
         assert (classOne.constraint is null);
         assert (classOne.templateParameters is null);
 
         auto classTwo = p.parseClassDeclaration();
-        assert (classTwo.name == "ClassTwo", classTwo.name.value);
+        assert (classTwo.name.text == "ClassTwo", classTwo.name.text);
         assert (classTwo.baseClassList !is null);
         assert (classTwo.baseClassList.items.length == 1,
             to!string(classTwo.baseClassList.items.length));
@@ -1246,7 +1215,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
             to!string(classTwo.structBody.declarations.length));
 
         auto classThree = p.parseClassDeclaration();
-        assert (classThree.name == "ClassThree", classThree.name.value);
+        assert (classThree.name.text == "ClassThree", classThree.name.text);
         assert (classThree.templateParameters !is null);
         assert (classThree.templateParameters.templateParameterList.items.length == 2);
         assert (classThree.baseClassList !is null);
@@ -1255,7 +1224,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
             to!string(classThree.structBody.declarations.length));
 
         //auto classFour = p.parseClassDeclaration();
-        //assert (classFour.name == "ClassFour", classFour.name.value);
+        //assert (classFour.name == "ClassFour", classFour.name.text);
         //assert (classFour.templateParameters !is null);
         //assert (classFour.baseClassList !is null);
         //assert (classFour.constraint !is null);
@@ -1284,36 +1253,36 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         auto shift = parseShiftExpression();
         if (!moreTokens())
             return shift;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case is_:
+        case tok!"is":
             node.identityExpression = parseIdentityExpression(shift);
             break;
-        case in_:
+        case tok!"in":
             node.inExpression = parseInExpression(shift);
             break;
-        case not:
-            if (peekIs(is_))
+        case tok!"!":
+            if (peekIs(tok!"is"))
                 node.identityExpression = parseIdentityExpression(shift);
-            else if (peekIs(in_))
+            else if (peekIs(tok!"in"))
                 node.inExpression = parseInExpression(shift);
             break;
-        case less:
-        case lessEqual:
-        case greater:
-        case greaterEqual:
-        case unordered:
-        case notLessEqualGreater:
-        case lessOrGreater:
-        case lessEqualGreater:
-        case notGreater:
-        case notGreaterEqual:
-        case notLess:
-        case notLessEqual:
+        case tok!"<":
+        case tok!"<=":
+        case tok!">":
+        case tok!">=":
+        case tok!"!<>=":
+        case tok!"!<>":
+        case tok!"<>":
+        case tok!"<>=":
+        case tok!"!>":
+        case tok!"!>=":
+        case tok!"!<":
+        case tok!"!<=":
             node.relExpression = parseRelExpression(shift);
             break;
-        case equal:
-        case notEqual:
+        case tok!"==":
+        case tok!"!=":
             node.equalExpression = parseEqualExpression(shift);
             break;
         default:
@@ -1338,13 +1307,13 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         auto node = new CompileCondition;
         switch (current.type)
         {
-        case TokenType.version_:
+        case tok!"version":
             node.versionCondition = parseVersionCondition();
             break;
-        case TokenType.debug_:
+        case tok!"debug":
             node.debugCondition = parseDebugCondition();
             break;
-        case TokenType.static_:
+        case tok!"static":
             node.staticIfCondition = parseStaticIfCondition();
             break;
         default:
@@ -1369,7 +1338,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         auto node = new ConditionalDeclaration;
         node.compileCondition = parseCompileCondition();
 
-        if (currentIs(TokenType.colon))
+        if (currentIs(tok!":"))
         {
             advance();
             while (isDeclaration())
@@ -1381,7 +1350,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         if (dec is null) return null;
         node.trueDeclarations ~= dec;
 
-        if(currentIs(TokenType.else_))
+        if(currentIs(tok!"else"))
             advance();
         else
             return node;
@@ -1405,7 +1374,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         auto node = new ConditionalStatement;
         node.compileCondition = parseCompileCondition();
         node.trueStatement = parseDeclarationOrStatement();
-        if (currentIs(TokenType.else_))
+        if (currentIs(tok!"else"))
         {
             advance();
             node.falseStatement = parseDeclarationOrStatement();
@@ -1424,10 +1393,10 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Constraint;
-        if (expect(TokenType.if_) is null) return null;
-        if (expect(TokenType.lParen) is null) return null;
+        if (expect(tok!"if") is null) return null;
+        if (expect(tok!"(") is null) return null;
         node.expression = parseExpression();
-        if (expect(TokenType.rParen) is null) return null;
+        if (expect(tok!")") is null) return null;
         return node;
     }
 
@@ -1442,12 +1411,14 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         Constructor node = new Constructor;
-        auto t = expect(TokenType.this_);
+        node.comment = comment;
+        comment = null;
+        auto t = expect(tok!"this");
         if (t is null) return null;
-        node.location = t.startIndex;
+        node.location = t.index;
         auto p = peekPastParens();
         bool isTemplate = false;
-        if (p !is null && p.type == TokenType.lParen)
+        if (p !is null && p.type == tok!"(")
         {
             isTemplate = true;
             node.templateParameters = parseTemplateParameters();
@@ -1458,10 +1429,10 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         while(moreTokens() && currentIsMemberFunctionAttribute())
             node.memberFunctionAttributes ~= parseMemberFunctionAttribute();
 
-        if (isTemplate && currentIs(TokenType.if_))
+        if (isTemplate && currentIs(tok!"if"))
             node.constraint = parseConstraint();
 
-        if (currentIs(TokenType.semicolon))
+        if (currentIs(tok!";"))
             advance();
         else
         {
@@ -1482,15 +1453,15 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     ContinueStatement parseContinueStatement()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        if (expect(TokenType.continue_) is null) return null;
+        if (expect(tok!"continue") is null) return null;
         auto node = new ContinueStatement;
         switch (current.type)
         {
-        case TokenType.identifier:
+        case tok!"identifier":
             node.label = advance();
-            if (expect(TokenType.semicolon) is null) return null;
+            if (expect(tok!";") is null) return null;
             break;
-        case TokenType.semicolon:
+        case tok!";":
             advance();
             break;
         default:
@@ -1511,18 +1482,18 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new DebugCondition;
-        if (expect(TokenType.debug_) is null) return null;
-        if (currentIs(TokenType.lParen))
+        if (expect(tok!"debug") is null) return null;
+        if (currentIs(tok!"("))
         {
             advance();
-            if (currentIsOneOf(TokenType.intLiteral, TokenType.identifier))
+            if (currentIsOneOf(tok!"intLiteral", tok!"identifier"))
                 node.identifierOrInteger = advance();
             else
             {
                 error(`Integer literal or identifier expected`);
                 return null;
             }
-            if (expect(TokenType.rParen) is null) return null;
+            if (expect(tok!")") is null) return null;
         }
         return node;
     }
@@ -1538,16 +1509,16 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new DebugSpecification;
-        if (expect(TokenType.debug_) is null) return null;
-        if (expect(TokenType.assign) is null) return null;
-        if (currentIsOneOf(TokenType.identifier, TokenType.intLiteral))
+        if (expect(tok!"debug") is null) return null;
+        if (expect(tok!"=") is null) return null;
+        if (currentIsOneOf(tok!"identifier", tok!"intLiteral"))
             node.identifierOrInteger = advance();
         else
         {
             error("Integer literal or identifier expected");
             return null;
         }
-        if (expect(TokenType.semicolon) is null) return null;
+        if (expect(tok!";") is null) return null;
         return node;
     }
 
@@ -1590,7 +1561,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Declaration;
-
+        comment = current.comment;
         do
         {
             if (!isAttribute())
@@ -1601,43 +1572,43 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
                 error("attribute is null");
                 break;
             }
-            if (currentIs(TokenType.colon))
-			{
+            if (currentIs(tok!":"))
+            {
                 node.attributeDeclaration = parseAttributeDeclaration(attr);
-				return node;
-			}
+                return node;
+            }
             else
                 node.attributes ~= attr;
         } while (moreTokens());
 
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case semicolon:
+        case tok!";":
             // http://d.puremagic.com/issues/show_bug.cgi?id=4559
             warn("Empty declaration");
             advance();
             break;
-        case lBrace:
+        case tok!"{":
             advance();
-            while (moreTokens() && !currentIs(rBrace))
+            while (moreTokens() && !currentIs(tok!"}"))
             {
                 auto declaration = parseDeclaration();
                 if (declaration !is null)
                     node.declarations ~= declaration;
             }
-            if (expect(TokenType.rBrace) is null) return null;
+            if (expect(tok!"}") is null) return null;
             break;
-        case alias_:
-            if (startsWith(alias_, identifier, this_))
+        case tok!"alias":
+            if (startsWith(tok!"alias", tok!"identifier", tok!"this"))
                 node.aliasThisDeclaration = parseAliasThisDeclaration();
             else
                 node.aliasDeclaration = parseAliasDeclaration();
             break;
-        case class_:
+        case tok!"class":
             node.classDeclaration = parseClassDeclaration();
             break;
-        case this_:
-            if (startsWith(this_, lParen, this_))
+        case tok!"this":
+            if (startsWith(tok!"this", tok!"(", tok!"this"))
             {
                 node.postblit = parsePostblit();
                 if (node.postblit is null) return null;
@@ -1648,35 +1619,35 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
                 if (node.constructor is null) return null;
             }
             break;
-        case tilde:
+        case tok!"~":
             node.destructor = parseDestructor();
             if (node.destructor is null) return null;
             break;
-        case enum_:
-            if (startsWith(TokenType.enum_, TokenType.identifier, TokenType.lParen))
-                goto case template_;
+        case tok!"enum":
+            if (startsWith(tok!"enum", tok!"identifier", tok!"("))
+                goto case tok!"template";
             node.enumDeclaration = parseEnumDeclaration();
             if (node.enumDeclaration is null) return null;
             break;
-        case import_:
+        case tok!"import":
             node.importDeclaration = parseImportDeclaration();
             if (node.importDeclaration is null) return null;
             break;
-        case interface_:
+        case tok!"interface":
             node.interfaceDeclaration = parseInterfaceDeclaration();
             if (node.interfaceDeclaration is null) return null;
             break;
-        case mixin_:
-            if (peekIs(TokenType.template_))
+        case tok!"mixin":
+            if (peekIs(tok!"template"))
                 node.mixinTemplateDeclaration = parseMixinTemplateDeclaration();
             else
             {
                 auto b = setBookmark();
                 advance();
-                if (currentIs(TokenType.lParen))
+                if (currentIs(tok!"("))
                 {
                     auto t = peekPastParens();
-                    if (t !is null && t.type == TokenType.semicolon)
+                    if (t !is null && t.type == tok!";")
                     {
                         goToBookmark(b);
                         node.mixinDeclaration = parseMixinDeclaration();
@@ -1696,51 +1667,51 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
                 }
             }
             break;
-        case pragma_:
+        case tok!"pragma":
             node.pragmaDeclaration = parsePragmaDeclaration();
             break;
-        case shared_:
-            if (startsWith(shared_, static_, this_))
+        case tok!"shared":
+            if (startsWith(tok!"shared", tok!"static", tok!"this"))
                 node.sharedStaticConstructor = parseSharedStaticConstructor();
-            else if (startsWith(shared_, static_, tilde))
+            else if (startsWith(tok!"shared", tok!"static", tok!"~"))
                 node.sharedStaticDestructor = parseSharedStaticDestructor();
             else
                 goto type;
             break;
-        case static_:
-            if (peekIs(this_))
+        case tok!"static":
+            if (peekIs(tok!"this"))
                 node.staticConstructor = parseStaticConstructor();
-            else if (peekIs(tilde))
+            else if (peekIs(tok!"~"))
                 node.staticDestructor = parseStaticDestructor();
-            else if (peekIs(if_))
+            else if (peekIs(tok!"if"))
                 node.conditionalDeclaration = parseConditionalDeclaration();
-            else if (peekIs(assert_))
+            else if (peekIs(tok!"assert"))
                 node.staticAssertDeclaration = parseStaticAssertDeclaration();
             else
                 goto type;
             break;
-        case struct_:
+        case tok!"struct":
             node.structDeclaration = parseStructDeclaration();
             break;
-        case template_:
+        case tok!"template":
             node.templateDeclaration = parseTemplateDeclaration();
             break;
-        case union_:
+        case tok!"union":
             node.unionDeclaration = parseUnionDeclaration();
             break;
-        case invariant_:
+        case tok!"invariant":
             node.invariant_ = parseInvariant();
             break;
-        case unittest_:
+        case tok!"unittest":
             node.unittest_ = parseUnittest();
             break;
-        case identifier:
+        case tok!"identifier":
             if (node.attributes.length > 0
                 && node.attributes[$ - 1].storageClass !is null)
             {
-                if (peekIs(assign))
+                if (peekIs(tok!"="))
                     node.variableDeclaration = parseVariableDeclaration(null, true);
-                else if (peekIs(lParen))
+                else if (peekIs(tok!"("))
                     node.functionDeclaration = parseFunctionDeclaration(null, true);
                 else
                     goto type;
@@ -1748,28 +1719,28 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
             else
                 goto type;
             break;
-        case const_:
-        case immutable_:
-        case inout_:
-        case scope_:
-        case typeof_:
-        mixin (BASIC_TYPE_CASE_RANGE);
+        case tok!"const":
+        case tok!"immutable":
+        case tok!"inout":
+        case tok!"scope":
+        case tok!"typeof":
+        mixin (BASIC_TYPE_CASES);
         type:
             Type type = parseType();
-            if (!currentIs(identifier))
+            if (!currentIs(tok!"identifier"))
             {
                 error("Identifier expected");
                 return null;
             }
-            if (peekIs(lParen))
+            if (peekIs(tok!"("))
                 node.functionDeclaration = parseFunctionDeclaration(type);
             else
                 node.variableDeclaration = parseVariableDeclaration(type);
             break;
-        case version_:
-            if (peekIs(lParen))
+        case tok!"version":
+            if (peekIs(tok!"("))
                 node.conditionalDeclaration = parseConditionalDeclaration();
-            else if (peekIs(assign))
+            else if (peekIs(tok!"="))
                 node.versionSpecification = parseVersionSpecification();
             else
             {
@@ -1777,7 +1748,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
                 return null;
             }
             break;
-        case debug_:
+        case tok!"debug":
             node.conditionalDeclaration = parseConditionalDeclaration();
             if (node.conditionalDeclaration is null) return null;
             break;
@@ -1801,7 +1772,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new DeclarationsAndStatements;
-        while (!currentIsOneOf(TokenType.rBrace) && moreTokens())
+        while (!currentIsOneOf(tok!"}") && moreTokens())
         {
             auto dos = parseDeclarationOrStatement();
             if (dos !is null)
@@ -1828,12 +1799,12 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         // Declarations are resolved by the declarations taking precedence."
         if (isDeclaration())
         {
-            trace("\033[01;36mparsing declaration");
+            trace("\033[01;36mparsing declaration\033[0m");
             node.declaration = parseDeclaration();
         }
         else
         {
-            trace("\033[01;36mparsing statement");
+            trace("\033[01;36mparsing statement\033[0m");
             node.statement = parseStatement();
         }
 
@@ -1856,15 +1827,15 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Declarator;
-        auto id = expect(TokenType.identifier);
+        auto id = expect(tok!"identifier");
         if (id is null) return null;
         node.name = *id;
-        if (currentIsOneOf(TokenType.lBracket, TokenType.star))
+        if (currentIsOneOf(tok!"[", tok!"*"))
         {
             error("C-style variable declarations are not supported.");
             return null;
         }
-        if (currentIs(TokenType.assign))
+        if (currentIs(tok!"="))
         {
             advance();
             node.initializer = parseInitializer();
@@ -1883,8 +1854,8 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new DefaultStatement;
-        if (expect(TokenType.default_) is null) return null;
-        if (expect(TokenType.colon) is null) return null;
+        if (expect(tok!"default") is null) return null;
+        if (expect(tok!":") is null) return null;
         node.declarationsAndStatements = parseDeclarationsAndStatements();
         return node;
     }
@@ -1900,7 +1871,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new DeleteExpression;
-        if (expect(TokenType.delete_) is null) return null;
+        if (expect(tok!"delete") is null) return null;
         node.unaryExpression = parseUnaryExpression();
         return node;
     }
@@ -1916,12 +1887,12 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Deprecated;
-        if (expect(TokenType.deprecated_) is null) return null;
-        if (currentIs(TokenType.lParen))
+        if (expect(tok!"deprecated") is null) return null;
+        if (currentIs(tok!"("))
         {
             advance();
             node.assignExpression = parseAssignExpression();
-            if (expect(TokenType.rParen) is null) return null;
+            if (expect(tok!")") is null) return null;
         }
         return node;
     }
@@ -1937,11 +1908,13 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Destructor;
-        if (expect(TokenType.tilde) is null) return null;
-        if (expect(TokenType.this_) is null) return null;
-        if (expect(TokenType.lParen) is null) return null;
-        if (expect(TokenType.rParen) is null) return null;
-        if (currentIs(TokenType.semicolon))
+        node.comment = comment;
+        comment = null;
+        if (expect(tok!"~") is null) return null;
+        if (expect(tok!"this") is null) return null;
+        if (expect(tok!"(") is null) return null;
+        if (expect(tok!")") is null) return null;
+        if (currentIs(tok!";"))
             advance();
         else
             node.functionBody = parseFunctionBody();
@@ -1970,13 +1943,13 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new DoStatement;
-        if (expect(TokenType.do_) is null) return null;
+        if (expect(tok!"do") is null) return null;
         node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault();
-        if (expect(TokenType.while_) is null) return null;
-        if (expect(TokenType.lParen) is null) return null;
+        if (expect(tok!"while") is null) return null;
+        if (expect(tok!"(") is null) return null;
         node.expression = parseExpression();
-        if (expect(TokenType.rParen) is null) return null;
-        if (expect(TokenType.semicolon) is null) return null;
+        if (expect(tok!")") is null) return null;
+        if (expect(tok!";") is null) return null;
         return node;
     }
 
@@ -1992,21 +1965,21 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         EnumBody node = new EnumBody;
-        if (!currentIs(TokenType.semicolon))
+        if (!currentIs(tok!";"))
         {
-            auto open = expect (TokenType.lBrace);
+            auto open = expect (tok!"{");
             if (open is null) goto ret;
-            node.startLocation = open.startIndex;
+            node.startLocation = open.index;
             while (moreTokens())
             {
-                if (!currentIsOneOf(TokenType.comma, TokenType.rBrace))
+                if (!currentIsOneOf(tok!",", tok!"}"))
                     node.enumMembers ~= parseEnumMember();
-                else if (currentIs(TokenType.comma))
+                else if (currentIs(tok!","))
                 {
                     advance();
                     continue;
                 }
-                else if (currentIs(TokenType.rBrace))
+                else if (currentIs(tok!"}"))
                     break;
                 else
                 {
@@ -2014,9 +1987,9 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
                     goto ret;
                 }
             }
-            auto close = expect (TokenType.rBrace);
+            auto close = expect (tok!"}");
             if (close !is null)
-                node.endLocation = close.startIndex;
+                node.endLocation = close.index;
         }
     ret:
         return node;
@@ -2033,12 +2006,14 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new EnumDeclaration;
-        if (expect(TokenType.enum_) is null) return null;
-        if (currentIs(TokenType.identifier))
+        if (expect(tok!"enum") is null) return null;
+        if (currentIs(tok!"identifier"))
             node.name = advance();
 		else
 			node.name.line = tokens[index - 1].line; // preserve line number if anonymous
-        if (currentIs(TokenType.colon))
+        node.comment = comment;
+        comment = null;
+        if (currentIs(tok!":"))
         {
             advance();
             node.type = parseType();
@@ -2059,11 +2034,12 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new EnumMember;
-        if (currentIs(TokenType.identifier))
+        node.comment = current.comment;
+        if (currentIs(tok!"identifier"))
         {
-            if (peekIsOneOf(TokenType.comma, TokenType.rBrace))
+            if (peekIsOneOf(tok!",", tok!"}"))
                 node.name = advance();
-            else if (peekIs(TokenType.assign))
+            else if (peekIs(tok!"="))
             {
                 node.name = advance();
                 goto assign;
@@ -2076,7 +2052,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     type:
             node.type = parseType();
     assign:
-            expect(TokenType.assign);
+            expect(tok!"=");
             node.assignExpression = parseAssignExpression();
         }
         return node;
@@ -2094,7 +2070,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new EqualExpression;
         node.left = shift is null ? parseShiftExpression() : shift;
-        if (currentIsOneOf(TokenType.equal, TokenType.notEqual))
+        if (currentIsOneOf(tok!"==", tok!"!="))
             node.operator = advance().type;
         node.right = parseShiftExpression();
         return node;
@@ -2125,7 +2101,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new ExpressionStatement;
         node.expression = expression is null ? parseExpression() : expression;
-        if (expect(TokenType.semicolon) is null) return null;
+        if (expect(tok!";") is null) return null;
         return node;
     }
 
@@ -2140,7 +2116,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new FinalSwitchStatement;
-        if (expect(TokenType.final_) is null) return null;
+        if (expect(tok!"final") is null) return null;
         node.switchStatement = parseSwitchStatement();
         if (node.switchStatement is null) return null;
         return node;
@@ -2157,7 +2133,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Finally;
-        if (expect(TokenType.finally_) is null) return null;
+        if (expect(tok!"finally") is null) return null;
         node.declarationOrStatement = parseDeclarationOrStatement();
         return node;
     }
@@ -2173,29 +2149,29 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new ForStatement;
-        if (expect(TokenType.for_) is null) return null;
-        node.startIndex = current().startIndex;
-        if (expect(TokenType.lParen) is null) return null;
+        if (expect(tok!"for") is null) return null;
+        node.startIndex = current().index;
+        if (expect(tok!"(") is null) return null;
 
-        if (currentIs(TokenType.semicolon))
+        if (currentIs(tok!";"))
             advance();
         else
             node.initialization = parseDeclarationOrStatement();
 
-        if (currentIs(TokenType.semicolon))
+        if (currentIs(tok!";"))
             advance();
         else
             node.test = parseExpressionStatement();
 
-        if (!currentIs(TokenType.rParen))
+        if (!currentIs(tok!")"))
             node.increment = parseExpression();
 
-        if (expect(TokenType.rParen) is null) return null;
-		if (currentIs(TokenType.rBrace))
-		{
-			error("Statement expected", false);
-			return node; // this line makes DCD better
-		}
+        if (expect(tok!")") is null) return null;
+        if (currentIs(tok!"}"))
+        {
+            error("Statement expected", false);
+            return node; // this line makes DCD better
+        }
         node.declarationOrStatement = parseDeclarationOrStatement();
         if (node.declarationOrStatement is null) return null;
         return node;
@@ -2213,22 +2189,22 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         ForeachStatement node = new ForeachStatement;
-        if (currentIsOneOf(TokenType.foreach_, TokenType.foreach_reverse_))
+        if (currentIsOneOf(tok!"foreach", tok!"foreach_reverse"))
             node.type = advance().type;
         else
         {
             error(`"foreach" or "foreach_reverse" expected`);
             return null;
         }
-        node.startIndex = current().startIndex;
-        if (expect(TokenType.lParen) is null) return null;
+        node.startIndex = current().index;
+        if (expect(tok!"(") is null) return null;
         ForeachTypeList feType = parseForeachTypeList();
         bool canBeRange = feType.items.length == 1;
 
-        if (expect(TokenType.semicolon) is null) return null;
+        if (expect(tok!";") is null) return null;
         node.low = parseExpression();
         if (node.low is null) return null;
-        if (currentIs(TokenType.dotdot))
+        if (currentIs(tok!".."))
         {
             if (!canBeRange)
             {
@@ -2244,8 +2220,8 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         {
             node.foreachTypeList = feType;
         }
-        if (expect(TokenType.rParen) is null) return null;
-        if (currentIs(TokenType.rBrace))
+        if (expect(tok!")") is null) return null;
+        if (currentIs(tok!"}"))
         {
             error("Statement expected", false);
             return node; // this line makes DCD better
@@ -2266,20 +2242,20 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new ForeachType;
-        if (currentIsOneOf(TokenType.ref_, TokenType.const_, TokenType.immutable_,
-            TokenType.shared_, TokenType.inout_))
+        if (currentIsOneOf(tok!"ref", tok!"const", tok!"immutable",
+            tok!"shared", tok!"inout"))
         {
             trace("\033[01;36mType constructor");
             if ((node.typeConstructors = parseTypeConstructors()) is null)
                 return null;
         }
-        if (currentIs(TokenType.identifier) && peekIsOneOf(TokenType.comma, TokenType.semicolon))
+        if (currentIs(tok!"identifier") && peekIsOneOf(tok!",", tok!";"))
         {
             node.identifier = advance();
             return node;
         }
         if ((node.type = parseType()) is null) return null;
-        auto ident = expect(TokenType.identifier);
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.identifier = *ident;
         return node;
@@ -2310,13 +2286,13 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     FunctionAttribute parseFunctionAttribute(bool validate = true)
     {
         auto node = new FunctionAttribute;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case at:
+        case tok!"@":
             node.atAttribute = parseAtAttribute();
             break;
-        case pure_:
-        case nothrow_:
+        case tok!"pure":
+        case tok!"nothrow":
             node.token = advance();
             break;
         default:
@@ -2338,25 +2314,25 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
     FunctionBody parseFunctionBody()
     {
         auto node = new FunctionBody;
-        if (currentIs(TokenType.semicolon))
+        if (currentIs(tok!";"))
         {
             advance();
             return node;
         }
-        else if (currentIs(TokenType.lBrace))
+        else if (currentIs(tok!"{"))
             node.blockStatement = parseBlockStatement();
         else
         {
-            if (currentIs(TokenType.in_))
+            if (currentIs(tok!"in"))
             {
                 node.inStatement = parseInStatement();
-                if (currentIs(TokenType.out_))
+                if (currentIs(tok!"out"))
                     node.outStatement = parseOutStatement();
             }
-            else if (currentIs(TokenType.out_))
+            else if (currentIs(tok!"out"))
             {
                 node.outStatement = parseOutStatement();
-                if (currentIs(TokenType.in_))
+                if (currentIs(tok!"in"))
                     node.inStatement = parseInStatement();
             }
             node.bodyStatement = parseBodyStatement();
@@ -2425,7 +2401,7 @@ body {} // six
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new FunctionCallExpression;
         node.unaryExpression = unary is null ? parseUnaryExpression() : unary;
-        if (currentIs(TokenType.not))
+        if (currentIs(tok!"!"))
             node.templateArguments = parseTemplateArguments();
         node.arguments = parseArguments();
         return node;
@@ -2443,7 +2419,7 @@ body {} // six
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new FunctionCallStatement;
         node.functionCallExpression = parseFunctionCallExpression();
-        if (expect(TokenType.semicolon) is null) return null;
+        if (expect(tok!";") is null) return null;
         return node;
     }
 
@@ -2458,6 +2434,8 @@ body {} // six
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new FunctionDeclaration;
+        node.comment = comment;
+        comment = null;
 
         if (isAuto)
             goto functionName;
@@ -2468,34 +2446,34 @@ body {} // six
         node.returnType = type is null ? parseType() : type;
 
     functionName:
-        auto ident = expect(TokenType.identifier);
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
 
         node.name = *ident;
 
-        if (!currentIs(TokenType.lParen))
+        if (!currentIs(tok!"("))
         {
             error(`"(" expected`);
             return null;
         }
 
-        assert (currentIs(TokenType.lParen));
+        assert (currentIs(tok!"("));
         auto p = peekPastParens();
-        bool isTemplate = p !is null && p.type == TokenType.lParen;
+        bool isTemplate = p !is null && p.type == tok!"(";
 
         if (isTemplate)
             node.templateParameters = parseTemplateParameters();
 
         node.parameters = parseParameters();
-		if (node.parameters is null) return null;
+        if (node.parameters is null) return null;
 
         while(moreTokens() && currentIsMemberFunctionAttribute())
             node.memberFunctionAttributes ~= parseMemberFunctionAttribute();
 
-        if (isTemplate && currentIs(TokenType.if_))
+        if (isTemplate && currentIs(tok!"if"))
             node.constraint = parseConstraint();
 
-        if (currentIs(TokenType.semicolon))
+        if (currentIs(tok!";"))
             advance();
         else
             node.functionBody = parseFunctionBody();
@@ -2513,17 +2491,17 @@ body {} // six
     FunctionLiteralExpression parseFunctionLiteralExpression()
     {
         auto node = new FunctionLiteralExpression;
-        if (currentIsOneOf(TokenType.function_, TokenType.delegate_))
+        if (currentIsOneOf(tok!"function", tok!"delegate"))
         {
             node.functionOrDelegate = advance().type;
-            if (!currentIsOneOf(TokenType.lParen, TokenType.in_, TokenType.body_,
-                TokenType.out_, TokenType.rBrace))
+            if (!currentIsOneOf(tok!"(", tok!"in", tok!"body",
+                tok!"out", tok!"}"))
             {
                 node.type = parseType();
                 if (node.type is null) return null;
             }
         }
-        if (currentIs(TokenType.lParen))
+        if (currentIs(tok!"("))
         {
             node.parameters = parseParameters();
             if (node.parameters is null) return null;
@@ -2551,23 +2529,23 @@ body {} // six
     GotoStatement parseGotoStatement()
     {
         auto node = new GotoStatement;
-        if (expect(TokenType.goto_) is null) return null;
-        with (TokenType) switch (current.type)
+        if (expect(tok!"goto") is null) return null;
+        switch (current.type)
         {
-        case identifier:
-        case default_:
+        case tok!"identifier":
+        case tok!"default":
             node.label = advance();
             break;
-        case case_:
+        case tok!"case":
             node.label = advance();
-            if (!currentIs(semicolon))
+            if (!currentIs(tok!";"))
                 node.expression = parseExpression();
             break;
         default:
             error(`Identifier, "default", or "case" expected`);
             return null;
         }
-        if (expect(TokenType.semicolon) is null) return null;
+        if (expect(tok!";") is null) return null;
         return node;
     }
 
@@ -2583,10 +2561,10 @@ body {} // six
         auto node = new IdentifierChain;
         while (moreTokens())
         {
-            auto ident = expect(TokenType.identifier);
+            auto ident = expect(tok!"identifier");
             if (ident is null) return null;
             node.identifiers ~= *ident;
-            if (currentIs(TokenType.dot))
+            if (currentIs(tok!"."))
             {
                 advance();
                 continue;
@@ -2609,10 +2587,10 @@ body {} // six
         auto node = new IdentifierList;
         do
         {
-            auto ident = expect(TokenType.identifier);
+            auto ident = expect(tok!"identifier");
             if (ident is null) return null;
             node.identifiers ~= *ident;
-            if (currentIs(TokenType.comma))
+            if (currentIs(tok!","))
             {
                 advance();
                 continue;
@@ -2636,7 +2614,7 @@ body {} // six
         while (moreTokens())
         {
             node.identifiersOrTemplateInstances ~= parseIdentifierOrTemplateInstance();
-            if (!currentIs(TokenType.dot))
+            if (!currentIs(tok!"."))
                 break;
             else
                 advance();
@@ -2656,15 +2634,15 @@ body {} // six
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new IdentifierOrTemplateInstance;
-        if (peekIs(TokenType.not) && !startsWith(TokenType.identifier,
-            TokenType.not, TokenType.is_)
-            && !startsWith(TokenType.identifier, TokenType.not, TokenType.in_))
+        if (peekIs(tok!"!") && !startsWith(tok!"identifier",
+            tok!"!", tok!"is")
+            && !startsWith(tok!"identifier", tok!"!", tok!"in"))
         {
             node.templateInstance = parseTemplateInstance();
         }
         else
         {
-            auto ident = expect(TokenType.identifier);
+            auto ident = expect(tok!"identifier");
             if (ident is null) return null;
             node.identifier = *ident;
         }
@@ -2683,12 +2661,12 @@ body {} // six
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new IdentityExpression;
         node.left = shift is null ? parseShiftExpression() : shift;
-        if (currentIs(TokenType.not))
+        if (currentIs(tok!"!"))
         {
             advance();
             node.negated = true;
         }
-        if (expect(TokenType.is_) is null) return null;
+        if (expect(tok!"is") is null) return null;
         node.right = parseShiftExpression();
         return node;
     }
@@ -2708,25 +2686,25 @@ body {} // six
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new IfStatement;
-        if (expect(TokenType.if_) is null) return null;
-        node.startIndex = current().startIndex;
-        if (expect(TokenType.lParen) is null) return null;
+        if (expect(tok!"if") is null) return null;
+        node.startIndex = current().index;
+        if (expect(tok!"(") is null) return null;
 
-        if (currentIs(TokenType.auto_))
+        if (currentIs(tok!"auto"))
         {
             advance();
-            auto i = expect(TokenType.identifier);
+            auto i = expect(tok!"identifier");
             if (i !is null)
                 node.identifier = *i;
-            expect(TokenType.assign);
+            expect(tok!"=");
             node.expression = parseExpression();
         }
         else
         {
             auto b = setBookmark();
             auto t = parseType();
-            if (t is null || !currentIs(TokenType.identifier)
-				|| !peekIs(TokenType.assign))
+            if (t is null || !currentIs(tok!"identifier")
+                || !peekIs(tok!"="))
             {
                 goToBookmark(b);
                 node.expression = parseExpression();
@@ -2735,22 +2713,22 @@ body {} // six
             {
                 goToBookmark(b);
                 node.type = parseType();
-                auto i = expect(TokenType.identifier);
+                auto i = expect(tok!"identifier");
                 if (i !is null)
                     node.identifier = *i;
-                expect(TokenType.assign);
+                expect(tok!"=");
                 node.expression = parseExpression();
             }
         }
 
-        if (expect(TokenType.rParen) is null) return null;
-        if (currentIs(TokenType.rBrace))
-		{
-			error("Statement expected", false);
-			return node; // this line makes DCD better
-		}
+        if (expect(tok!")") is null) return null;
+        if (currentIs(tok!"}"))
+        {
+            error("Statement expected", false);
+            return node; // this line makes DCD better
+        }
         node.thenStatement = parseDeclarationOrStatement();
-        if (currentIs(TokenType.else_))
+        if (currentIs(tok!"else"))
         {
             advance();
             node.elseStatement = parseDeclarationOrStatement();
@@ -2768,13 +2746,13 @@ body {} // six
     ImportBind parseImportBind()
     {
         auto node = new ImportBind;
-        auto ident = expect(TokenType.identifier);
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.left = *ident;
-        if (currentIs(TokenType.assign))
+        if (currentIs(tok!"="))
         {
             advance();
-            auto id = expect(TokenType.identifier);
+            auto id = expect(tok!"identifier");
             if (id is null) return null;
             node.right = *id;
         }
@@ -2792,11 +2770,11 @@ body {} // six
     {
         auto node = new ImportBindings;
         node.singleImport = singleImport is null ? parseSingleImport() : singleImport;
-        if (expect(TokenType.colon) is null) return null;
+        if (expect(tok!":") is null) return null;
         while (moreTokens())
         {
             node.importBinds ~= parseImportBind();
-            if (currentIs(TokenType.comma))
+            if (currentIs(tok!","))
                 advance();
             else
                 break;
@@ -2815,14 +2793,14 @@ body {} // six
     ImportDeclaration parseImportDeclaration()
     {
         auto node = new ImportDeclaration;
-        if (expect(TokenType.import_) is null) return null;
+        if (expect(tok!"import") is null) return null;
         SingleImport si = parseSingleImport();
-        if (currentIs(TokenType.colon))
+        if (currentIs(tok!":"))
             node.importBindings = parseImportBindings(si);
         else
         {
             node.singleImports ~= si;
-            if (currentIs(TokenType.comma))
+            if (currentIs(tok!","))
             {
                 advance();
                 while (moreTokens())
@@ -2830,7 +2808,7 @@ body {} // six
                     auto single = parseSingleImport();
                     if (single is null)
                         return null;
-                    if (currentIs(TokenType.colon))
+                    if (currentIs(tok!":"))
                     {
                         node.importBindings = parseImportBindings(single);
                         break;
@@ -2838,7 +2816,7 @@ body {} // six
                     else
                     {
                         node.singleImports ~= single;
-                        if (currentIs(TokenType.comma))
+                        if (currentIs(tok!","))
                             advance();
                         else
                             break;
@@ -2846,7 +2824,7 @@ body {} // six
                 }
             }
         }
-        if (expect(TokenType.semicolon) is null) return null;
+        if (expect(tok!";") is null) return null;
         return node;
     }
 
@@ -2913,10 +2891,10 @@ import core.stdc.stdio, std.string : KeepTerminator;
     ImportExpression parseImportExpression()
     {
         auto node = new ImportExpression;
-        if (expect(TokenType.import_) is null) return null;
-        if (expect(TokenType.lParen) is null) return null;
+        if (expect(tok!"import") is null) return null;
+        if (expect(tok!"(") is null) return null;
         node.assignExpression = parseAssignExpression();
-        if (expect(TokenType.rParen) is null) return null;
+        if (expect(tok!")") is null) return null;
         return node;
     }
 
@@ -2932,9 +2910,9 @@ import core.stdc.stdio, std.string : KeepTerminator;
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new IndexExpression;
         node.unaryExpression = unaryExpression is null ? parseUnaryExpression() : unaryExpression;
-        if (expect(TokenType.lBracket) is null) return null;
+        if (expect(tok!"[") is null) return null;
         node.argumentList = parseArgumentList();
-        if (expect(TokenType.rBracket) is null) return null;
+        if (expect(tok!"]") is null) return null;
         return node;
     }
 
@@ -2949,12 +2927,12 @@ import core.stdc.stdio, std.string : KeepTerminator;
     {
         auto node = new InExpression;
         node.left = shift is null ? parseShiftExpression() : shift;
-        if (currentIs(TokenType.not))
-		{
-			node.negated = true;
-			advance();
-		}
-        if (expect(TokenType.in_) is null) return null;
+        if (currentIs(tok!"!"))
+        {
+            node.negated = true;
+            advance();
+        }
+        if (expect(tok!"in") is null) return null;
         node.right = parseShiftExpression();
         return node;
     }
@@ -2969,7 +2947,7 @@ import core.stdc.stdio, std.string : KeepTerminator;
     InStatement parseInStatement()
     {
         auto node = new InStatement;
-        if (expect(TokenType.in_) is null) return null;
+        if (expect(tok!"in") is null) return null;
         node.blockStatement = parseBlockStatement();
         return node;
     }
@@ -2985,12 +2963,12 @@ import core.stdc.stdio, std.string : KeepTerminator;
     Initialize parseInitialize()
     {
         auto node = new Initialize;
-        if (!currentIs(TokenType.semicolon))
+        if (!currentIs(tok!";"))
         {
             node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault();
             if (node.statementNoCaseNoDefault is null) return null;
         }
-        else if (expect(TokenType.semicolon) is null)
+        else if (expect(tok!";") is null)
             return null;
         return node;
     }
@@ -3006,7 +2984,7 @@ import core.stdc.stdio, std.string : KeepTerminator;
     Initializer parseInitializer()
     {
         auto node = new Initializer;
-        if (currentIs(TokenType.void_))
+        if (currentIs(tok!"void"))
             advance();
         else
             node.nonVoidInitializer = parseNonVoidInitializer();
@@ -3023,17 +3001,19 @@ import core.stdc.stdio, std.string : KeepTerminator;
     InterfaceDeclaration parseInterfaceDeclaration()
     {
         auto node = new InterfaceDeclaration;
-        if (expect(TokenType.interface_) is null) return null;
-        auto ident = expect(TokenType.identifier);
+        if (expect(tok!"interface") is null) return null;
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.name = *ident;
-        if (currentIs(TokenType.lParen))
+        node.comment = comment;
+        comment = null;
+        if (currentIs(tok!"("))
         {
             node.templateParameters = parseTemplateParameters();
-            if (currentIs(TokenType.if_))
+            if (currentIs(tok!"if"))
                 node.constraint = parseConstraint();
         }
-        if (currentIs(TokenType.colon))
+        if (currentIs(tok!":"))
         {
             advance();
             node.baseClassList = parseBaseClassList();
@@ -3055,7 +3035,7 @@ interface "Four"
 
         InterfaceDeclaration one = p.parseInterfaceDeclaration();
         assert (one !is null);
-        assert (one.identifier == "One");
+        assert (one.name.text == "One");
         assert (one.constraint is null);
         assert (one.templateParameters is null);
         assert (one.structBody !is null);
@@ -3064,7 +3044,7 @@ interface "Four"
 
         InterfaceDeclaration two = p.parseInterfaceDeclaration();
         assert (two !is null);
-        assert (two.identifier == "Two");
+        assert (two.name.text == "Two");
         assert (two.constraint is null);
         assert (two.templateParameters is null);
         assert (two.structBody !is null);
@@ -3073,7 +3053,7 @@ interface "Four"
 
         InterfaceDeclaration three = p.parseInterfaceDeclaration();
         assert (three !is null);
-        assert (three.identifier == "Three");
+        assert (three.name.text == "Three");
         assert (three.constraint !is null);
         assert (three.templateParameters !is null);
         assert (three.structBody !is null);
@@ -3097,11 +3077,11 @@ interface "Four"
     Invariant parseInvariant()
     {
         auto node = new Invariant;
-        if (expect(TokenType.invariant_) is null) return null;
-        if (currentIs(TokenType.lParen))
+        if (expect(tok!"invariant") is null) return null;
+        if (currentIs(tok!"("))
         {
             advance();
-            if (expect(TokenType.rParen) is null) return null;
+            if (expect(tok!")") is null) return null;
         }
         if ((node.blockStatement = parseBlockStatement()) is null) return null;
         return node;
@@ -3143,26 +3123,36 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new IsExpression;
-        if (expect(TokenType.is_) is null) return null;
-        if (expect(TokenType.lParen) is null) return null;
+        if (expect(tok!"is") is null) return null;
+        if (expect(tok!"(") is null) return null;
         node.type = parseType();
         if (node.type is null) return null;
-        if (currentIs(TokenType.identifier))
+        if (currentIs(tok!"identifier"))
             node.identifier = advance();
-        if (currentIsOneOf(TokenType.equal, TokenType.colon))
+        if (currentIsOneOf(tok!"==", tok!":"))
         {
             node.equalsOrColon = advance().type;
             node.typeSpecialization = parseTypeSpecialization();
-            if (currentIs(TokenType.comma))
+            if (currentIs(tok!","))
             {
                 advance();
                 node.templateParameterList = parseTemplateParameterList();
             }
         }
-        if (expect(TokenType.rParen) is null) return null;
+        if (expect(tok!")") is null) return null;
         return node;
     }
 
+	unittest
+    {
+        auto sourceCode = q{is ( x : uybte)}c;
+        Parser p = getParserForUnittest(sourceCode, "parseIsExpression");
+        auto isExp1 = p.parseIsExpression();
+        assert (isExp1 !is null);
+        assert (p.errorCount == 0);
+		stderr.writeln("Unittest for parseIsExpression passed.");
+    }
+
     /**
      * Parses a KeyValuePair
      *
@@ -3175,7 +3165,7 @@ invariant() foo();
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new KeyValuePair;
         node.key = parseAssignExpression();
-        if (expect(TokenType.colon) is null) return null;
+        if (expect(tok!":") is null) return null;
         node.value = parseAssignExpression();
         return node;
     }
@@ -3196,10 +3186,10 @@ invariant() foo();
             auto kvPair = parseKeyValuePair();
             if (kvPair !is null)
                 node.keyValuePairs ~= kvPair;
-            if (currentIs(TokenType.comma))
+            if (currentIs(tok!","))
             {
                 advance();
-                if (currentIs(TokenType.rBracket))
+                if (currentIs(tok!"]"))
                     break;
             }
             else
@@ -3219,10 +3209,10 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new LabeledStatement;
-        auto ident = expect(TokenType.identifier);
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.identifier = *ident;
-        expect(TokenType.colon);
+        expect(tok!":");
         node.declarationOrStatement = parseDeclarationOrStatement();
         return node;
     }
@@ -3241,14 +3231,14 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new LambdaExpression;
-        if (currentIsOneOf(TokenType.function_, TokenType.delegate_))
+        if (currentIsOneOf(tok!"function", tok!"delegate"))
         {
             node.functionType = advance().type;
             goto lParen;
         }
-        else if (currentIs(TokenType.identifier))
+        else if (currentIs(tok!"identifier"))
             node.identifier = advance();
-        else if (currentIs(TokenType.lParen))
+        else if (currentIs(tok!"("))
         {
         lParen:
             node.parameters = parseParameters();
@@ -3267,7 +3257,7 @@ invariant() foo();
             return null;
         }
 
-        if (expect(TokenType.goesTo) is null) return null;
+        if (expect(tok!"=>") is null) return null;
 
         if ((node.assignExpression = parseAssignExpression()) is null)
             return null;
@@ -3286,7 +3276,7 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new LastCatch;
-        if (expect(TokenType.catch_) is null) return null;
+        if (expect(tok!"catch") is null) return null;
         if ((node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault()) is null)
             return null;
         return node;
@@ -3303,17 +3293,17 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new LinkageAttribute;
-        expect(TokenType.extern_);
-        expect(TokenType.lParen);
-        auto ident = expect(TokenType.identifier);
+        expect(tok!"extern");
+        expect(tok!"(");
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.identifier = *ident;
-        if (currentIs(TokenType.increment))
+        if (currentIs(tok!"++"))
         {
             advance();
             node.hasPlusPlus = true;
         }
-        expect(TokenType.rParen);
+        expect(tok!")");
         return node;
     }
 
@@ -3332,17 +3322,17 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new MemberFunctionAttribute;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case at:
+        case tok!"@":
             node.atAttribute = parseAtAttribute();
             break;
-        case immutable_:
-        case inout_:
-        case shared_:
-        case const_:
-        case pure_:
-        case nothrow_:
+        case tok!"immutable":
+        case tok!"inout":
+        case tok!"shared":
+        case tok!"const":
+        case tok!"pure":
+        case tok!"nothrow":
             node.tokenType = advance().type;
             break;
         default:
@@ -3363,16 +3353,16 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new MixinDeclaration;
-        if (peekIs(TokenType.identifier))
+        if (peekIs(tok!"identifier"))
             node.templateMixinExpression = parseTemplateMixinExpression();
-        else if (peekIs(TokenType.lParen))
+        else if (peekIs(tok!"("))
             node.mixinExpression = parseMixinExpression();
         else
         {
             error(`"(" or identifier expected`);
             return null;
         }
-        expect(TokenType.semicolon);
+        expect(tok!";");
         return node;
     }
 
@@ -3387,10 +3377,10 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new MixinExpression;
-        expect(TokenType.mixin_);
-        expect(TokenType.lParen);
+        expect(tok!"mixin");
+        expect(tok!"(");
         node.assignExpression = parseAssignExpression();
-        expect(TokenType.rParen);
+        expect(tok!")");
         return node;
     }
 
@@ -3405,7 +3395,7 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new MixinTemplateDeclaration;
-        if (expect(TokenType.mixin_) is null) return null;
+        if (expect(tok!"mixin") is null) return null;
         node.templateDeclaration = parseTemplateDeclaration();
         if (node.templateDeclaration is null) return null;
         return node;
@@ -3423,10 +3413,10 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new MixinTemplateName;
-        if (currentIs(TokenType.typeof_))
+        if (currentIs(tok!"typeof"))
         {
             node.typeofExpression = parseTypeofExpression();
-            expect(TokenType.dot);
+            expect(tok!".");
             node.identifierOrTemplateChain = parseIdentifierOrTemplateChain();
         }
         else
@@ -3434,6 +3424,10 @@ invariant() foo();
         return node;
     }
 
+    unittest
+    {
+    }
+
     /**
      * Parses a Module
      *
@@ -3443,10 +3437,11 @@ invariant() foo();
      */
     Module parseModule()
     {
+        mixin(traceEnterAndExit!(__FUNCTION__));
         Module m = new Module;
-        if (currentIs(TokenType.scriptLine))
+        if (currentIs(tok!"scriptLine"))
             advance();
-        if (currentIs(TokenType.module_))
+        if (currentIs(tok!"module"))
             m.moduleDeclaration = parseModuleDeclaration();
         while (moreTokens())
         {
@@ -3467,9 +3462,9 @@ invariant() foo();
     ModuleDeclaration parseModuleDeclaration()
     {
         auto node = new ModuleDeclaration;
-        expect(TokenType.module_);
+        expect(tok!"module");
         node.moduleName = parseIdentifierChain();
-        expect(TokenType.semicolon);
+        expect(tok!";");
         return node;
     }
 
@@ -3484,7 +3479,7 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         return parseLeftAssocBinaryExpression!(MulExpression, PowExpression,
-            TokenType.star, TokenType.div, TokenType.mod)();
+            tok!"*", tok!"/", tok!"%")();
     }
 
     /**
@@ -3497,11 +3492,11 @@ invariant() foo();
     NewAnonClassExpression parseNewAnonClassExpression()
     {
         auto node = new NewAnonClassExpression;
-        expect(TokenType.new_);
-        if (currentIs(TokenType.lParen))
+        expect(tok!"new");
+        if (currentIs(tok!"("))
             node.allocatorArguments = parseArguments();
-        expect(TokenType.class_);
-        if (!currentIs(TokenType.lBrace))
+        expect(tok!"class");
+        if (!currentIs(tok!"{"))
             node.baseClassList = parseBaseClassList();
         node.structBody = parseStructBody();
         return node;
@@ -3518,19 +3513,19 @@ invariant() foo();
     NewExpression parseNewExpression()
     {
         auto node = new NewExpression;
-        if (peekIsOneOf(TokenType.class_, TokenType.lParen))
+        if (peekIsOneOf(tok!"class", tok!"("))
             node.newAnonClassExpression = parseNewAnonClassExpression();
         else
         {
-            expect(TokenType.new_);
+            expect(tok!"new");
             node.type = parseType();
-            if (currentIs(TokenType.lBracket))
+            if (currentIs(tok!"["))
             {
                 advance();
                 node.assignExpression = parseAssignExpression();
-                expect(TokenType.rBracket);
+                expect(tok!"]");
             }
-            else if (currentIs(TokenType.lParen))
+            else if (currentIs(tok!"("))
                 node.arguments = parseArguments();
         }
         return node;
@@ -3569,62 +3564,62 @@ invariant() foo();
     StatementNoCaseNoDefault parseStatementNoCaseNoDefault()
     {
         auto node = new StatementNoCaseNoDefault;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case lBrace:
+        case tok!"{":
             node.blockStatement = parseBlockStatement();
             break;
-        case if_:
+        case tok!"if":
             node.ifStatement = parseIfStatement();
             break;
-        case while_:
+        case tok!"while":
             node.whileStatement = parseWhileStatement();
             break;
-        case do_:
+        case tok!"do":
             node.doStatement = parseDoStatement();
             break;
-        case for_:
+        case tok!"for":
             node.forStatement = parseForStatement();
             break;
-        case foreach_:
-        case foreach_reverse_:
+        case tok!"foreach":
+        case tok!"foreach_reverse":
             node.foreachStatement = parseForeachStatement();
             break;
-        case switch_:
+        case tok!"switch":
             node.switchStatement = parseSwitchStatement();
             break;
-        case continue_:
+        case tok!"continue":
             node.continueStatement = parseContinueStatement();
             break;
-        case break_:
+        case tok!"break":
             node.breakStatement = parseBreakStatement();
             break;
-        case return_:
+        case tok!"return":
             node.returnStatement = parseReturnStatement();
             break;
-        case goto_:
+        case tok!"goto":
             node.gotoStatement = parseGotoStatement();
             break;
-        case with_:
+        case tok!"with":
             node.withStatement = parseWithStatement();
             break;
-        case synchronized_:
+        case tok!"synchronized":
             node.synchronizedStatement = parseSynchronizedStatement();
             break;
-        case try_:
+        case tok!"try":
             node.tryStatement = parseTryStatement();
             break;
-        case throw_:
+        case tok!"throw":
             node.throwStatement = parseThrowStatement();
             break;
-        case scope_:
+        case tok!"scope":
             node.scopeGuardStatement = parseScopeGuardStatement();
             break;
-        case asm_:
+        case tok!"asm":
             node.asmStatement = parseAsmStatement();
             break;
-        case final_:
-            if (peekIs(switch_))
+        case tok!"final":
+            if (peekIs(tok!"switch"))
             {
                 node.finalSwitchStatement = parseFinalSwitchStatement();
                 break;
@@ -3634,33 +3629,33 @@ invariant() foo();
                 error(`"switch" expected`);
                 return null;
             }
-        case debug_:
-            if (peekIs(TokenType.assign))
+        case tok!"debug":
+            if (peekIs(tok!"="))
                 node.debugSpecification = parseDebugSpecification();
             else
                 node.conditionalStatement = parseConditionalStatement();
             break;
-        case version_:
-            if (peekIs(TokenType.assign))
+        case tok!"version":
+            if (peekIs(tok!"="))
                 node.versionSpecification = parseVersionSpecification();
             else
                 node.conditionalStatement = parseConditionalStatement();
             break;
-        case static_:
-            if (peekIs(TokenType.if_))
+        case tok!"static":
+            if (peekIs(tok!"if"))
                 node.conditionalStatement = parseConditionalStatement();
-            else if (peekIs(TokenType.assert_))
+            else if (peekIs(tok!"assert"))
                 node.staticAssertStatement = parseStaticAssertStatement();
             break;
-        case identifier:
-            if (peekIs(TokenType.colon))
+        case tok!"identifier":
+            if (peekIs(tok!":"))
             {
                 node.labeledStatement = parseLabeledStatement();
                 break;
             }
             goto default;
-        case delete_:
-        case assert_:
+        case tok!"delete":
+        case tok!"assert":
         default:
             node.expressionStatement = parseExpressionStatement();
             break;
@@ -3681,16 +3676,16 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new NonVoidInitializer;
-        if (currentIs(TokenType.lBrace))
+        if (currentIs(tok!"{"))
             node.structInitializer = parseStructInitializer();
-        else if (currentIs(TokenType.lBracket))
+        else if (currentIs(tok!"["))
         {
             auto b = peekPastBrackets();
-            if (b !is null && (b.type == TokenType.comma
-                || b.type == TokenType.rParen
-                || b.type == TokenType.rBracket
-                || b.type == TokenType.rBrace
-                || b.type == TokenType.semicolon))
+            if (b !is null && (b.type == tok!","
+                || b.type == tok!")"
+                || b.type == tok!"]"
+                || b.type == tok!"}"
+                || b.type == tok!";"))
             {
                 node.arrayInitializer = parseArrayInitializer();
             }
@@ -3727,7 +3722,7 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         return parseLeftAssocBinaryExpression!(OrExpression, XorExpression,
-            TokenType.bitOr)();
+            tok!"|")();
     }
 
     /**
@@ -3742,7 +3737,7 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         return parseLeftAssocBinaryExpression!(OrOrExpression, AndAndExpression,
-            TokenType.logicOr)();
+            tok!"||")();
     }
 
     /**
@@ -3755,14 +3750,14 @@ invariant() foo();
     OutStatement parseOutStatement()
     {
         auto node = new OutStatement;
-        expect(TokenType.out_);
-        if (currentIs(TokenType.lParen))
+        expect(tok!"out");
+        if (currentIs(tok!"("))
         {
             advance();
-            auto ident = expect(TokenType.identifier);
+            auto ident = expect(tok!"identifier");
             if (ident is null) return null;
             node.parameter = *ident;
-            expect(TokenType.rParen);
+            expect(tok!")");
         }
         node.blockStatement = parseBlockStatement();
         return node;
@@ -3781,34 +3776,34 @@ invariant() foo();
         auto node = new Parameter;
         while (moreTokens())
         {
-            TokenType type = parseParameterAttribute(false);
-            if (type == TokenType.invalid)
+            IdType type = parseParameterAttribute(false);
+            if (type == tok!"")
                 break;
             else
                 node.parameterAttributes ~= type;
         }
         node.type = parseType();
         if (node.type is null) return null;
-        if (currentIs(TokenType.identifier))
+        if (currentIs(tok!"identifier"))
         {
             node.name = advance();
-            if (currentIs(TokenType.vararg))
+            if (currentIs(tok!"..."))
             {
                 advance();
                 node.vararg = true;
             }
-            else if (currentIs(TokenType.assign))
+            else if (currentIs(tok!"="))
             {
                 advance();
                 node.default_ = parseAssignExpression();
             }
         }
-        else if (currentIs(TokenType.vararg))
+        else if (currentIs(tok!"..."))
         {
             node.vararg = true;
             advance();
         }
-        else if (currentIs(TokenType.assign))
+        else if (currentIs(tok!"="))
         {
             advance();
             node.default_ = parseAssignExpression();
@@ -3830,30 +3825,32 @@ invariant() foo();
      *     | $(LITERAL 'auto')
      *     ;)
      */
-    TokenType parseParameterAttribute(bool validate = false)
+    IdType parseParameterAttribute(bool validate = false)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case immutable_:
-        case shared_:
-        case const_:
-        case inout_:
-            if (peekIs(TokenType.lParen))
-                return invalid;
+        case tok!"immutable":
+        case tok!"shared":
+        case tok!"const":
+        case tok!"inout":
+            if (peekIs(tok!"("))
+                return tok!"";
             else
-                goto case auto_;
-        case final_:
-        case in_:
-        case lazy_:
-        case out_:
-        case ref_:
-        case scope_:
-        case auto_:
+                goto _auto;
+        case tok!"final":
+        case tok!"in":
+        case tok!"lazy":
+        case tok!"out":
+        case tok!"ref":
+        case tok!"scope":
+        _auto:
+        case tok!"auto":
             return advance().type;
         default:
-            if (validate) error("Parameter attribute expected");
-            return invalid;
+            if (validate)
+                error("Parameter attribute expected");
+            return tok!"";
         }
     }
 
@@ -3870,10 +3867,10 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Parameters;
-        if (expect(TokenType.lParen) is null) return null;
-        if (currentIs(TokenType.rParen))
+        if (expect(tok!"(") is null) return null;
+        if (currentIs(tok!")"))
             goto end;
-        if (currentIs(TokenType.vararg))
+        if (currentIs(tok!"..."))
         {
             advance();
             node.hasVarargs = true;
@@ -3881,25 +3878,26 @@ invariant() foo();
         }
         while (moreTokens())
         {
-            if (currentIs(TokenType.vararg))
+            if (currentIs(tok!"..."))
             {
                 advance();
                 node.hasVarargs = true;
                 break;
             }
-            if (currentIs(TokenType.rParen))
+            if (currentIs(tok!")"))
                 break;
             auto param = parseParameter();
             if (param is null)
                 return null;
             node.parameters ~= param;
-            if (currentIs(TokenType.comma))
+            if (currentIs(tok!","))
                 advance();
             else
                 break;
         }
     end:
-        if (expect(TokenType.rParen) is null) return null;
+        if (expect(tok!")") is null)
+            return null;
         return node;
     }
 
@@ -3915,7 +3913,7 @@ q{(int a, ...)
         Parameters params1 = p.parseParameters();
         assert (params1.hasVarargs);
         assert (params1.parameters.length == 1);
-        assert (params1.parameters[0].name == "a");
+        assert (params1.parameters[0].name.text == "a");
 
         Parameters params2 = p.parseParameters();
         assert (params2.parameters.length == 1);
@@ -3940,11 +3938,11 @@ q{(int a, ...)
     Postblit parsePostblit()
     {
         auto node = new Postblit;
-        expect(TokenType.this_);
-        expect(TokenType.lParen);
-        expect(TokenType.this_);
-        expect(TokenType.rParen);
-        if (currentIs(TokenType.semicolon))
+        expect(tok!"this");
+        expect(tok!"(");
+        expect(tok!"this");
+        expect(tok!")");
+        if (currentIs(tok!";"))
             advance();
         else
             node.functionBody = parseFunctionBody();
@@ -3977,7 +3975,7 @@ q{(int a, ...)
     ExpressionNode parsePowExpression()
     {
         return parseLeftAssocBinaryExpression!(PowExpression, UnaryExpression,
-            TokenType.pow)();
+            tok!"^^")();
     }
 
     /**
@@ -3991,7 +3989,7 @@ q{(int a, ...)
     {
         auto node = new PragmaDeclaration;
         node.pragmaExpression = parsePragmaExpression();
-        expect(TokenType.semicolon);
+        expect(tok!";");
         return node;
     }
 
@@ -4005,17 +4003,17 @@ q{(int a, ...)
     PragmaExpression parsePragmaExpression()
     {
         auto node = new PragmaExpression;
-        expect(TokenType.pragma_);
-        expect(TokenType.lParen);
-        auto ident = expect(TokenType.identifier);
+        expect(tok!"pragma");
+        expect(tok!"(");
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.identifier = *ident;
-        if (currentIs(TokenType.comma))
+        if (currentIs(tok!","))
         {
             advance();
             node.argumentList = parseArgumentList();
         }
-        expect(TokenType.rParen);
+        expect(tok!")");
         return node;
     }
 
@@ -4030,7 +4028,7 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new PreIncDecExpression;
-        if (currentIsOneOf(TokenType.increment, TokenType.decrement))
+        if (currentIsOneOf(tok!"++", tok!"--"))
             advance();
         else
         {
@@ -4086,72 +4084,73 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new PrimaryExpression;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case dot:
+        case tok!".":
             node.dot = advance();
             goto case;
-        case identifier:
-            if (peekIs(TokenType.goesTo))
+        case tok!"identifier":
+            if (peekIs(tok!"=>"))
                 node.lambdaExpression = parseLambdaExpression();
             else
                 node.identifierOrTemplateInstance = parseIdentifierOrTemplateInstance();
             break;
-        mixin (BASIC_TYPE_CASE_RANGE);
+        mixin (BASIC_TYPE_CASES);
             node.basicType = advance();
-            expect(dot);
-            auto t = expect(identifier);
+            expect(tok!".");
+            auto t = expect(tok!"identifier");
             if (t !is null)
                 node.primary = *t;
             break;
-        case function_:
-        case delegate_:
-            if (peekIs(lParen))
+        case tok!"function":
+        case tok!"delegate":
+            if (peekIs(tok!"("))
             {
                 auto b = setBookmark();
                 advance(); // function | delegate
                 skipParens();
-                if (currentIs(goesTo))
+                if (currentIs(tok!"=>"))
                 {
                     goToBookmark(b);
                     node.lambdaExpression = parseLambdaExpression();
+                    break;
                 }
                 else
                     goToBookmark(b);
             }
             goto case;
-        case lBrace:
-        case in_:
-        case out_:
-        case body_:
+        case tok!"{":
+        case tok!"in":
+        case tok!"out":
+        case tok!"body":
             node.functionLiteralExpression = parseFunctionLiteralExpression();
             break;
-        case typeof_:
+        case tok!"typeof":
             node.typeofExpression = parseTypeofExpression();
             break;
-        case typeid_:
+        case tok!"typeid":
             node.typeidExpression = parseTypeidExpression();
             break;
-        case vector:
+        case tok!"__vector":
             node.vector = parseVector();
             break;
-        case lBracket:
+        case tok!"[":
             if (isAssociativeArrayLiteral())
                 node.assocArrayLiteral = parseAssocArrayLiteral();
             else
                 node.arrayLiteral = parseArrayLiteral();
             break;
-        case lParen:
+        case tok!"(":
             auto b = setBookmark();
             skipParens();
             while (isAttribute())
                 parseAttribute();
-            if (currentIs(goesTo))
+            if (currentIs(tok!"=>"))
             {
                 goToBookmark(b);
                 node.lambdaExpression = parseLambdaExpression();
             }
-            else if (currentIs(lBrace))
+            else if (currentIs(tok!"{"))
             {
                 goToBookmark(b);
                 node.functionLiteralExpression = parseFunctionLiteralExpression();
@@ -4161,42 +4160,42 @@ q{(int a, ...)
                 goToBookmark(b);
                 advance();
                 node.expression = parseExpression();
-                expect(TokenType.rParen);
+                expect(tok!")");
             }
             break;
-        case is_:
+        case tok!"is":
             node.isExpression = parseIsExpression();
             break;
-        case traits:
+        case tok!"__traits":
             node.traitsExpression = parseTraitsExpression();
             break;
-        case mixin_:
+        case tok!"mixin":
             node.mixinExpression = parseMixinExpression();
             break;
-        case import_:
+        case tok!"import":
             node.importExpression = parseImportExpression();
             break;
-        case dollar:
-        case this_:
-        case super_:
-        case null_:
-        case true_:
-        case false_:
-        mixin (SPECIAL_CASE_RANGE);
-        mixin (LITERAL_CASE_RANGE);
-            if (currentIsOneOf(stringLiteral, wstringLiteral, dstringLiteral))
+        case tok!"$":
+        case tok!"this":
+        case tok!"super":
+        case tok!"null":
+        case tok!"true":
+        case tok!"false":
+        mixin (SPECIAL_CASES);
+        mixin (LITERAL_CASES);
+            if (currentIsOneOf(tok!"stringLiteral", tok!"wstringLiteral", tok!"dstringLiteral"))
             {
                 node.primary = advance();
                 bool alreadyWarned = false;
-                while (currentIsOneOf(stringLiteral, wstringLiteral,
-                    dstringLiteral))
+                while (currentIsOneOf(tok!"stringLiteral", tok!"wstringLiteral",
+                    tok!"dstringLiteral"))
                 {
                     if (!alreadyWarned)
                     {
                         warn("Implicit concatenation of string literals");
                         alreadyWarned = true;
                     }
-                    node.primary.value ~= advance().value;
+                    node.primary.text~= advance().text;
                 }
             }
             else
@@ -4221,16 +4220,16 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Register;
-        auto ident = expect(TokenType.identifier);
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.identifier = *ident;
-        if (currentIs(TokenType.lParen))
+        if (currentIs(tok!"("))
         {
             advance();
-            auto intLit = expect(TokenType.intLiteral);
+            auto intLit = expect(tok!"intLiteral");
             if (intLit is null) return null;
             node.intLiteral = *intLit;
-            expect(TokenType.rParen);
+            expect(tok!")");
         }
         return node;
     }
@@ -4260,12 +4259,9 @@ q{(int a, ...)
     ExpressionNode parseRelExpression(ExpressionNode shift = null)
     {
         return parseLeftAssocBinaryExpression!(RelExpression, ShiftExpression,
-            TokenType.less, TokenType.lessEqual, TokenType.greater,
-            TokenType.greaterEqual, TokenType.unordered,
-            TokenType.notLessEqualGreater, TokenType.lessOrGreater,
-            TokenType.lessEqualGreater, TokenType.notGreater,
-            TokenType.notGreaterEqual, TokenType.notLess,
-            TokenType.notLessEqual)(shift);
+            tok!"<", tok!"<=", tok!">", tok!">=", tok!"!<>=", tok!"!<>",
+            tok!"<>", tok!"<>=", tok!"!>", tok!"!>=", tok!"!>=", tok!"!<",
+            tok!"!<=")(shift);
     }
 
     /**
@@ -4279,10 +4275,10 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new ReturnStatement;
-        if (expect(TokenType.return_) is null) return null;
-        if (!currentIs(TokenType.semicolon))
+        if (expect(tok!"return") is null) return null;
+        if (!currentIs(tok!";"))
             node.expression = parseExpression();
-        if (expect(TokenType.semicolon) is null) return null;
+        if (expect(tok!";") is null) return null;
         return node;
     }
 
@@ -4297,12 +4293,12 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new ScopeGuardStatement;
-        expect(TokenType.scope_);
-        expect(TokenType.lParen);
-        auto ident = expect(TokenType.identifier);
+        expect(tok!"scope");
+        expect(tok!"(");
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.identifier = *ident;
-        expect(TokenType.rParen);
+        expect(tok!")");
         node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault();
         return node;
     }
@@ -4317,15 +4313,8 @@ q{(int a, ...)
     SharedStaticConstructor parseSharedStaticConstructor()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new SharedStaticConstructor;
-        if (expect(TokenType.shared_) is null) return null;
-        if (expect(TokenType.static_) is null) return null;
-        if (expect(TokenType.this_) is null) return null;
-        if (expect(TokenType.lParen) is null) return null;
-        if (expect(TokenType.rParen) is null) return null;
-        node.functionBody = parseFunctionBody();
-		if (node.functionBody is null) return null;
-        return node;
+        mixin (simpleParse!(SharedStaticConstructor, tok!"shared", tok!"static",
+            tok!"this", tok!"(", tok!")", "functionBody|parseFunctionBody"));
     }
 
     /**
@@ -4338,15 +4327,9 @@ q{(int a, ...)
     SharedStaticDestructor parseSharedStaticDestructor()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new SharedStaticDestructor;
-        expect(TokenType.shared_);
-        expect(TokenType.static_);
-        expect(TokenType.tilde);
-        expect(TokenType.this_);
-        expect(TokenType.lParen);
-        expect(TokenType.rParen);
-        node.functionBody = parseFunctionBody();
-        return node;
+        mixin (simpleParse!(SharedStaticDestructor, tok!"shared", tok!"static",
+            tok!"~", tok!"this", tok!"(", tok!")",
+            "functionBody|parseFunctionBody"));
     }
 
     /**
@@ -4361,8 +4344,7 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         return parseLeftAssocBinaryExpression!(ShiftExpression, AddExpression,
-            TokenType.shiftLeft, TokenType.shiftRight,
-            TokenType.unsignedShiftRight)();
+            tok!"<<", tok!">>", tok!">>>")();
     }
 
     /**
@@ -4376,14 +4358,14 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new SingleImport;
-        if (startsWith(TokenType.identifier, TokenType.assign))
+        if (startsWith(tok!"identifier", tok!"="))
         {
             node.rename = advance();
             advance(); // =
         }
         node.identifierChain = parseIdentifierChain();
-		if (node.identifierChain is null)
-			return null;
+        if (node.identifierChain is null)
+            return null;
         return node;
     }
 
@@ -4400,14 +4382,14 @@ q{(int a, ...)
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new SliceExpression;
         node.unaryExpression = unary is null ? parseUnaryExpression() : unary;
-        if (expect(TokenType.lBracket) is null) return null;
-        if (!currentIs(TokenType.rBracket))
+        if (expect(tok!"[") is null) return null;
+        if (!currentIs(tok!"]"))
         {
             node.lower = parseAssignExpression();
-            expect(TokenType.dotdot);
+            expect(tok!"..");
             node.upper = parseAssignExpression();
         }
-        if (expect(TokenType.rBracket) is null) return null;
+        if (expect(tok!"]") is null) return null;
         return node;
     }
 
@@ -4427,15 +4409,15 @@ q{(int a, ...)
         auto node = new Statement;
         switch (current.type)
         {
-        case TokenType.case_:
+        case tok!"case":
             advance();
             auto argumentList = parseArgumentList();
-            if (argumentList.items.length == 1 && startsWith(TokenType.colon, TokenType.dotdot))
+            if (argumentList.items.length == 1 && startsWith(tok!":", tok!".."))
                 node.caseRangeStatement = parseCaseRangeStatement(argumentList.items[0]);
             else
                 node.caseStatement = parseCaseStatement(argumentList);
             break;
-        case TokenType.default_:
+        case tok!"default":
             node.defaultStatement = parseDefaultStatement();
             break;
         default:
@@ -4455,10 +4437,8 @@ q{(int a, ...)
     StaticAssertDeclaration parseStaticAssertDeclaration()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new StaticAssertDeclaration;
-        node.staticAssertStatement = parseStaticAssertStatement();
-        if (node.staticAssertStatement is null) return null;
-        return node;
+        mixin (simpleParse!(StaticAssertDeclaration,
+            "staticAssertStatement|parseStaticAssertStatement"));
     }
 
 
@@ -4472,12 +4452,8 @@ q{(int a, ...)
     StaticAssertStatement parseStaticAssertStatement()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new StaticAssertStatement;
-        if (expect(TokenType.static_) is null) return null;
-        node.assertExpression = parseAssertExpression();
-        if (node.assertExpression is null) return null;
-        if (expect(TokenType.semicolon) is null) return null;
-        return node;
+        mixin (simpleParse!(StaticAssertStatement,
+            tok!"static", "assertExpression|parseAssertExpression", tok!";"));
     }
 
     /**
@@ -4490,13 +4466,8 @@ q{(int a, ...)
     StaticConstructor parseStaticConstructor()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new StaticConstructor;
-        expect(TokenType.static_);
-        expect(TokenType.this_);
-        expect(TokenType.lParen);
-        expect(TokenType.rParen);
-        node.functionBody = parseFunctionBody();
-        return node;
+        mixin (simpleParse!(StaticConstructor, tok!"static", tok!"this",
+            tok!"(", tok!")", "functionBody|parseFunctionBody"));
     }
 
     /**
@@ -4509,14 +4480,8 @@ q{(int a, ...)
     StaticDestructor parseStaticDestructor()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new StaticDestructor;
-        expect(TokenType.static_);
-        expect(TokenType.tilde);
-        expect(TokenType.this_);
-        expect(TokenType.lParen);
-        expect(TokenType.rParen);
-        node.functionBody = parseFunctionBody();
-        return node;
+        mixin (simpleParse!(StaticDestructor, tok!"static", tok!"~", tok!"this",
+            tok!"(", tok!")", "functionBody|parseFunctionBody"));
     }
 
     /**
@@ -4529,13 +4494,8 @@ q{(int a, ...)
     StaticIfCondition parseStaticIfCondition()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new StaticIfCondition;
-        expect(TokenType.static_);
-        expect(TokenType.if_);
-        expect(TokenType.lParen);
-        node.assignExpression = parseAssignExpression();
-        expect(TokenType.rParen);
-        return node;
+        mixin (simpleParse!(StaticIfCondition, tok!"static", tok!"if", tok!"(",
+            "assignExpression|parseAssignExpression", tok!")"));
     }
 
     /**
@@ -4563,32 +4523,32 @@ q{(int a, ...)
     StorageClass parseStorageClass()
     {
         auto node = new StorageClass;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case at:
+        case tok!"@":
             node.atAttribute = parseAtAttribute();
             if (node.atAttribute is null) return null;
             break;
-        case deprecated_:
+        case tok!"deprecated":
             node.deprecated_ = parseDeprecated();
             break;
-        case const_:
-        case immutable_:
-        case inout_:
-        case shared_:
-        case abstract_:
-        case auto_:
-        case enum_:
-        case extern_:
-        case final_:
-        case nothrow_:
-        case override_:
-        case pure_:
-        case ref_:
-        case gshared:
-        case scope_:
-        case static_:
-        case synchronized_:
+        case tok!"const":
+        case tok!"immutable":
+        case tok!"inout":
+        case tok!"shared":
+        case tok!"abstract":
+        case tok!"auto":
+        case tok!"enum":
+        case tok!"extern":
+        case tok!"final":
+        case tok!"nothrow":
+        case tok!"override":
+        case tok!"pure":
+        case tok!"ref":
+        case tok!"__gshared":
+        case tok!"scope":
+        case tok!"static":
+        case tok!"synchronized":
             node.token = advance();
             break;
         default:
@@ -4609,16 +4569,16 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new StructBody;
-        auto start = expect(TokenType.lBrace);
-        if (start !is null) node.startLocation = start.startIndex;
-        while (!currentIs(TokenType.rBrace) && moreTokens())
+        auto start = expect(tok!"{");
+        if (start !is null) node.startLocation = start.index;
+        while (!currentIs(tok!"}") && moreTokens())
         {
             auto dec = parseDeclaration();
             if (dec !is null)
                 node.declarations ~= dec;
         }
-        auto end = expect(TokenType.rBrace);
-        if (end !is null) node.endLocation = end.startIndex;
+        auto end = expect(tok!"}");
+        if (end !is null) node.endLocation = end.index;
         return node;
     }
 
@@ -4633,24 +4593,26 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new StructDeclaration;
-        expect(TokenType.struct_);
-        if (currentIs(TokenType.identifier))
+        expect(tok!"struct");
+        if (currentIs(tok!"identifier"))
         {
             node.name = advance();
         }
+        node.comment = comment;
+        comment = null;
 
-        if (currentIs(TokenType.lParen))
+        if (currentIs(tok!"("))
         {
             node.templateParameters = parseTemplateParameters();
-            if (tokens[index] == TokenType.if_)
+            if (tokens[index] == tok!"if")
                 node.constraint = parseConstraint();
             node.structBody = parseStructBody();
         }
-        else if (currentIs(TokenType.lBrace))
+        else if (currentIs(tok!"{"))
         {
             node.structBody = parseStructBody();
         }
-        else if (currentIs(TokenType.semicolon))
+        else if (currentIs(tok!";"))
             advance();
         else
         {
@@ -4671,9 +4633,9 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new StructInitializer;
-        expect(TokenType.lBrace);
+        expect(tok!"{");
         node.structMemberInitializers = parseStructMemberInitializers();
-        expect(TokenType.rBrace);
+        expect(tok!"}");
         return node;
     }
 
@@ -4688,7 +4650,7 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new StructMemberInitializer;
-        if (startsWith(TokenType.identifier, TokenType.colon))
+        if (startsWith(tok!"identifier", tok!":"))
         {
             node.identifier = tokens[index++];
             index++;
@@ -4711,10 +4673,10 @@ q{(int a, ...)
         do
         {
             auto structMemberInitializer = parseStructMemberInitializer();
-            if (currentIs(TokenType.comma))
+            if (currentIs(tok!","))
             {
                 advance();
-                if (!currentIs(TokenType.rBrace))
+                if (!currentIs(tok!"}"))
                     continue;
                 else
                     break;
@@ -4736,10 +4698,10 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new SwitchStatement;
-        expect(TokenType.switch_);
-        expect(TokenType.lParen);
+        expect(tok!"switch");
+        expect(tok!"(");
         node.expression = parseExpression();
-        expect(TokenType.rParen);
+        expect(tok!")");
         node.statement = parseStatement();
         return node;
     }
@@ -4755,7 +4717,7 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Symbol;
-        if (currentIs(TokenType.dot))
+        if (currentIs(tok!"."))
 		{
 			node.dot = true;
 			advance();
@@ -4775,12 +4737,12 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new SynchronizedStatement;
-        expect(TokenType.synchronized_);
-        if (currentIs(TokenType.lParen))
+        expect(tok!"synchronized");
+        if (currentIs(tok!"("))
         {
-            expect(TokenType.lParen);
+            expect(tok!"(");
             node.expression = parseExpression();
-            expect(TokenType.rParen);
+            expect(tok!")");
         }
         node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault();
         return node;
@@ -4797,11 +4759,11 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateAliasParameter;
-        expect(TokenType.alias_);
-        if (currentIs(TokenType.identifier))
+        expect(tok!"alias");
+        if (currentIs(tok!"identifier"))
         {
-            if (peekIsOneOf(TokenType.comma, TokenType.rParen, TokenType.assign,
-                TokenType.colon))
+            if (peekIsOneOf(tok!",", tok!")", tok!"=",
+                tok!":"))
             {
                 node.identifier = advance();
             }
@@ -4809,12 +4771,12 @@ q{(int a, ...)
         else
         {
             if ((node.type = parseType()) is null) return null;
-            auto ident = expect(TokenType.identifier);
+            auto ident = expect(tok!"identifier");
             if (ident is null) return null;
             node.identifier = *ident;
         }
 
-        if (currentIs(TokenType.colon))
+        if (currentIs(tok!":"))
         {
             advance();
             if (isType())
@@ -4822,7 +4784,7 @@ q{(int a, ...)
             else
                 node.colonExpression = parseAssignExpression();
         }
-        if (currentIs(TokenType.assign))
+        if (currentIs(tok!"="))
         {
             advance();
             if (isType())
@@ -4847,7 +4809,7 @@ q{(int a, ...)
         auto node = new TemplateArgument;
         auto b = setBookmark();
         auto t = parseType();
-        if (t !is null && currentIsOneOf(TokenType.comma, TokenType.rParen))
+        if (t !is null && currentIsOneOf(tok!",", tok!")"))
         {
             goToBookmark(b);
             node.type = parseType();
@@ -4884,13 +4846,13 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateArguments;
-        expect(TokenType.not);
-        if (currentIs(TokenType.lParen))
+        expect(tok!"!");
+        if (currentIs(tok!"("))
         {
             advance();
-            if (!currentIs(TokenType.rParen))
+            if (!currentIs(tok!")"))
                 node.templateArgumentList = parseTemplateArgumentList();
-            expect(TokenType.rParen);
+            expect(tok!")");
         }
         else
             node.templateSingleArgument = parseTemplateSingleArgument();
@@ -4909,26 +4871,28 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateDeclaration;
-        if (currentIs(TokenType.enum_))
+        node.comment = comment;
+        comment = null;
+        if (currentIs(tok!"enum"))
         {
             node.eponymousTemplateDeclaration = parseEponymousTemplateDeclaration();
             return node;
         }
-        expect(TokenType.template_);
-        auto ident = expect(TokenType.identifier);
+        expect(tok!"template");
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.name = *ident;
         node.templateParameters = parseTemplateParameters();
-        if (currentIs(TokenType.if_))
+        if (currentIs(tok!"if"))
             node.constraint = parseConstraint();
-        if (expect(TokenType.lBrace) is null) return null;
-        while (moreTokens() && !currentIs(TokenType.rBrace))
+        if (expect(tok!"{") is null) return null;
+        while (moreTokens() && !currentIs(tok!"}"))
         {
             auto decl = parseDeclaration();
             if (decl !is null)
                 node.declarations ~= decl;
         }
-        expect(TokenType.rBrace);
+        expect(tok!"}");
         return node;
     }
 
@@ -4943,14 +4907,14 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new EponymousTemplateDeclaration;
-        expect(TokenType.enum_);
-        auto ident = expect(TokenType.identifier);
+        expect(tok!"enum");
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.name = *ident;
         node.templateParameters = parseTemplateParameters();
-        expect(TokenType.assign);
+        expect(tok!"=");
         node.assignExpression = parseAssignExpression();
-        expect(TokenType.semicolon);
+        expect(tok!";");
         return node;
     }
 
@@ -4965,7 +4929,7 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateInstance;
-        auto ident = expect(TokenType.identifier);
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.identifier = *ident;
         node.templateArguments = parseTemplateArguments();
@@ -4985,11 +4949,11 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateMixinExpression;
-        if (expect(TokenType.mixin_) is null) return null;
+        if (expect(tok!"mixin") is null) return null;
         node.mixinTemplateName = parseMixinTemplateName();
-        if (currentIs(TokenType.not))
+        if (currentIs(tok!"!"))
             node.templateArguments = parseTemplateArguments();
-        if (currentIs(TokenType.identifier))
+        if (currentIs(tok!"identifier"))
             node.identifier = advance();
         return node;
     }
@@ -5009,20 +4973,20 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateParameter;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case alias_:
+        case tok!"alias":
             node.templateAliasParameter = parseTemplateAliasParameter();
             break;
-        case identifier:
-            if(peekIs(vararg))
+        case tok!"identifier":
+            if(peekIs(tok!"..."))
                 node.templateTupleParameter = parseTemplateTupleParameter();
-            else if (peekIsOneOf(colon, assign, comma, rParen))
+            else if (peekIsOneOf(tok!":", tok!"=", tok!",", tok!")"))
                 node.templateTypeParameter = parseTemplateTypeParameter();
             else
                 node.templateValueParameter = parseTemplateValueParameter();
             break;
-        case this_:
+        case tok!"this":
             node.templateThisParameter = parseTemplateThisParameter();
             break;
         default:
@@ -5056,10 +5020,10 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateParameters;
-        if (expect(TokenType.lParen) is null) return null;
-        if (!currentIs(TokenType.rParen))
+        if (expect(tok!"(") is null) return null;
+        if (!currentIs(tok!")"))
             node.templateParameterList = parseTemplateParameterList();
-        if (expect(TokenType.rParen) is null) return null;
+        if (expect(tok!")") is null) return null;
         return node;
     }
 
@@ -5093,16 +5057,16 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateSingleArgument;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case true_:
-        case false_:
-        case null_:
-        case this_:
-        case identifier:
-        mixin (SPECIAL_CASE_RANGE);
-        mixin (LITERAL_CASE_RANGE);
-        mixin (BASIC_TYPE_CASE_RANGE);
+        case tok!"true":
+        case tok!"false":
+        case tok!"null":
+        case tok!"this":
+        case tok!"identifier":
+        mixin (SPECIAL_CASES);
+        mixin (LITERAL_CASES);
+        mixin (BASIC_TYPE_CASES);
             node.token = advance();
             break;
         default:
@@ -5123,7 +5087,7 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateThisParameter;
-        expect(TokenType.this_);
+        expect(tok!"this");
         node.templateTypeParameter = parseTemplateTypeParameter();
         return node;
     }
@@ -5139,11 +5103,11 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateTupleParameter;
-        auto i = expect(TokenType.identifier);
+        auto i = expect(tok!"identifier");
         if (i is null)
             return null;
         node.identifier = *i;
-        if (expect(TokenType.vararg) is null) return null;
+        if (expect(tok!"...") is null) return null;
         return node;
     }
 
@@ -5158,15 +5122,15 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateTypeParameter;
-        auto ident = expect(TokenType.identifier);
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.identifier = *ident;
-        if (currentIs(TokenType.colon))
+        if (currentIs(tok!":"))
         {
             advance();
             node.colonType = parseType();
         }
-        if (currentIs(TokenType.assign))
+        if (currentIs(tok!"="))
         {
             advance();
             node.assignType = parseType();
@@ -5186,15 +5150,15 @@ q{(int a, ...)
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateValueParameter;
         if ((node.type = parseType()) is null) return null;
-        auto ident = expect(TokenType.identifier);
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.identifier = *ident;
-        if (currentIs(TokenType.colon))
+        if (currentIs(tok!":"))
         {
             advance();
             if ((node.expression = parseExpression()) is null) return null;
         }
-        if (currentIs(TokenType.assign))
+        if (currentIs(tok!"="))
         {
             if ((node.templateValueParameterDefault = parseTemplateValueParameterDefault()) is null)
                 return null;
@@ -5213,14 +5177,14 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TemplateValueParameterDefault;
-        expect(TokenType.assign);
-        with (TokenType) switch (current.type)
+        expect(tok!"=");
+        switch (current.type)
         {
-        case specialFile:
-        case specialModule:
-        case specialLine:
-        case specialFunction:
-        case specialPrettyFunction:
+        case tok!"__FILE__":
+        case tok!"__MODULE__":
+        case tok!"__LINE__":
+        case tok!"__FUNCTION__":
+        case tok!"__PRETTY_FUNCTION__":
             node.token = advance();
             break;
         default:
@@ -5242,11 +5206,11 @@ q{(int a, ...)
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TernaryExpression;
         node.orOrExpression = parseOrOrExpression();
-        if (currentIs(TokenType.ternary))
+        if (currentIs(tok!"?"))
         {
             advance();
             node.expression = parseExpression();
-            expect(TokenType.colon);
+            if (expect(tok!":") is null) return null;
             node.ternaryExpression = parseTernaryExpression();
         }
         return node;
@@ -5263,9 +5227,9 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new ThrowStatement;
-        expect(TokenType.throw_);
+        expect(tok!"throw");
         node.expression = parseExpression();
-        expect(TokenType.semicolon);
+        expect(tok!";");
         return node;
     }
 
@@ -5280,17 +5244,17 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TraitsExpression;
-        if (expect(TokenType.traits) is null) return null;
-        if (expect(TokenType.lParen) is null) return null;
-        auto ident = expect(TokenType.identifier);
+        if (expect(tok!"__traits") is null) return null;
+        if (expect(tok!"(") is null) return null;
+        auto ident = expect(tok!"identifier");
         if (ident is null) return null;
         node.identifier = *ident;
-        if (currentIs(TokenType.comma))
+        if (currentIs(tok!","))
         {
             advance();
             if ((node.templateArgumentList = parseTemplateArgumentList()) is null) return null;
         }
-        if (expect(TokenType.rParen) is null) return null;
+        if (expect(tok!")") is null) return null;
         return node;
     }
 
@@ -5305,11 +5269,11 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TryStatement;
-        expect(TokenType.try_);
+        expect(tok!"try");
         node.declarationOrStatement = parseDeclarationOrStatement();
-        if (currentIs(TokenType.catch_))
+        if (currentIs(tok!"catch"))
             node.catches = parseCatches();
-        if (currentIs(TokenType.finally_))
+        if (currentIs(tok!"finally"))
             node.finally_ = parseFinally();
         return node;
     }
@@ -5327,14 +5291,14 @@ q{(int a, ...)
         auto node = new Type;
         switch(current.type)
         {
-        case TokenType.const_:
-        case TokenType.immutable_:
-        case TokenType.inout_:
-        case TokenType.shared_:
-        case TokenType.scope_:
-        case TokenType.pure_:
-        case TokenType.nothrow_:
-            if (!peekIs(TokenType.lParen))
+        case tok!"const":
+        case tok!"immutable":
+        case tok!"inout":
+        case tok!"shared":
+        case tok!"scope":
+        case tok!"pure":
+        case tok!"nothrow":
+            if (!peekIs(tok!"("))
                 node.typeConstructors = parseTypeConstructors();
             break;
         default:
@@ -5343,12 +5307,12 @@ q{(int a, ...)
         node.type2 = parseType2();
         if (node.type2 is null)
             return null;
-        loop: while (moreTokens()) with (TokenType) switch (current.type)
+        loop: while (moreTokens()) switch (current.type)
         {
-        case TokenType.star:
-        case TokenType.lBracket:
-        case TokenType.delegate_:
-        case TokenType.function_:
+        case tok!"*":
+        case tok!"[":
+        case tok!"delegate":
+        case tok!"function":
             auto suffix = parseTypeSuffix();
             if (suffix !is null)
                 node.typeSuffixes ~= suffix;
@@ -5375,40 +5339,40 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new Type2;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-            case identifier:
-            case dot:
-                if ((node.symbol = parseSymbol()) is null)
-                    return null;
-                break;
-            case bool_: .. case wchar_:
-                if ((node.builtinType = parseBasicType()) == TokenType.invalid)
-                    return null;
-                break;
-            case typeof_:
-                if ((node.typeofExpression = parseTypeofExpression()) is null)
-                    return null;
-                if (currentIs(TokenType.dot))
-                {
-                    advance();
-                    node.identifierOrTemplateChain = parseIdentifierOrTemplateChain();
-                    if (node.identifierOrTemplateChain is null)
-                        return null;
-                }
-                break;
-            case const_:
-            case immutable_:
-            case inout_:
-            case shared_:
-                node.typeConstructor = advance().type;
-                if (expect(TokenType.lParen) is null) return null;
-                if ((node.type = parseType()) is null) return null;
-                if (expect(TokenType.rParen) is null) return null;
-                break;
-            default:
-                error("Basic type, type constructor, symbol, or typeof expected");
+        case tok!"identifier":
+        case tok!".":
+            if ((node.symbol = parseSymbol()) is null)
                 return null;
+            break;
+        mixin (BASIC_TYPE_CASES);
+            if ((node.builtinType = parseBasicType()) == tok!"")
+                return null;
+            break;
+        case tok!"typeof":
+            if ((node.typeofExpression = parseTypeofExpression()) is null)
+                return null;
+            if (currentIs(tok!"."))
+            {
+                advance();
+                node.identifierOrTemplateChain = parseIdentifierOrTemplateChain();
+                if (node.identifierOrTemplateChain is null)
+                    return null;
+            }
+            break;
+        case tok!"const":
+        case tok!"immutable":
+        case tok!"inout":
+        case tok!"shared":
+            node.typeConstructor = advance().type;
+            if (expect(tok!"(") is null) return null;
+            if ((node.type = parseType()) is null) return null;
+            if (expect(tok!")") is null) return null;
+            break;
+        default:
+            error("Basic type, type constructor, symbol, or typeof expected");
+            return null;
         }
         return node;
     }
@@ -5424,26 +5388,26 @@ q{(int a, ...)
      *     | $(LITERAL 'scope')
      *     ;)
      */
-    TokenType parseTypeConstructor(bool validate = true)
+    IdType parseTypeConstructor(bool validate = true)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case const_:
-        case immutable_:
-        case inout_:
-        case shared_:
-        case scope_:
-        case ref_:
-        case pure_:
-        case nothrow_:
-            if (!peekIs(lParen))
+        case tok!"const":
+        case tok!"immutable":
+        case tok!"inout":
+        case tok!"shared":
+        case tok!"scope":
+        case tok!"ref":
+        case tok!"pure":
+        case tok!"nothrow":
+            if (!peekIs(tok!"("))
                 return advance().type;
             goto default;
         default:
             if (validate)
                 error(`"const", "immutable", "inout", "shared", or "scope" expected`);
-            return invalid;
+            return tok!"";
         }
     }
 
@@ -5454,14 +5418,14 @@ q{(int a, ...)
      *     $(RULE typeConstructor)+
      *     ;)
      */
-    TokenType[] parseTypeConstructors()
+    IdType[] parseTypeConstructors()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        TokenType[] r;
+        IdType[] r;
         while (moreTokens())
         {
-            TokenType type = parseTypeConstructor(false);
-            if (type == TokenType.invalid)
+            IdType type = parseTypeConstructor(false);
+            if (type == tok!"")
                 break;
             else
                 r ~= type;
@@ -5495,24 +5459,24 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TypeSpecialization;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case struct_:
-        case union_:
-        case class_:
-        case interface_:
-        case enum_:
-        case function_:
-        case delegate_:
-        case super_:
-        case return_:
-        case typedef_:
-        case parameters:
-        case const_:
-        case immutable_:
-        case inout_:
-        case shared_:
-            if (peekIsOneOf(rParen, comma))
+        case tok!"struct":
+        case tok!"union":
+        case tok!"class":
+        case tok!"interface":
+        case tok!"enum":
+        case tok!"function":
+        case tok!"delegate":
+        case tok!"super":
+        case tok!"return":
+        case tok!"typedef":
+        case tok!"__parameters":
+        case tok!"const":
+        case tok!"immutable":
+        case tok!"inout":
+        case tok!"shared":
+            if (peekIsOneOf(tok!")", tok!","))
                 node.token = advance();
             else
                 goto default;
@@ -5539,23 +5503,23 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TypeSuffix;
-        with (TokenType) switch(current.type)
+        switch(current.type)
         {
-        case star:
+        case tok!"*":
             node.star = true;
             advance();
             return node;
-        case lBracket:
+        case tok!"[":
             node.array = true;
             advance();
-            if (currentIs(rBracket))
+            if (currentIs(tok!"]"))
             {
                 advance();
                 return node;
             }
             auto bookmark = setBookmark();
             auto type = parseType();
-            if (type !is null && currentIs(rBracket))
+            if (type !is null && currentIs(tok!"]"))
             {
                 goToBookmark(bookmark);
                 type = parseType();
@@ -5566,17 +5530,17 @@ q{(int a, ...)
                 goToBookmark(bookmark);
                 node.low = parseAssignExpression();
                 if (node.low is null) return null;
-                if (currentIs(dotdot))
+                if (currentIs(tok!".."))
                 {
                     advance();
                     node.high = parseAssignExpression();
                     if (node.high is null) return null;
                 }
             }
-            if (expect(TokenType.rBracket) is null) return null;
+            if (expect(tok!"]") is null) return null;
             return node;
-        case delegate_:
-        case function_:
+        case tok!"delegate":
+        case tok!"function":
             node.delegateOrFunction = advance();
             node.parameters = parseParameters();
             while (currentIsMemberFunctionAttribute())
@@ -5599,11 +5563,11 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TypeidExpression;
-        expect(TokenType.typeid_);
-        expect(TokenType.lParen);
+        expect(tok!"typeid");
+        expect(tok!"(");
         auto b = setBookmark();
         auto t = parseType();
-        if (t is null || !currentIs(TokenType.rParen))
+        if (t is null || !currentIs(tok!")"))
         {
             goToBookmark(b);
             node.expression = parseExpression();
@@ -5615,7 +5579,7 @@ q{(int a, ...)
             node.type = parseType();
             if (node.type is null) return null;
         }
-        expect(TokenType.rParen);
+        expect(tok!")");
         return node;
     }
 
@@ -5630,13 +5594,13 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new TypeofExpression;
-        expect(TokenType.typeof_);
-        expect(TokenType.lParen);
-        if (currentIs(TokenType.return_))
+        expect(tok!"typeof");
+        expect(tok!"(");
+        if (currentIs(tok!"return"))
             node.return_ = advance();
         else
             node.expression = parseExpression();
-        expect(TokenType.rParen);
+        expect(tok!")");
         return node;
     }
 
@@ -5670,42 +5634,42 @@ q{(int a, ...)
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new UnaryExpression;
-        with(TokenType) switch (current.type)
+        switch (current.type)
         {
-        case amp:
-        case not:
-        case star:
-        case plus:
-        case minus:
-        case tilde:
-        case increment:
-        case decrement:
+        case tok!"&":
+        case tok!"!":
+        case tok!"*":
+        case tok!"+":
+        case tok!"-":
+        case tok!"~":
+        case tok!"++":
+        case tok!"--":
             node.prefix = advance();
             node.unaryExpression = parseUnaryExpression();
             break;
-        case new_:
+        case tok!"new":
             node.newExpression = parseNewExpression();
             break;
-        case delete_:
+        case tok!"delete":
             node.deleteExpression = parseDeleteExpression();
             break;
-        case cast_:
+        case tok!"cast":
             node.castExpression = parseCastExpression();
             break;
-        case assert_:
+        case tok!"assert":
             node.assertExpression = parseAssertExpression();
             break;
-        case lParen:
+        case tok!"(":
             // handle (type).identifier
             auto b = setBookmark();
             skipParens();
-            if (startsWith(dot, identifier))
+            if (startsWith(tok!".", tok!"identifier"))
             {
                 // go back to the (
                 index = b;
                 advance();
                 auto t = parseType();
-                if (t is null || !currentIs(rParen))
+                if (t is null || !currentIs(tok!")"))
                 {
                     goToBookmark(b);
                     goto default;
@@ -5727,32 +5691,33 @@ q{(int a, ...)
             break;
         }
 
-        loop: while (moreTokens()) with (TokenType) switch (current.type)
+        loop: while (moreTokens()) switch (current.type)
         {
-        case not:
-            if (peekIs(is_))
+        case tok!"!":
+            if (peekIs(tok!"is"))
                 break loop;
             index++;
-            bool jump =  (currentIs(TokenType.lParen) && peekPastParens().type == TokenType.lParen)
-                || peekIs(TokenType.lParen);
+            bool jump =  (currentIs(tok!"(") && peekPastParens().type == tok!"(")
+                || peekIs(tok!"(");
             index--;
             if (jump)
-                goto case lParen;
+                goto case tok!"(";
             else
                 break loop;
-        case lParen:
-            auto n = new UnaryExpression();
-            n.functionCallExpression = parseFunctionCallExpression(node);
-            node = n;
+
+        case tok!"(":
+            auto newUnary = new UnaryExpression();
+            newUnary.functionCallExpression = parseFunctionCallExpression(node);
+            node = newUnary;
             break;
-        case increment:
-        case decrement:
+        case tok!"++":
+        case tok!"--":
             auto n = new UnaryExpression();
             n.unaryExpression = node;
             n.suffix = advance();
             node = n;
             break;
-        case lBracket:
+        case tok!"[":
             auto n = new UnaryExpression;
             if (isSliceExpression())
                 n.sliceExpression = parseSliceExpression(node);
@@ -5760,7 +5725,7 @@ q{(int a, ...)
                 n.indexExpression = parseIndexExpression(node);
             node = n;
             break;
-        case dot:
+        case tok!".":
             advance();
             auto n = new UnaryExpression();
             n.unaryExpression = node;
@@ -5797,17 +5762,17 @@ q{doStuff(5)}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new UnionDeclaration;
-		// grab line number even if it's anonymous
-		auto l = expect(TokenType.union_).line;
+        // grab line number even if it's anonymous
+        auto l = expect(tok!"union").line;
         bool templated = false;
-        if (currentIs(TokenType.identifier))
+        if (currentIs(tok!"identifier"))
         {
             node.name = advance();
-            if (currentIs(TokenType.lParen))
+            if (currentIs(tok!"("))
             {
                 templated = true;
                 node.templateParameters = parseTemplateParameters();
-                if (currentIs(TokenType.if_))
+                if (currentIs(tok!"if"))
                     node.constraint = parseConstraint();
                 node.structBody = parseStructBody();
             }
@@ -5816,9 +5781,9 @@ q{doStuff(5)}c;
         }
         else
         {
-			node.name.line = l;
+            node.name.line = l;
     semiOrStructBody:
-            if (currentIs(TokenType.semicolon))
+            if (currentIs(tok!";"))
                 advance();
             else
                 node.structBody = parseStructBody();
@@ -5836,10 +5801,7 @@ q{doStuff(5)}c;
     Unittest parseUnittest()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new Unittest;
-        expect(TokenType.unittest_);
-        node.blockStatement = parseBlockStatement();
-        return node;
+        mixin (simpleParse!(Unittest, tok!"unittest", "blockStatement|parseBlockStatement", true));
     }
 
     /**
@@ -5867,12 +5829,12 @@ q{doStuff(5)}c;
             auto declarator = parseDeclarator();
             if (declarator is null) return null;
             node.declarators ~= declarator;
-            if (moreTokens() && currentIs(TokenType.comma))
+            if (moreTokens() && currentIs(tok!","))
                 advance();
             else
                 break;
         }
-        expect(TokenType.semicolon);
+        expect(tok!";");
         return node;
     }
 
@@ -5886,12 +5848,7 @@ q{doStuff(5)}c;
     Vector parseVector()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new Vector;
-        if (expect(TokenType.vector) is null) return null;
-        if (expect(TokenType.lParen) is null) return null;
-        if ((node.type = parseType()) is null) return null;
-        if (expect(TokenType.rParen) is null) return null;
-        return node;
+        mixin (simpleParse!(Vector, tok!"__vector", tok!"(", "type|parseType", tok!")"));
     }
 
     /**
@@ -5905,10 +5862,9 @@ q{doStuff(5)}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new VersionCondition;
-        if (expect(TokenType.version_) is null) return null;
-        if (expect(TokenType.lParen) is null) return null;
-        if (currentIsOneOf(TokenType.intLiteral, TokenType.identifier,
-            TokenType.unittest_, TokenType.assert_))
+        mixin (expectSequence!(tok!"version", tok!"("));
+        if (currentIsOneOf(tok!"intLiteral", tok!"identifier",
+            tok!"unittest", tok!"assert"))
         {
             node.token = advance();
         }
@@ -5917,7 +5873,7 @@ q{doStuff(5)}c;
             error(`Expected an integer literal, an identifier, "assert", or "unittest"`);
             return null;
         }
-        expect(TokenType.rParen);
+        expect(tok!")");
         return node;
     }
 
@@ -5932,15 +5888,14 @@ q{doStuff(5)}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new VersionSpecification;
-        expect(TokenType.version_);
-        expect(TokenType.assign);
-        if (!currentIsOneOf(TokenType.identifier, TokenType.intLiteral))
+        mixin (expectSequence!(tok!"version", tok!"="));
+        if (!currentIsOneOf(tok!"identifier", tok!"intLiteral"))
         {
             error("Identifier or integer literal expected");
             return null;
         }
         node.token = advance();
-        expect(TokenType.semicolon);
+        expect(tok!";");
         return node;
     }
 
@@ -5955,12 +5910,12 @@ q{doStuff(5)}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new WhileStatement;
-        expect(TokenType.while_);
-        node.startIndex = current().startIndex;
-        expect(TokenType.lParen);
+        expect(tok!"while");
+        node.startIndex = current().index;
+        expect(tok!"(");
         node.expression = parseExpression();
-        expect(TokenType.rParen);
-        if (currentIs(TokenType.rBrace))
+        expect(tok!")");
+        if (currentIs(tok!"}"))
         {
             error("Statement expected", false);
             return node; // this line makes DCD better
@@ -5979,14 +5934,9 @@ q{doStuff(5)}c;
     WithStatement parseWithStatement()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        auto node = new WithStatement;
-        if (expect(TokenType.with_) is null) return null;
-        if (expect(TokenType.lParen) is null) return null;
-        if ((node.expression = parseExpression()) is null) return null;
-        if (expect(TokenType.rParen) is null) return null;
-        node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault();
-        if (node.statementNoCaseNoDefault is null) return null;
-        return node;
+        mixin (simpleParse!(WithStatement, tok!"with", tok!"(",
+            "expression|parseExpression", tok!")",
+            "statementNoCaseNoDefault|parseStatementNoCaseNoDefault"));
     }
 
     /**
@@ -6001,7 +5951,7 @@ q{doStuff(5)}c;
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         return parseLeftAssocBinaryExpression!(XorExpression, AndExpression,
-            TokenType.xor)();
+            tok!"^")();
     }
 
     /**
@@ -6024,14 +5974,14 @@ q{doStuff(5)}c;
      * The parameters are the file name, line number, column number,
      * and the error or warning message.
      */
-    void function(string, int, int, string) messageFunction;
+    void function(string, size_t, size_t, string) messageFunction;
 
     bool isSliceExpression()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
-        if (startsWith(TokenType.lBracket, TokenType.rBracket))
+        if (startsWith(tok!"[", tok!"]"))
             return true;
-        return hasMagicDelimiter!(TokenType.dotdot)();
+        return hasMagicDelimiter!(tok!"..")();
     }
 
     void setTokens(const(Token)[] tokens)
@@ -6045,14 +5995,14 @@ protected:
     {
         switch (current.type)
         {
-        case TokenType.const_:
-            return peekIsOneOf(TokenType.shared_, TokenType.rParen);
-        case TokenType.immutable_:
-            return peekIs(TokenType.rParen);
-        case TokenType.inout_:
-            return peekIsOneOf(TokenType.shared_, TokenType.rParen);
-        case TokenType.shared_:
-            return peekIsOneOf(TokenType.const_, TokenType.inout_, TokenType.rParen);
+        case tok!"const":
+            return peekIsOneOf(tok!"shared", tok!")");
+        case tok!"immutable":
+            return peekIs(tok!")");
+        case tok!"inout":
+            return peekIsOneOf(tok!"shared", tok!")");
+        case tok!"shared":
+            return peekIsOneOf(tok!"const", tok!"inout", tok!")");
         default:
             return false;
         }
@@ -6060,24 +6010,22 @@ protected:
 
     bool isAssociativeArrayLiteral()
     {
-        return hasMagicDelimiter!(TokenType.colon)();
+        return hasMagicDelimiter!(tok!":")();
     }
 
-
-
     bool hasMagicDelimiter(alias T)()
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto i = index;
         scope(exit) index = i;
-        assert(currentIs(TokenType.lBracket));
+        assert(currentIs(tok!"["));
         advance();
-        while (moreTokens()) with (TokenType) switch (current.type)
+        while (moreTokens()) switch (current.type)
         {
-        case lBrace: skipBraces(); break;
-        case lParen: skipParens(); break;
-        case lBracket: skipBrackets(); break;
-        case rBracket: return false;
+        case tok!"{": skipBraces(); break;
+        case tok!"(": skipParens(); break;
+        case tok!"[": skipBrackets(); break;
+        case tok!"]": return false;
         case T: return true;
         default: advance(); break;
         }
@@ -6088,15 +6036,15 @@ protected:
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         if (!moreTokens()) return false;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case final_:
-            return !peekIs(switch_);
-        case debug_:
-        case version_:
-            return peekIs(assign);
-        case synchronized_:
-            if (peekIs(lParen))
+        case tok!"final":
+            return !peekIs(tok!"switch");
+        case tok!"debug":
+        case tok!"version":
+            return peekIs(tok!"=");
+        case tok!"synchronized":
+            if (peekIs(tok!"("))
                 return false;
             else
             {
@@ -6105,61 +6053,61 @@ protected:
                 advance();
                 return isDeclaration();
             }
-        case static_:
-            if (peekIs(if_))
+        case tok!"static":
+            if (peekIs(tok!"if"))
                 return false;
             goto case;
-        case scope_:
-            if (peekIs(lParen))
+        case tok!"scope":
+            if (peekIs(tok!"("))
                 return false;
             goto case;
-        case const_:
-        case immutable_:
-        case inout_:
-        case shared_:
-        case gshared:
-        case alias_:
-        case class_:
-        case enum_:
-        case interface_:
-        case struct_:
-        case union_:
-        case unittest_:
-        case auto_:
-        case ref_:
-        case at:
-        case align_:
-        case deprecated_:
-        case private_:
-        case package_:
-        case protected_:
-        case public_:
-        case export_:
-        case extern_:
-        case override_:
-        case abstract_:
-        case pure_:
-        case nothrow_:
-        mixin(BASIC_TYPE_CASE_RANGE);
+        case tok!"const":
+        case tok!"immutable":
+        case tok!"inout":
+        case tok!"shared":
+        case tok!"__gshared":
+        case tok!"alias":
+        case tok!"class":
+        case tok!"enum":
+        case tok!"interface":
+        case tok!"struct":
+        case tok!"union":
+        case tok!"unittest":
+        case tok!"auto":
+        case tok!"ref":
+        case tok!"@":
+        case tok!"align":
+        case tok!"deprecated":
+        case tok!"private":
+        case tok!"package":
+        case tok!"protected":
+        case tok!"public":
+        case tok!"export":
+        case tok!"extern":
+        case tok!"override":
+        case tok!"abstract":
+        case tok!"pure":
+        case tok!"nothrow":
+        mixin(BASIC_TYPE_CASES);
             return true;
-        case case_:
-        case default_:
-        case return_:
-        case if_:
-        case while_:
-        case do_:
-        case for_:
-        case foreach_:
-        case switch_:
-        case continue_:
-        case break_:
-        case goto_:
-        case try_:
-        case throw_:
-        case asm_:
-        case foreach_reverse_:
-        case lBrace:
-        case assert_:
+        case tok!"case":
+        case tok!"default":
+        case tok!"return":
+        case tok!"if":
+        case tok!"while":
+        case tok!"do":
+        case tok!"for":
+        case tok!"foreach":
+        case tok!"switch":
+        case tok!"continue":
+        case tok!"break":
+        case tok!"goto":
+        case tok!"try":
+        case tok!"throw":
+        case tok!"asm":
+        case tok!"foreach_reverse":
+        case tok!"{":
+        case tok!"assert":
             return false;
         default:
             break;
@@ -6192,59 +6140,59 @@ protected:
         auto b = setBookmark();
         scope (exit) goToBookmark(b);
         if (parseType() is null) return false;
-        if (currentIsOneOf(TokenType.comma, TokenType.rParen)) return true;
+        if (currentIsOneOf(tok!",", tok!")")) return true;
         return false;
     }
 
     bool isAttribute()
     {
         if (!moreTokens()) return false;
-        with (TokenType) switch (current.type)
+        switch (current.type)
         {
-        case const_:
-        case immutable_:
-        case inout_:
-        case scope_:
-            return !peekIs(TokenType.lParen);
-        case static_:
-            return !peekIsOneOf(assert_, this_, if_, tilde);
-        case shared_:
-            return !(startsWith(shared_, static_, this_)
-                || startsWith(shared_, static_, tilde)
-                || peekIs(TokenType.lParen));
-        case enum_:
-            if (peekIsOneOf(lBrace, colon, semicolon))
+        case tok!"const":
+        case tok!"immutable":
+        case tok!"inout":
+        case tok!"scope":
+            return !peekIs(tok!"(");
+        case tok!"static":
+            return !peekIsOneOf(tok!"assert", tok!"this", tok!"if", tok!"~");
+        case tok!"shared":
+            return !(startsWith(tok!"shared", tok!"static", tok!"this")
+                || startsWith(tok!"shared", tok!"static", tok!"~")
+                || peekIs(tok!"("));
+        case tok!"enum":
+            if (peekIsOneOf(tok!"{", tok!":", tok!";"))
                 return false;
-            else if (peekIs(TokenType.identifier))
+            else if (peekIs(tok!"identifier"))
             {
-                if (startsWith(TokenType.enum_, TokenType.identifier, TokenType.lParen))
+                if (startsWith(tok!"enum", tok!"identifier", tok!"("))
                     return false;
                 auto b = setBookmark();
                 scope(exit) goToBookmark(b);
                 advance();
-                if (peekIsOneOf(lBrace, colon, semicolon))
+                if (peekIsOneOf(tok!"{", tok!":", tok!";"))
                     return false;
                 return true;
             }
             return true;
-        case deprecated_:
-        case private_:
-        case package_:
-        case protected_:
-        case public_:
-        case export_:
-        case final_:
-        case synchronized_:
-        case override_:
-        case abstract_:
-        case auto_:
-        case gshared:
-        case pure_:
-        case nothrow_:
-        case at:
-        case ref_:
-        case extern_:
-        case align_:
+        case tok!"deprecated":
+        case tok!"private":
+        case tok!"package":
+        case tok!"protected":
+        case tok!"public":
+        case tok!"export":
+        case tok!"final":
+        case tok!"synchronized":
+        case tok!"override":
+        case tok!"abstract":
+        case tok!"auto":
+        case tok!"__gshared":
+        case tok!"pure":
+        case tok!"nothrow":
+        case tok!"@":
+        case tok!"ref":
+        case tok!"extern":
+        case tok!"align":
             return true;
         default:
             return false;
@@ -6255,13 +6203,13 @@ protected:
     {
         switch (current.type)
         {
-        case TokenType.const_:
-        case TokenType.immutable_:
-        case TokenType.inout_:
-        case TokenType.shared_:
-        case TokenType.at:
-        case TokenType.pure_:
-        case TokenType.nothrow_:
+        case tok!"const":
+        case tok!"immutable":
+        case tok!"inout":
+        case tok!"shared":
+        case tok!"@":
+        case tok!"pure":
+        case tok!"nothrow":
             return true;
         default:
             return false;
@@ -6293,11 +6241,11 @@ protected:
         while (moreTokens())
         {
             mixin ("node.items ~= parse" ~ ItemType.stringof ~ "();");
-            if (currentIs(TokenType.comma))
+            if (currentIs(tok!","))
             {
                 advance();
-                if (allowTrailingComma && currentIsOneOf(TokenType.rParen,
-                    TokenType.rBrace, TokenType.rBracket))
+                if (allowTrailingComma && currentIsOneOf(tok!")",
+                    tok!"}", tok!"]"))
                 {
                     break;
                 }
@@ -6332,18 +6280,18 @@ protected:
             auto column = index < tokens.length ? tokens[index].column : tokens[$ - 1].column;
             auto line = index < tokens.length ? tokens[index].line : tokens[$ - 1].line;
             if (messageFunction is null)
-			{
+            {
                 stderr.writefln("%s(%d:%d)[error]: %s", fileName, line, column, message);
-			}
+            }
             else
                 messageFunction(fileName, line, column, message);
         }
         while (shouldAdvance && moreTokens())
         {
-            if (currentIsOneOf(TokenType.semicolon, TokenType.rBrace,
-                TokenType.rParen, TokenType.rBracket))
+            if (currentIsOneOf(tok!";", tok!"}",
+                tok!")", tok!"]"))
             {
-				advance();
+                advance();
                 break;
             }
             else
@@ -6353,7 +6301,7 @@ protected:
 
     void skip(alias O, alias C)()
     {
-        assert(currentIs(O), current().value);
+        assert(currentIs(O), current().text);
         advance();
         int depth = 1;
         while (moreTokens())
@@ -6379,17 +6327,17 @@ protected:
 
     void skipBraces()
     {
-        skip!(TokenType.lBrace, TokenType.rBrace)();
+        skip!(tok!"{", tok!"}")();
     }
 
     void skipParens()
     {
-        skip!(TokenType.lParen, TokenType.rParen)();
+        skip!(tok!"(", tok!")")();
     }
 
     void skipBrackets()
     {
-        skip!(TokenType.lBracket, TokenType.rBracket)();
+        skip!(tok!"[", tok!"]")();
     }
 
     const(Token)* peek() const
@@ -6426,25 +6374,25 @@ protected:
 
     const(Token)* peekPastParens() const nothrow
     {
-        return peekPast!(TokenType.lParen, TokenType.rParen)();
+        return peekPast!(tok!"(", tok!")")();
     }
 
     const(Token)* peekPastBrackets() const nothrow
     {
-        return peekPast!(TokenType.lBracket, TokenType.rBracket)();
+        return peekPast!(tok!"[", tok!"]")();
     }
 
     const(Token)* peekPastBraces() const nothrow
     {
-        return peekPast!(TokenType.lBrace, TokenType.rBrace)();
+        return peekPast!(tok!"{", tok!"}")();
     }
 
-    bool peekIs(TokenType t) const nothrow
+    bool peekIs(IdType t) const nothrow
     {
         return index + 1 < tokens.length && tokens[index + 1].type == t;
     }
 
-    bool peekIsOneOf(TokenType[] types...) const nothrow
+    bool peekIsOneOf(IdType[] types...) const nothrow
     {
         if (index + 1 >= tokens.length) return false;
         return canFind(types, tokens[index + 1].type);
@@ -6454,20 +6402,22 @@ protected:
      * Returns a token of the specified type if it was the next token, otherwise
      * calls the error function and returns null.
      */
-    const(Token)* expect(TokenType type)
+    const(Token)* expect(IdType type)
     {
         if (index < tokens.length && tokens[index].type == type)
             return &tokens[index++];
         else
         {
-            string tokenString = getTokenValue(type) is null
-                ? to!string(type) : getTokenValue(type);
+            string tokenString = str(type) is null
+                ? to!string(type) : str(type);
             bool shouldNotAdvance = index < tokens.length
-                && (tokens[index].type == TokenType.rParen
-                || tokens[index].type == TokenType.semicolon
-                || tokens[index].type == TokenType.rBrace);
-            error("Expected " ~  tokenString ~ " instead of "
-                ~ (index < tokens.length ? tokens[index].value : "EOF"),
+                && (tokens[index].type == tok!")"
+                || tokens[index].type == tok!";"
+                || tokens[index].type == tok!"}");
+            auto token = (index < tokens.length
+                ? (tokens[index].text is null ? str(tokens[index].type) : tokens[index].text)
+                : "EOF");
+            error("Expected " ~  tokenString ~ " instead of " ~ token,
                 !shouldNotAdvance);
             return null;
         }
@@ -6492,7 +6442,7 @@ protected:
     /**
      * Returns: true if the current token has the given type
      */
-    bool currentIs(TokenType type) const
+    bool currentIs(IdType type) const
     {
         return index < tokens.length && tokens[index] == type;
     }
@@ -6500,14 +6450,14 @@ protected:
     /**
      * Returns: true if the current token is one of the given types
      */
-    bool currentIsOneOf(TokenType[] types...) const
+    bool currentIsOneOf(IdType[] types...) const
     {
         if (index >= tokens.length)
             return false;
         return canFind(types, current.type);
     }
 
-    bool startsWith(TokenType[] types...) const nothrow
+    bool startsWith(IdType[] types...) const nothrow
     {
         if (index + types.length >= tokens.length)
             return false;
@@ -6542,38 +6492,81 @@ protected:
     }
 
     version (unittest) static void doNothingErrorFunction(string fileName,
-        int line, int column, string message) {}
+        size_t line, size_t column, string message) {}
 
     version (unittest) static Parser getParserForUnittest(string sourceCode,
         string testName)
     {
-        LexerConfig config;
-        auto r = byToken(cast(const(ubyte)[]) sourceCode, config);
-        Parser p;
-        //p.messageFunction = &doNothingErrorFunction;
+        auto r = byToken(cast(ubyte[]) sourceCode);
+        Parser p = new Parser;
+        p.messageFunction = &doNothingErrorFunction;
         p.fileName = testName ~ ".d";
         p.tokens = r.array();
         return p;
     }
 
+    template simpleParse(NodeType, parts ...)
+    {
+        static if (__traits(hasMember, NodeType, "comment"))
+            enum simpleParse = "auto node = new " ~ NodeType.stringof ~ ";\n"
+                ~ "node.comment = comment;\n"
+                ~ "comment = null;\n"
+                ~ simpleParseItems!(parts)
+                ~ "\nreturn node;\n";
+        else
+            enum simpleParse = "auto node = new " ~ NodeType.stringof ~ ";\n"
+                ~ simpleParseItems!(parts)
+                ~ "\nreturn node;\n";
+    }
+
+    template simpleParseItems(items ...)
+    {
+        static if (items.length > 1)
+            enum simpleParseItems = simpleParseItem!(items[0]) ~ "\n"
+                ~ simpleParseItems!(items[1 .. $]);
+        else static if (items.length == 1)
+            enum simpleParseItems = simpleParseItem!(items[0]);
+        else
+            enum simpleParseItems = "";
+    }
+
+    template simpleParseItem(alias item)
+    {
+        static if (is (typeof(item) == string))
+            enum simpleParseItem = "if ((node." ~ item[0 .. item.countUntil("|")]
+                ~ " = " ~ item[item.countUntil("|") + 1 .. $] ~ "()) is null) return null;";
+        else
+            enum simpleParseItem = "if (expect(" ~ item.stringof ~ ") is null) return null;";
+    }
+
+    template expectSequence(sequence ...)
+    {
+        static if (sequence.length == 1)
+            enum expectSequence = "if (expect(" ~ sequence[0].stringof ~ ") is null) return null;";
+        else
+            enum expectSequence = "if (expect(" ~ sequence[0].stringof ~ ") is null) return null;\n"
+                ~ expectSequence!(sequence[1..$]);
+    }
+
     template traceEnterAndExit(string fun)
     {
-        enum traceEnterAndExit = `version (std_parser_verbose) trace("`
-            ~ "\033[01;32m" ~ fun ~ "\033[0m" ~ ` ");`
-            ~ `version (std_parser_verbose) scope(exit) trace("`
-            ~ "\033[01;31m" ~ fun ~ "\033[0m" ~ ` ");`;
+        enum traceEnterAndExit = `version (std_parser_verbose) { _traceDepth++; trace("`
+            ~ `\033[01;32m` ~ fun ~ `\033[0m"); }`
+            ~ `version (std_parser_verbose) scope(exit) { trace("`
+            ~ `\033[01;31m` ~ fun ~ `\033[0m"); _traceDepth--; }`;
     }
 
     version (std_parser_verbose)
     {
         void trace(lazy string message)
         {
+			auto depth = format("%4d ", _traceDepth);
             if (suppressMessages > 0)
                 return;
             if (index < tokens.length)
-                writeln(message, "(", current.line, ":", current.column, ")");
+                writeln(depth, message, "(", current.line, ":", current.column, ")");
             else
-                writeln(message, "(EOF:0)");
+                writeln(depth, message, "(EOF:0)");
         }
     }
     else
@@ -6581,10 +6574,64 @@ protected:
         void trace(lazy string message) {}
     }
 
-    enum string BASIC_TYPE_CASE_RANGE = q{case bool_: .. case wchar_:};
-    enum string LITERAL_CASE_RANGE = q{case doubleLiteral: .. case wstringLiteral:};
-    enum string SPECIAL_CASE_RANGE = q{case specialDate: .. case specialPrettyFunction:};
+    enum string BASIC_TYPE_CASES = q{
+        case tok!"int":
+        case tok!"uint":
+        case tok!"double":
+        case tok!"idouble":
+        case tok!"float":
+        case tok!"ifloat":
+        case tok!"short":
+        case tok!"ushort":
+        case tok!"long":
+        case tok!"ulong":
+        case tok!"char":
+        case tok!"wchar":
+        case tok!"dchar":
+        case tok!"bool":
+        case tok!"void":
+        case tok!"cent":
+        case tok!"ucent":
+        case tok!"real":
+        case tok!"ireal":
+        case tok!"byte":
+        case tok!"ubyte":
+        case tok!"cdouble":
+        case tok!"cfloat":
+        case tok!"creal":
+    };
+    enum string LITERAL_CASES = q{
+        case tok!"doubleLiteral":
+        case tok!"floatLiteral":
+        case tok!"idoubleLiteral":
+        case tok!"ifloatLiteral":
+        case tok!"intLiteral":
+        case tok!"longLiteral":
+        case tok!"realLiteral":
+        case tok!"irealLiteral":
+        case tok!"uintLiteral":
+        case tok!"ulongLiteral":
+        case tok!"stringLiteral":
+        case tok!"wstringLiteral":
+        case tok!"dstringLiteral":
+        case tok!"characterLiteral":
+    };
+    enum string SPECIAL_CASES = q{
+        case tok!"__DATE__":
+        case tok!"__EOF__":
+        case tok!"__FILE__":
+        case tok!"__FUNCTION__":
+        case tok!"__LINE__":
+        case tok!"__MODULE__":
+        case tok!"__PRETTY_FUNCTION__":
+        case tok!"__TIME__":
+        case tok!"__TIMESTAMP__":
+        case tok!"__VENDOR__":
+        case tok!"__VERSION__":
+    };
     const(Token)[] tokens;
     int suppressMessages;
     size_t index;
+    int _traceDepth;
+    string comment;
 }
diff --git a/stdx/lexer.d b/stdx/lexer.d
new file mode 100644
index 0000000..537e6a1
--- /dev/null
+++ b/stdx/lexer.d
@@ -0,0 +1,300 @@
+// Written in the D programming language
+
+/**
+ * This module contains a range-based _lexer generator.
+ *
+ * Copyright: Brian Schott 2013
+ * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0)
+ * Authors: Brian Schott, with ideas shamelessly stolen from Andrei Alexandrescu
+ * Source: $(PHOBOSSRC std/_lexer.d)
+ */
+
+module stdx.lexer;
+
+import std.typecons;
+import std.algorithm;
+import std.range;
+import std.traits;
+import std.conv;
+import std.math;
+import dpick.buffer.buffer;
+import dpick.buffer.traits;
+
+template TokenIdType(alias staticTokens, alias dynamicTokens,
+	alias possibleDefaultTokens)
+{
+  static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= ubyte.max)
+		alias TokenIdType = ubyte;
+	else static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= ushort.max)
+		alias TokenIdType = ushort;
+	else static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= uint.max)
+		alias TokenIdType = uint;
+	else
+		static assert (false);
+}
+
+string tokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens, alias possibleDefaultTokens)(IdType type) @property
+{
+	if (type == 0)
+		return "!ERROR!";
+	else if (type < staticTokens.length + 1)
+		return staticTokens[type - 1];
+	else if (type < staticTokens.length + possibleDefaultTokens.length + 1)
+		return possibleDefaultTokens[type - staticTokens.length - 1];
+    else if (type < staticTokens.length + possibleDefaultTokens.length + dynamicTokens.length + 1)
+		return dynamicTokens[type - staticTokens.length - possibleDefaultTokens.length - 1];
+	else
+		return null;
+}
+
+template TokenId(IdType, alias staticTokens, alias dynamicTokens,
+	alias possibleDefaultTokens, string symbol)
+{
+	static if (symbol == "")
+	{
+	  enum id = 0;
+		alias id TokenId;
+	}
+	else static if (symbol == "\0")
+	{
+		enum id = 1 + staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length;
+		alias id TokenId;
+	}
+	else
+	{
+		enum i = staticTokens.countUntil(symbol);
+		static if (i >= 0)
+		{
+			enum id = i + 1;
+			alias id TokenId;
+		}
+		else
+		{
+			enum ii = possibleDefaultTokens.countUntil(symbol);
+			static if (ii >= 0)
+			{
+				enum id = ii + staticTokens.length + 1;
+				static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol);
+				alias id TokenId;
+			}
+			else
+			{
+				enum dynamicId = dynamicTokens.countUntil(symbol);
+				enum id = dynamicId >= 0
+					? i + staticTokens.length + possibleDefaultTokens.length + dynamicId + 1
+					: -1;
+				static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol);
+				alias id TokenId;
+			}
+		}
+	}
+}
+
+struct TokenStructure(IDType, string extraFields = "")
+{
+	bool opEquals(IDType type) const pure nothrow @safe
+	{
+		return this.type == type;
+	}
+
+	this(IDType type)
+	{
+		this.type = type;
+	}
+
+	this(IDType type, string text, size_t line, size_t column, size_t index)
+	{
+		this.text = text;
+		this.line = line;
+		this.column = column;
+		this.type = type;
+		this.index = index;
+	}
+
+	string text;
+	size_t line;
+	size_t column;
+	size_t index;
+	IDType type;
+	mixin (extraFields);
+}
+
+mixin template Lexer(R, IDType, Token, alias defaultTokenFunction,
+	alias staticTokens, alias dynamicTokens, alias pseudoTokens,
+	alias pseudoTokenHandlers, alias possibleDefaultTokens)
+{
+	static string generateCaseStatements(string[] tokens, size_t offset = 0)
+	{
+		string code;
+		for (size_t i = 0; i < tokens.length; i++)
+		{
+			auto indent = "";
+			foreach (k; 0 .. offset)
+				indent ~= "    ";
+			size_t j = i + 1;
+
+			if (offset < tokens[i].length)
+			{
+				while (j < tokens.length && offset < tokens[j].length
+					&& tokens[i][offset] == tokens[j][offset]) j++;
+				code ~= indent ~ "case " ~ text(cast(ubyte) tokens[i][offset]) ~ ":\n";
+				if (i + 1 >= j)
+				{
+					if (offset + 1 == tokens[i].length)
+						code ~= generateLeaf(tokens[i], indent ~ "    ");
+					else
+					{
+						code ~= indent ~ "    if (range.lookahead(" ~ text(tokens[i].length) ~ ").length == 0)\n";
+						code ~= indent ~ "        goto outer_default;\n";
+						code ~= indent ~ "    if (range.lookahead(" ~ text(tokens[i].length) ~ ") == \"" ~ escape(tokens[i]) ~ "\")\n";
+						code ~= indent ~ "    {\n";
+						code ~= generateLeaf(tokens[i], indent ~ "        ");
+						code ~= indent ~ "    }\n";
+						code ~= indent ~ "    else\n";
+						code ~= indent ~ "        goto outer_default;\n";
+					}
+				}
+				else
+				{
+					code ~= indent ~ "    if (range.lookahead(" ~ text(offset + 2) ~ ").length == 0)\n";
+					code ~= indent ~ "    {\n";
+					code ~= generateLeaf(tokens[i][0 .. offset + 1], indent ~ "        ");
+					code ~= indent ~ "    }\n";
+					code ~= indent ~ "    switch (range.lookahead(" ~ text(offset + 2) ~ ")[" ~ text(offset + 1) ~ "])\n";
+					code ~= indent ~ "    {\n";
+					code ~= generateCaseStatements(tokens[i .. j], offset + 1);
+					code ~= indent ~ "    default:\n";
+					code ~= generateLeaf(tokens[i][0 .. offset + 1], indent ~ "        ");
+					code ~= indent ~ "    }\n";
+				}
+			}
+			i = j - 1;
+		}
+		return code;
+	}
+
+	static string generateLeaf(string token, string indent)
+	{
+		static assert (pseudoTokenHandlers.length % 2 == 0,
+			"Each pseudo-token must have a matching function name.");
+		string code;
+		if (staticTokens.countUntil(token) >= 0)
+		{
+			if (token.length == 1)
+				code ~= indent ~ "range.popFront();\n";
+			else
+				code ~= indent ~ "range.popFrontN(" ~ text(token.length) ~ ");\n";
+			code ~= indent ~ "return Token(tok!\"" ~ escape(token) ~ "\", null, range.line, range.column, range.index);\n";
+		}
+		else if (pseudoTokens.countUntil(token) >= 0)
+			code ~= indent ~ "return " ~ pseudoTokenHandlers[pseudoTokenHandlers.countUntil(token) + 1] ~ "();\n";
+		else if (possibleDefaultTokens.countUntil(token) >= 0)
+		{
+			code ~= indent ~ "if (range.lookahead(" ~ text(token.length + 1) ~ ").length == 0 || isSeparating(range.lookahead(" ~ text(token.length + 1) ~ ")[" ~ text(token.length) ~ "]))\n";
+			code ~= indent ~ "{\n";
+			if (token.length == 1)
+				code ~= indent ~ "    range.popFront();\n";
+			else
+				code ~= indent ~ "    range.popFrontN(" ~ text(token.length) ~ ");\n";
+			code ~= indent ~ "    return Token(tok!\"" ~ escape(token) ~"\", null, range.line, range.column, range.index);\n";
+			code ~= indent ~ "}\n";
+			code ~= indent ~ "else\n";
+			code ~= indent ~ "    goto outer_default;\n";
+		}
+		else
+			code ~= indent ~ "goto outer_default;\n";
+		return code;
+	}
+
+	const(Token) front() pure nothrow const @property
+	{
+		return _front;
+	}
+
+	void _popFront() pure
+	{
+		_front = advance();
+	}
+
+	bool empty() pure const nothrow @property
+	{
+		return _front.type == tok!"\0";
+	}
+
+	static string escape(string input)
+	{
+		string rVal;
+		foreach (ubyte c; cast(ubyte[]) input)
+		{
+			switch (c)
+			{
+			case '\\': rVal ~= `\\`; break;
+			case '"': rVal ~= `\"`; break;
+			case '\'': rVal ~= `\'`; break;
+			case '\t': rVal ~= `\t`; break;
+			case '\n': rVal ~= `\n`; break;
+			case '\r': rVal ~= `\r`; break;
+			default: rVal ~= c; break;
+			}
+		}
+		return rVal;
+	}
+
+	Token advance() pure
+	{
+		if (range.empty)
+			return Token(tok!"\0");
+		lexerLoop: switch (range.front)
+		{
+		mixin(generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens))));
+//		pragma(msg, generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens))));
+		outer_default:
+		default:
+			return defaultTokenFunction();
+		}
+	}
+
+	/**
+	 * This only exists because the real array() can't be called at compile-time
+	 */
+	static T[] stupidToArray(R, T = ElementType!R)(R range)
+	{
+		T[] rVal;
+		foreach (v; range)
+			rVal ~= v;
+		return rVal;
+	}
+
+	LexerRange!(typeof(buffer(R.init))) range;
+	Token _front;
+}
+
+struct LexerRange(BufferType) if (isBuffer!BufferType)
+{
+	this(BufferType r)
+	{
+		this.range = r;
+		index = 0;
+		column = 1;
+		line = 1;
+	}
+
+	void popFront() pure
+	{
+		index++;
+		column++;
+		range.popFront();
+	}
+
+	void incrementLine() pure nothrow
+	{
+		column = 1;
+		line++;
+	}
+
+	BufferType range;
+	alias range this;
+	size_t index;
+	size_t column;
+	size_t line;
+}
diff --git a/style.d b/style.d
new file mode 100644
index 0000000..322bad6
--- /dev/null
+++ b/style.d
@@ -0,0 +1,102 @@
+//          Copyright Brian Schott (Sir Alaran) 2014.
+// Distributed under the Boost Software License, Version 1.0.
+//    (See accompanying file LICENSE_1_0.txt or copy at
+//          http://www.boost.org/LICENSE_1_0.txt)
+
+module style;
+
+import stdx.d.ast;
+import stdx.d.lexer;
+import stdx.d.parser;
+import std.stdio;
+import std.regex;
+import std.array;
+import std.conv;
+
+void doNothing(string, size_t, size_t, string) {}
+
+void styleCheck(File output, string[] fileNames)
+{
+	foreach (fileName; fileNames)
+	{
+		File f = File(fileName);
+		auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
+		f.rawRead(bytes);
+		auto tokens = byToken(bytes);
+		Module m = parseModule(tokens.array, fileName, &doNothing);
+		auto checker = new StyleChecker;
+		checker.fileName = fileName;
+		checker.visit(m);
+	}
+}
+
+class StyleChecker : ASTVisitor
+{
+	enum varFunNameRegex = `^([\p{Ll}_][_\w\d]*|[\p{Lu}\d_]+)$`;
+	enum aggregateNameRegex = `^\p{Lu}[\w\d]*$`;
+	enum moduleNameRegex = `^\p{Ll}+$`;
+	
+	override void visit(ModuleDeclaration dec)
+	{
+		foreach (part; dec.moduleName.identifiers)
+		{
+			if (part.text.matchFirst(moduleNameRegex).length == 0)
+				writeln(fileName, "(", part.line, ":", part.column, ") ",
+					"Module/package name ", part.text, " does not match style guidelines");
+		}
+	}
+	
+	override void visit(Declarator dec)
+	{
+		checkLowercaseName("Variable", dec.name);
+	}
+	
+	override void visit(FunctionDeclaration dec)
+	{
+		checkLowercaseName("Function", dec.name);
+	}
+	
+	void checkLowercaseName(string type, ref Token name)
+	{
+		if (name.text.matchFirst(varFunNameRegex).length == 0)
+			writeln(fileName, "(", name.line, ":", name.column, ") ",
+				type, " name ", name.text, " does not match style guidelines");
+	}
+	
+	override void visit(ClassDeclaration dec)
+	{
+		checkAggregateName("Class", dec.name);
+		dec.accept(this);
+	}
+	
+	override void visit(InterfaceDeclaration dec)
+	{
+		checkAggregateName("Interface", dec.name);
+		dec.accept(this);
+	}
+	
+	override void visit(EnumDeclaration dec)
+	{
+		if (dec.name.text is null || dec.name.text.length == 0)
+			return;
+		checkAggregateName("Enum", dec.name);
+		dec.accept(this);
+	}
+	
+	override void visit(StructDeclaration dec)
+	{
+		checkAggregateName("Struct", dec.name);
+		dec.accept(this);
+	}
+	
+	void checkAggregateName(string aggregateType, ref Token name)
+	{
+		if (name.text.matchFirst(aggregateNameRegex).length == 0)
+			writeln(fileName, "(", name.line, ":", name.column, ") ",
+				aggregateType, " name ", name.text,
+				" does not match style guidelines");
+	}
+
+	alias ASTVisitor.visit visit;
+	string fileName;
+}