More implementation of the lexer

codegen.d

@@ -49,13 +49,15 @@ string printCaseStatements(K, V)(TrieNode!(K,V) node, string indentString)
 		caseStatement ~= k;
 		caseStatement ~= "':\n";
 		caseStatement ~= indentString;
-		caseStatement ~= "\tcurrentToken.value ~= '";
+		caseStatement ~= "\tcurrent.value ~= '";
 		caseStatement ~= k;
 		caseStatement ~= "';\n";
 		caseStatement ~= indentString;
-		caseStatement ~= "\tcurrentToken.lineNumber = lineNumber;\n";
+		caseStatement ~= "\tcurrent.lineNumber = lineNumber;\n";
 		caseStatement ~= indentString;
-		caseStatement ~= "\t++endIndex;\n";
+		caseStatement ~= "\t++index;\n";
+		caseStatement ~= indentString;
+		caseStatement ~= "\tinput.popFront();\n";
 		if (v.children.length > 0)
 		{
 			caseStatement ~= indentString;
@@ -63,21 +65,21 @@ string printCaseStatements(K, V)(TrieNode!(K,V) node, string indentString)
 			caseStatement ~= indentString;
 			caseStatement ~= "\t{\n";
 			caseStatement ~= indentString;
-			caseStatement ~= "\t\tcurrentToken.type = " ~ node.children[k].value;
+			caseStatement ~= "\t\tcurrent.type = " ~ node.children[k].value;
 			caseStatement ~= ";\n";
 			caseStatement ~= indentString;
 			caseStatement ~= "\t\tbreak;\n";
 			caseStatement ~= indentString;
 			caseStatement ~= "\t}\n";
 			caseStatement ~= indentString;
-			caseStatement ~= "\tswitch (inputString[endIndex])\n";
+			caseStatement ~= "\tswitch (input.front)\n";
 			caseStatement ~= indentString;
 			caseStatement ~= "\t{\n";
 			caseStatement ~= printCaseStatements(v, indentString ~ "\t");
 			caseStatement ~= indentString;
 			caseStatement ~= "\tdefault:\n";
 			caseStatement ~= indentString;
-			caseStatement ~= "\t\tcurrentToken.type = ";
+			caseStatement ~= "\t\tcurrent.type = ";
 			caseStatement ~= v.value;
 			caseStatement ~= ";\n";
 			caseStatement ~= indentString;
@@ -90,7 +92,7 @@ string printCaseStatements(K, V)(TrieNode!(K,V) node, string indentString)
 		else
 		{
 			caseStatement ~= indentString;
-			caseStatement ~= "\tcurrentToken.type = ";
+			caseStatement ~= "\tcurrent.type = ";
 			caseStatement ~= v.value;
 			caseStatement ~= ";\n";
 			caseStatement ~= indentString;

langutils.d

@@ -629,18 +629,46 @@ pure bool isIdentifierOrType(ref const Token t)
 		&& TokenType.TYPES_END);
 }
 
+/**
+ * Token structure
+ */
 struct Token
 {
+	/// The token type
 	TokenType type;
+
+	/// The representation of the token in the original source code
 	string value;
+
+	/// The number of the line the token is on
 	uint lineNumber;
-	size_t startIndex;
+
+	/// The character index of the start of the token in the original text
+	uint startIndex;
+
+	/**
+	 * Check to see if the token is of the same type and has the same string
+	 * representation as the given token
+	 */
 	bool opEquals(ref const(Token) other) const
 	{
 		return other.type == type && other.value == value;
 	}
+
+	/**
+	 * Checks to see if the token's string representation is equal to the given
+	 * string
+	 */
 	bool opEquals(string range) const { return range == value; }
+
+	/**
+	 * Checks to see if the token is of the given type
+	 */
 	bool opEquals(TokenType t) const { return type == t; }
+
+	/**
+	 * Comparison operator orders by start index
+	 */
 	int opCmp(size_t i) const
 	{
 		if (startIndex < i) return -1;