Merge branch 'range-based-lexer' of https://github.com/Hackerpilot/Dscanner into range-based-lexer

build.sh

@@ -1,2 +1,2 @@
-dmd *.d std/d/*.d -release -noboundscheck -O -w -wi -m64 -property -ofdscanner -L-lsqlite3 #-inline
-#dmd *.d std/d/*.d -g -m64 -w -wi -property -ofdscanner -L-lsqlite3 #-unittest
+#dmd *.d std/d/*.d -release -noboundscheck -O -w -wi -m64 -property -ofdscanner -L-lsqlite3 #-inline
+dmd *.d std/d/*.d -g -m64 -w -wi -property -ofdscanner -L-lsqlite3 #-unittest

circularbuffer.d

@@ -32,7 +32,7 @@ public:
 		}
 	}
 
-    override T front() const @property
+	override T front() @property
 	{
 		return data[index];
 	}
@@ -94,6 +94,7 @@ public:
 			result = dg(front);
 			if (result)
 				break;
+            popFront();
 		}
 		return result;
 	}
@@ -107,6 +108,7 @@ public:
 			result = dg(i, front);
 			if (result)
 				break;
+            popFront();
 		}
 		return result;
 	}

highlighter.d

@@ -10,8 +10,6 @@ import std.stdio;
 import std.array;
 import std.d.lexer;
 
-import langutils;
-
 void writeSpan(string cssClass, string value)
 {
 	stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`);

langutils.d

@@ -6,6 +6,7 @@
 module langutils;
 
 import std.array;
+import std.algorithm;
 import std.d.lexer;
 
 
@@ -43,7 +44,7 @@ string combineTokens(ref const Token[] tokens)
 	return app.data;
 }
 
-pure string getTypeFromToken(const Token t)
+pure nothrow string getTypeFromToken(const Token t)
 {
 	switch (t.type)
 	{
@@ -73,8 +74,14 @@ pure string getTypeFromToken(const Token t)
 	}
 }
 
-pure bool isIdentifierOrType(inout Token t)
+pure bool isIdentifierOrType(const Token t)
 {
 	return t.type == TokenType.Identifier || (t.type > TokenType.TYPES_BEGIN
 		&& TokenType.TYPES_END);
 }
+
+pure bool isDocComment(ref const Token t)
+{
+    return t.value.startsWith("///") || t.value.startsWith("/**")
+        || t.value.startsWith("/++");
+}

main.d

@@ -15,6 +15,7 @@ import std.parallelism;
 import std.path;
 import std.regex;
 import std.stdio;
+import std.range;
 import std.d.lexer;
 
 import autocomplete;
@@ -28,7 +29,7 @@ import circularbuffer;
 
 immutable size_t CIRC_BUFF_SIZE = 4;
 
-pure bool isLineOfCode(TokenType t)
+pure nothrow bool isLineOfCode(TokenType t)
 {
 	switch(t)
 	{
@@ -138,36 +139,21 @@ int main(string[] args)
 	{
 		if (args.length == 1)
 		{
-			auto f = appender!string();
-			char[] buf;
-			while (stdin.readln(buf))
-				f.put(buf);
-			writeln(f.data.byToken().count!(a => isLineOfCode(a.type))());
+			writeln(stdin.byLine(KeepTerminator.yes).join().byToken().count!(a => isLineOfCode(a.type))());
 		}
 		else
 		{
-			writeln(args[1..$].map!(a => a.readText().byToken())().joiner()
-				.count!(a => isLineOfCode(a.type))());
+			writeln(args[1..$].map!(a => File(a).byLine(KeepTerminator.yes).join().byToken(a))()
+                .joiner().count!(a => isLineOfCode(a.type))());
 		}
 		return 0;
 	}
 
 	if (highlight)
 	{
-		if (args.length == 1)
-		{
-			auto f = appender!string();
-			char[] buf;
-			while (stdin.readln(buf))
-				f.put(buf);
-			highlighter.highlight(f.data.byToken("stdin", IterationStyle.Everything,
-				TokenStyle.Source));
-		}
-		else
-		{
-			highlighter.highlight(args[1].readText().byToken(args[1],
-				IterationStyle.Everything, TokenStyle.Source));
-		}
+        File f = args.length == 1 ? stdin : File(args[1]);
+        highlighter.highlight(f.byLine(KeepTerminator.yes).join().byToken(
+            "", IterationStyle.Everything, TokenStyle.Source));
 		return 0;
 	}
 
@@ -213,20 +199,9 @@ int main(string[] args)
 	if (json)
 	{
 		CircularBuffer!(Token) tokens;
-		if (args.length == 1)
-		{
-			// Read from stdin
-			auto f = appender!string();
-			char[] buf;
-			while (stdin.readln(buf))
-				f.put(buf);
-			tokens = new CircularBuffer!(Token)(CIRC_BUFF_SIZE, byToken!string(f.data));
-		}
-		else
-		{
-			// read given file
-			tokens = new CircularBuffer!(Token)(CIRC_BUFF_SIZE, byToken!string(readText(args[1])));
-		}
+        File f = args.length == 1 ? stdin : File(args[1]);
+        tokens = new CircularBuffer!(Token)(CIRC_BUFF_SIZE,
+            f.byLine(KeepTerminator.yes).join().byToken!(char[])());
 		auto mod = parseModule(tokens);
 		mod.writeJSONTo(stdout);
 		return 0;

parser.d

@@ -26,6 +26,7 @@ public:
 	this(InputRange!Token tokens, TokenType open, TokenType close)
 	{
 		super(0, tokens);
+        this.range = tokens;
 		this.open = open;
 		this.close = close;
 	}
@@ -35,7 +36,7 @@ public:
 		return _empty;
 	}
 
-	override Token front() const @property
+	override Token front() @property
 	{
 		return range.front;
 	}
@@ -47,14 +48,20 @@ public:
 			++depth;
 		else if (range.front == close)
 			--depth;
-		_empty = depth == 0;
+		_empty = depth == 0 || range.empty;
+	}
+
+    invariant()
+    {
+        assert (range);
+        assert (depth >= 0);
     }
 
 private:
 	int depth;
 	TokenType open;
 	TokenType close;
-	TokenBuffer range;
+	InputRange!(Token) range;
 	bool _empty;
 }
 

std/d/lexer.d

@@ -164,16 +164,16 @@ enum IterationStyle
 	IncludeComments = 0b0001,
 	/// Includes whitespace
 	IncludeWhitespace = 0b0010,
-	/// Include $(LINK2 http://dlang.org/lex.html#Special%20Tokens%20Sequence, special token sequences)
+	/// Include $(LINK2 http://dlang.org/lex.html#specialtokens, special tokens)
 	IncludeSpecialTokens = 0b0100,
-	/// Do not terminate iteration upon reaching the ___EOF__ token
+	/// Do not stop iteration on reaching the ___EOF__ token
 	IgnoreEOF = 0b1000,
-	/// Include everything, including the __EOF__ token.
+	/// Include everything
 	Everything = IncludeComments | IncludeWhitespace | IgnoreEOF
 }
 
 /**
- * Configuration of the string lexing style. These flags may be combined with a
+ * Configuration of the token lexing style. These flags may be combined with a
  * bitwise or.
  */
 enum TokenStyle : uint
@@ -2486,7 +2486,6 @@ pure nothrow TokenType lookupTokenType(const string input)
 		default: break;
 		}
 		break;
-
 	case 6:
 		switch (input)
 		{