Re-introduce string cache and clean up some code

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "datapicked"]
 	path = datapicked
-	url = ./datapicked/
+	url = https://github.com/blackwhale/datapicked.git

ctags.d

@@ -19,12 +19,14 @@ void doNothing(string, size_t, size_t, string, bool) {}
 void printCtags(File output, string[] fileNames)
 {
 	string[] tags;
+	LexerConfig config;
+	StringCache cache;
 	foreach (fileName; fileNames)
 	{
 		File f = File(fileName);
 		auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
 		f.rawRead(bytes);
-		auto tokens = byToken(bytes);
+		auto tokens = byToken(bytes, config, cache);
 		Module m = parseModule(tokens.array, fileName, &doNothing);
 		auto printer = new CTagsPrinter;
 		printer.fileName = fileName;
@@ -131,7 +133,7 @@ class CTagsPrinter : ASTVisitor
 		}
 		dec.accept(this);
 	}
-	
+
 	alias ASTVisitor.visit visit;
 
 	string fileName;

main.d

@@ -15,6 +15,7 @@ import std.path;
 import std.regex;
 import std.stdio;
 import std.range;
+import stdx.lexer;
 import stdx.d.lexer;
 import stdx.d.parser;
 import dpick.buffer.buffer;
@@ -92,32 +93,33 @@ int main(string[] args)
 		return 1;
 	}
 
-	if (highlight)
+	StringCache cache;
-	{
+
-		bool usingStdin = args.length == 1;
+	if (tokenDump || highlight)
-		ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
-        LexerConfig config;
-        config.whitespaceBehavior = WhitespaceBehavior.include;
-        config.stringBehavior = StringBehavior.source;
-        config.commentBehavior = CommentBehavior.include;
-		auto tokens = byToken(bytes, config);
-		highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]);
-		return 0;
-	}
-	else if (tokenDump)
 	{
 		bool usingStdin = args.length == 1;
 		ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
 		LexerConfig config;
-        config.whitespaceBehavior = WhitespaceBehavior.skip;
+		config.whitespaceBehavior = WhitespaceBehavior.include;
-        config.stringBehavior = StringBehavior.source;
+		config.stringBehavior = StringBehavior.source;
-        config.commentBehavior = CommentBehavior.attach;
+		config.commentBehavior = CommentBehavior.include;
-		auto tokens = byToken(bytes, config);
+		auto tokens = byToken(bytes, config, cache);
-		foreach (ref token; tokens)
+		if (highlight)
 		{
-			writeln("«", token.text is null ? str(token.type) : token.text,
+			highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]);
-				" ", token.index, " ", token.line, " ", token.column, " ",
+			return 0;
-                token.comment, "»");
+		}
+		else if (tokenDump)
+		{
+			while (!tokens.empty)
+			{
+				auto token = tokens.front();
+				tokens.popFront();
+				writeln("«", token.text is null ? str(token.type) : token.text,
+					"» ", token.index, " ", token.line, " ", token.column, " ",
+					token.comment);
+			}
+			return 0;
 		}
 	}
 	else if (ctags)
@@ -135,11 +137,11 @@ int main(string[] args)
 		{
 			if (usingStdin)
 			{
-                LexerConfig config;
+				LexerConfig config;
-                config.whitespaceBehavior = WhitespaceBehavior.include;
+				config.whitespaceBehavior = WhitespaceBehavior.include;
-                config.stringBehavior = StringBehavior.source;
+				config.stringBehavior = StringBehavior.source;
-                config.commentBehavior = CommentBehavior.include;
+				config.commentBehavior = CommentBehavior.include;
-				auto tokens = byToken(readStdin(), config);
+				auto tokens = byToken(readStdin(), config, cache);
 				if (tokenCount)
 					printTokenCount(stdout, "stdin", tokens);
 				else
@@ -159,32 +161,26 @@ int main(string[] args)
 				writefln("total:\t%d", count);
 			}
 		}
-		else if (syntaxCheck)
+		else if (syntaxCheck || imports || ast || outline)
-		{
-			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
-			parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
-		}
-		else if (imports)
 		{
 			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
 			auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
-			auto visitor = new ImportPrinter;
+			if (imports)
-			visitor.visit(mod);
+			{
-		}
+				auto visitor = new ImportPrinter;
-		else if (ast)
+				visitor.visit(mod);
-		{
+			}
-			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
+			else if (ast)
-			auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
+			{
-			auto printer = new XMLPrinter;
+				auto printer = new XMLPrinter;
-			printer.output = stdout;
+				printer.output = stdout;
-			printer.visit(mod);
+				printer.visit(mod);
-		}
+			}
-		else if (outline)
+			else if (outline)
-		{
+			{
-			auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
+				auto outliner = new Outliner(stdout);
-			auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
+				outliner.visit(mod);
-			auto outliner = new Outliner(stdout);
+			}
-			outliner.visit(mod);
 		}
 	}
 	return 0;

stdx/d/lexer.d

@@ -6,6 +6,7 @@ import std.array;
 import std.algorithm;
 import std.range;
 import stdx.lexer;
+public import stdx.lexer : StringCache;
 
 private enum staticTokens = [
 	",", ".", "..", "...", "/", "/=", "!", "!<", "!<=", "!<>", "!<>=", "!=",
@@ -114,12 +115,19 @@ public struct LexerConfig
 public auto byToken(R)(R range)
 {
     LexerConfig config;
-    return byToken(range, config);
+	StringCache cache;
+    return byToken(range, config, cache);
 }
 
-public auto byToken(R)(R range, const LexerConfig config)
+public auto byToken(R)(R range, StringCache cache)
 {
-	return DLexer!(R)(range, config);
+	LexerConfig config;
+	return DLexer!(R)(range, config, cache);
+}
+
+public auto byToken(R)(R range, const LexerConfig config, StringCache cache)
+{
+	return DLexer!(R)(range, config, cache);
 }
 
 unittest
@@ -431,7 +439,7 @@ public struct DLexer(R)
 
 	private alias typeof(range).Mark Mark;
 
-	this(R range, const LexerConfig config)
+	this(R range, const LexerConfig config, StringCache cache)
 	{
 		this.range = LexerRange!(typeof(buffer(range)))(buffer(range));
         this.config = config;
@@ -448,7 +456,7 @@ public struct DLexer(R)
     {
         _popFront();
         string comment = null;
-        switch (_front.type)
+        switch (front.type)
         {
             case tok!"comment":
                 if (config.commentBehavior == CommentBehavior.attach)
@@ -573,7 +581,7 @@ public struct DLexer(R)
 				break loop;
 			}
 		} while (!range.empty);
-		return Token(tok!"whitespace", cast(string) range.slice(mark), line,
+		return Token(tok!"whitespace", cache.cacheGet(range.slice(mark)), line,
 			column, index);
 	}
 
@@ -654,7 +662,7 @@ public struct DLexer(R)
 				break hexLoop;
 			}
 		}
-		return Token(type, cast(string) range.slice(mark), line, column,
+		return Token(type, cache.cacheGet(range.slice(mark)), line, column,
 			index);
 	}
 
@@ -685,7 +693,7 @@ public struct DLexer(R)
 				break binaryLoop;
 			}
 		}
-		return Token(type, cast(string) range.slice(mark), line, column,
+		return Token(type, cache.cacheGet(range.slice(mark)), line, column,
 			index);
 	}
 
@@ -769,7 +777,7 @@ public struct DLexer(R)
 				break decimalLoop;
 			}
 		}
-		return Token(type, cast(string) range.slice(mark), line, column,
+		return Token(type, cache.cacheGet(range.slice(mark)), line, column,
 			index);
 	}
 
@@ -878,7 +886,7 @@ public struct DLexer(R)
 		mixin (tokenStart);
 		while (!range.empty && !isNewline)
 			range.popFront();
-		return Token(tok!"scriptLine", cast(string) range.slice(mark),
+		return Token(tok!"scriptLine", cache.cacheGet(range.slice(mark)),
 			line, column, index);
 	}
 
@@ -887,7 +895,7 @@ public struct DLexer(R)
 		mixin (tokenStart);
 		while (!range.empty && !isNewline)
 			range.popFront();
-		return Token(tok!"specialTokenSequence", cast(string) range.slice(mark),
+		return Token(tok!"specialTokenSequence", cache.cacheGet(range.slice(mark)),
 			line, column, index);
 	}
 
@@ -911,7 +919,7 @@ public struct DLexer(R)
 			else
 				popFrontWhitespaceAware();
 		}
-		return Token(type, cast(string) range.slice(mark), line, column,
+		return Token(type, cache.cacheGet(range.slice(mark)), line, column,
 			index);
 	}
 
@@ -927,7 +935,7 @@ public struct DLexer(R)
 				break;
 			range.popFront();
 		}
-		return Token(type, cast(string) range.slice(mark), line, column,
+		return Token(type, cache.cacheGet(range.slice(mark)), line, column,
 			index);
 	}
 
@@ -961,7 +969,7 @@ public struct DLexer(R)
 			else
 				popFrontWhitespaceAware();
 		}
-		return Token(type, cast(string) range.slice(mark), line, column,
+		return Token(type, cache.cacheGet(range.slice(mark)), line, column,
 			index);
 	}
 
@@ -990,7 +998,7 @@ public struct DLexer(R)
 		}
 		IdType type = tok!"stringLiteral";
 		lexStringSuffix(type);
-		return Token(type, cast(string) range.slice(mark), line, column,
+		return Token(type, cache.cacheGet(range.slice(mark)), line, column,
 			index);
 	}
 
@@ -1044,7 +1052,7 @@ public struct DLexer(R)
 			}
 		}
 		lexStringSuffix(type);
-		return Token(type, cast(string) range.slice(mark), line, column,
+		return Token(type, cache.cacheGet(range.slice(mark)), line, column,
 			index);
 	}
 
@@ -1131,7 +1139,7 @@ public struct DLexer(R)
 		}
 		IdType type = tok!"stringLiteral";
 		lexStringSuffix(type);
-		return Token(type, cast(string) range.slice(mark), line, column, index);
+		return Token(type, cache.cacheGet(range.slice(mark)), line, column, index);
 	}
 
 	Token lexHeredocString() pure nothrow
@@ -1149,7 +1157,7 @@ public struct DLexer(R)
 		auto app = appender!string();
 		app.put("q{");
 		int depth = 1;
-		
+
 		LexerConfig c = config;
 		scope(exit) config = c;
 		config.whitespaceBehavior = WhitespaceBehavior.include;
@@ -1180,7 +1188,8 @@ public struct DLexer(R)
 		}
 		IdType type = tok!"stringLiteral";
 		lexStringSuffix(type);
-		return Token(type, app.data, line, column, index);
+		return Token(type, cache.cacheGet(cast(const(ubyte)[]) app.data), line,
+			column, index);
 	}
 
 	Token lexHexString() pure nothrow
@@ -1216,7 +1225,7 @@ public struct DLexer(R)
 
 		IdType type = tok!"stringLiteral";
 		lexStringSuffix(type);
-		return Token(type, cast(string) range.slice(mark), line, column,
+		return Token(type, cache.cacheGet(range.slice(mark)), line, column,
 			index);
 	}
 
@@ -1325,7 +1334,7 @@ public struct DLexer(R)
 		else if (range.front == '\'')
 		{
 			range.popFront();
-			return Token(tok!"characterLiteral", cast(string) range.slice(mark),
+			return Token(tok!"characterLiteral", cache.cacheGet(range.slice(mark)),
 				line, column, index);
 		}
 		else if (range.front & 0x80)
@@ -1343,7 +1352,7 @@ public struct DLexer(R)
 		if (range.front == '\'')
 		{
 			range.popFront();
-			return Token(tok!"characterLiteral", cast(string) range.slice(mark),
+			return Token(tok!"characterLiteral", cache.cacheGet(range.slice(mark)),
 				line, column, index);
 		}
 		else
@@ -1360,7 +1369,7 @@ public struct DLexer(R)
 		{
 			range.popFront();
 		}
-		return Token(tok!"identifier", cast(string) range.slice(mark), line,
+		return Token(tok!"identifier", cache.cacheGet(range.slice(mark)), line,
 			column, index);
 	}
 
@@ -1400,7 +1409,7 @@ public struct DLexer(R)
 		range.popFront();
 		range.popFront();
 		range.incrementLine();
-		return Token(tok!"whitespace", cast(string) range.slice(mark), line,
+		return Token(tok!"whitespace", cache.cacheGet(range.slice(mark)), line,
 			column, index);
 	}
 
@@ -1431,7 +1440,7 @@ public struct DLexer(R)
 		size_t index = range.index;
 		size_t column = range.column;
 		size_t line = range.line;
-		auto mark = range.mark();
+		const mark = range.mark();
 	};
 
 	void error(...) pure {
@@ -1442,5 +1451,6 @@ public struct DLexer(R)
 
 	}
 
-    LexerConfig config;
+	StringCache cache;
+	LexerConfig config;
 }

stdx/lexer.d

@@ -41,7 +41,7 @@ string tokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens
 		return staticTokens[type - 1];
 	else if (type < staticTokens.length + possibleDefaultTokens.length + 1)
 		return possibleDefaultTokens[type - staticTokens.length - 1];
-    else if (type < staticTokens.length + possibleDefaultTokens.length + dynamicTokens.length + 1)
+	else if (type < staticTokens.length + possibleDefaultTokens.length + dynamicTokens.length + 1)
 		return dynamicTokens[type - staticTokens.length - possibleDefaultTokens.length - 1];
 	else
 		return null;
@@ -52,7 +52,7 @@ template TokenId(IdType, alias staticTokens, alias dynamicTokens,
 {
 	static if (symbol == "")
 	{
-	  enum id = 0;
+		enum id = 0;
 		alias id TokenId;
 	}
 	else static if (symbol == "\0")
@@ -272,6 +272,8 @@ mixin template Lexer(R, IDType, Token, alias defaultTokenFunction,
 	Token _front;
 }
 
+debug = 1;
+
 struct LexerRange(BufferType) if (isBuffer!BufferType)
 {
 	this(BufferType r)
@@ -301,3 +303,193 @@ struct LexerRange(BufferType) if (isBuffer!BufferType)
 	size_t column;
 	size_t line;
 }
+
+struct StringCache
+{
+public:
+
+	string cacheGet(const(ubyte[]) bytes) pure nothrow @safe
+	{
+		return get(cache(bytes));
+	}
+
+	size_t cache(const(ubyte)[] bytes) pure nothrow @safe
+	in
+	{
+		assert (bytes.length > 0);
+	}
+	body
+	{
+		immutable uint hash = hashBytes(bytes);
+		const(Item)* found = find(bytes, hash);
+		if (found is null)
+			return intern(bytes, hash);
+		return found.index;
+	}
+
+	string get(size_t index) const pure nothrow @safe
+	in
+	{
+		assert (items.length > index);
+		assert (items[index] !is null);
+	}
+	body
+	{
+		return items[index].str;
+	}
+
+private:
+
+	size_t intern(const(ubyte)[] bytes, uint hash) pure nothrow @safe
+	{
+		Item* item = new Item;
+		item.hash = hash;
+		item.str = allocate(bytes);
+		item.index = items.length;
+		items ~= item;
+		buckets[hash % bucketCount] ~= item;
+		return item.index;
+	}
+
+	const(Item)* find(const(ubyte)[] bytes, uint hash) pure nothrow const @safe
+	{
+		immutable size_t index = hash % buckets.length;
+		foreach (item; buckets[index])
+		{
+			if (item.hash == hash && bytes.equal(item.str))
+				return item;
+		}
+		return null;
+	}
+
+	string allocate(const(ubyte)[] bytes) pure nothrow @trusted
+	out (rVal)
+	{
+		assert (rVal == bytes);
+	}
+	body
+	{
+		import core.memory;
+		if (bytes.length > (pageSize / 4))
+		{
+			ubyte* memory = cast(ubyte*) GC.malloc(bytes.length, GC.BlkAttr.NO_SCAN);
+			memory[0 .. bytes.length] = bytes[];
+			return cast(string) memory[0..bytes.length];
+		}
+		foreach (ref block; blocks)
+		{
+			immutable size_t endIndex = block.used + bytes.length;
+			if (endIndex > block.bytes.length)
+				continue;
+			block.bytes[block.used .. endIndex] = bytes[];
+			string slice = cast(string) block.bytes[block.used .. endIndex];
+			block.used = endIndex;
+			return slice;
+		}
+		blocks.length = blocks.length + 1;
+		blocks[$ - 1].bytes = (cast(ubyte*) GC.malloc(pageSize, GC.BlkAttr.NO_SCAN))[0 .. pageSize];
+		blocks[$ - 1].bytes[0 .. bytes.length] = bytes[];
+		blocks[$ - 1].used = bytes.length;
+		return cast(string) blocks[$ - 1].bytes[0 .. bytes.length];
+	}
+
+	Item*[] items;
+	Item*[][bucketCount] buckets;
+	Block[] blocks;
+
+	struct Item
+	{
+		size_t index;
+		string str;
+		uint hash;
+	}
+
+	struct Block
+	{
+		ubyte[] bytes;
+		size_t used;
+	}
+
+	static uint hashBytes(const(ubyte)[] data) pure nothrow @safe
+    {
+        uint hash = 0;
+        foreach (b; data)
+        {
+            hash ^= sbox[b];
+            hash *= 3;
+        }
+        return hash;
+    }
+
+	enum pageSize = 4096 * 1024;
+	enum bucketCount = 2048;
+
+	static enum uint[] sbox = [
+		0xF53E1837, 0x5F14C86B, 0x9EE3964C, 0xFA796D53,
+		0x32223FC3, 0x4D82BC98, 0xA0C7FA62, 0x63E2C982,
+		0x24994A5B, 0x1ECE7BEE, 0x292B38EF, 0xD5CD4E56,
+		0x514F4303, 0x7BE12B83, 0x7192F195, 0x82DC7300,
+		0x084380B4, 0x480B55D3, 0x5F430471, 0x13F75991,
+		0x3F9CF22C, 0x2FE0907A, 0xFD8E1E69, 0x7B1D5DE8,
+		0xD575A85C, 0xAD01C50A, 0x7EE00737, 0x3CE981E8,
+		0x0E447EFA, 0x23089DD6, 0xB59F149F, 0x13600EC7,
+		0xE802C8E6, 0x670921E4, 0x7207EFF0, 0xE74761B0,
+		0x69035234, 0xBFA40F19, 0xF63651A0, 0x29E64C26,
+		0x1F98CCA7, 0xD957007E, 0xE71DDC75, 0x3E729595,
+		0x7580B7CC, 0xD7FAF60B, 0x92484323, 0xA44113EB,
+		0xE4CBDE08, 0x346827C9, 0x3CF32AFA, 0x0B29BCF1,
+		0x6E29F7DF, 0xB01E71CB, 0x3BFBC0D1, 0x62EDC5B8,
+		0xB7DE789A, 0xA4748EC9, 0xE17A4C4F, 0x67E5BD03,
+		0xF3B33D1A, 0x97D8D3E9, 0x09121BC0, 0x347B2D2C,
+		0x79A1913C, 0x504172DE, 0x7F1F8483, 0x13AC3CF6,
+		0x7A2094DB, 0xC778FA12, 0xADF7469F, 0x21786B7B,
+		0x71A445D0, 0xA8896C1B, 0x656F62FB, 0x83A059B3,
+		0x972DFE6E, 0x4122000C, 0x97D9DA19, 0x17D5947B,
+		0xB1AFFD0C, 0x6EF83B97, 0xAF7F780B, 0x4613138A,
+		0x7C3E73A6, 0xCF15E03D, 0x41576322, 0x672DF292,
+		0xB658588D, 0x33EBEFA9, 0x938CBF06, 0x06B67381,
+		0x07F192C6, 0x2BDA5855, 0x348EE0E8, 0x19DBB6E3,
+		0x3222184B, 0xB69D5DBA, 0x7E760B88, 0xAF4D8154,
+		0x007A51AD, 0x35112500, 0xC9CD2D7D, 0x4F4FB761,
+		0x694772E3, 0x694C8351, 0x4A7E3AF5, 0x67D65CE1,
+		0x9287DE92, 0x2518DB3C, 0x8CB4EC06, 0xD154D38F,
+		0xE19A26BB, 0x295EE439, 0xC50A1104, 0x2153C6A7,
+		0x82366656, 0x0713BC2F, 0x6462215A, 0x21D9BFCE,
+		0xBA8EACE6, 0xAE2DF4C1, 0x2A8D5E80, 0x3F7E52D1,
+		0x29359399, 0xFEA1D19C, 0x18879313, 0x455AFA81,
+		0xFADFE838, 0x62609838, 0xD1028839, 0x0736E92F,
+		0x3BCA22A3, 0x1485B08A, 0x2DA7900B, 0x852C156D,
+		0xE8F24803, 0x00078472, 0x13F0D332, 0x2ACFD0CF,
+		0x5F747F5C, 0x87BB1E2F, 0xA7EFCB63, 0x23F432F0,
+		0xE6CE7C5C, 0x1F954EF6, 0xB609C91B, 0x3B4571BF,
+		0xEED17DC0, 0xE556CDA0, 0xA7846A8D, 0xFF105F94,
+		0x52B7CCDE, 0x0E33E801, 0x664455EA, 0xF2C70414,
+		0x73E7B486, 0x8F830661, 0x8B59E826, 0xBB8AEDCA,
+		0xF3D70AB9, 0xD739F2B9, 0x4A04C34A, 0x88D0F089,
+		0xE02191A2, 0xD89D9C78, 0x192C2749, 0xFC43A78F,
+		0x0AAC88CB, 0x9438D42D, 0x9E280F7A, 0x36063802,
+		0x38E8D018, 0x1C42A9CB, 0x92AAFF6C, 0xA24820C5,
+		0x007F077F, 0xCE5BC543, 0x69668D58, 0x10D6FF74,
+		0xBE00F621, 0x21300BBE, 0x2E9E8F46, 0x5ACEA629,
+		0xFA1F86C7, 0x52F206B8, 0x3EDF1A75, 0x6DA8D843,
+		0xCF719928, 0x73E3891F, 0xB4B95DD6, 0xB2A42D27,
+		0xEDA20BBF, 0x1A58DBDF, 0xA449AD03, 0x6DDEF22B,
+		0x900531E6, 0x3D3BFF35, 0x5B24ABA2, 0x472B3E4C,
+		0x387F2D75, 0x4D8DBA36, 0x71CB5641, 0xE3473F3F,
+		0xF6CD4B7F, 0xBF7D1428, 0x344B64D0, 0xC5CDFCB6,
+		0xFE2E0182, 0x2C37A673, 0xDE4EB7A3, 0x63FDC933,
+		0x01DC4063, 0x611F3571, 0xD167BFAF, 0x4496596F,
+		0x3DEE0689, 0xD8704910, 0x7052A114, 0x068C9EC5,
+		0x75D0E766, 0x4D54CC20, 0xB44ECDE2, 0x4ABC653E,
+		0x2C550A21, 0x1A52C0DB, 0xCFED03D0, 0x119BAFE2,
+		0x876A6133, 0xBC232088, 0x435BA1B2, 0xAE99BBFA,
+		0xBB4F08E4, 0xA62B5F49, 0x1DA4B695, 0x336B84DE,
+		0xDC813D31, 0x00C134FB, 0x397A98E6, 0x151F0E64,
+		0xD9EB3E69, 0xD3C7DF60, 0xD2F2C336, 0x2DDD067B,
+		0xBD122835, 0xB0B3BD3A, 0xB0D54E46, 0x8641F1E4,
+		0xA0B38F96, 0x51D39199, 0x37A6AD75, 0xDF84EE41,
+		0x3C034CBA, 0xACDA62FC, 0x11923B8B, 0x45EF170A,
+	];
+}
+
+

style.d

@@ -13,6 +13,8 @@ import std.regex;
 import std.array;
 import std.conv;
 
+// TODO: Warn on assigning to non-ref foreach item.
+
 void doNothing(string, size_t, size_t, string, bool) {}
 
 void styleCheck(File output, string[] fileNames)
@@ -35,7 +37,7 @@ class StyleChecker : ASTVisitor
 	enum varFunNameRegex = `^([\p{Ll}_][_\w\d]*|[\p{Lu}\d_]+)$`;
 	enum aggregateNameRegex = `^\p{Lu}[\w\d]*$`;
 	enum moduleNameRegex = `^\p{Ll}+$`;
-	
+
 	override void visit(ModuleDeclaration dec)
 	{
 		foreach (part; dec.moduleName.identifiers)
@@ -45,36 +47,36 @@ class StyleChecker : ASTVisitor
 					"Module/package name ", part.text, " does not match style guidelines");
 		}
 	}
-	
+
 	override void visit(Declarator dec)
 	{
 		checkLowercaseName("Variable", dec.name);
 	}
-	
+
 	override void visit(FunctionDeclaration dec)
 	{
 		checkLowercaseName("Function", dec.name);
 	}
-	
+
 	void checkLowercaseName(string type, ref Token name)
 	{
 		if (name.text.matchFirst(varFunNameRegex).length == 0)
 			writeln(fileName, "(", name.line, ":", name.column, ") ",
 				type, " name ", name.text, " does not match style guidelines");
 	}
-	
+
 	override void visit(ClassDeclaration dec)
 	{
 		checkAggregateName("Class", dec.name);
 		dec.accept(this);
 	}
-	
+
 	override void visit(InterfaceDeclaration dec)
 	{
 		checkAggregateName("Interface", dec.name);
 		dec.accept(this);
 	}
-	
+
 	override void visit(EnumDeclaration dec)
 	{
 		if (dec.name.text is null || dec.name.text.length == 0)
@@ -82,13 +84,13 @@ class StyleChecker : ASTVisitor
 		checkAggregateName("Enum", dec.name);
 		dec.accept(this);
 	}
-	
+
 	override void visit(StructDeclaration dec)
 	{
 		checkAggregateName("Struct", dec.name);
 		dec.accept(this);
 	}
-	
+
 	void checkAggregateName(string aggregateType, ref Token name)
 	{
 		if (name.text.matchFirst(aggregateNameRegex).length == 0)