Update dparse and dsymbol

dsymbol

@@ -1 +1 @@
-Subproject commit f32e35a5d2827ccbd3b6842e8d257dd84a5acb71
+Subproject commit 3bc0ee1d6a403fc59c5f5926af72f5be59269a36

libdparse

@@ -1 +1 @@
-Subproject commit adb134c054ffcb05c74d7033e59617dedba1493b
+Subproject commit 1bdfe95a212ada95fec973f0335a42e495841b4a

makefile

@@ -19,8 +19,8 @@ INCLUDE_PATHS = \
 	-Ilibdparse/experimental_allocator/src\
 	-Idsymbol/src -Icontainers/src
 VERSIONS =
-DEBUG_VERSIONS = -version=std_parser_verbose
+DEBUG_VERSIONS = -version=dparse_verbose
-DMD_FLAGS = -w -O -inline -J. -od${OBJ_DIR} -version=StdLoggerDisableWarning
+DMD_FLAGS = -w -inline -release -O -J. -od${OBJ_DIR} -version=StdLoggerDisableWarning
 DMD_TEST_FLAGS = -w -g -unittest -J.
 
 all: dmdbuild

src/analysis/helpers.d

@@ -10,9 +10,12 @@ import std.traits;
 import std.stdio;
 
 import dparse.ast;
+import dparse.rollback_allocator;
 import analysis.config;
 import analysis.run;
 import analysis.base;
+import std.experimental.allocator.mallocator;
+import std.experimental.allocator;
 
 S between(S)(S value, S before, S after) if (isSomeString!S)
 {
@@ -21,24 +24,18 @@ S between(S)(S value, S before, S after) if (isSomeString!S)
 
 S before(S)(S value, S separator) if (isSomeString!S)
 {
-	auto i = indexOf(value, separator);
+	immutable i = indexOf(value, separator);
-
 	if (i == -1)
 		return value;
-
 	return value[0 .. i];
 }
 
 S after(S)(S value, S separator) if (isSomeString!S)
 {
-	auto i = indexOf(value, separator);
+	immutable i = indexOf(value, separator);
-
 	if (i == -1)
 		return "";
-
+	return value[i + separator.length .. $];
-	size_t start = i + separator.length;
-
-	return value[start .. $];
 }
 
 /**
@@ -49,15 +46,15 @@ S after(S)(S value, S separator) if (isSomeString!S)
 void assertAnalyzerWarnings(string code, const StaticAnalysisConfig config,
 		string file = __FILE__, size_t line = __LINE__)
 {
-	import analysis.run : ParseAllocator, parseModule;
+	import analysis.run : parseModule;
 	import dparse.lexer : StringCache, Token;
 
 	StringCache cache = StringCache(StringCache.defaultBucketCount);
-	ParseAllocator p = new ParseAllocator;
+	RollbackAllocator r;
 	const(Token)[] tokens;
-	const(Module) m = parseModule(file, cast(ubyte[]) code, p, cache, false, tokens);
+	const(Module) m = parseModule(file, cast(ubyte[]) code, &r, cache, false, tokens);
 
-	auto moduleCache = ModuleCache(p);
+	auto moduleCache = ModuleCache(new CAllocatorImpl!Mallocator);
 
 	// Run the code and get any warnings
 	MessageSet rawWarnings = analyze("test", m, config, moduleCache, tokens);

src/analysis/run.d

@@ -14,6 +14,7 @@ import std.array;
 import dparse.lexer;
 import dparse.parser;
 import dparse.ast;
+import dparse.rollback_allocator;
 import std.typecons : scoped;
 
 import std.experimental.allocator : CAllocatorImpl;
@@ -115,9 +116,9 @@ void generateReport(string[] fileNames, const StaticAnalysisConfig config,
 			continue;
 		auto code = uninitializedArray!(ubyte[])(to!size_t(f.size));
 		f.rawRead(code);
-		ParseAllocator p = new ParseAllocator;
+		RollbackAllocator r;
 		const(Token)[] tokens;
-		const Module m = parseModule(fileName, code, p, cache, true, tokens, &lineOfCodeCount);
+		const Module m = parseModule(fileName, code, &r, cache, true, tokens, &lineOfCodeCount);
 		stats.visit(m);
 		MessageSet results = analyze(fileName, m, config, moduleCache, tokens, true);
 		foreach (result; results[])
@@ -154,11 +155,11 @@ bool analyze(string[] fileNames, const StaticAnalysisConfig config,
 			continue;
 		auto code = uninitializedArray!(ubyte[])(to!size_t(f.size));
 		f.rawRead(code);
-		ParseAllocator p = new ParseAllocator;
+		RollbackAllocator r;
 		uint errorCount = 0;
 		uint warningCount = 0;
 		const(Token)[] tokens;
-		const Module m = parseModule(fileName, code, p, cache, false, tokens,
+		const Module m = parseModule(fileName, code, &r, cache, false, tokens,
 				null, &errorCount, &warningCount);
 		assert(m);
 		if (errorCount > 0 || (staticAnalyze && warningCount > 0))
@@ -173,7 +174,7 @@ bool analyze(string[] fileNames, const StaticAnalysisConfig config,
 	return hasErrors;
 }
 
-const(Module) parseModule(string fileName, ubyte[] code, ParseAllocator p,
+const(Module) parseModule(string fileName, ubyte[] code, RollbackAllocator* p,
 		ref StringCache cache, bool report, ref const(Token)[] tokens,
 		ulong* linesOfCode = null, uint* errorCount = null, uint* warningCount = null)
 {

src/ctags.d

@@ -8,6 +8,7 @@ module ctags;
 import dparse.parser;
 import dparse.lexer;
 import dparse.ast;
+import dparse.rollback_allocator;
 import std.algorithm;
 import std.range;
 import std.stdio;
@@ -15,6 +16,7 @@ import std.array;
 import std.conv;
 import std.typecons;
 import containers.ttree;
+import std.string;
 
 /**
  * Prints CTAGS information to the given file.
@@ -30,13 +32,14 @@ void printCtags(File output, string[] fileNames)
 	StringCache cache = StringCache(StringCache.defaultBucketCount);
 	foreach (fileName; fileNames)
 	{
+		RollbackAllocator rba;
 		File f = File(fileName);
 		if (f.size == 0)
 			continue;
 		auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
 		f.rawRead(bytes);
 		auto tokens = getTokensForParser(bytes, config, &cache);
-		Module m = parseModule(tokens.array, fileName, null, &doNothing);
+		Module m = parseModule(tokens.array, fileName, &rba, &doNothing);
 
 		auto printer = new CTagsPrinter(&tags);
 		printer.fileName = fileName;

src/etags.d

@@ -9,12 +9,14 @@ module etags;
 import dparse.parser;
 import dparse.lexer;
 import dparse.ast;
+import dparse.rollback_allocator;
 import std.algorithm;
 import std.range;
 import std.stdio;
 import std.path;
 import std.array;
 import std.conv;
+import std.string;
 
 // Prefix tags with their module name.	Seems like correct behavior, but just
 // in case, make it an option.
@@ -39,13 +41,14 @@ void printEtags(File output, bool tagAll, string[] fileNames)
 	StringCache cache = StringCache(StringCache.defaultBucketCount);
 	foreach (fileName; fileNames)
 	{
+		RollbackAllocator rba;
 		File f = File(fileName);
 		if (f.size == 0)
 			continue;
 		auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
 		f.rawRead(bytes);
 		auto tokens = getTokensForParser(bytes, config, &cache);
-		Module m = parseModule(tokens.array, fileName, null, &doNothing);
+		Module m = parseModule(tokens.array, fileName, &rba, &doNothing);
 
 		auto printer = new EtagsPrinter;
 		printer.moduleName = m.moduleFullName(fileName);

src/main.d

@@ -16,6 +16,7 @@ import std.range;
 import std.experimental.lexer;
 import dparse.lexer;
 import dparse.parser;
+import dparse.rollback_allocator;
 
 import highlighter;
 import stats;
@@ -257,6 +258,7 @@ else
 		else if (imports)
 		{
 			string[] fileNames = usingStdin ? ["stdin"] : args[1 .. $];
+			RollbackAllocator rba;
 			LexerConfig config;
 			config.stringBehavior = StringBehavior.source;
 			auto visitor = new ImportPrinter;
@@ -265,7 +267,7 @@ else
 				config.fileName = name;
 				auto tokens = getTokensForParser(usingStdin ? readStdin()
 						: readFile(name), config, &cache);
-				auto mod = parseModule(tokens, name, null, &doNothing);
+				auto mod = parseModule(tokens, name, &rba, &doNothing);
 				visitor.visit(mod);
 			}
 			foreach (imp; visitor.imports[])
@@ -274,12 +276,13 @@ else
 		else if (ast || outline)
 		{
 			string fileName = usingStdin ? "stdin" : args[1];
+			RollbackAllocator rba;
 			LexerConfig config;
 			config.fileName = fileName;
 			config.stringBehavior = StringBehavior.source;
 			auto tokens = getTokensForParser(usingStdin ? readStdin()
 					: readFile(args[1]), config, &cache);
-			auto mod = parseModule(tokens, fileName, null, &doNothing);
+			auto mod = parseModule(tokens, fileName, &rba, &doNothing);
 
 			if (ast)
 			{

src/symbol_finder.d

@@ -9,6 +9,7 @@ import std.stdio : File;
 import dparse.lexer;
 import dparse.parser;
 import dparse.ast;
+import dparse.rollback_allocator;
 import std.stdio;
 import std.file : isFile;
 
@@ -29,7 +30,8 @@ void findDeclarationOf(File output, string symbolName, string[] fileNames)
 		auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
 		f.rawRead(bytes);
 		auto tokens = getTokensForParser(bytes, config, &cache);
-		Module m = parseModule(tokens.array, fileName, null, &doNothing);
+		RollbackAllocator rba;
+		Module m = parseModule(tokens.array, fileName, &rba, &doNothing);
 		visitor.fileName = fileName;
 		visitor.visit(m);
 	}