Updated DScanner and msgpack-d

.gitignore

@@ -1,3 +1,10 @@
 # Windows binaries
 *.exe
 *.obj
+
+# *nix binaries
+dcd-client
+dcd-server
+
+# Perf reports
+perf.data

astconverter.d

@@ -116,7 +116,7 @@ final class FirstPass : ASTVisitor
 	override void visit(FunctionDeclaration dec)
 	{
 //		Log.trace(__FUNCTION__, " ", typeof(dec).stringof);
-		SemanticSymbol* symbol = new SemanticSymbol(dec.name.text.dup,
+		SemanticSymbol* symbol = new SemanticSymbol(getCached(dec.name.text),
 			CompletionKind.functionName, symbolFile, dec.name.index);
 		processParameters(symbol, dec.returnType, symbol.acSymbol.name,
 			dec.parameters, dec.comment);
@@ -169,7 +169,7 @@ final class FirstPass : ASTVisitor
 		foreach (declarator; dec.declarators)
 		{
 			SemanticSymbol* symbol = new SemanticSymbol(
-				declarator.name.text.dup,
+				getCached(declarator.name.text),
 				CompletionKind.variableName,
 				symbolFile,
 				declarator.name.index);
@@ -185,7 +185,7 @@ final class FirstPass : ASTVisitor
 		if (aliasDeclaration.initializers.length == 0)
 		{
 			SemanticSymbol* symbol = new SemanticSymbol(
-				aliasDeclaration.name.text.dup,
+				getCached(aliasDeclaration.name.text),
 				CompletionKind.aliasName,
 				symbolFile,
 				aliasDeclaration.name.index);
@@ -199,7 +199,7 @@ final class FirstPass : ASTVisitor
 			foreach (initializer; aliasDeclaration.initializers)
 			{
 				SemanticSymbol* symbol = new SemanticSymbol(
-					initializer.name.text.dup,
+					getCached(initializer.name.text),
 					CompletionKind.aliasName,
 					symbolFile,
 					initializer.name.index);
@@ -214,7 +214,7 @@ final class FirstPass : ASTVisitor
 	override void visit(AliasThisDeclaration dec)
 	{
 //		Log.trace(__FUNCTION__, " ", typeof(dec).stringof);
-		currentSymbol.aliasThis ~= dec.identifier.text.dup;
+		currentSymbol.aliasThis ~= getCached(dec.identifier.text);
 	}
 
 	override void visit(Declaration dec)
@@ -256,7 +256,7 @@ final class FirstPass : ASTVisitor
 	{
 		assert (currentSymbol);
 //		Log.trace(__FUNCTION__, " ", typeof(dec).stringof);
-		SemanticSymbol* symbol = new SemanticSymbol(dec.name.text.dup,
+		SemanticSymbol* symbol = new SemanticSymbol(getCached(dec.name.text),
 			CompletionKind.enumName, symbolFile, dec.name.index);
 		symbol.type = dec.type;
 		symbol.parent = currentSymbol;
@@ -270,7 +270,7 @@ final class FirstPass : ASTVisitor
 	override void visit(EnumMember member)
 	{
 //		Log.trace(__FUNCTION__, " ", typeof(member).stringof);
-		SemanticSymbol* symbol = new SemanticSymbol(member.name.text.dup,
+		SemanticSymbol* symbol = new SemanticSymbol(getCached(member.name.text),
 			CompletionKind.enumMember, symbolFile, member.name.index);
 		symbol.type = member.type;
 		symbol.parent = currentSymbol;
@@ -281,7 +281,7 @@ final class FirstPass : ASTVisitor
 	{
 //		Log.trace(__FUNCTION__, " ", typeof(dec).stringof);
 		foreach (Token t; dec.moduleName.identifiers)
-			moduleName ~= t.text.dup;
+			moduleName ~= getCached(t.text);
 	}
 
 	// creates scopes for
@@ -326,8 +326,8 @@ final class FirstPass : ASTVisitor
 		foreach (bind; importDeclaration.importBindings.importBinds)
 		{
 			Tuple!(string, string) bindTuple;
-			bindTuple[0] = bind.left.text.dup;
-			bindTuple[1] = bind.right == tok!"" ? null : bind.right.text.dup;
+			bindTuple[0] = getCached(bind.left.text);
+			bindTuple[1] = bind.right == tok!"" ? null : getCached(bind.right.text);
 			info.importedSymbols ~= bindTuple;
 		}
 		info.isPublic = protection == tok!"public";
@@ -377,7 +377,7 @@ private:
 	void visitAggregateDeclaration(AggType)(AggType dec, CompletionKind kind)
 	{
 //		Log.trace("visiting aggregate declaration ", dec.name.text);
-		SemanticSymbol* symbol = new SemanticSymbol(dec.name.text.dup,
+		SemanticSymbol* symbol = new SemanticSymbol(getCached(dec.name.text),
 			kind, symbolFile, dec.name.index);
 		symbol.acSymbol.parts ~= classSymbols;
 		symbol.parent = currentSymbol;
@@ -409,7 +409,7 @@ private:
 	{
 		SemanticSymbol* symbol = new SemanticSymbol("~this",
 			CompletionKind.functionName, symbolFile, location);
-		symbol.acSymbol.callTip = formatComment(doc) ~ "~this()";
+		symbol.acSymbol.callTip = /*formatComment(doc) ~*/ "~this()";
 		symbol.protection = protection;
 		symbol.parent = currentSymbol;
 		currentSymbol.addChild(symbol);
@@ -428,7 +428,7 @@ private:
 		{
 			foreach (Parameter p; parameters.parameters)
 			{
-				SemanticSymbol* parameter = new SemanticSymbol(p.name.text.dup,
+				SemanticSymbol* parameter = new SemanticSymbol(getCached(p.name.text),
 					CompletionKind.variableName, symbolFile, size_t.max);
 				parameter.type = p.type;
 				symbol.addChild(parameter);
@@ -460,8 +460,11 @@ private:
 		string parameterString = parameters is null ? "()"
 			: formatNode(parameters);
 		if (returnType is null)
-			return "%s%s%s".format(formatComment(doc), name, parameterString);
-		return "%s%s %s%s".format(formatComment(doc), formatNode(returnType), name, parameterString);
+			return "%s%s".format(name, parameterString);
+		return "%s %s%s".format(formatNode(returnType), name, parameterString);
+//		if (returnType is null)
+//			return "%s%s%s".format(formatComment(doc), name, parameterString);
+//		return "%s%s %s%s".format(formatComment(doc), formatNode(returnType), name, parameterString);
 	}
 
 	/// Current protection type
@@ -708,7 +711,7 @@ private:
 		return s;
 	}
 
-	static string[] expandSymbol(const IdentifierOrTemplateChain chain) pure
+	static string[] expandSymbol(const IdentifierOrTemplateChain chain)
 	{
 		string[] strings = new string[chain.identifiersOrTemplateInstances.length];
 		for (size_t i = 0; i != chain.identifiersOrTemplateInstances.length; ++i)
@@ -716,9 +719,9 @@ private:
 			auto identOrTemplate = chain.identifiersOrTemplateInstances[i];
 			if (identOrTemplate is null)
 				continue;
-			strings[i] = identOrTemplate.templateInstance is null ?
-				identOrTemplate.identifier.text.dup
-				: identOrTemplate.identifier.text.dup;
+			strings[i] = getCached(identOrTemplate.templateInstance is null ?
+				identOrTemplate.identifier.text
+				: identOrTemplate.templateInstance.identifier.text);
 		}
 		return strings;
 	}
@@ -864,9 +867,9 @@ string[] iotcToStringArray(const IdentifierOrTemplateChain iotc)
 	foreach (ioti; iotc.identifiersOrTemplateInstances)
 	{
 		if (ioti.identifier != tok!"")
-			parts ~= ioti.identifier.text.dup;
+			parts ~= getCached(ioti.identifier.text);
 		else
-			parts ~= ioti.templateInstance.identifier.text.dup;
+			parts ~= getCached(ioti.templateInstance.identifier.text);
 	}
 	return parts;
 }
@@ -918,12 +921,18 @@ string formatComment(string comment)
 		.replaceAll(regex("\n"), `\n`).outdent();
 }
 
+string getCached(string s)
+{
+	return s.length == 0 ? ""
+		: ModuleCache.stringCache.cacheGet(cast(const(ubyte)[]) s);
+}
+
 //unittest
 //{
 //	auto comment1 = "/**\n * This is some text\n */";
 //	auto result1 = formatComment(comment1);
 //	assert (result1 == `This is some text\n\n`, result1);
-//	
+//
 //	auto comment2 = "///some\n///text";
 //	auto result2 = formatComment(comment2);
 //	assert (result2 == `some\ntext\n\n`, result2);

autocomplete.d

@@ -46,7 +46,8 @@ AutocompleteResponse findDeclaration(const AutocompleteRequest request)
 	AutocompleteResponse response;
 	LexerConfig config;
 	config.fileName = "stdin";
-	auto tokens = byToken(cast(ubyte[]) request.sourceCode, config);
+	StringCache* cache = new StringCache(StringCache.defaultBucketCount);
+	auto tokens = byToken(cast(ubyte[]) request.sourceCode, config, cache);
 	const(Token)[] tokenArray = void;
 	try {
 		tokenArray = tokens.array();
@@ -256,7 +257,9 @@ AutocompleteResponse complete(const AutocompleteRequest request)
 
 	LexerConfig config;
 	config.fileName = "stdin";
-	auto tokens = byToken(cast(ubyte[]) request.sourceCode, config);
+	StringCache* cache = new StringCache(StringCache.defaultBucketCount);
+	auto tokens = byToken(cast(ubyte[]) request.sourceCode, config,
+		cache);
 	const(Token)[] tokenArray = void;
 	try {
 		tokenArray = tokens.array();

build.sh

@@ -24,13 +24,10 @@ dmd \
 	dscanner/stdx/d/lexer.d\
 	dscanner/stdx/d/entities.d\
 	dscanner/formatter.d\
-	datapicked/dpick/buffer/buffer.d\
-	datapicked/dpick/buffer/traits.d\
 	msgpack-d/src/msgpack.d\
 	-Imsgpack-d/src\
 	-Idscanner\
-	-Idatapicked\
 	-wi\
-	-g\
+	-O -release -noboundscheck -inline\
 	-ofdcd-server
-	
+

datapicked

@@ -1 +0,0 @@
-Subproject commit f63a843e9c0ce8db7fd897684fe323697255d87d

dscanner

@@ -1 +1 @@
-Subproject commit c08602e0b7f1423e7430284f74183e81a7845e59
+Subproject commit c01c51a61ea4f2e6a94a64396be0abd9e8249bab

modulecache.d

@@ -20,6 +20,7 @@ module modulecache;
 
 import std.file;
 import std.datetime;
+import stdx.lexer;
 import stdx.d.lexer;
 import stdx.d.parser;
 import stdx.d.ast;
@@ -126,7 +127,8 @@ struct ModuleCache
 
 			LexerConfig config;
 			config.fileName = location;
-			auto tokens = source.byToken(config).array();
+			StringCache* cache = new StringCache(StringCache.defaultBucketCount);
+			auto tokens = source.byToken(config, cache).array();
 			symbols = convertAstToSymbols(tokens, location);
 
 			// Parsing allocates a lot of AST nodes. We can greatly reduce the
@@ -190,6 +192,13 @@ struct ModuleCache
 		return cast(const(string[])) importPaths;
 	}
 
+	static this()
+	{
+		stringCache = new StringCache(StringCache.defaultBucketCount);
+	}
+
+	static StringCache* stringCache;
+
 private:
 
 	/**

server.d

@@ -98,6 +98,7 @@ int main(string[] args)
 	Log.info("Startup completed in ", sw.peek().to!("msecs", float), " milliseconds");
 	ModuleCache.estimateMemory();
 
+
     // No relative paths
 	version (Posix) chdir("/");