Able to autocomplete some symbols from other modules now. Still crashes a lot

actypes.d

@@ -18,6 +18,7 @@
 
 module actypes;
 
+import stdx.d.lexer;
 import stdx.d.ast;
 import std.algorithm;
 import std.stdio;
@@ -161,6 +162,34 @@ public:
         return null;
     }
 
+	void resolveSymbolTypes()
+	{
+		foreach (s; symbols.filter!(a => a.kind == CompletionKind.variableName)())
+		{
+			Type type = s.type;
+			if (type.typeSuffixes.length == 0)
+			{
+				if (type.type2.builtinType != TokenType.invalid)
+				{
+					s.resolvedType = findSymbolInCurrentScope(s.location,
+						getTokenValue(type.type2.builtinType));
+				}
+				else if (type.type2.symbol !is null)
+				{
+					Symbol sym = type.type2.symbol;
+					if (sym.identifierOrTemplateChain.identifiersOrTemplateInstances.length != 1)
+						return;
+					s.resolvedType = findSymbolInCurrentScope(s.location,
+						sym.identifierOrTemplateChain.identifiersOrTemplateInstances[0].identifier.value);
+				}
+			}
+		}
+		foreach (c; children)
+		{
+			c.resolveSymbolTypes();
+		}
+	}
+
 	/**
 	 * Index of the opening brace
 	 */

acvisitor.d

@@ -26,8 +26,9 @@ import std.stdio;
 
 import actypes;
 import messages;
+import modulecache;
 
-class AutoCompleteVisitor : ASTVisitor
+class AutocompleteVisitor : ASTVisitor
 {
 	alias ASTVisitor.visit visit;
 
@@ -35,6 +36,7 @@ class AutoCompleteVisitor : ASTVisitor
 	{
 		auto symbol = new ACSymbol;
 		symbol.name = dec.name.value;
+		symbol.location = dec.name.startIndex;
 		symbol.kind = CompletionKind.structName;
 		mixin (visitAndAdd);
 	}
@@ -43,6 +45,7 @@ class AutoCompleteVisitor : ASTVisitor
 	{
 		auto symbol = new ACSymbol;
 		symbol.name = dec.name.value;
+		symbol.location = dec.name.startIndex;
 		symbol.kind = CompletionKind.className;
 		mixin (visitAndAdd);
 	}
@@ -51,6 +54,7 @@ class AutoCompleteVisitor : ASTVisitor
 	{
 		auto symbol = new ACSymbol;
 		symbol.name = dec.name.value;
+		symbol.location = dec.name.startIndex;
 		symbol.kind = CompletionKind.interfaceName;
 		mixin (visitAndAdd);
 	}
@@ -59,6 +63,8 @@ class AutoCompleteVisitor : ASTVisitor
 	{
 		auto s = scope_;
 		scope_ = new Scope(structBody.startLocation, structBody.endLocation);
+		scope_.symbols ~= new ACSymbol("this", CompletionKind.variableName,
+			parentSymbol);
 		scope_.parent = s;
 		structBody.accept(this);
 		scope_ = s;
@@ -68,15 +74,16 @@ class AutoCompleteVisitor : ASTVisitor
 	{
 		auto symbol = new ACSymbol;
 		symbol.name = dec.name.value;
+		symbol.location = dec.name.startIndex;
 		symbol.kind = CompletionKind.enumName;
 		mixin (visitAndAdd);
 	}
 
 	override void visit(FunctionDeclaration dec)
 	{
-		writeln("Found function declaration ", dec.name.value);
 		auto symbol = new ACSymbol;
 		symbol.name = dec.name.value;
+		symbol.location = dec.name.startIndex;
 		symbol.kind = CompletionKind.functionName;
 		mixin (visitAndAdd);
 	}
@@ -86,7 +93,7 @@ class AutoCompleteVisitor : ASTVisitor
 		auto s = new ACSymbol;
 		s.kind = CompletionKind.enumMember;
 		s.name = member.name.value;
-//		writeln("Added enum member ", s.name);
+		s.location = member.name.startIndex;
 		if (parentSymbol !is null)
 			parentSymbol.parts ~= s;
 	}
@@ -95,10 +102,10 @@ class AutoCompleteVisitor : ASTVisitor
 	{
 		foreach (d; dec.declarators)
 		{
-			writeln("Found variable declaration ", d.name.value);
 			auto symbol = new ACSymbol;
 			symbol.type = dec.type;
 			symbol.name = d.name.value;
+			symbol.location = d.name.startIndex;
 			symbol.kind = CompletionKind.variableName;
 			if (parentSymbol is null)
 				symbols ~= symbol;
@@ -110,19 +117,22 @@ class AutoCompleteVisitor : ASTVisitor
 
 	override void visit(ImportDeclaration dec)
 	{
+		if (!currentFile) return;
 		foreach (singleImport; dec.singleImports)
 		{
-			imports ~= convertChainToImportPath(singleImport.identifierChain);
+			scope_.symbols ~= ModuleCache.getSymbolsInModule(
+				convertChainToImportPath(singleImport.identifierChain));
 		}
 		if (dec.importBindings !is null)
 		{
-			imports ~= convertChainToImportPath(dec.importBindings.singleImport.identifierChain);
+			scope_.symbols ~= ModuleCache.getSymbolsInModule(
+				convertChainToImportPath(
+					dec.importBindings.singleImport.identifierChain));
 		}
 	}
 
 	override void visit(BlockStatement blockStatement)
 	{
-		writeln("Processing block statement");
 		auto s = scope_;
 		scope_ = new Scope(blockStatement.startLocation,
 			blockStatement.endLocation);
@@ -157,6 +167,7 @@ class AutoCompleteVisitor : ASTVisitor
 	ACSymbol parentSymbol;
 	Scope scope_;
 	string[] imports = ["object"];
+	bool currentFile = false;
 
 private:
 	static enum string visitAndAdd = q{
@@ -174,10 +185,11 @@ private:
 
 void doesNothing(string, int, int, string) {}
 
-AutoCompleteVisitor processModule(const(Token)[] tokens)
+AutocompleteVisitor processModule(const(Token)[] tokens)
 {
-	Module mod = parseModule(tokens, "", null/*&doesNothing*/);
+	Module mod = parseModule(tokens, "", &doesNothing);
-	auto visitor = new AutoCompleteVisitor;
+	auto visitor = new AutocompleteVisitor;
+	visitor.currentFile = true;
 	visitor.visit(mod);
 	return visitor;
 }

build.sh

@@ -1,2 +1,2 @@
-dmd client.d messages.d msgpack-d/src/msgpack.d -Imsgpack-d/src -ofdcd-client
+dmd -wi client.d messages.d msgpack-d/src/msgpack.d -Imsgpack-d/src -ofdcd-client
-dmd server.d actypes.d messages.d constants.d acvisitor.d autocomplete.d ../dscanner/stdx/d/ast.d ../dscanner/stdx/d/parser.d ../dscanner/stdx/d/lexer.d ../dscanner/stdx/d/entities.d msgpack-d/src/msgpack.d -Imsgpack-d/src -I../dscanner/ -ofdcd-server
+dmd -wi server.d modulecache.d actypes.d messages.d constants.d acvisitor.d autocomplete.d ../dscanner/stdx/d/ast.d ../dscanner/stdx/d/parser.d ../dscanner/stdx/d/lexer.d ../dscanner/stdx/d/entities.d msgpack-d/src/msgpack.d -Imsgpack-d/src -I../dscanner/ -ofdcd-server

client.d

@@ -32,11 +32,12 @@ int main(string[] args)
     string[] importPaths;
     ushort port = 9166;
     bool help;
+	bool shutdown;
 
     try
     {
         getopt(args, "cursorPos|c", &cursorPos, "I", &importPaths,
-            "port|p", &port, "help|h", &help);
+			"port|p", &port, "help|h", &help, "shutdown", &shutdown);
     }
     catch (Exception e)
     {
@@ -49,6 +50,23 @@ int main(string[] args)
         return 0;
 	}
 
+	if (shutdown)
+	{
+		AutocompleteRequest request;
+		request.kind = RequestKind.shutdown;
+		auto socket = new TcpSocket(AddressFamily.INET);
+		scope (exit) { socket.shutdown(SocketShutdown.BOTH); socket.close(); }
+		socket.connect(new InternetAddress("127.0.0.1", port));
+		socket.blocking = true;
+		socket.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1);
+		ubyte[] message = msgpack.pack(request);
+		ubyte[] messageBuffer = new ubyte[message.length + message.length.sizeof];
+		auto messageLength = message.length;
+		messageBuffer[0 .. 8] = (cast(ubyte*) &messageLength)[0 .. 8];
+		messageBuffer[8 .. $] = message[];
+		return socket.send(messageBuffer) == messageBuffer.length ? 0 : 1;
+    }
+
     // cursor position is a required argument
     if (cursorPos == size_t.max)
     {
@@ -118,24 +136,32 @@ void printHelp(string programName)
 `
     Usage: %1$s --cursorPos NUMBER [options] [FILENAME]
        or: %1$s -cNUMBER [options] [FILENAME]
+       or: %1$s --clearCache
+       or: %1$s --shutdown
 
     A file name is optional. If it is given, autocomplete information will be
     given for the file specified. If it is missing, input will be read from
     stdin instead.
 
-    Source code is assumed to be UTF-8 encoded.
+    Source code is assumed to be UTF-8 encoded and must not exceed 4 megabytes.
-
-Mandatory Arguments:
-    --cursorPos | -c position
-        Provides auto-completion at the given cursor position. The cursor
-        position is measured in bytes from the beginning of the source code.
 
 Options:
     --help | -h
         Displays this help message
 
+    --cursorPos | -c position
+        Provides auto-completion at the given cursor position. The cursor
+        position is measured in bytes from the beginning of the source code.
+
+    --clearCache
+        Instructs the server to clear out its autocompletion cache.
+
+    --shutdown
+        Instructs the server to shut down.
+
     -IPATH
-        Includes PATH in the listing of paths that are searched for file imports
+        Instructs the server to add PATH to its list of paths searced for
+        imported modules.
 
     --port PORTNUMBER | -pPORTNUMBER
         Uses PORTNUMBER to communicate with the server instead of the default

messages.d

@@ -87,7 +87,8 @@ enum RequestKind
 {
 	autocomplete,
 	clearCache,
-	addImport
+	addImport,
+	shutdown
 }
 
 /**

modulecache.d

@@ -20,8 +20,14 @@ module modulecache;
 
 import std.file;
 import std.datetime;
+import stdx.d.lexer;
+import stdx.d.parser;
+import stdx.d.ast;
+import std.stdio;
+import std.array;
 
 import acvisitor;
+import actypes;
 
 struct CacheEntry
 {
@@ -39,58 +45,80 @@ struct ModuleCache
 	/**
 	 * Clears the completion cache
 	 */
-	void clear()
+	static void clear()
 	{
-		cache = [];
+		cache.clear();
 	}
 
 	/**
 	 * Adds the given path to the list of directories checked for imports
 	 */
-	void addImportPath(string path)
+	static void addImportPath(string path)
 	{
 		importPaths ~= path;
 	}
 
 	/**
 	 * Params:
-	 *     moduleName = the name of the module in "a.b.c" form
+	 *     moduleName = the name of the module in "a/b.d" form
 	 * Returns:
 	 *     The symbols defined in the given module
 	 */
-	ACSymbol[] getSymbolsInModule(string moduleName)
+	static ACSymbol[] getSymbolsInModule(string moduleName)
 	{
 		string location = resolveImportLoctation(moduleName);
+		if (location is null)
+			return [];
 		if (!needsReparsing(location))
-			return;
+			return cache[location].symbols;
 
+		File f = File(location);
+		ubyte[] source = uninitializedArray!(ubyte[])(f.size);
+		f.rawRead(source);
+
+		LexerConfig config;
+		auto tokens = source.byToken(config).array();
 		Module mod = parseModule(tokens, location, &doesNothing);
 		auto visitor = new AutocompleteVisitor;
 		visitor.visit(mod);
-		cache[location].mod = visitor.symbols;
+		SysTime access;
+		SysTime modification;
+		getTimes(location, access, modification);
+		if (location !in cache)
+			cache[location] = CacheEntry.init;
+		cache[location].modificationTime = modification;
+		cache[location].symbols = visitor.symbols;
+		return cache[location].symbols;
 	}
 
 	/**
 	 * Params:
-	 *     moduleName the name of the module being imported, in "a.b.c" style
+	 *     moduleName the name of the module being imported, in "a/b/c.d" style
 	 * Returns:
 	 *     The absolute path to the file that contains the module, or null if
 	 *     not found.
 	 */
-	string resolveImportLoctation(string moduleName)
+	static string resolveImportLoctation(string moduleName)
 	{
+		writeln("Resolving location of ", moduleName);
 		foreach (path; importPaths)
 		{
-			string filePath = path ~ "/" ~ imp;
+			string filePath = path ~ "/" ~ moduleName;
 			if (filePath.exists())
 				return filePath;
 			filePath ~= "i"; // check for x.di if x.d isn't found
 			if (filePath.exists())
 				return filePath;
 		}
+		writeln("Could not find ", moduleName);
 		return null;
 	}
 
+	static const(string[]) getImportPaths()
+	{
+		return cast(const(string[])) importPaths;
+	}
+
 private:
 
 	/**
@@ -99,7 +127,7 @@ private:
 	 * Returns:
 	 *     true  if the module needs to be reparsed, false otherwise
 	 */
-    bool needsReparsing(string mod)
+    static bool needsReparsing(string mod)
     {
         if (!exists(mod) || mod !in cache)
             return true;
@@ -110,8 +138,8 @@ private:
     }
 
 	// Mapping of file paths to their cached symbols.
-    CacheEntry[string] cache;
+    static CacheEntry[string] cache;
 
 	// Listing of paths to check for imports
-	string[] importPaths;
+	static string[] importPaths;
 }

server.d

@@ -21,11 +21,13 @@ module server;
 import std.socket;
 import std.stdio;
 import std.getopt;
+import std.algorithm;
 
 import msgpack;
 
 import messages;
 import autocomplete;
+import modulecache;
 
 int main(string[] args)
 {
@@ -44,6 +46,18 @@ int main(string[] args)
         return 1;
     }
 
+	// begin hack
+	importPaths ~= "/home/alaran/src/dcd";
+	importPaths ~= "/home/alaran/src/dscanner";
+	importPaths ~= "/usr/include/d2/core";
+	importPaths ~= "/usr/include/d2/phobos";
+	importPaths ~= "/usr/include/d2/druntime/import";
+	// end hack
+
+	foreach (path; importPaths)
+		ModuleCache.addImportPath(path);
+	writeln(ModuleCache.getImportPaths());
+
     auto socket = new TcpSocket(AddressFamily.INET);
     socket.blocking = true;
     socket.bind(new InternetAddress("127.0.0.1", port));
@@ -84,11 +98,16 @@ int main(string[] args)
 		msgpack.unpack(buffer[8 .. bytesReceived], request);
 		if (request.kind == RequestKind.addImport)
 		{
-
+			//ModuleCache.addImportPath();
 		}
         else if (request.kind == RequestKind.clearCache)
 		{
-
+			ModuleCache.clear();
+		}
+		else if (request.kind == RequestKind.shutdown)
+		{
+			writeln("Shutting down.");
+			break;
 		}
 		else
         {