From 72ad182f2120cbf64d57283f4d9e3f2649cf6292 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Tue, 10 Sep 2013 22:50:16 -0700 Subject: [PATCH 01/27] Fixed cache clearing --- modulecache.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulecache.d b/modulecache.d index 46d8585..95c53d0 100644 --- a/modulecache.d +++ b/modulecache.d @@ -51,7 +51,7 @@ struct ModuleCache */ static void clear() { - cache.clear(); + cache = cache.init; } /** From e2be6948e9bd82a239c3bb6bd0ad16c6f766f174 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Wed, 9 Oct 2013 02:16:11 -0700 Subject: [PATCH 02/27] 0.2.0 development work --- actypes.d | 443 +++++++++++++++------------------ acvisitor.d | 570 ------------------------------------------ astconverter.d | 656 +++++++++++++++++++++++++++++++++++++++++++++++++ autocomplete.d | 235 +++++------------- build.sh | 28 ++- constants.d | 87 +++++++ dscanner | 2 +- messages.d | 13 +- modulecache.d | 41 ++-- semantic.d | 61 +++++ server.d | 54 +++- stupidlog.d | 72 ++++++ 12 files changed, 1247 insertions(+), 1015 deletions(-) delete mode 100644 acvisitor.d create mode 100644 astconverter.d create mode 100644 semantic.d create mode 100644 stupidlog.d diff --git a/actypes.d b/actypes.d index ef15612..02e3aa2 100644 --- a/actypes.d +++ b/actypes.d @@ -24,13 +24,12 @@ import std.algorithm; import std.stdio; import std.array; import messages; -import autocomplete; import std.array; /** * Any special information about a variable declaration symbol. */ -enum SymbolQualifier +enum SymbolQualifier : ubyte { /// _none none, @@ -45,12 +44,10 @@ enum SymbolQualifier /** * Autocompletion symbol */ -class ACSymbol +struct ACSymbol { public: - this() {} - /** * Params: * name = the symbol's name @@ -77,23 +74,34 @@ public: * kind = the symbol's completion kind * resolvedType = the resolved type of the symbol */ - this(string name, CompletionKind kind, ACSymbol resolvedType) + this(string name, CompletionKind kind, ACSymbol* type) { this.name = name; this.kind = kind; - this.resolvedType = resolvedType; + this.type = type; } + /** + * Comparison operator sorts based on the name field + */ + int opCmp(string str) const + { + if (str < this.name) return -1; + if (str > this.name) return 1; + return 0; + } + + /// ditto + int opCmp(const(ACSymbol)* other) const + { + return this.opCmp(other.name); + } + /** * Symbols that compose this symbol, such as enum members, class variables, * methods, etc. */ - ACSymbol[] parts; - - /** - * Listing of superclasses - */ - string[] superClasses; + ACSymbol*[] parts; /** * Symbol's name @@ -101,14 +109,14 @@ public: string name; /** - * Symbol's location in bytes + * The symbol that represents the type. */ - size_t location; + ACSymbol* type; /** - * Any special information about this symbol + * Calltip to display if this is a function */ - SymbolQualifier qualifier; + string callTip; /** * The kind of symbol @@ -116,250 +124,207 @@ public: CompletionKind kind; /** - * The return type if this is a function, or the element type if this is an - * array or associative array, or the variable type if this is a variable. - * This field is null if this symbol is a class + * Symbol qualifier */ - Type type; + SymbolQualifier qualifier; /** - * The symbol that represents the type. _resolvedType is an autocomplete - * class, type is an AST class, so after a module is parsed the symbols - * need to be post-processed to tie variable declarations to the symbols - * that actually contain the correct autocomplete information. + * Symbol location */ - ACSymbol resolvedType; + size_t location; /** - * Calltip to display if this is a function + * Module containing the symbol. */ - string calltip; + string symbolFile; - /** - * Finds symbol parts by name - */ - ACSymbol[] getPartsByName(string name) + const(ACSymbol)*[] getPartsByName(string name) const { - return parts.filter!(a => a.name == name).array; + return cast(typeof(return)) parts.filter!(a => a.name == name).array; + } + + /** + * Sorts the parts array, and the parts array of each part, and so on. + */ + void sortParts() + { + parts.sort(); + foreach (p; parts) + p.sortParts(); + } +} + +mixin template scopeImplementation(ScopeType) +{ + ScopeType* parent; + ScopeType*[] children; + size_t startLocation; + size_t endLocation; + + ScopeType* getScopeByCursor(size_t cursorPosition) const + { + if (cursorPosition < startLocation) return null; + if (cursorPosition > endLocation) return null; + foreach (child; children) + { + auto childScope = child.getScopeByCursor(cursorPosition); + if (childScope !is null) + return childScope; + } + return cast(typeof(return)) &this; } } +struct Scope +{ + mixin scopeImplementation!(typeof(this)); + + ACSymbol*[] getSymbolsInCursorScope(size_t cursorPosition) const + { + auto s = getScopeByCursor(cursorPosition); + if (s is null) + return []; + return cast(typeof(return)) s.symbols; + } + + const(ACSymbol)*[] getSymbolsByName(string name) const + { + const(ACSymbol)*[] retVal = cast(typeof(return)) symbols.filter!(a => a.name == name).array(); + if (retVal.length > 0) + return retVal; + if (parent is null) + return []; + return parent.getSymbolsByName(name); + } + + const(ACSymbol)*[] getSymbolsByNameAndCursor(string name, size_t cursorPosition) const + { + auto s = getScopeByCursor(cursorPosition); + if (s is null) + return []; + return s.getSymbolsByName(name); + } + + ACSymbol*[] symbols; +} + + /** - * Scope such as a block statement, struct body, etc. + * Initializes builtin types and the various properties of builtin types */ -class Scope +static this() { -public: + // TODO: make sure all parts are sorted. + auto bool_ = new ACSymbol("bool", CompletionKind.keyword); + auto int_ = new ACSymbol("int", CompletionKind.keyword); + auto long_ = new ACSymbol("long", CompletionKind.keyword); + auto byte_ = new ACSymbol("byte", CompletionKind.keyword); + auto char_ = new ACSymbol("char", CompletionKind.keyword); + auto dchar_ = new ACSymbol("dchar", CompletionKind.keyword); + auto short_ = new ACSymbol("short", CompletionKind.keyword); + auto ubyte_ = new ACSymbol("ubyte", CompletionKind.keyword); + auto uint_ = new ACSymbol("uint", CompletionKind.keyword); + auto ulong_ = new ACSymbol("ulong", CompletionKind.keyword); + auto ushort_ = new ACSymbol("ushort", CompletionKind.keyword); + auto wchar_ = new ACSymbol("wchar", CompletionKind.keyword); - /** - * Params: - * start = the index of the opening brace - * end = the index of the closing brace - */ - this(size_t start, size_t end) + auto alignof_ = new ACSymbol("alignof", CompletionKind.keyword, ulong_); + auto mangleof_ = new ACSymbol("mangleof", CompletionKind.keyword); + auto sizeof_ = new ACSymbol("sizeof", CompletionKind.keyword, ulong_); + auto stringof_ = new ACSymbol("stringof", CompletionKind.keyword); + + arraySymbols ~= alignof_; + arraySymbols ~= new ACSymbol("dup", CompletionKind.keyword); + arraySymbols ~= new ACSymbol("idup", CompletionKind.keyword); + arraySymbols ~= new ACSymbol("init", CompletionKind.keyword); + arraySymbols ~= new ACSymbol("length", CompletionKind.keyword, ulong_); + arraySymbols ~= mangleof_; + arraySymbols ~= new ACSymbol("ptr", CompletionKind.keyword); + arraySymbols ~= new ACSymbol("reverse", CompletionKind.keyword); + arraySymbols ~= sizeof_; + arraySymbols ~= new ACSymbol("sort", CompletionKind.keyword); + arraySymbols ~= stringof_; + arraySymbols.sort(); + + assocArraySymbols ~= alignof_; + assocArraySymbols ~= new ACSymbol("byKey", CompletionKind.keyword); + assocArraySymbols ~= new ACSymbol("byValue", CompletionKind.keyword); + assocArraySymbols ~= new ACSymbol("dup", CompletionKind.keyword); + assocArraySymbols ~= new ACSymbol("get", CompletionKind.keyword); + assocArraySymbols ~= new ACSymbol("init", CompletionKind.keyword); + assocArraySymbols ~= new ACSymbol("keys", CompletionKind.keyword); + assocArraySymbols ~= new ACSymbol("length", CompletionKind.keyword, ulong_); + assocArraySymbols ~= mangleof_; + assocArraySymbols ~= new ACSymbol("rehash", CompletionKind.keyword); + assocArraySymbols ~= sizeof_; + assocArraySymbols ~= stringof_; + assocArraySymbols ~= new ACSymbol("values", CompletionKind.keyword); + assocArraySymbols.sort(); + + foreach (s; [bool_, int_, long_, byte_, char_, dchar_, short_, ubyte_, uint_, + ulong_, ushort_, wchar_]) { - this.start = start; - this.end = end; + s.parts ~= new ACSymbol("init", CompletionKind.keyword, s); + s.parts ~= new ACSymbol("min", CompletionKind.keyword, s); + s.parts ~= new ACSymbol("max", CompletionKind.keyword, s); + s.parts ~= alignof_; + s.parts ~= sizeof_; + s.parts ~= stringof_; + s.parts ~= mangleof_; + s.sortParts(); } - /** - * Gets all symbols in the scope that contains the cursor as well as its - * parent scopes. - */ - ACSymbol[] getSymbolsInCurrentScope(size_t cursorPosition) + auto cdouble_ = new ACSymbol("cdouble", CompletionKind.keyword); + auto cent_ = new ACSymbol("cent", CompletionKind.keyword); + auto cfloat_ = new ACSymbol("cfloat", CompletionKind.keyword); + auto creal_ = new ACSymbol("creal", CompletionKind.keyword); + auto double_ = new ACSymbol("double", CompletionKind.keyword); + auto float_ = new ACSymbol("float", CompletionKind.keyword); + auto idouble_ = new ACSymbol("idouble", CompletionKind.keyword); + auto ifloat_ = new ACSymbol("ifloat", CompletionKind.keyword); + auto ireal_ = new ACSymbol("ireal", CompletionKind.keyword); + auto real_ = new ACSymbol("real", CompletionKind.keyword); + auto ucent_ = new ACSymbol("ucent", CompletionKind.keyword); + + foreach (s; [cdouble_, cent_, cfloat_, creal_, double_, float_, + idouble_, ifloat_, ireal_, real_, ucent_]) { - Scope s = findCurrentScope(cursorPosition); - if (s is null) - return []; - else - return s.getSymbols(); + s.parts ~= alignof_; + s.parts ~= new ACSymbol("dig", CompletionKind.keyword, s); + s.parts ~= new ACSymbol("epsilon", CompletionKind.keyword, s); + s.parts ~= new ACSymbol("infinity", CompletionKind.keyword, s); + s.parts ~= new ACSymbol("init", CompletionKind.keyword, s); + s.parts ~= mangleof_; + s.parts ~= new ACSymbol("mant_dig", CompletionKind.keyword, int_); + s.parts ~= new ACSymbol("max", CompletionKind.keyword, s); + s.parts ~= new ACSymbol("max_10_exp", CompletionKind.keyword, int_); + s.parts ~= new ACSymbol("max_exp", CompletionKind.keyword, int_); + s.parts ~= new ACSymbol("min", CompletionKind.keyword, s); + s.parts ~= new ACSymbol("min_exp", CompletionKind.keyword, int_); + s.parts ~= new ACSymbol("min_10_exp", CompletionKind.keyword, int_); + s.parts ~= new ACSymbol("min_normal", CompletionKind.keyword, s); + s.parts ~= new ACSymbol("nan", CompletionKind.keyword, s); + s.parts ~= sizeof_; + s.parts ~= stringof_; + s.sortParts(); } - /** - * Gets all symbols in this scope and its parent scopes. - */ - ACSymbol[] getSymbols() - { - ACSymbol[] rVal; - rVal ~= symbols; - if (parent !is null) - rVal ~= parent.getSymbols(); - return rVal; - } + ireal_.parts ~= new ACSymbol("im", CompletionKind.keyword, real_); + ifloat_.parts ~= new ACSymbol("im", CompletionKind.keyword, float_); + idouble_.parts ~= new ACSymbol("im", CompletionKind.keyword, double_); + ireal_.parts ~= new ACSymbol("re", CompletionKind.keyword, real_); + ifloat_.parts ~= new ACSymbol("re", CompletionKind.keyword, float_); + idouble_.parts ~= new ACSymbol("re", CompletionKind.keyword, double_); - /** - * Finds the scope containing the cursor position, then searches for a - * symbol with the given name. - */ - ACSymbol[] findSymbolsInCurrentScope(size_t cursorPosition, string name) - { - auto s = findCurrentScope(cursorPosition); - if (s is null) - { - writeln("Could not find scope"); - return []; - } - else - return s.findSymbolsInScope(name); - } + auto void_ = new ACSymbol("void", CompletionKind.keyword); - /** - * Returns: the innermost Scope that contains the given cursor position. - */ - Scope findCurrentScope(size_t cursorPosition) - { - if (start != size_t.max && (cursorPosition < start || cursorPosition > end)) - return null; - foreach (sc; children) - { - auto s = sc.findCurrentScope(cursorPosition); - if (s is null) - continue; - else - return s; - } - return this; - } - - /** - * Finds a symbol with the given name in this scope or one of its parent - * scopes. - */ - ACSymbol[] findSymbolsInScope(string name) - { - ACSymbol[] currentMatches = symbols.filter!(a => a.name == name)().array(); - if (currentMatches.length == 0 && parent !is null) - return parent.findSymbolsInScope(name); - return currentMatches; - } - - /** - * Fills in the $(D resolvedType) fields of the symbols in this scope and - * all child scopes. - */ - void resolveSymbolTypes() - { - // We only care about resolving types of variables, all other symbols - // don't have any indirection - foreach (ref s; symbols.filter!(a => (a.kind == CompletionKind.variableName - || a.kind == CompletionKind.functionName || a.kind == CompletionKind.memberVariableName - || a.kind == CompletionKind.enumMember || a.kind == CompletionKind.aliasName) - && a.resolvedType is null)()) - { - //writeln("Resolving type of symbol ", s.name); - Type type = s.type; - if (type is null) - { - //writeln("Could not find it due to null type"); - continue; - } - if (type.type2.builtinType != TokenType.invalid) - { - // This part is easy. Autocomplete properties of built-in types - auto foundSymbols = findSymbolsInScope(getTokenValue(type.type2.builtinType)); - s.resolvedType = foundSymbols[0]; - } - else if (type.type2.symbol !is null) - { - // Look up a type by its name for cases like class, enum, - // interface, struct, or union members. - - // TODO: Does not work with qualified names or template instances - Symbol sym = type.type2.symbol; - if (sym.identifierOrTemplateChain.identifiersOrTemplateInstances.length != 1) - { - writeln("Could not resolve type"); - continue; - } - ACSymbol[] resolvedType = findSymbolsInCurrentScope(s.location, - sym.identifierOrTemplateChain.identifiersOrTemplateInstances[0].identifier.value); - if (resolvedType.length > 0 && (resolvedType[0].kind == CompletionKind.interfaceName - || resolvedType[0].kind == CompletionKind.className - || resolvedType[0].kind == CompletionKind.aliasName - || resolvedType[0].kind == CompletionKind.unionName - || resolvedType[0].kind == CompletionKind.structName)) - { -// writeln("Type resolved to ", resolvedType[0].name, " which has kind ", -// resolvedType[0].kind, " and call tip ", resolvedType[0].calltip); - s.resolvedType = resolvedType[0]; - } - } - - foreach (suffix; type.typeSuffixes) - { - //writeln("Handling type suffix"); - // Handle type suffixes for declarations, e.g.: - // int[] a; - // SomeClass[string] b; - // double function(double, double) c; - auto sym = s.resolvedType; - s.resolvedType = new ACSymbol; - s.resolvedType.resolvedType = sym; - if (suffix.array) - { - if (suffix.type !is null) - { - // assocative array - s.resolvedType.qualifier = SymbolQualifier.assocArray; - s.resolvedType.parts ~= assocArraySymbols; - } - else - { - // normal array - s.resolvedType.qualifier = SymbolQualifier.array; - s.resolvedType.parts ~= arraySymbols; - } - } - else if (suffix.delegateOrFunction.type != TokenType.invalid) - { - s.resolvedType.qualifier = SymbolQualifier.func; - } - } - } - - foreach (c; children) - { - c.resolveSymbolTypes(); - } - - foreach (ref ACSymbol c; symbols.filter!(a => a.kind == CompletionKind.className - || a.kind == CompletionKind.interfaceName)) - { - foreach (string sc; c.superClasses) - { - //writeln("Adding inherited fields from ", sc); - ACSymbol[] s = findSymbolsInScope(sc); - if (s.length > 0) - { - foreach (part; s[0].parts) - c.parts ~= part; - } - } - } - } - - /** - * Index of the opening brace - */ - size_t start = size_t.max; - - /** - * Index of the closing brace - */ - size_t end = size_t.max; - - /** - * Symbols contained in this scope - */ - ACSymbol[] symbols; - - /** - * The parent scope - */ - Scope parent; - - /** - * Child scopes - */ - Scope[] children; + builtinSymbols = [bool_, int_, long_, byte_, char_, dchar_, short_, ubyte_, uint_, + ulong_, ushort_, wchar_, cdouble_, cent_, cfloat_, creal_, double_, + float_, idouble_, ifloat_, ireal_, real_, ucent_, void_]; } + +ACSymbol*[] builtinSymbols; +ACSymbol*[] arraySymbols; +ACSymbol*[] assocArraySymbols; +ACSymbol*[] classSymbols; +ACSymbol*[] structSymbols; diff --git a/acvisitor.d b/acvisitor.d deleted file mode 100644 index 89baff2..0000000 --- a/acvisitor.d +++ /dev/null @@ -1,570 +0,0 @@ -/** - * This file is part of DCD, a development tool for the D programming language. - * Copyright (C) 2013 Brian Schott - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -module acvisitor; - -import std.file; -import stdx.d.parser; -import stdx.d.ast; -import stdx.d.lexer; -import std.stdio; -import std.algorithm; -import std.path; -import std.range; -import std.conv; - -import actypes; -import messages; -import modulecache; -import autocomplete; - -/** - * Converts an AST into a simple symbol and scope heirarchy so that the - * autocompletion coed can do its job more easily. - */ -class AutocompleteVisitor : ASTVisitor -{ - alias ASTVisitor.visit visit; - - override void visit(Unittest dec) - { -// writeln("Unitttest visit"); - auto symbol = new ACSymbol("*unittest*"); - auto p = parentSymbol; - parentSymbol = symbol; - dec.accept(this); - parentSymbol = p; - } - - override void visit(StructDeclaration dec) - { -// writeln("StructDeclaration visit"); - auto symbol = new ACSymbol; - symbol.name = dec.name.value; - symbol.location = dec.name.startIndex; - symbol.kind = CompletionKind.structName; - mixin (visitAndAdd); - } - - override void visit(ClassDeclaration dec) - { -// writeln("ClassDeclaration visit"); - auto symbol = new ACSymbol; - symbol.name = dec.name.value; - symbol.location = dec.name.startIndex; - symbol.kind = CompletionKind.className; - mixin (visitAndAdd); - } - - override void visit(ForStatement forStatement) - { - if (forStatement.declarationOrStatement is null) goto visitBody; - if (forStatement.declarationOrStatement.declaration is null) goto visitBody; - if (forStatement.declarationOrStatement.declaration.variableDeclaration is null) goto visitBody; - if (forStatement.statementNoCaseNoDefault is null) goto visitBody; - if (forStatement.statementNoCaseNoDefault.blockStatement is null) goto visitBody; - -// writeln("Visiting for statement"); - - ACSymbol[] symbols; - VariableDeclaration varDec = forStatement.declarationOrStatement.declaration.variableDeclaration; - Type t = varDec.type; - foreach (Declarator declarator; varDec.declarators) - { - ACSymbol symbol = new ACSymbol(); - symbol.name = declarator.name.value; - symbol.type = t; - symbol.kind = CompletionKind.variableName; - symbols ~= symbol; - //writeln("For statement variable ", symbol.name, " of type ", symbol.type, " added."); - } - BlockStatement block = forStatement.statementNoCaseNoDefault.blockStatement; - auto s = new Scope(forStatement.startIndex, - block.endLocation); - s.parent = scope_; - scope_.children ~= s; - auto p = scope_; - scope_ = s; - - foreach (symbol; symbols) - { - //writeln("added ", symbol.name, " to scope"); - symbol.location = scope_.start; - scope_.symbols ~= symbol; - - } - if (block.declarationsAndStatements !is null) - { - //writeln("visiting body"); - visit(block.declarationsAndStatements); - } - scope_ = p; - return; - - visitBody: -// writeln("visiting body"); - if (forStatement.statementNoCaseNoDefault !is null) - visit(forStatement.statementNoCaseNoDefault); - } - - override void visit(ForeachStatement statement) - { - ACSymbol[] symbols; - - if (statement.foreachTypeList is null) - { - statement.statementNoCaseNoDefault.accept(this); - } - else if (statement.foreachType !is null) - { - ACSymbol loopVariable = new ACSymbol(statement.foreachType.identifier.value); - loopVariable.type = statement.foreachType.type; - loopVariable.kind = CompletionKind.variableName; - symbols ~= loopVariable; - } - else foreach (ForeachType feType; statement.foreachTypeList.items.filter!(a => a.type !is null)) - { - ACSymbol loopVariable = new ACSymbol(feType.identifier.value); - loopVariable.type = feType.type; - loopVariable.kind = CompletionKind.variableName; - symbols ~= loopVariable; - } - - if (statement.statementNoCaseNoDefault !is null - && statement.statementNoCaseNoDefault.blockStatement !is null) - { - BlockStatement block = statement.statementNoCaseNoDefault.blockStatement; - auto s = scope_; - scope_ = new Scope(statement.startIndex, - block.endLocation); - scope_.parent = s; - foreach (symbol; symbols) - { - symbol.location = block.startLocation; - scope_.symbols ~= symbol; - } - if (block.declarationsAndStatements !is null) - block.declarationsAndStatements.accept(this); - s.children ~= scope_; - scope_ = s; - } - } - - override void visit(InterfaceDeclaration dec) - { -// writeln("InterfaceDeclaration visit"); - auto symbol = new ACSymbol; - symbol.name = dec.name.value; - symbol.location = dec.name.startIndex; - symbol.kind = CompletionKind.interfaceName; - mixin (visitAndAdd); - } - - override void visit(StructBody structBody) - { -// writeln("StructBody visit"); - auto s = scope_; - scope_ = new Scope(structBody.startLocation, structBody.endLocation); - scope_.symbols ~= new ACSymbol("this", CompletionKind.variableName, - parentSymbol); - scope_.parent = s; - s.children ~= scope_; - structBody.accept(this); - scope_ = s; - } - - override void visit(EnumDeclaration dec) - { - // TODO: Set enum type based on initializer of first member -// writeln("EnumDeclaration visit"); - auto symbol = new ACSymbol; - symbol.name = dec.name.value; - symbol.location = dec.name.startIndex; - symbol.kind = CompletionKind.enumName; - auto type = dec.type; - auto p = parentSymbol; - parentSymbol = symbol; - - if (dec.enumBody !is null) - { - Scope enumBodyScope = new Scope(dec.enumBody.startLocation, - dec.enumBody.endLocation); - foreach (member; dec.enumBody.enumMembers) - { - auto s = new ACSymbol; - s.kind = CompletionKind.enumMember; - s.name = member.name.value; - s.location = member.name.startIndex; - if (type is null) - s.resolvedType = scope_.findSymbolsInScope("int")[0]; - else - s.type = type; - if (parentSymbol !is null) - parentSymbol.parts ~= s; - enumBodyScope.symbols ~= s; - } - enumBodyScope.parent = scope_; - scope_.children ~= enumBodyScope; - } - - parentSymbol = p; - if (parentSymbol is null) - symbols ~= symbol; - else - parentSymbol.parts ~= symbol; - scope_.symbols ~= symbol; - } - - override void visit(Constructor dec) - { - ACSymbol symbol = new ACSymbol("*constructor*"); - symbol.location = dec.location; - symbol.kind = CompletionKind.functionName; - //symbol.type = dec.returnType; - - ACSymbol[] parameterSymbols; - if (dec.parameters !is null) - { - foreach (parameter; dec.parameters.parameters) - { -// writeln("Adding parameter ", parameter.name.value); - ACSymbol paramSymbol = new ACSymbol; - paramSymbol.name = parameter.name.value; - paramSymbol.type = parameter.type; - paramSymbol.kind = CompletionKind.variableName; - paramSymbol.location = parameter.name.startIndex; - parameterSymbols ~= paramSymbol; - } - } - - if (dec.parameters !is null && parentSymbol !is null) - { - symbol.calltip = format("%s this%s", parentSymbol.name, - formatNode(dec.parameters)); - } - auto p = parentSymbol; - parentSymbol = symbol; - - BlockStatement functionBody = dec.functionBody is null ? null - : (dec.functionBody.bodyStatement !is null - ? dec.functionBody.bodyStatement.blockStatement : dec.functionBody.blockStatement); - - if (functionBody !is null) - { - auto s = scope_; - scope_ = new Scope(functionBody.startLocation, - functionBody.endLocation); - scope_.parent = s; - foreach (parameterSymbol; parameterSymbols) - { - parameterSymbol.location = functionBody.startLocation; - scope_.symbols ~= parameterSymbol; - } - if (functionBody.declarationsAndStatements !is null) - functionBody.declarationsAndStatements.accept(this); - s.children ~= scope_; - scope_ = s; - } - - parentSymbol = p; - if (parentSymbol is null) - symbols ~= symbol; - else - parentSymbol.parts ~= symbol; - scope_.symbols ~= symbol; - } - - override void visit(FunctionDeclaration dec) - { - ACSymbol symbol = new ACSymbol; - symbol.name = dec.name.value; - symbol.location = dec.name.startIndex; - symbol.kind = CompletionKind.functionName; - symbol.type = dec.returnType; - - ACSymbol[] parameterSymbols; - if (dec.parameters !is null) - { - foreach (parameter; dec.parameters.parameters) - { -// writeln("Adding parameter ", parameter.name.value); - ACSymbol paramSymbol = new ACSymbol; - paramSymbol.name = parameter.name.value; - paramSymbol.type = parameter.type; - paramSymbol.kind = CompletionKind.variableName; - paramSymbol.location = parameter.name.startIndex; - parameterSymbols ~= paramSymbol; - } - } - - if (dec.parameters !is null) - { - string returnType; - if (dec.returnType !is null) - returnType = formatNode(dec.returnType); - else - { - if (dec.hasAuto) - { - returnType = "auto"; - if (dec.hasRef) - returnType = "auto ref"; - } - else if (dec.hasRef) - returnType = "ref"; - } - symbol.calltip = formatCalltip(dec.returnType, dec.name.value, dec.parameters); - } - auto p = parentSymbol; - parentSymbol = symbol; - - BlockStatement functionBody = dec.functionBody is null ? null - : (dec.functionBody.bodyStatement !is null - ? dec.functionBody.bodyStatement.blockStatement : dec.functionBody.blockStatement); - - if (functionBody !is null) - { - auto s = scope_; - scope_ = new Scope(functionBody.startLocation, - functionBody.endLocation); - scope_.parent = s; - foreach (parameterSymbol; parameterSymbols) - { - parameterSymbol.location = functionBody.startLocation; - scope_.symbols ~= parameterSymbol; - } - if (functionBody.declarationsAndStatements !is null) - functionBody.declarationsAndStatements.accept(this); - s.children ~= scope_; - scope_ = s; - } - - parentSymbol = p; - if (parentSymbol is null) - symbols ~= symbol; - else - parentSymbol.parts ~= symbol; - scope_.symbols ~= symbol; - } - - override void visit(VariableDeclaration dec) - { -// writeln("VariableDeclaration visit"); - foreach (d; dec.declarators) - { - ACSymbol symbol = new ACSymbol; - if (dec.type.typeSuffixes.length > 0 - && dec.type.typeSuffixes[$-1].delegateOrFunction != TokenType.invalid) - { - TypeSuffix suffix = dec.type.typeSuffixes[$ - 1]; - dec.type.typeSuffixes = dec.type.typeSuffixes[0 .. $ - 1]; - symbol.calltip = formatCalltip(dec.type, - suffix.delegateOrFunction.value, suffix.parameters); - } - symbol.kind = CompletionKind.variableName; - - symbol.type = dec.type; - symbol.name = d.name.value; - symbol.location = d.name.startIndex; - - if (parentSymbol is null) - symbols ~= symbol; - else - parentSymbol.parts ~= symbol; - scope_.symbols ~= symbol; - } - } - - override void visit(AliasDeclaration dec) - { - if (dec.type is null) foreach (aliasPart; dec.initializers) - { - ACSymbol aliasSymbol = new ACSymbol; - aliasSymbol.kind = CompletionKind.aliasName; - aliasSymbol.location = aliasPart.name.startIndex; - aliasSymbol.type = aliasPart.type; - if (aliasPart.type.typeSuffixes.length > 0 - && aliasPart.type.typeSuffixes[$-1].delegateOrFunction != TokenType.invalid) - { - TypeSuffix suffix = aliasPart.type.typeSuffixes[$ - 1]; - aliasPart.type.typeSuffixes = aliasPart.type.typeSuffixes[0 .. $ - 1]; - aliasSymbol.calltip = formatCalltip(dec.type, suffix.delegateOrFunction.value, suffix.parameters); - } - if (parentSymbol is null) - symbols ~= aliasSymbol; - else - parentSymbol.parts ~= aliasSymbol; - scope_.symbols ~= aliasSymbol; - } - else - { -// writeln("Visiting alias declaration ", dec.name.value); - ACSymbol aliasSymbol = new ACSymbol; - aliasSymbol.kind = CompletionKind.aliasName; - aliasSymbol.name = dec.name.value; - aliasSymbol.type = dec.type; - if (dec.type.typeSuffixes.length > 0 - && dec.type.typeSuffixes[$-1].delegateOrFunction != TokenType.invalid) - { - TypeSuffix suffix = dec.type.typeSuffixes[$ - 1]; - dec.type.typeSuffixes = dec.type.typeSuffixes[0 .. $ - 1]; - aliasSymbol.calltip = "%s %s%s".format(formatNode(dec.type), - suffix.delegateOrFunction.value, - formatNode(suffix.parameters)); - } - aliasSymbol.location = dec.name.startIndex; - if (parentSymbol is null) - symbols ~= aliasSymbol; - else - parentSymbol.parts ~= aliasSymbol; - scope_.symbols ~= aliasSymbol; - } - - } - - override void visit(ImportDeclaration dec) - { - // TODO: handle public imports - if (!currentFile) return; - foreach (singleImport; dec.singleImports.filter!(a => a !is null - && a.identifierChain !is null)) - { - scope_.symbols ~= ModuleCache.getSymbolsInModule( - convertChainToImportPath(singleImport.identifierChain)); - } - if (dec.importBindings !is null - && dec.importBindings.singleImport.identifierChain !is null) - { - ACSymbol[] importedSymbols = ModuleCache.getSymbolsInModule( - convertChainToImportPath(dec.importBindings.singleImport.identifierChain)); - foreach (ImportBind b; dec.importBindings.importBinds) - { - if (b.right == TokenType.invalid) - { - // Selecive import - foreach (ACSymbol symbol; importedSymbols.filter!(a => a.name == b.left)) - scope_.symbols ~= symbol; - } - else - { - // renamed selective import - foreach (ACSymbol symbol; importedSymbols.filter!(a => a.name == b.right)) - { - ACSymbol s = new ACSymbol; - s.kind = symbol.kind; - s.location = symbol.location; - s.name = b.left.value; - s.parts = symbol.parts; - s.qualifier = symbol.qualifier; - s.resolvedType = symbol.resolvedType; - s.superClasses = symbol.superClasses; - s.type = symbol.type; - scope_.symbols ~= s; - } - } - } - } - } - - override void visit(BaseClassList classList) - { - if (parentSymbol is null) - return; - foreach (BaseClass bc; classList.items) - { - if (bc.identifierOrTemplateChain is null) - continue; - if (bc.identifierOrTemplateChain.identifiersOrTemplateInstances.length != 1) - continue; - IdentifierOrTemplateInstance i = bc.identifierOrTemplateChain.identifiersOrTemplateInstances[0]; - if (i is null || i.identifier == TokenType.invalid) - continue; - parentSymbol.superClasses ~= i.identifier.value; - } - } - - override void visit(BlockStatement blockStatement) - { - auto s = scope_; - scope_ = new Scope(blockStatement.startLocation, - blockStatement.endLocation); - scope_.parent = s; - blockStatement.accept(this); - s.children ~= scope_; - scope_ = s; - } - - override void visit(Module mod) - { - scope_ = new Scope(0, size_t.max); - scope_.symbols ~= builtinSymbols; - mod.accept(this); - } - - private static string convertChainToImportPath(IdentifierChain chain) - { - return to!string(chain.identifiers.map!(a => a.value).join(dirSeparator).array) ~ ".d"; - } - - ACSymbol[] symbols; - ACSymbol parentSymbol; - Scope scope_; - string[] imports = ["object"]; - bool currentFile = false; - -private: - - static string formatCalltip(Type returnType, string name, Parameters parameters, - string doc = null) - { - return "%s %s%s".format(formatNode(returnType), name, formatNode(parameters)); - } - - static string formatNode(T)(T node) - { - if (node is null) return ""; - import formatter; - auto app = appender!(char[])(); - auto f = new Formatter!(typeof(app))(app); - f.format(node); - return to!string(app.data); - } - - static enum string visitAndAdd = q{ - auto p = parentSymbol; - parentSymbol = symbol; - dec.accept(this); - parentSymbol = p; - if (parentSymbol is null) - symbols ~= symbol; - else - parentSymbol.parts ~= symbol; - scope_.symbols ~= symbol; - }; -} - -void doesNothing(string, int, int, string) {} - -AutocompleteVisitor processModule(const(Token)[] tokens) -{ - Module mod = parseModule(tokens, "", &doesNothing); - auto visitor = new AutocompleteVisitor; - visitor.currentFile = true; - visitor.visit(mod); - return visitor; -} diff --git a/astconverter.d b/astconverter.d new file mode 100644 index 0000000..e00c4bd --- /dev/null +++ b/astconverter.d @@ -0,0 +1,656 @@ +/******************************************************************************* + * Authors: Brian Schott + * Copyright: Brian Schott + * Date: Sep 21 2013 + * + * License: + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + ******************************************************************************/ + +module astconverter; + +import std.array; +import std.conv; +import std.range; +import std.algorithm; + +import stdx.d.ast; +import stdx.d.lexer; +import stdx.d.parser; + +import actypes; +import messages; +import semantic; +import stupidlog; + +enum SemanticType +{ + partial, + full +} + +/** + * Basic visitor class used for caching completions of imported modules. + */ +class SemanticVisitor : ASTVisitor +{ + this (SemanticType sType) + { + this.semanticType = sType; + } + + override void visit(Constructor con) + { +// Log.trace(__FUNCTION__, " ", typeof(con).stringof); + visitFunctionDeclaration(con); + } + + override void visit(SharedStaticConstructor con) + { +// Log.trace(__FUNCTION__, " ", typeof(con).stringof); + visitFunctionDeclaration(con); + } + + override void visit(StaticConstructor con) + { +// Log.trace(__FUNCTION__, " ", typeof(con).stringof); + visitFunctionDeclaration(con); + } + + override void visit(Destructor des) + { +// Log.trace(__FUNCTION__, " ", typeof(des).stringof); + visitFunctionDeclaration(des); + } + + override void visit(SharedStaticDestructor des) + { +// Log.trace(__FUNCTION__, " ", typeof(des).stringof); + visitFunctionDeclaration(des); + } + + override void visit(StaticDestructor des) + { +// Log.trace(__FUNCTION__, " ", typeof(des).stringof); + visitFunctionDeclaration(des); + } + + override void visit(FunctionDeclaration dec) + { +// Log.trace(__FUNCTION__, " ", typeof(dec).stringof); + visitFunctionDeclaration(dec); + } + + override void visit(ClassDeclaration dec) + { +// Log.trace(__FUNCTION__, " ", typeof(dec).stringof); + visitAggregateDeclaration(dec, CompletionKind.className); + } + + override void visit(InterfaceDeclaration dec) + { +// Log.trace(__FUNCTION__, " ", typeof(dec).stringof); + visitAggregateDeclaration(dec, CompletionKind.interfaceName); + } + + override void visit(UnionDeclaration dec) + { +// Log.trace(__FUNCTION__, " ", typeof(dec).stringof); + visitAggregateDeclaration(dec, CompletionKind.unionName); + } + + override void visit(StructDeclaration dec) + { +// Log.trace(__FUNCTION__, " ", typeof(dec).stringof); + visitAggregateDeclaration(dec, CompletionKind.structName); + } + + override void visit(BaseClass bc) + { +// Log.trace(__FUNCTION__, " ", typeof(bc).stringof); + currentSymbol.baseClasses ~= iotcToStringArray(bc.identifierOrTemplateChain); + } + + override void visit(VariableDeclaration dec) + { + assert (currentSymbol); +// Log.trace(__FUNCTION__, " ", typeof(dec).stringof); + Type t = dec.type; + foreach (declarator; dec.declarators) + { + SemanticSymbol* symbol = new SemanticSymbol; + symbol.type = t; + symbol.protection = protection; + symbol.kind = CompletionKind.variableName; + symbol.name = declarator.name.value.dup; + symbol.location = declarator.name.startIndex; + symbol.parent = currentSymbol; + currentSymbol.children ~= symbol; + } + } + + override void visit(AliasThisDeclaration dec) + { +// Log.trace(__FUNCTION__, " ", typeof(dec).stringof); + currentSymbol.aliasThis ~= dec.identifier.value.dup; + } + + override void visit(Declaration dec) + { +// Log.trace(__FUNCTION__, " ", typeof(dec).stringof); + if (dec.attributeDeclaration !is null + && isProtection(dec.attributeDeclaration.attribute.attribute)) + { + protection = dec.attributeDeclaration.attribute.attribute; + return; + } + TokenType p = protection; + foreach (Attribute attr; dec.attributes) + { + if (isProtection(attr.attribute)) + p = attr.attribute; + } + dec.accept(this); + protection = p; + } + + override void visit(Module mod) + { +// Log.trace(__FUNCTION__, " ", typeof(mod).stringof); + rootSymbol = new SemanticSymbol; + rootSymbol.kind = CompletionKind.moduleName; + rootSymbol.startLocation = 0; + rootSymbol.endLocation = size_t.max; + currentSymbol = rootSymbol; + currentScope = new Scope(); + currentScope.startLocation = 0; + currentScope.endLocation = size_t.max; + mod.accept(this); + } + + override void visit(EnumDeclaration dec) + { + assert (currentSymbol); +// Log.trace(__FUNCTION__, " ", typeof(dec).stringof); + SemanticSymbol* symbol = new SemanticSymbol; + symbol.name = dec.name.value.dup; + symbol.location = dec.name.startIndex; + symbol.kind = CompletionKind.enumName; + symbol.type = dec.type; + symbol.parent = currentSymbol; + currentSymbol = symbol; + if (dec.enumBody !is null) + dec.enumBody.accept(this); + currentSymbol = symbol.parent; + currentSymbol.children ~= symbol; + } + + override void visit(EnumMember member) + { +// Log.trace(__FUNCTION__, " ", typeof(member).stringof); + SemanticSymbol* symbol = new SemanticSymbol; + symbol.kind = CompletionKind.enumMember; + symbol.name = member.name.value.dup; + symbol.location = member.name.startIndex; + symbol.type = member.type; + symbol.parent = currentSymbol; + currentSymbol.children ~= symbol; + } + + override void visit(ModuleDeclaration dec) + { +// Log.trace(__FUNCTION__, " ", typeof(dec).stringof); + foreach (Token t; dec.moduleName.identifiers) + moduleName ~= t.value.dup; + } + + // creates scopes for + override void visit(StructBody structBody) + { +// Log.trace(__FUNCTION__, " ", typeof(structBody).stringof); + Scope* s = new Scope; + s.startLocation = structBody.startLocation; + s.endLocation = structBody.endLocation; +// Log.trace("Added scope ", s.startLocation, " ", s.endLocation); + s.parent = currentScope; + currentScope = s; + foreach (dec; structBody.declarations) + visit(dec); + currentScope = s.parent; + currentScope.children ~= s; + } + + // Create scope for block statements + override void visit(BlockStatement blockStatement) + { + Log.trace(__FUNCTION__, " ", typeof(blockStatement).stringof); + Scope* s = new Scope; + s.startLocation = blockStatement.startLocation; + s.endLocation = blockStatement.endLocation; + + if (currentSymbol.kind == CompletionKind.functionName) + { + foreach (child; currentSymbol.children) + { + Log.trace("Setting ", child.name, " location"); + child.location = s.startLocation + 1; + } + } +// Log.trace("Added scope ", s.startLocation, " ", s.endLocation); + s.parent = currentScope; + if (blockStatement.declarationsAndStatements !is null) + { + currentScope = s; + visit (blockStatement.declarationsAndStatements); + currentScope = s.parent; + } + currentScope.children ~= s; + } + + alias ASTVisitor.visit visit; + +private: + + void visitAggregateDeclaration(AggType)(AggType dec, CompletionKind kind) + { + SemanticSymbol* symbol = new SemanticSymbol; +// Log.trace("visiting aggregate declaration ", dec.name.value); + symbol.name = dec.name.value.dup; + symbol.location = dec.name.startIndex; + symbol.kind = kind; + symbol.parent = currentSymbol; + symbol.protection = protection; + currentSymbol = symbol; + dec.accept(this); + currentSymbol = symbol.parent; + currentSymbol.children ~= symbol; + } + + void visitFunctionDeclaration(DeclarationType)(DeclarationType dec) + { + SemanticSymbol* symbol = new SemanticSymbol; + + static if (is (DeclarationType == FunctionDeclaration)) + { + symbol.name = dec.name.value.dup; + symbol.location = dec.name.startIndex; + } + else static if (is (DeclarationType == Destructor) + || is (DeclarationType == StaticDestructor) + || is (DeclarationType == SharedStaticDestructor)) + { + symbol.name = "*destructor*"; + symbol.location = dec.location; + } + else + { + symbol.name = "*constructor*"; + symbol.location = dec.location; + } + + static if (is (DeclarationType == Destructor) + || is (DeclarationType == StaticDestructor) + || is (DeclarationType == SharedStaticDestructor)) + { + symbol.callTip = "~this()"; + } + else static if (is (DeclarationType == StaticConstructor) + || is (DeclarationType == SharedStaticConstructor)) + { + symbol.callTip = "this()"; + } + else + { + string parameterString; + if (dec.parameters !is null) + { + parameterString = formatNode(dec.parameters); + foreach (Parameter p; dec.parameters.parameters) + { + SemanticSymbol* parameter = new SemanticSymbol; + parameter.name = p.name.value.dup; + parameter.type = p.type; + parameter.kind = CompletionKind.variableName; + parameter.startLocation = p.name.startIndex; + symbol.children ~= parameter; + Log.trace("Parameter ", parameter.name, " added to ", symbol.name); + } + } + else + parameterString = "()"; + + static if (is (DeclarationType == FunctionDeclaration)) + symbol.callTip = "%s %s%s".format(formatNode(dec.returnType), + dec.name.value, parameterString); + else + symbol.callTip = "this%s".format(parameterString); + } + + symbol.protection = protection; + symbol.kind = CompletionKind.functionName; + symbol.parent = currentSymbol; + currentSymbol = symbol; + + if (dec.functionBody !is null) + { + dec.functionBody.accept(this); + } + currentSymbol = symbol.parent; + currentSymbol.children ~= symbol; + } + + static string[] iotcToStringArray(const IdentifierOrTemplateChain iotc) + { + string[] parts; + foreach (ioti; iotc.identifiersOrTemplateInstances) + { + if (ioti.identifier != TokenType.invalid) + parts ~= ioti.identifier.value.dup; + else + parts ~= ioti.templateInstance.identifier.value.dup; + } + return parts; + } + + static string formatCalltip(Type returnType, string name, Parameters parameters, + string doc = null) + { + return "%s %s%s".format(formatNode(returnType), name, formatNode(parameters)); + } + + static string formatNode(T)(T node) + { + if (node is null) return ""; + import formatter; + auto app = appender!(char[])(); + auto f = new Formatter!(typeof(app))(app); + f.format(node); + return to!string(app.data); + } + + /// Current protection type + TokenType protection; + + /// Current symbol + SemanticSymbol* currentSymbol; + + /// The module + SemanticSymbol* rootSymbol; + + /// Package and module name + string[] moduleName; + + /// Current scope + Scope* currentScope; + + SemanticType semanticType; +} + + +struct SemanticConverter +{ +public: + + this(const(SemanticSymbol)* symbol, Scope* scopes) + { + this.sc = scopes; + this.symbol = symbol; + } + + void convertModule() + { + convertSemanticSymbol(symbol); + assert (current !is null); + } + + ACSymbol* convertSemanticSymbol(const(SemanticSymbol)* symbol) + { + ACSymbol* s = null; + with (CompletionKind) final switch (symbol.kind) + { + case moduleName: + s = convertAggregateDeclaration(symbol); + current = s; + break; + case className: + case interfaceName: + case structName: + case unionName: + case enumName: + s = convertAggregateDeclaration(symbol); + break; + case enumMember: + s = convertEnumMember(symbol); + break; + case variableName: + case memberVariableName: + s = convertVariableDeclaration(symbol); + break; + case functionName: + s = convertFunctionDeclaration(symbol); + break; + case aliasName: + s = convertAliasDeclaration(symbol); + break; + case packageName: + assert (false, "Not implemented"); + case keyword: + case array: + case assocArray: + case dummy: + assert (false, "This should never be reached"); + } + if (sc !is null && symbol.kind != CompletionKind.moduleName) + { + sc.getScopeByCursor(s.location).symbols ~= s; +// Log.trace("Set scope location"); + } + return s; + } + + ACSymbol* convertAliasDeclaration(const(SemanticSymbol)* symbol) + { + ACSymbol* ac = new ACSymbol; + ac.name = symbol.name; + ac.kind = symbol.kind; + ac.location = symbol.location; + // TODO: Set type + return ac; + } + + ACSymbol* convertVariableDeclaration(const(SemanticSymbol)* symbol) + { + ACSymbol* ac = new ACSymbol; + ac.name = symbol.name; + ac.kind = CompletionKind.variableName; + ac.location = symbol.location; + if (symbol.type !is null) + ac.type = resolveType(symbol.type); + // TODO: Handle auto + return ac; + } + + ACSymbol* convertEnumMember(const(SemanticSymbol)* symbol) + { + ACSymbol* ac = new ACSymbol; + ac.name = symbol.name; + ac.kind = symbol.kind; + ac.location = symbol.location; + // TODO: type + return ac; + } + + ACSymbol* convertFunctionDeclaration(const(SemanticSymbol)* symbol) + { +// Log.trace("Converted ", symbol.name, " ", symbol.kind, " ", symbol.location); + ACSymbol* ac = new ACSymbol; + ac.name = symbol.name; + ac.kind = symbol.kind; + ac.location = symbol.location; + ac.callTip = symbol.callTip; + if (symbol.type !is null) + ac.type = resolveType(symbol.type); + return ac; + } + + ACSymbol* convertAggregateDeclaration(const(SemanticSymbol)* symbol) + { +// Log.trace("Converted ", symbol.name, " ", symbol.kind, " ", symbol.location); + ACSymbol* ac = new ACSymbol; + ac.name = symbol.name; + ac.kind = symbol.kind; + ac.location = symbol.location; + if (symbol.kind == CompletionKind.className + || symbol.kind == CompletionKind.structName) + { + ACSymbol* thisSymbol = new ACSymbol("this", + CompletionKind.variableName, ac); + ac.parts ~= thisSymbol; + } + auto temp = current; + current = ac; + foreach (child; symbol.children) + current.parts ~= convertSemanticSymbol(child); + current = temp; + return ac; + } + +private: + + + ACSymbol* resolveType(const Type t) + in + { + assert (t !is null); + assert (t.type2 !is null); + } + body + { + ACSymbol* s; + if (t.type2.builtinType != TokenType.invalid) + s = convertBuiltinType(t.type2); + else if (t.type2.typeConstructor != TokenType.invalid) + s = resolveType(t.type2.type); + else if (t.type2.symbol !is null) + { + if (t.type2.symbol.dot) + Log.trace("TODO: global scoped symbol handling"); + string[] symbolParts = expandSymbol( + t.type2.symbol.identifierOrTemplateChain); + + } + foreach (suffix; t.typeSuffixes) + s = processSuffix(s, suffix); + return null; + } + + static string[] expandSymbol(const IdentifierOrTemplateChain chain) + { + string[] strings = new string[chain.identifiersOrTemplateInstances.length]; + for (size_t i = 0; i != chain.identifiersOrTemplateInstances.length; ++i) + { + auto identOrTemplate = chain.identifiersOrTemplateInstances[i]; + strings[i] = identOrTemplate.templateInstance is null ? + identOrTemplate.identifier.value.dup + : identOrTemplate.identifier.value.dup; + } + return strings; + } + + static ACSymbol* processSuffix(ACSymbol* symbol, const TypeSuffix suffix) + { + if (suffix.star) + return symbol; + if (suffix.array || suffix.type) + { + ACSymbol* s = new ACSymbol; + s.parts = arraySymbols; + s.type = symbol; + s.qualifier = suffix.array ? SymbolQualifier.array : SymbolQualifier.assocArray; + return s; + } + if (suffix.parameters) + { + Log.error("TODO: Function type suffix"); + return null; + } + return null; + } + + static ACSymbol* convertBuiltinType(const Type2 type2) + { + string stringRepresentation = getTokenValue(type2.builtinType); + if (stringRepresentation is null) return null; + // TODO: Make this use binary search instead + foreach (s; builtinSymbols) + if (s.name == stringRepresentation) + return s; + return null; + } + + ACSymbol* current; + Scope* sc; + const(SemanticSymbol)* symbol; +} + +const(ACSymbol)*[] convertAstToSymbols(Module m) +{ + SemanticVisitor visitor = new SemanticVisitor(SemanticType.partial); + visitor.visit(m); + SemanticConverter converter = SemanticConverter(visitor.rootSymbol, null); + converter.convertModule(); + return cast(typeof(return)) converter.current.parts; +} + +const(Scope)* generateAutocompleteTrees(const(Token)[] tokens) +{ + Module m = parseModule(tokens, null); + SemanticVisitor visitor = new SemanticVisitor(SemanticType.full); + visitor.visit(m); + SemanticConverter converter = SemanticConverter(visitor.rootSymbol, + visitor.currentScope); + converter.convertModule(); + return converter.sc; +} + +version(unittest) Module parseTestCode(string code) +{ + LexerConfig config; + const(Token)[] tokens = byToken(cast(ubyte[]) code, config); + Parser p = new Parser; + p.fileName = "unittest"; + p.tokens = tokens; + Module m = p.parseModule(); + assert (p.errorCount == 0); + assert (p.warningCount == 0); + return m; +} + +unittest +{ + auto source = q{ + module foo; + + struct Bar { int x; int y;} + }c; + Module m = parseTestCode(source); + BasicSemanticVisitor visitor = new BasicSemanticVisitor; + visitor.visit(m); + assert (visitor.rootSymbol !is null); + assert (visitor.rootSymbol.name == "foo"); + SemanticSymbol* bar = visitor.root.getPartByName("Bar"); + assert (bar !is null); + assert (bar.getPartByName("x") !is null); + assert (bar.getPartByName("y") !is null); +} diff --git a/autocomplete.d b/autocomplete.d index e5be929..80632e2 100644 --- a/autocomplete.d +++ b/autocomplete.d @@ -34,19 +34,27 @@ import stdx.d.parser; import std.string; import messages; -import acvisitor; import actypes; import constants; import modulecache; +import astconverter; +import stupidlog; AutocompleteResponse complete(AutocompleteRequest request, string[] importPaths) { - writeln("Got a completion request"); + Log.info("Got a completion request"); AutocompleteResponse response; LexerConfig config; - auto tokens = request.sourceCode.byToken(config); - auto tokenArray = tokens.array(); + config.fileName = "stdin"; + auto tokens = byToken(cast(ubyte[]) request.sourceCode, config); + const(Token)[] tokenArray = void; + try { + tokenArray = tokens.array(); + } catch (Exception e) { + Log.error("Could not provide autocomplete due to lexing exception: ", e.msg); + return response; + } auto sortedTokens = assumeSorted(tokenArray); string partial; @@ -56,7 +64,7 @@ AutocompleteResponse complete(AutocompleteRequest request, string[] importPaths) if (beforeTokens.length >= 1 && beforeTokens[$ - 1] == TokenType.identifier) { - //writeln("partial completion"); + Log.trace("partial completion"); partial = beforeTokens[$ - 1].value; tokenType = beforeTokens[$ - 1].type; beforeTokens = beforeTokens[0 .. $ - 1]; @@ -92,10 +100,10 @@ AutocompleteResponse complete(AutocompleteRequest request, string[] importPaths) case TokenType.identifier: case TokenType.rParen: case TokenType.rBracket: - auto visitor = processModule(tokenArray); + const(Scope)* completionScope = generateAutocompleteTrees(tokenArray); auto expression = getExpression(beforeTokens[0 .. $ - 1]); - response.setCompletions(visitor, expression, request.cursorPosition, - CompletionType.calltips); + response.setCompletions(completionScope, expression, + request.cursorPosition, CompletionType.calltips); break; default: break; @@ -144,10 +152,10 @@ dotCompletion: case TokenType.rParen: case TokenType.rBracket: case TokenType.this_: - auto visitor = processModule(tokenArray); + const(Scope)* completionScope = generateAutocompleteTrees(tokenArray); auto expression = getExpression(beforeTokens); - response.setCompletions(visitor, expression, request.cursorPosition, - CompletionType.identifiers, partial); + response.setCompletions(completionScope, expression, + request.cursorPosition, CompletionType.identifiers, partial); break; case TokenType.lParen: case TokenType.lBrace: @@ -164,7 +172,7 @@ dotCompletion: } void setCompletions(T)(ref AutocompleteResponse response, - AutocompleteVisitor visitor, T tokens, size_t cursorPosition, + const(Scope)* completionScope, T tokens, size_t cursorPosition, CompletionType completionType, string partial = null) { // Autocomplete module imports instead of symbols @@ -175,14 +183,11 @@ void setCompletions(T)(ref AutocompleteResponse response, return; } - visitor.scope_.resolveSymbolTypes(); - // Handle the simple case where we get all symbols in scope and filter it // based on the currently entered text. if (partial !is null && tokens.length == 0) { -// writeln("Showing all symbols in current scope that start with ", partial); - foreach (s; visitor.scope_.getSymbolsInCurrentScope(cursorPosition) + foreach (s; completionScope.getSymbolsInCursorScope(cursorPosition) .filter!(a => a.name.toUpper().startsWith(partial.toUpper()))) { response.completionKinds ~= s.kind; @@ -196,10 +201,11 @@ void setCompletions(T)(ref AutocompleteResponse response, return; // Find the symbol corresponding to the beginning of the chain - ACSymbol[] symbols = visitor.scope_.findSymbolsInCurrentScope(cursorPosition, tokens[0].value); + const(ACSymbol)*[] symbols = completionScope.getSymbolsByNameAndCursor( + tokens[0].value, cursorPosition); if (symbols.length == 0) { - writeln("Could not find declaration of ", tokens[0].value); + Log.trace("Could not find declaration of ", tokens[0].value); return; } @@ -209,11 +215,9 @@ void setCompletions(T)(ref AutocompleteResponse response, || symbols[0].kind == CompletionKind.aliasName || symbols[0].kind == CompletionKind.enumMember) { - symbols = symbols[0].resolvedType is null ? [] : [symbols[0].resolvedType]; + symbols = symbols[0].type is null ? [] : [symbols[0].type]; if (symbols.length == 0) - { return; - } } loop: for (size_t i = 1; i < tokens.length; i++) @@ -236,40 +240,40 @@ void setCompletions(T)(ref AutocompleteResponse response, } with (TokenType) switch (tokens[i].type) { - case TokenType.int_: - case TokenType.uint_: - case TokenType.long_: - case TokenType.ulong_: - case TokenType.char_: - case TokenType.wchar_: - case TokenType.dchar_: - case TokenType.bool_: - case TokenType.byte_: - case TokenType.ubyte_: - case TokenType.short_: - case TokenType.ushort_: - case TokenType.cent_: - case TokenType.ucent_: - case TokenType.float_: - case TokenType.ifloat_: - case TokenType.cfloat_: - case TokenType.idouble_: - case TokenType.cdouble_: - case TokenType.double_: - case TokenType.real_: - case TokenType.ireal_: - case TokenType.creal_: + case int_: + case uint_: + case long_: + case ulong_: + case char_: + case wchar_: + case dchar_: + case bool_: + case byte_: + case ubyte_: + case short_: + case ushort_: + case cent_: + case ucent_: + case float_: + case ifloat_: + case cfloat_: + case idouble_: + case cdouble_: + case double_: + case real_: + case ireal_: + case creal_: case this_: symbols = symbols[0].getPartsByName(getTokenValue(tokens[i].type)); if (symbols.length == 0) break loop; break; case identifier: -// writeln("looking for ", tokens[i].value, " in ", symbols[0].name); + Log.trace("looking for ", tokens[i].value, " in ", symbols[0].name); symbols = symbols[0].getPartsByName(tokens[i].value); if (symbols.length == 0) { -// writeln("Couldn't find it."); + Log.trace("Couldn't find it."); break loop; } if (symbols[0].kind == CompletionKind.variableName @@ -279,7 +283,7 @@ void setCompletions(T)(ref AutocompleteResponse response, && (completionType == CompletionType.identifiers || i + 1 < tokens.length))) { - symbols = symbols[0].resolvedType is null ? [] : [symbols[0].resolvedType]; + symbols = symbols[0].type is null ? [] : [symbols[0].type]; } if (symbols.length == 0) break loop; @@ -287,7 +291,7 @@ void setCompletions(T)(ref AutocompleteResponse response, && (completionType == CompletionType.identifiers || i + 1 < tokens.length)) { - symbols = symbols[0].resolvedType is null ? [] : [symbols[0].resolvedType]; + symbols = symbols[0].type is null ? [] : [symbols[0].type]; } if (symbols.length == 0) break loop; @@ -308,12 +312,12 @@ void setCompletions(T)(ref AutocompleteResponse response, p.setTokens(tokens[h .. i].array()); if (!p.isSliceExpression()) { - symbols = symbols[0].resolvedType is null ? [] : [symbols[0].resolvedType]; + symbols = symbols[0].type is null ? [] : [symbols[0].type]; } } else if (symbols[0].qualifier == SymbolQualifier.assocArray) { - symbols = symbols[0].resolvedType is null ? [] :[symbols[0].resolvedType]; + symbols = symbols[0].type is null ? [] :[symbols[0].type]; skip(); } else @@ -322,14 +326,14 @@ void setCompletions(T)(ref AutocompleteResponse response, skip(); Parser p; p.setTokens(tokens[h .. i].array()); - ACSymbol[] overloads; + const(ACSymbol)*[] overloads; if (p.isSliceExpression()) overloads = symbols[0].getPartsByName("opSlice"); else overloads = symbols[0].getPartsByName("opIndex"); if (overloads.length > 0) { - symbols = overloads[0].resolvedType is null ? [] : [overloads[0].resolvedType]; + symbols = overloads[0].type is null ? [] : [overloads[0].type]; } else return; @@ -344,7 +348,7 @@ void setCompletions(T)(ref AutocompleteResponse response, if (symbols.length == 0) { - writeln("Could not get completions"); + Log.error("Could not get completions"); return; } if (completionType == CompletionType.identifiers) @@ -354,7 +358,7 @@ void setCompletions(T)(ref AutocompleteResponse response, && (partial is null ? true : a.name.toUpper().startsWith(partial.toUpper())) && !response.completions.canFind(a.name))) { - //writeln("Adding ", s.name, " to the completion list"); + Log.trace("Adding ", s.name, " to the completion list"); response.completionKinds ~= s.kind; response.completions ~= s.name; } @@ -362,9 +366,9 @@ void setCompletions(T)(ref AutocompleteResponse response, } else if (completionType == CompletionType.calltips) { - //writeln("Showing call tips for ", symbols[0].name, " of type ", symbols[0].kind); + Log.trace("Showing call tips for ", symbols[0].name, " of type ", symbols[0].kind); if (symbols[0].kind != CompletionKind.functionName - && symbols[0].calltip is null) + && symbols[0].callTip is null) { auto call = symbols[0].getPartsByName("opCall"); if (call.length == 0) @@ -385,8 +389,8 @@ void setCompletions(T)(ref AutocompleteResponse response, response.completionType = CompletionType.calltips; foreach (symbol; symbols) { -// writeln("Adding calltip ", symbol.calltip); - response.completions ~= symbol.calltip; + Log.trace("Adding calltip ", symbol.callTip); + response.completions ~= symbol.callTip; } } @@ -522,118 +526,3 @@ void setImportCompletions(T)(T tokens, ref AutocompleteResponse response) } } } - -/** - * Initializes builtin types and the various properties of builtin types - */ -static this() -{ - auto bool_ = new ACSymbol("bool", CompletionKind.keyword); - auto int_ = new ACSymbol("int", CompletionKind.keyword); - auto long_ = new ACSymbol("long", CompletionKind.keyword); - auto byte_ = new ACSymbol("byte", CompletionKind.keyword); - auto char_ = new ACSymbol("char", CompletionKind.keyword); - auto dchar_ = new ACSymbol("dchar", CompletionKind.keyword); - auto short_ = new ACSymbol("short", CompletionKind.keyword); - auto ubyte_ = new ACSymbol("ubyte", CompletionKind.keyword); - auto uint_ = new ACSymbol("uint", CompletionKind.keyword); - auto ulong_ = new ACSymbol("ulong", CompletionKind.keyword); - auto ushort_ = new ACSymbol("ushort", CompletionKind.keyword); - auto wchar_ = new ACSymbol("wchar", CompletionKind.keyword); - - auto alignof_ = new ACSymbol("alignof", CompletionKind.keyword, ulong_); - auto mangleof_ = new ACSymbol("mangleof", CompletionKind.keyword); - auto sizeof_ = new ACSymbol("sizeof", CompletionKind.keyword, ulong_); - auto stringof_ = new ACSymbol("stringof", CompletionKind.keyword); - - arraySymbols ~= alignof_; - arraySymbols ~= new ACSymbol("dup", CompletionKind.keyword); - arraySymbols ~= new ACSymbol("idup", CompletionKind.keyword); - arraySymbols ~= new ACSymbol("init", CompletionKind.keyword); - arraySymbols ~= new ACSymbol("length", CompletionKind.keyword, ulong_); - arraySymbols ~= mangleof_; - arraySymbols ~= new ACSymbol("ptr", CompletionKind.keyword); - arraySymbols ~= new ACSymbol("reverse", CompletionKind.keyword); - arraySymbols ~= sizeof_; - arraySymbols ~= new ACSymbol("sort", CompletionKind.keyword); - arraySymbols ~= stringof_; - - assocArraySymbols ~= alignof_; - assocArraySymbols ~= new ACSymbol("byKey", CompletionKind.keyword); - assocArraySymbols ~= new ACSymbol("byValue", CompletionKind.keyword); - assocArraySymbols ~= new ACSymbol("dup", CompletionKind.keyword); - assocArraySymbols ~= new ACSymbol("get", CompletionKind.keyword); - assocArraySymbols ~= new ACSymbol("init", CompletionKind.keyword); - assocArraySymbols ~= new ACSymbol("keys", CompletionKind.keyword); - assocArraySymbols ~= new ACSymbol("length", CompletionKind.keyword, ulong_); - assocArraySymbols ~= mangleof_; - assocArraySymbols ~= new ACSymbol("rehash", CompletionKind.keyword); - assocArraySymbols ~= sizeof_; - assocArraySymbols ~= stringof_; - assocArraySymbols ~= new ACSymbol("values", CompletionKind.keyword); - - foreach (s; [bool_, int_, long_, byte_, char_, dchar_, short_, ubyte_, uint_, - ulong_, ushort_, wchar_]) - { - s.parts ~= new ACSymbol("init", CompletionKind.keyword, s); - s.parts ~= new ACSymbol("min", CompletionKind.keyword, s); - s.parts ~= new ACSymbol("max", CompletionKind.keyword, s); - s.parts ~= alignof_; - s.parts ~= sizeof_; - s.parts ~= stringof_; - s.parts ~= mangleof_; - } - - auto cdouble_ = new ACSymbol("cdouble", CompletionKind.keyword); - auto cent_ = new ACSymbol("cent", CompletionKind.keyword); - auto cfloat_ = new ACSymbol("cfloat", CompletionKind.keyword); - auto creal_ = new ACSymbol("creal", CompletionKind.keyword); - auto double_ = new ACSymbol("double", CompletionKind.keyword); - auto float_ = new ACSymbol("float", CompletionKind.keyword); - auto idouble_ = new ACSymbol("idouble", CompletionKind.keyword); - auto ifloat_ = new ACSymbol("ifloat", CompletionKind.keyword); - auto ireal_ = new ACSymbol("ireal", CompletionKind.keyword); - auto real_ = new ACSymbol("real", CompletionKind.keyword); - auto ucent_ = new ACSymbol("ucent", CompletionKind.keyword); - - foreach (s; [cdouble_, cent_, cfloat_, creal_, double_, float_, - idouble_, ifloat_, ireal_, real_, ucent_]) - { - s.parts ~= alignof_; - s.parts ~= new ACSymbol("dig", CompletionKind.keyword, s); - s.parts ~= new ACSymbol("epsilon", CompletionKind.keyword, s); - s.parts ~= new ACSymbol("infinity", CompletionKind.keyword, s); - s.parts ~= new ACSymbol("init", CompletionKind.keyword, s); - s.parts ~= mangleof_; - s.parts ~= new ACSymbol("mant_dig", CompletionKind.keyword, int_); - s.parts ~= new ACSymbol("max", CompletionKind.keyword, s); - s.parts ~= new ACSymbol("max_10_exp", CompletionKind.keyword, int_); - s.parts ~= new ACSymbol("max_exp", CompletionKind.keyword, int_); - s.parts ~= new ACSymbol("min", CompletionKind.keyword, s); - s.parts ~= new ACSymbol("min_exp", CompletionKind.keyword, int_); - s.parts ~= new ACSymbol("min_10_exp", CompletionKind.keyword, int_); - s.parts ~= new ACSymbol("min_normal", CompletionKind.keyword, s); - s.parts ~= new ACSymbol("nan", CompletionKind.keyword, s); - s.parts ~= sizeof_; - s.parts ~= stringof_; - } - - ireal_.parts ~= new ACSymbol("im", CompletionKind.keyword, real_); - ifloat_.parts ~= new ACSymbol("im", CompletionKind.keyword, float_); - idouble_.parts ~= new ACSymbol("im", CompletionKind.keyword, double_); - ireal_.parts ~= new ACSymbol("re", CompletionKind.keyword, real_); - ifloat_.parts ~= new ACSymbol("re", CompletionKind.keyword, float_); - idouble_.parts ~= new ACSymbol("re", CompletionKind.keyword, double_); - - auto void_ = new ACSymbol("void", CompletionKind.keyword); - - builtinSymbols = [bool_, int_, long_, byte_, char_, dchar_, short_, ubyte_, uint_, - ulong_, ushort_, wchar_, cdouble_, cent_, cfloat_, creal_, double_, - float_, idouble_, ifloat_, ireal_, real_, ucent_, void_]; -} - -ACSymbol[] builtinSymbols; -ACSymbol[] arraySymbols; -ACSymbol[] assocArraySymbols; -ACSymbol[] classSymbols; -ACSymbol[] structSymbols; diff --git a/build.sh b/build.sh index 67ca672..f7d84a2 100755 --- a/build.sh +++ b/build.sh @@ -1,2 +1,26 @@ -dmd -wi client.d messages.d msgpack-d/src/msgpack.d -Imsgpack-d/src -ofdcd-client -dmd -wi -g server.d modulecache.d actypes.d messages.d constants.d acvisitor.d autocomplete.d dscanner/stdx/d/ast.d dscanner/stdx/d/parser.d dscanner/stdx/d/lexer.d dscanner/stdx/d/entities.d dscanner/formatter.d msgpack-d/src/msgpack.d -Imsgpack-d/src -Idscanner/ -ofdcd-server +dmd -wi client.d\ + messages.d\ + msgpack-d/src/msgpack.d\ + -Imsgpack-d/src\ + -ofdcd-client + +dmd \ + actypes.d\ + astconverter.d\ + autocomplete.d\ + constants.d\ + messages.d\ + modulecache.d\ + semantic.d\ + server.d\ + stupidlog.d\ + dscanner/stdx/d/ast.d\ + dscanner/stdx/d/parser.d\ + dscanner/stdx/d/lexer.d\ + dscanner/stdx/d/entities.d\ + dscanner/formatter.d\ + msgpack-d/src/msgpack.d\ + -Imsgpack-d/src\ + -Idscanner\ + -wi\ + -ofdcd-server diff --git a/constants.d b/constants.d index 9c78779..d7b4f37 100644 --- a/constants.d +++ b/constants.d @@ -53,7 +53,9 @@ immutable string[] traits = [ "getMember", "getOverloads", "getProtection", + "getUnitTests", "getVirtualFunctions", + "getVirtualIndex", "getVirtualMethods", "hasMember", "identifier", @@ -68,6 +70,7 @@ immutable string[] traits = [ "isLazy", "isNested", "isOut", + "isOverrideFunction", "isPOD", "isRef", "isSame", @@ -199,3 +202,87 @@ immutable string[] structProperties = [ "sizeof", "stringof" ]; + +immutable string[] predefinedVersions; + +static this() +{ + version(AArch64) predefinedVersions ~= "AArch64"; + version(AIX) predefinedVersions ~= "AIX"; + version(all) predefinedVersions ~= "all"; + version(Alpha) predefinedVersions ~= "Alpha"; + version(Alpha_HardFloat) predefinedVersions ~= "Alpha_HardFloat"; + version(Alpha_SoftFloat) predefinedVersions ~= "Alpha_SoftFloat"; + version(Android) predefinedVersions ~= "Android"; + version(ARM) predefinedVersions ~= "ARM"; + version(ARM_HardFloat) predefinedVersions ~= "ARM_HardFloat"; + version(ARM_SoftFloat) predefinedVersions ~= "ARM_SoftFloat"; + version(ARM_SoftFP) predefinedVersions ~= "ARM_SoftFP"; + version(ARM_Thumb) predefinedVersions ~= "ARM_Thumb"; + version(assert) predefinedVersions ~= "assert"; + version(BigEndian) predefinedVersions ~= "BigEndian"; + version(BSD) predefinedVersions ~= "BSD"; + version(Cygwin) predefinedVersions ~= "Cygwin"; + version(D_Coverage) predefinedVersions ~= "D_Coverage"; + version(D_Ddoc) predefinedVersions ~= "D_Ddoc"; + version(D_HardFloat) predefinedVersions ~= "D_HardFloat"; + version(DigitalMars) predefinedVersions ~= "DigitalMars"; + version(D_InlineAsm_X86) predefinedVersions ~= "D_InlineAsm_X86"; + version(D_InlineAsm_X86_64) predefinedVersions ~= "D_InlineAsm_X86_64"; + version(D_LP64) predefinedVersions ~= "D_LP64"; + version(D_NoBoundsChecks) predefinedVersions ~= "D_NoBoundsChecks"; + version(D_PIC) predefinedVersions ~= "D_PIC"; + version(DragonFlyBSD) predefinedVersions ~= "DragonFlyBSD"; + version(D_SIMD) predefinedVersions ~= "D_SIMD"; + version(D_SoftFloat) predefinedVersions ~= "D_SoftFloat"; + version(D_Version2) predefinedVersions ~= "D_Version2"; + version(D_X32) predefinedVersions ~= "D_X32"; + version(FreeBSD) predefinedVersions ~= "FreeBSD"; + version(GNU) predefinedVersions ~= "GNU"; + version(Haiku) predefinedVersions ~= "Haiku"; + version(HPPA) predefinedVersions ~= "HPPA"; + version(HPPA64) predefinedVersions ~= "HPPA64"; + version(Hurd) predefinedVersions ~= "Hurd"; + version(IA64) predefinedVersions ~= "IA64"; + version(LDC) predefinedVersions ~= "LDC"; + version(linux) predefinedVersions ~= "linux"; + version(LittleEndian) predefinedVersions ~= "LittleEndian"; + version(MIPS32) predefinedVersions ~= "MIPS32"; + version(MIPS64) predefinedVersions ~= "MIPS64"; + version(MIPS_EABI) predefinedVersions ~= "MIPS_EABI"; + version(MIPS_HardFloat) predefinedVersions ~= "MIPS_HardFloat"; + version(MIPS_N32) predefinedVersions ~= "MIPS_N32"; + version(MIPS_N64) predefinedVersions ~= "MIPS_N64"; + version(MIPS_O32) predefinedVersions ~= "MIPS_O32"; + version(MIPS_O64) predefinedVersions ~= "MIPS_O64"; + version(MIPS_SoftFloat) predefinedVersions ~= "MIPS_SoftFloat"; + version(NetBSD) predefinedVersions ~= "NetBSD"; + version(none) predefinedVersions ~= "none"; + version(OpenBSD) predefinedVersions ~= "OpenBSD"; + version(OSX) predefinedVersions ~= "OSX"; + version(Posix) predefinedVersions ~= "Posix"; + version(PPC) predefinedVersions ~= "PPC"; + version(PPC64) predefinedVersions ~= "PPC64"; + version(PPC_HardFloat) predefinedVersions ~= "PPC_HardFloat"; + version(PPC_SoftFloat) predefinedVersions ~= "PPC_SoftFloat"; + version(S390) predefinedVersions ~= "S390"; + version(S390X) predefinedVersions ~= "S390X"; + version(SDC) predefinedVersions ~= "SDC"; + version(SH) predefinedVersions ~= "SH"; + version(SH64) predefinedVersions ~= "SH64"; + version(SkyOS) predefinedVersions ~= "SkyOS"; + version(Solaris) predefinedVersions ~= "Solaris"; + version(SPARC) predefinedVersions ~= "SPARC"; + version(SPARC64) predefinedVersions ~= "SPARC64"; + version(SPARC_HardFloat) predefinedVersions ~= "SPARC_HardFloat"; + version(SPARC_SoftFloat) predefinedVersions ~= "SPARC_SoftFloat"; + version(SPARC_V8Plus) predefinedVersions ~= "SPARC_V8Plus"; + version(SysV3) predefinedVersions ~= "SysV3"; + version(SysV4) predefinedVersions ~= "SysV4"; + version(unittest) predefinedVersions ~= "unittest"; + version(Win32) predefinedVersions ~= "Win32"; + version(Win64) predefinedVersions ~= "Win64"; + version(Windows) predefinedVersions ~= "Windows"; + version(X86) predefinedVersions ~= "X86"; + version(X86_64) predefinedVersions ~= "X86_64"; +} diff --git a/dscanner b/dscanner index cd3bb40..982510c 160000 --- a/dscanner +++ b/dscanner @@ -1 +1 @@ -Subproject commit cd3bb40730a34cb31a0f35149369973cda359719 +Subproject commit 982510c50c9b59c70619a80c771b202c5db12be1 diff --git a/messages.d b/messages.d index 613eeec..39f8276 100644 --- a/messages.d +++ b/messages.d @@ -23,6 +23,10 @@ module messages; */ enum CompletionKind : char { + /// Invalid completion kind. This is used internally and will never + /// be returned in a completion response. + dummy = '?', + /// class names className = 'c', @@ -86,11 +90,18 @@ enum CompletionType : string calltips = "calltips" } -enum RequestKind +/** + * Request kind + */ +enum RequestKind : ubyte { + /// Autocompletion autocomplete, + /// Clear the completion cache clearCache, + /// Add import directory to server addImport, + /// Shut down the server shutdown } diff --git a/modulecache.d b/modulecache.d index 95c53d0..956f63e 100644 --- a/modulecache.d +++ b/modulecache.d @@ -29,14 +29,20 @@ import std.path; import std.algorithm; import std.conv; -import acvisitor; import actypes; -import autocomplete; +import semantic; +import astconverter; +import stupidlog; struct CacheEntry { - ACSymbol[] symbols; + const(ACSymbol)*[] symbols; SysTime modificationTime; + void opAssign(ref const CacheEntry other) + { + this.symbols = cast(typeof(symbols)) other.symbols; + this.modificationTime = other.modificationTime; + } } /** @@ -64,8 +70,11 @@ struct ModuleCache importPaths ~= path; foreach (fileName; dirEntries(path, "*.{d,di}", SpanMode.depth)) { - writeln("Loading and caching completions for ", fileName); + Log.info("Loading and caching completions for ", fileName); getSymbolsInModule(fileName); + import core.memory; + GC.collect(); + GC.minimize(); } } @@ -75,16 +84,16 @@ struct ModuleCache * Returns: * The symbols defined in the given module */ - static ACSymbol[] getSymbolsInModule(string moduleName) + static const(ACSymbol)*[] getSymbolsInModule(string moduleName) { - writeln("Getting symbols for module ", moduleName); +// Log.info("Getting symbols for module ", moduleName); string location = resolveImportLoctation(moduleName); if (location is null) return []; if (!needsReparsing(location)) return cache[location].symbols; - auto visitor = new AutocompleteVisitor; + const(ACSymbol)*[] symbols; try { File f = File(location); @@ -92,25 +101,23 @@ struct ModuleCache f.rawRead(source); LexerConfig config; + config.fileName = location; auto tokens = source.byToken(config).array(); Module mod = parseModule(tokens, location, &doesNothing); - visitor.visit(mod); - visitor.scope_.resolveSymbolTypes(); + symbols = convertAstToSymbols(mod); } catch (Exception ex) { - writeln("Couln't parse ", location, " due to exception: ", ex.msg); + Log.error("Couln't parse ", location, " due to exception: ", ex.msg); return []; } SysTime access; SysTime modification; getTimes(location, access, modification); - if (location !in cache) - cache[location] = CacheEntry.init; - cache[location].modificationTime = modification; - cache[location].symbols = visitor.symbols; - return cache[location].symbols; + CacheEntry c = CacheEntry(symbols, modification); + cache[location] = c; + return symbols; } /** @@ -122,7 +129,7 @@ struct ModuleCache */ static string resolveImportLoctation(string moduleName) { -// writeln("Resolving location of ", moduleName); +// Log.trace("Resolving location of ", moduleName); if (isRooted(moduleName)) return moduleName; @@ -168,3 +175,5 @@ private: // Listing of paths to check for imports static string[] importPaths; } + +private void doesNothing(string a, int b, int c, string d) {} diff --git a/semantic.d b/semantic.d new file mode 100644 index 0000000..60dabb5 --- /dev/null +++ b/semantic.d @@ -0,0 +1,61 @@ +/** + * This file is part of DCD, a development tool for the D programming language. + * Copyright (C) 2013 Brian Schott + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module semantic; + +import messages; +import actypes; +import stdx.d.ast; +import stdx.d.lexer; + +/** + * Intermediate form between ACSymbol and the AST classes. Stores enough + * information to resolve things like base classes and alias this. + */ +struct SemanticSymbol +{ +public: + /// Symbol name + string name; + + /// Base classes + string[][] baseClasses; + + /// Completion kind + CompletionKind kind; + + /// Variable type or function return type + Type type; + + /// Function call tip. Null if this is not a function + string callTip; + + /// Alias this symbols + string[] aliasThis; + + /// MixinTemplates + string[] mixinTemplates; + + /// Protection level for this symobol + TokenType protection; + + /// Symbol location + size_t location; + + mixin scopeImplementation!(SemanticSymbol); +} diff --git a/server.d b/server.d index 59d09d7..4bf0ca1 100644 --- a/server.d +++ b/server.d @@ -26,12 +26,16 @@ import std.path; import std.file; import std.array; import std.process; +import std.datetime; import msgpack; import messages; import autocomplete; import modulecache; +import stupidlog; +import actypes; +import core.memory; enum CONFIG_FILE_NAME = "dcd.conf"; @@ -42,9 +46,15 @@ version(OSX) version = useXDG; int main(string[] args) { + + Log.info("Starting up..."); + StopWatch sw = StopWatch(AutoStart.yes); // No relative paths version (Posix) chdir("/"); + Log.output = stdout; + Log.level = LogLevel.trace; + ushort port = 9166; bool help; string[] importPaths; @@ -75,30 +85,47 @@ int main(string[] args) socket.listen(0); scope (exit) { - writeln("Shutting down sockets..."); + Log.info("Shutting down sockets..."); socket.shutdown(SocketShutdown.BOTH); socket.close(); - writeln("Sockets shut down."); + Log.info("Sockets shut down."); } foreach (path; importPaths) ModuleCache.addImportPath(path); - writeln("Import directories: ", ModuleCache.getImportPaths()); + Log.info("Import directories: ", ModuleCache.getImportPaths()); ubyte[] buffer = new ubyte[1024 * 1024 * 4]; // 4 megabytes should be enough for anybody... - writeln("Startup complete"); + sw.stop(); + Log.info("Startup completed in ", sw.peek().to!("msecs", float), " milliseconds"); while (true) { auto s = socket.accept(); s.blocking = true; + + if (s.remoteAddress.toHostNameString() != "localhost") + { + Log.error("Warning: Connection attempt from ", + s.remoteAddress.toHostNameString(), "ignored. DCD only accepts " + ~ " requests from localhost"); + s.shutdown(SocketShutdown.BOTH); + s.close(); + continue; + } + + // TODO: Only process connections from localhost + scope (exit) { s.shutdown(SocketShutdown.BOTH); s.close(); } ptrdiff_t bytesReceived = s.receive(buffer); + + auto requestWatch = StopWatch(AutoStart.yes); + size_t messageLength; // bit magic! (cast(ubyte*) &messageLength)[0..size_t.sizeof] = buffer[0..size_t.sizeof]; @@ -115,7 +142,7 @@ int main(string[] args) if (bytesReceived == Socket.ERROR) { - writeln("Socket recieve failed"); + Log.error("Socket recieve failed"); break; } @@ -131,12 +158,12 @@ int main(string[] args) } else if (request.kind == RequestKind.clearCache) { - writeln("Clearing cache."); + Log.info("Clearing cache."); ModuleCache.clear(); } else if (request.kind == RequestKind.shutdown) { - writeln("Shutting down."); + Log.info("Shutting down."); break; } else @@ -145,6 +172,7 @@ int main(string[] args) ubyte[] responseBytes = msgpack.pack(response); assert(s.send(responseBytes) == responseBytes.length); } + Log.info("Request processed in ", requestWatch.peek().to!("msecs", float), " milliseconds"); } return 0; } @@ -178,11 +206,11 @@ void warnAboutOldConfigLocation() version (linux) if ("~/.config/dcd".expandTilde().exists() && "~/.config/dcd".expandTilde().isFile()) { - writeln("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); - writeln("!! Upgrade warning:"); - writeln("!! '~/.config/dcd' should be moved to '$XDG_CONFIG_HOME/dcd/dcd.conf'"); - writeln("!! or '$HOME/.config/dcd/dcd.conf'"); - writeln("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); + Log.error("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); + Log.error("!! Upgrade warning:"); + Log.error("!! '~/.config/dcd' should be moved to '$XDG_CONFIG_HOME/dcd/dcd.conf'"); + Log.error("!! or '$HOME/.config/dcd/dcd.conf'"); + Log.error("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); } } @@ -192,7 +220,7 @@ string[] loadConfiguredImportDirs() immutable string configLocation = getConfigurationLocation(); if (!configLocation.exists()) return []; - writeln("Loading configuration from ", configLocation); + Log.info("Loading configuration from ", configLocation); File f = File(configLocation, "rt"); return f.byLine(KeepTerminator.no).map!(a => a.idup).filter!(a => a.exists()).array(); } diff --git a/stupidlog.d b/stupidlog.d new file mode 100644 index 0000000..86ca842 --- /dev/null +++ b/stupidlog.d @@ -0,0 +1,72 @@ +/******************************************************************************* + * Authors: Brian Schott + * Copyright: Brian Schott + * Date: Oct 5 2013 + * + * License: + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + ******************************************************************************/ + +module stupidlog; + +import std.stdio; +import core.vararg; + +enum LogLevel : uint +{ + fatal = 0, + error, + info, + trace +} + +struct Log +{ + static void trace(T...)(lazy string message, T args) + { + if (level < LogLevel.trace) return; + if (output is stdout) + output.writeln("[\033[01;36mtrace\033[0m] " ~ message, args); + else + output.writeln("[trace] " ~ message, args); + } + + static void info(T...)(lazy string message, T args) + { + if (level < LogLevel.info) return; + if (output is stdout) + output.writeln("[\033[01;32minfo\033[0m ] " ~ message, args); + else + output.writeln("[info ] " ~ message, args); + } + + static void error(T...)(lazy string message, T args) + { + if (level < LogLevel.error) return; + if (output is stdout) + output.writeln("[\033[01;31merror\033[0m] " ~ message, args); + else + output.writeln("[error] " ~ message, args); + } + + static void fatal(T...)(lazy string message, T args) + { + if (output is stdout) + output.writeln("[\033[01;35mfatal\033[0m] " ~ message, args); + else + output.writeln("[fatal] " ~ message, args); + } + static LogLevel level; + static File output; +} From 072932b6ff562a3f69cde399659ec6feb63e0575 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Sat, 12 Oct 2013 12:34:14 -0700 Subject: [PATCH 03/27] Work in progress --- actypes.d | 4 ++++ astconverter.d | 59 ++++++++++++++++++++++++++++++-------------------- notes.txt | 31 ++++++++++++++++++++++++++ semantic.d | 22 +++++++++---------- stupidlog.d | 24 ++++++++++---------- 5 files changed, 93 insertions(+), 47 deletions(-) create mode 100644 notes.txt diff --git a/actypes.d b/actypes.d index 02e3aa2..e4949ae 100644 --- a/actypes.d +++ b/actypes.d @@ -138,6 +138,9 @@ public: */ string symbolFile; + /** + * Gets all parts whose name matches the given string. + */ const(ACSymbol)*[] getPartsByName(string name) const { return cast(typeof(return)) parts.filter!(a => a.name == name).array; @@ -206,6 +209,7 @@ struct Scope } ACSymbol*[] symbols; + string[][] imports; } diff --git a/astconverter.d b/astconverter.d index e00c4bd..eed5309 100644 --- a/astconverter.d +++ b/astconverter.d @@ -18,6 +18,28 @@ * along with this program. If not, see . ******************************************************************************/ +/** + * AST conversion takes place in several steps + * 1. AST is converted to a tree of SemanicSymbols, a tree of ACSymbols, and a + * tree of scopes. The following fields are set on the symbols: + * * name + * * location + * * alias this + * * base class names + * * protection level + * * symbol kind + * * function call tip + * * symbol file path + * Import statements are recorded in the scope tree. + * 2. Scope tree is traversed and all imports are resolved by adding appropriate + * ACSymbol instances. + * 3. Semantic symbol tree is traversed + * * types are resolved + * * base classes are resolved + * * mixin templates are resolved + * * alias this is resolved + */ + module astconverter; import std.array; @@ -34,22 +56,8 @@ import messages; import semantic; import stupidlog; -enum SemanticType +class FirstPass : ASTVisitor { - partial, - full -} - -/** - * Basic visitor class used for caching completions of imported modules. - */ -class SemanticVisitor : ASTVisitor -{ - this (SemanticType sType) - { - this.semanticType = sType; - } - override void visit(Constructor con) { // Log.trace(__FUNCTION__, " ", typeof(con).stringof); @@ -130,13 +138,13 @@ class SemanticVisitor : ASTVisitor foreach (declarator; dec.declarators) { SemanticSymbol* symbol = new SemanticSymbol; - symbol.type = t; - symbol.protection = protection; + symbol.acSymbol.type = t; symbol.kind = CompletionKind.variableName; symbol.name = declarator.name.value.dup; symbol.location = declarator.name.startIndex; + symbol.protection = protection; symbol.parent = currentSymbol; - currentSymbol.children ~= symbol; + currentSymbol.addChild(symbol); } } @@ -234,7 +242,7 @@ class SemanticVisitor : ASTVisitor // Create scope for block statements override void visit(BlockStatement blockStatement) { - Log.trace(__FUNCTION__, " ", typeof(blockStatement).stringof); +// Log.trace(__FUNCTION__, " ", typeof(blockStatement).stringof); Scope* s = new Scope; s.startLocation = blockStatement.startLocation; s.endLocation = blockStatement.endLocation; @@ -243,7 +251,7 @@ class SemanticVisitor : ASTVisitor { foreach (child; currentSymbol.children) { - Log.trace("Setting ", child.name, " location"); +// Log.trace("Setting ", child.name, " location"); child.location = s.startLocation + 1; } } @@ -324,7 +332,7 @@ private: parameter.kind = CompletionKind.variableName; parameter.startLocation = p.name.startIndex; symbol.children ~= parameter; - Log.trace("Parameter ", parameter.name, " added to ", symbol.name); +// Log.trace("Parameter ", parameter.name, " added to ", symbol.name); } } else @@ -393,8 +401,6 @@ private: /// Current scope Scope* currentScope; - - SemanticType semanticType; } @@ -412,6 +418,11 @@ public: { convertSemanticSymbol(symbol); assert (current !is null); + resolveTypes(current); + } + + void resolveTypes(const(ACSymbol*) symbol) + { } ACSymbol* convertSemanticSymbol(const(SemanticSymbol)* symbol) @@ -545,7 +556,7 @@ private: else if (t.type2.symbol !is null) { if (t.type2.symbol.dot) - Log.trace("TODO: global scoped symbol handling"); + Log.error("TODO: global scoped symbol handling"); string[] symbolParts = expandSymbol( t.type2.symbol.identifierOrTemplateChain); diff --git a/notes.txt b/notes.txt new file mode 100644 index 0000000..da15461 --- /dev/null +++ b/notes.txt @@ -0,0 +1,31 @@ +AST → Scopes + → SemanticSymbol → ACSymbol + +ModuleCache has ACSymbol tree +Completion Context has Scopes and ACSymbol instances + + + +Caching +======= + +# Load file +# Lex +# Parse +# BasicSemanticVisitor generates semantic object tree +# Semantic information used to create ACSymbol tree. +# ACSymbol tree stored in ModuleCache + +Completion +========== + +# Lex source +# Parse +# AST Visitor generates semantic objects and scope information +# Semantic informaton used to create ACSymbol tree. This pass is more involved + because it also looks at variables local to functions and other symbols + that would not be included in the module cache. +# Semantic and scope info used to associate ACSymbol nodes and scopes. ACSymbol + objects store their location information. Given a blank Scope tree, a method + can be wnitten to assign ACSymbol instances to the correct scope. +# Autocompletion will need to only look at this simplified Scope/ACSymbol tree. diff --git a/semantic.d b/semantic.d index 60dabb5..6c06000 100644 --- a/semantic.d +++ b/semantic.d @@ -30,21 +30,24 @@ import stdx.d.lexer; struct SemanticSymbol { public: - /// Symbol name - string name; + + void name(string n) @property { acSymbol.name = n; } + + void addChild(SemanticSymbol* child) + { + children ~= child; + acSymbol.parts ~= child.acSymbol; + } + + /// Autocompletion symbol + ACSymbol* acSymbol; /// Base classes string[][] baseClasses; - /// Completion kind - CompletionKind kind; - /// Variable type or function return type Type type; - /// Function call tip. Null if this is not a function - string callTip; - /// Alias this symbols string[] aliasThis; @@ -54,8 +57,5 @@ public: /// Protection level for this symobol TokenType protection; - /// Symbol location - size_t location; - mixin scopeImplementation!(SemanticSymbol); } diff --git a/stupidlog.d b/stupidlog.d index 86ca842..dc78a57 100644 --- a/stupidlog.d +++ b/stupidlog.d @@ -33,39 +33,39 @@ enum LogLevel : uint struct Log { - static void trace(T...)(lazy string message, T args) + static void trace(T...)(T args) { if (level < LogLevel.trace) return; if (output is stdout) - output.writeln("[\033[01;36mtrace\033[0m] " ~ message, args); + output.writeln("[\033[01;36mtrace\033[0m] ", args); else - output.writeln("[trace] " ~ message, args); + output.writeln("[trace] ", args); } - static void info(T...)(lazy string message, T args) + static void info(T...)(T args) { if (level < LogLevel.info) return; if (output is stdout) - output.writeln("[\033[01;32minfo\033[0m ] " ~ message, args); + output.writeln("[\033[01;32minfo\033[0m ] ", args); else - output.writeln("[info ] " ~ message, args); + output.writeln("[info ] ", args); } - static void error(T...)(lazy string message, T args) + static void error(T...)(T args) { if (level < LogLevel.error) return; if (output is stdout) - output.writeln("[\033[01;31merror\033[0m] " ~ message, args); + output.writeln("[\033[01;31merror\033[0m] ", args); else - output.writeln("[error] " ~ message, args); + output.writeln("[error] ", args); } - static void fatal(T...)(lazy string message, T args) + static void fatal(T...)(T args) { if (output is stdout) - output.writeln("[\033[01;35mfatal\033[0m] " ~ message, args); + output.writeln("[\033[01;35mfatal\033[0m] ", args); else - output.writeln("[fatal] " ~ message, args); + output.writeln("[fatal] ", args); } static LogLevel level; static File output; From 6b432053f01e279b90443ad648ab5c0357b9e37e Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Sat, 12 Oct 2013 19:46:47 +0000 Subject: [PATCH 04/27] It compiles again --- actypes.d | 30 +-- astconverter.d | 603 +++++++++++++++++++++++++------------------------ autocomplete.d | 6 +- build.sh | 1 + messages.d | 6 + modulecache.d | 2 +- semantic.d | 14 +- 7 files changed, 343 insertions(+), 319 deletions(-) diff --git a/actypes.d b/actypes.d index e4949ae..fd92767 100644 --- a/actypes.d +++ b/actypes.d @@ -157,14 +157,9 @@ public: } } -mixin template scopeImplementation(ScopeType) +struct Scope { - ScopeType* parent; - ScopeType*[] children; - size_t startLocation; - size_t endLocation; - - ScopeType* getScopeByCursor(size_t cursorPosition) const + Scope* getScopeByCursor(size_t cursorPosition) const { if (cursorPosition < startLocation) return null; if (cursorPosition > endLocation) return null; @@ -176,11 +171,6 @@ mixin template scopeImplementation(ScopeType) } return cast(typeof(return)) &this; } -} - -struct Scope -{ - mixin scopeImplementation!(typeof(this)); ACSymbol*[] getSymbolsInCursorScope(size_t cursorPosition) const { @@ -209,7 +199,21 @@ struct Scope } ACSymbol*[] symbols; - string[][] imports; + ImportInformation[] importInformation; + Scope* parent; + Scope*[] children; + size_t startLocation; + size_t endLocation; +} + +struct ImportInformation +{ + /// module relative path + string modulePath; + /// symbols to import from this module + string[string] importedSymbols; + /// true if the import is public + bool isPublic; } diff --git a/astconverter.d b/astconverter.d index eed5309..1a833d9 100644 --- a/astconverter.d +++ b/astconverter.d @@ -18,86 +18,102 @@ * along with this program. If not, see . ******************************************************************************/ -/** - * AST conversion takes place in several steps - * 1. AST is converted to a tree of SemanicSymbols, a tree of ACSymbols, and a - * tree of scopes. The following fields are set on the symbols: - * * name - * * location - * * alias this - * * base class names - * * protection level - * * symbol kind - * * function call tip - * * symbol file path - * Import statements are recorded in the scope tree. - * 2. Scope tree is traversed and all imports are resolved by adding appropriate - * ACSymbol instances. - * 3. Semantic symbol tree is traversed - * * types are resolved - * * base classes are resolved - * * mixin templates are resolved - * * alias this is resolved - */ - module astconverter; +import std.algorithm; import std.array; import std.conv; +import std.path; import std.range; -import std.algorithm; import stdx.d.ast; import stdx.d.lexer; import stdx.d.parser; import actypes; +import constants; import messages; import semantic; import stupidlog; -class FirstPass : ASTVisitor +/** + * First Pass handles the following: + * $(UL + * $(LI symbol name) + * $(LI symbol location) + * $(LI alias this locations) + * $(LI base class names) + * $(LI protection level) + * $(LI symbol kind) + * $(LI function call tip) + * $(LI symbol file path) + * ) + */ +final class FirstPass : ASTVisitor { + this(Module mod, string symbolFile) + { + this.symbolFile = symbolFile; + this.mod = mod; + } + + void run() + { + visit(mod); + } + override void visit(Constructor con) { // Log.trace(__FUNCTION__, " ", typeof(con).stringof); - visitFunctionDeclaration(con); + visitConstructor(con.location, con.parameters, con.functionBody); } override void visit(SharedStaticConstructor con) { // Log.trace(__FUNCTION__, " ", typeof(con).stringof); - visitFunctionDeclaration(con); + visitConstructor(con.location, null, con.functionBody); } override void visit(StaticConstructor con) { // Log.trace(__FUNCTION__, " ", typeof(con).stringof); - visitFunctionDeclaration(con); + visitConstructor(con.location, null, con.functionBody); } override void visit(Destructor des) { // Log.trace(__FUNCTION__, " ", typeof(des).stringof); - visitFunctionDeclaration(des); + visitDestructor(des.location, des.functionBody); } override void visit(SharedStaticDestructor des) { // Log.trace(__FUNCTION__, " ", typeof(des).stringof); - visitFunctionDeclaration(des); + visitDestructor(des.location, des.functionBody); } override void visit(StaticDestructor des) { // Log.trace(__FUNCTION__, " ", typeof(des).stringof); - visitFunctionDeclaration(des); + visitDestructor(des.location, des.functionBody); } override void visit(FunctionDeclaration dec) { // Log.trace(__FUNCTION__, " ", typeof(dec).stringof); - visitFunctionDeclaration(dec); + SemanticSymbol* symbol = new SemanticSymbol(dec.name.value.dup, + CompletionKind.functionName, symbolFile, dec.name.startIndex); + processParameters(symbol, dec.returnType, symbol.acSymbol.name, + dec.parameters); + symbol.protection = protection; + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol); + if (dec.functionBody !is null) + { + currentSymbol = symbol; + dec.functionBody.accept(this); + currentSymbol = symbol.parent; + } } override void visit(ClassDeclaration dec) @@ -137,11 +153,12 @@ class FirstPass : ASTVisitor Type t = dec.type; foreach (declarator; dec.declarators) { - SemanticSymbol* symbol = new SemanticSymbol; - symbol.acSymbol.type = t; - symbol.kind = CompletionKind.variableName; - symbol.name = declarator.name.value.dup; - symbol.location = declarator.name.startIndex; + SemanticSymbol* symbol = new SemanticSymbol( + declarator.name.value.dup, + CompletionKind.variableName, + symbolFile, + declarator.name.startIndex); + symbol.type = t; symbol.protection = protection; symbol.parent = currentSymbol; currentSymbol.addChild(symbol); @@ -176,14 +193,16 @@ class FirstPass : ASTVisitor override void visit(Module mod) { // Log.trace(__FUNCTION__, " ", typeof(mod).stringof); - rootSymbol = new SemanticSymbol; - rootSymbol.kind = CompletionKind.moduleName; - rootSymbol.startLocation = 0; - rootSymbol.endLocation = size_t.max; - currentSymbol = rootSymbol; +// + currentSymbol = new SemanticSymbol(null, CompletionKind.moduleName, + symbolFile); + rootSymbol = currentSymbol; + currentScope = new Scope(); currentScope.startLocation = 0; currentScope.endLocation = size_t.max; + moduleScope = currentScope; + mod.accept(this); } @@ -191,29 +210,25 @@ class FirstPass : ASTVisitor { assert (currentSymbol); // Log.trace(__FUNCTION__, " ", typeof(dec).stringof); - SemanticSymbol* symbol = new SemanticSymbol; - symbol.name = dec.name.value.dup; - symbol.location = dec.name.startIndex; - symbol.kind = CompletionKind.enumName; + SemanticSymbol* symbol = new SemanticSymbol(dec.name.value.dup, + CompletionKind.enumName, symbolFile, dec.name.startIndex); symbol.type = dec.type; symbol.parent = currentSymbol; currentSymbol = symbol; if (dec.enumBody !is null) dec.enumBody.accept(this); currentSymbol = symbol.parent; - currentSymbol.children ~= symbol; + currentSymbol.addChild(symbol); } override void visit(EnumMember member) { // Log.trace(__FUNCTION__, " ", typeof(member).stringof); - SemanticSymbol* symbol = new SemanticSymbol; - symbol.kind = CompletionKind.enumMember; - symbol.name = member.name.value.dup; - symbol.location = member.name.startIndex; + SemanticSymbol* symbol = new SemanticSymbol(member.name.value.dup, + CompletionKind.enumMember, symbolFile, member.name.startIndex); symbol.type = member.type; symbol.parent = currentSymbol; - currentSymbol.children ~= symbol; + currentSymbol.addChild(symbol); } override void visit(ModuleDeclaration dec) @@ -239,31 +254,63 @@ class FirstPass : ASTVisitor currentScope.children ~= s; } + override void visit(ImportDeclaration importDeclaration) + { + Log.trace(__FUNCTION__, " ImportDeclaration"); + + + foreach (single; importDeclaration.singleImports.filter!( + a => a !is null && a.identifierChain !is null)) + { + ImportInformation info; + info.modulePath = convertChainToImportPath(single.identifierChain); + currentScope.importInformation ~= info; + } + if (importDeclaration.importBindings is null) return; + if (importDeclaration.importBindings.singleImport.identifierChain is null) return; + ImportInformation info; + info.modulePath = convertChainToImportPath( + importDeclaration.importBindings.singleImport.identifierChain); + foreach (bind; importDeclaration.importBindings.importBinds) + { + if (bind.right == TokenType.invalid) + info.importedSymbols[bind.left.value] = null; + else + info.importedSymbols[bind.left.value] = bind.right.value; + } + currentScope.importInformation ~= info; + } + // Create scope for block statements override void visit(BlockStatement blockStatement) { // Log.trace(__FUNCTION__, " ", typeof(blockStatement).stringof); Scope* s = new Scope; + s.parent = currentScope; + currentScope.children ~= s; s.startLocation = blockStatement.startLocation; s.endLocation = blockStatement.endLocation; - if (currentSymbol.kind == CompletionKind.functionName) + if (currentSymbol.acSymbol.kind == CompletionKind.functionName) { foreach (child; currentSymbol.children) { -// Log.trace("Setting ", child.name, " location"); - child.location = s.startLocation + 1; + child.acSymbol.location = s.startLocation + 1; } } -// Log.trace("Added scope ", s.startLocation, " ", s.endLocation); - s.parent = currentScope; if (blockStatement.declarationsAndStatements !is null) { currentScope = s; visit (blockStatement.declarationsAndStatements); currentScope = s.parent; } - currentScope.children ~= s; + } + + override void visit(VersionCondition versionCondition) + { + // TODO: This is a bit of a hack + if (predefinedVersions.canFind(versionCondition.token.value)) + versionCondition.accept(this); } alias ASTVisitor.visit visit; @@ -272,109 +319,88 @@ private: void visitAggregateDeclaration(AggType)(AggType dec, CompletionKind kind) { - SemanticSymbol* symbol = new SemanticSymbol; + // Log.trace("visiting aggregate declaration ", dec.name.value); - symbol.name = dec.name.value.dup; - symbol.location = dec.name.startIndex; - symbol.kind = kind; + CompletionKind k; + static if (is (AggType == ClassDeclaration)) + k = CompletionKind.className; + else static if (is (AggType == InterfaceDeclaration)) + k = CompletionKind.interfaceName; + else static if (is (AggType == StructDeclaration)) + k = CompletionKind.structName; + else static if (is (AggType == UnionDeclaration)) + k = CompletionKind.unionName; + else static assert (false, "Unhandled aggregate type " ~ AggType.stringof); + + SemanticSymbol* symbol = new SemanticSymbol(dec.name.value.dup, + kind, symbolFile, dec.name.startIndex); symbol.parent = currentSymbol; symbol.protection = protection; currentSymbol = symbol; dec.accept(this); currentSymbol = symbol.parent; - currentSymbol.children ~= symbol; + currentSymbol.addChild(symbol); } - void visitFunctionDeclaration(DeclarationType)(DeclarationType dec) - { - SemanticSymbol* symbol = new SemanticSymbol; - - static if (is (DeclarationType == FunctionDeclaration)) - { - symbol.name = dec.name.value.dup; - symbol.location = dec.name.startIndex; - } - else static if (is (DeclarationType == Destructor) - || is (DeclarationType == StaticDestructor) - || is (DeclarationType == SharedStaticDestructor)) - { - symbol.name = "*destructor*"; - symbol.location = dec.location; - } - else - { - symbol.name = "*constructor*"; - symbol.location = dec.location; - } - - static if (is (DeclarationType == Destructor) - || is (DeclarationType == StaticDestructor) - || is (DeclarationType == SharedStaticDestructor)) - { - symbol.callTip = "~this()"; - } - else static if (is (DeclarationType == StaticConstructor) - || is (DeclarationType == SharedStaticConstructor)) - { - symbol.callTip = "this()"; - } - else - { - string parameterString; - if (dec.parameters !is null) - { - parameterString = formatNode(dec.parameters); - foreach (Parameter p; dec.parameters.parameters) - { - SemanticSymbol* parameter = new SemanticSymbol; - parameter.name = p.name.value.dup; - parameter.type = p.type; - parameter.kind = CompletionKind.variableName; - parameter.startLocation = p.name.startIndex; - symbol.children ~= parameter; -// Log.trace("Parameter ", parameter.name, " added to ", symbol.name); - } - } - else - parameterString = "()"; - - static if (is (DeclarationType == FunctionDeclaration)) - symbol.callTip = "%s %s%s".format(formatNode(dec.returnType), - dec.name.value, parameterString); - else - symbol.callTip = "this%s".format(parameterString); - } - - symbol.protection = protection; - symbol.kind = CompletionKind.functionName; - symbol.parent = currentSymbol; - currentSymbol = symbol; - - if (dec.functionBody !is null) - { - dec.functionBody.accept(this); - } - currentSymbol = symbol.parent; - currentSymbol.children ~= symbol; - } - - static string[] iotcToStringArray(const IdentifierOrTemplateChain iotc) + void visitConstructor(size_t location, Parameters parameters, + FunctionBody functionBody) { - string[] parts; - foreach (ioti; iotc.identifiersOrTemplateInstances) + SemanticSymbol* symbol = new SemanticSymbol("this", + CompletionKind.functionName, symbolFile, location); + processParameters(symbol, null, "this", parameters); + symbol.protection = protection; + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol); + if (functionBody !is null) { - if (ioti.identifier != TokenType.invalid) - parts ~= ioti.identifier.value.dup; - else - parts ~= ioti.templateInstance.identifier.value.dup; + currentSymbol = symbol; + functionBody.accept(this); + currentSymbol = symbol.parent; } - return parts; } - static string formatCalltip(Type returnType, string name, Parameters parameters, + void visitDestructor(size_t location, FunctionBody functionBody) + { + SemanticSymbol* symbol = new SemanticSymbol("~this", + CompletionKind.functionName, symbolFile, location); + symbol.acSymbol.callTip = "~this()"; + symbol.protection = protection; + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol); + if (functionBody !is null) + { + currentSymbol = symbol; + functionBody.accept(this); + currentSymbol = symbol.parent; + } + } + + void processParameters(SemanticSymbol* symbol, Type returnType, + string functionName, Parameters parameters) const + { + if (parameters !is null) + { + foreach (Parameter p; parameters.parameters) + { + SemanticSymbol* parameter = new SemanticSymbol(p.name.value.dup, + CompletionKind.variableName, symbolFile, p.name.startIndex); + parameter.type = p.type; + symbol.addChild(parameter); + parameter.parent = symbol; + } + } + symbol.acSymbol.callTip = formatCallTip(returnType, functionName, + parameters); + } + + static string formatCallTip(Type returnType, string name, Parameters parameters, string doc = null) { - return "%s %s%s".format(formatNode(returnType), name, formatNode(parameters)); + string parameterString = parameters is null ? "()" + : formatNode(parameters); + if (returnType is null) + return "%s%s".format(name, parameterString); + return "%s %s%s".format(formatNode(returnType), name, parameterString); } static string formatNode(T)(T node) @@ -401,144 +427,116 @@ private: /// Current scope Scope* currentScope; + + /// Module scope + Scope* moduleScope; + + /// Path to the file being converted + string symbolFile; + + Module mod; } - -struct SemanticConverter +/** + * Second pass handles the following: + * $(UL + * $(LI Import statements) + * $(LI assigning symbols to scopes) + * ) + */ +struct SecondPass { public: - this(const(SemanticSymbol)* symbol, Scope* scopes) + this(SemanticSymbol* rootSymbol, Scope* moduleScope) { - this.sc = scopes; - this.symbol = symbol; + this.rootSymbol = rootSymbol; + this.moduleScope = moduleScope; } - void convertModule() + void run() { - convertSemanticSymbol(symbol); - assert (current !is null); - resolveTypes(current); + assignToScopes(rootSymbol.acSymbol); + resolveImports(moduleScope); } - void resolveTypes(const(ACSymbol*) symbol) - { - } - - ACSymbol* convertSemanticSymbol(const(SemanticSymbol)* symbol) - { - ACSymbol* s = null; - with (CompletionKind) final switch (symbol.kind) - { - case moduleName: - s = convertAggregateDeclaration(symbol); - current = s; - break; - case className: - case interfaceName: - case structName: - case unionName: - case enumName: - s = convertAggregateDeclaration(symbol); - break; - case enumMember: - s = convertEnumMember(symbol); - break; - case variableName: - case memberVariableName: - s = convertVariableDeclaration(symbol); - break; - case functionName: - s = convertFunctionDeclaration(symbol); - break; - case aliasName: - s = convertAliasDeclaration(symbol); - break; - case packageName: - assert (false, "Not implemented"); - case keyword: - case array: - case assocArray: - case dummy: - assert (false, "This should never be reached"); - } - if (sc !is null && symbol.kind != CompletionKind.moduleName) - { - sc.getScopeByCursor(s.location).symbols ~= s; -// Log.trace("Set scope location"); - } - return s; - } - - ACSymbol* convertAliasDeclaration(const(SemanticSymbol)* symbol) - { - ACSymbol* ac = new ACSymbol; - ac.name = symbol.name; - ac.kind = symbol.kind; - ac.location = symbol.location; - // TODO: Set type - return ac; - } - - ACSymbol* convertVariableDeclaration(const(SemanticSymbol)* symbol) - { - ACSymbol* ac = new ACSymbol; - ac.name = symbol.name; - ac.kind = CompletionKind.variableName; - ac.location = symbol.location; - if (symbol.type !is null) - ac.type = resolveType(symbol.type); - // TODO: Handle auto - return ac; - } - - ACSymbol* convertEnumMember(const(SemanticSymbol)* symbol) - { - ACSymbol* ac = new ACSymbol; - ac.name = symbol.name; - ac.kind = symbol.kind; - ac.location = symbol.location; - // TODO: type - return ac; - } - - ACSymbol* convertFunctionDeclaration(const(SemanticSymbol)* symbol) - { -// Log.trace("Converted ", symbol.name, " ", symbol.kind, " ", symbol.location); - ACSymbol* ac = new ACSymbol; - ac.name = symbol.name; - ac.kind = symbol.kind; - ac.location = symbol.location; - ac.callTip = symbol.callTip; - if (symbol.type !is null) - ac.type = resolveType(symbol.type); - return ac; - } - - ACSymbol* convertAggregateDeclaration(const(SemanticSymbol)* symbol) - { -// Log.trace("Converted ", symbol.name, " ", symbol.kind, " ", symbol.location); - ACSymbol* ac = new ACSymbol; - ac.name = symbol.name; - ac.kind = symbol.kind; - ac.location = symbol.location; - if (symbol.kind == CompletionKind.className - || symbol.kind == CompletionKind.structName) - { - ACSymbol* thisSymbol = new ACSymbol("this", - CompletionKind.variableName, ac); - ac.parts ~= thisSymbol; - } - auto temp = current; - current = ac; - foreach (child; symbol.children) - current.parts ~= convertSemanticSymbol(child); - current = temp; - return ac; - } - private: + void assignToScopes(ACSymbol* currentSymbol) + { + moduleScope.getScopeByCursor(currentSymbol.location).symbols + ~= currentSymbol; + foreach (part; currentSymbol.parts) + assignToScopes(part); + } + + void resolveImports(Scope* currentScope) + { + foreach (importInfo; currentScope.importInformation) + { + // TODO: actually process imports here + } + } + + SemanticSymbol* rootSymbol; + Scope* moduleScope; +} + +/** + * Third pass handles the following: + * $(UL + * $(LI types) + * $(LI base classes) + * $(LI mixin templates) + * $(LI alias this) + * ) + */ +struct ThirdPass +{ +public: + this(SemanticSymbol* rootSymbol, Scope* moduleScope) + { + this.rootSymbol = rootSymbol; + this.moduleScope = moduleScope; + } + + void run() + { + thirdPass(rootSymbol); + } + +private: + + void thirdPass(SemanticSymbol* currentSymbol) + { + with (CompletionKind) final switch (currentSymbol.acSymbol.kind) + { + case className: + case interfaceName: + // resolve inheritance + goto case structName; + case structName: + case unionName: + break; + case variableName: + case memberVariableName: + break; + case functionName: + break; + case enumName: + break; + case keyword: + case enumMember: + case packageName: + case moduleName: + case dummy: + case array: + case assocArray: + case aliasName: + case templateName: + case mixinTemplateName: + } + } ACSymbol* resolveType(const Type t) in @@ -610,29 +608,30 @@ private: return null; } - ACSymbol* current; - Scope* sc; - const(SemanticSymbol)* symbol; + SemanticSymbol* rootSymbol; + Scope* moduleScope; } -const(ACSymbol)*[] convertAstToSymbols(Module m) +const(ACSymbol)*[] convertAstToSymbols(Module m, string symbolFile) { - SemanticVisitor visitor = new SemanticVisitor(SemanticType.partial); - visitor.visit(m); - SemanticConverter converter = SemanticConverter(visitor.rootSymbol, null); - converter.convertModule(); - return cast(typeof(return)) converter.current.parts; + FirstPass first = new FirstPass(m, symbolFile); + first.run(); + SecondPass second = SecondPass(first.rootSymbol, first.moduleScope); + second.run(); + ThirdPass third = ThirdPass(second.rootSymbol, second.moduleScope); + return cast(typeof(return)) third.rootSymbol.acSymbol.parts; } -const(Scope)* generateAutocompleteTrees(const(Token)[] tokens) +const(Scope)* generateAutocompleteTrees(const(Token)[] tokens, string symbolFile) { Module m = parseModule(tokens, null); - SemanticVisitor visitor = new SemanticVisitor(SemanticType.full); - visitor.visit(m); - SemanticConverter converter = SemanticConverter(visitor.rootSymbol, - visitor.currentScope); - converter.convertModule(); - return converter.sc; + FirstPass first = new FirstPass(m, symbolFile); + first.run(); + SecondPass second = SecondPass(first.rootSymbol, first.currentScope); + second.run(); + ThirdPass third = ThirdPass(second.rootSymbol, second.moduleScope); + third.run(); + return cast(typeof(return)) third.moduleScope; } version(unittest) Module parseTestCode(string code) @@ -648,20 +647,22 @@ version(unittest) Module parseTestCode(string code) return m; } -unittest -{ - auto source = q{ - module foo; +private: - struct Bar { int x; int y;} - }c; - Module m = parseTestCode(source); - BasicSemanticVisitor visitor = new BasicSemanticVisitor; - visitor.visit(m); - assert (visitor.rootSymbol !is null); - assert (visitor.rootSymbol.name == "foo"); - SemanticSymbol* bar = visitor.root.getPartByName("Bar"); - assert (bar !is null); - assert (bar.getPartByName("x") !is null); - assert (bar.getPartByName("y") !is null); +string[] iotcToStringArray(const IdentifierOrTemplateChain iotc) +{ + string[] parts; + foreach (ioti; iotc.identifiersOrTemplateInstances) + { + if (ioti.identifier != TokenType.invalid) + parts ~= ioti.identifier.value.dup; + else + parts ~= ioti.templateInstance.identifier.value.dup; + } + return parts; +} + +private static string convertChainToImportPath(IdentifierChain chain) +{ + return to!string(chain.identifiers.map!(a => a.value).join(dirSeparator).array) ~ ".d"; } diff --git a/autocomplete.d b/autocomplete.d index 80632e2..4ecdeef 100644 --- a/autocomplete.d +++ b/autocomplete.d @@ -100,7 +100,8 @@ AutocompleteResponse complete(AutocompleteRequest request, string[] importPaths) case TokenType.identifier: case TokenType.rParen: case TokenType.rBracket: - const(Scope)* completionScope = generateAutocompleteTrees(tokenArray); + const(Scope)* completionScope = generateAutocompleteTrees(tokenArray, + "stdin"); auto expression = getExpression(beforeTokens[0 .. $ - 1]); response.setCompletions(completionScope, expression, request.cursorPosition, CompletionType.calltips); @@ -152,7 +153,8 @@ dotCompletion: case TokenType.rParen: case TokenType.rBracket: case TokenType.this_: - const(Scope)* completionScope = generateAutocompleteTrees(tokenArray); + const(Scope)* completionScope = generateAutocompleteTrees(tokenArray, + "stdin"); auto expression = getExpression(beforeTokens); response.setCompletions(completionScope, expression, request.cursorPosition, CompletionType.identifiers, partial); diff --git a/build.sh b/build.sh index f7d84a2..1c0d5f6 100755 --- a/build.sh +++ b/build.sh @@ -23,4 +23,5 @@ dmd \ -Imsgpack-d/src\ -Idscanner\ -wi\ + -g\ -ofdcd-server diff --git a/messages.d b/messages.d index 39f8276..121adaa 100644 --- a/messages.d +++ b/messages.d @@ -71,6 +71,12 @@ enum CompletionKind : char /// alias name aliasName = 'l', + + /// template name + templateName = 't', + + /// mixin template name + mixinTemplateName = 'T' } /** diff --git a/modulecache.d b/modulecache.d index 956f63e..31587e4 100644 --- a/modulecache.d +++ b/modulecache.d @@ -105,7 +105,7 @@ struct ModuleCache auto tokens = source.byToken(config).array(); Module mod = parseModule(tokens, location, &doesNothing); - symbols = convertAstToSymbols(mod); + symbols = convertAstToSymbols(mod, location); } catch (Exception ex) { diff --git a/semantic.d b/semantic.d index 6c06000..36596e5 100644 --- a/semantic.d +++ b/semantic.d @@ -31,7 +31,15 @@ struct SemanticSymbol { public: - void name(string n) @property { acSymbol.name = n; } + @disable this(); + + this(string name, CompletionKind kind, string symbolFile, + size_t location = size_t.max) + { + acSymbol = new ACSymbol(name, kind); + acSymbol.location = location; + acSymbol.symbolFile = symbolFile; + } void addChild(SemanticSymbol* child) { @@ -57,5 +65,7 @@ public: /// Protection level for this symobol TokenType protection; - mixin scopeImplementation!(SemanticSymbol); + SemanticSymbol* parent; + + SemanticSymbol*[] children; } From 41dbdaed4a204e973b34877e36659e40b3407477 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Sun, 13 Oct 2013 17:03:44 +0000 Subject: [PATCH 05/27] More work on the new completion process. Fixed #55 --- actypes.d | 33 ++++------ astconverter.d | 160 ++++++++++++++++++++++++++++++++++++------------- modulecache.d | 47 +++++++++++---- semantic.d | 6 ++ server.d | 9 +-- 5 files changed, 174 insertions(+), 81 deletions(-) diff --git a/actypes.d b/actypes.d index fd92767..f788250 100644 --- a/actypes.d +++ b/actypes.d @@ -25,6 +25,7 @@ import std.stdio; import std.array; import messages; import std.array; +import std.typecons; /** * Any special information about a variable declaration symbol. @@ -74,7 +75,7 @@ public: * kind = the symbol's completion kind * resolvedType = the resolved type of the symbol */ - this(string name, CompletionKind kind, ACSymbol* type) + this(string name, CompletionKind kind, const(ACSymbol)* type) { this.name = name; this.kind = kind; @@ -101,7 +102,7 @@ public: * Symbols that compose this symbol, such as enum members, class variables, * methods, etc. */ - ACSymbol*[] parts; + const(ACSymbol)*[] parts; /** * Symbol's name @@ -111,7 +112,7 @@ public: /** * The symbol that represents the type. */ - ACSymbol* type; + const(ACSymbol)* type; /** * Calltip to display if this is a function @@ -145,16 +146,6 @@ public: { return cast(typeof(return)) parts.filter!(a => a.name == name).array; } - - /** - * Sorts the parts array, and the parts array of each part, and so on. - */ - void sortParts() - { - parts.sort(); - foreach (p; parts) - p.sortParts(); - } } struct Scope @@ -198,7 +189,7 @@ struct Scope return s.getSymbolsByName(name); } - ACSymbol*[] symbols; + const(ACSymbol)*[] symbols; ImportInformation[] importInformation; Scope* parent; Scope*[] children; @@ -211,7 +202,7 @@ struct ImportInformation /// module relative path string modulePath; /// symbols to import from this module - string[string] importedSymbols; + Tuple!(string, string)[] importedSymbols; /// true if the import is public bool isPublic; } @@ -279,7 +270,6 @@ static this() s.parts ~= sizeof_; s.parts ~= stringof_; s.parts ~= mangleof_; - s.sortParts(); } auto cdouble_ = new ACSymbol("cdouble", CompletionKind.keyword); @@ -314,7 +304,6 @@ static this() s.parts ~= new ACSymbol("nan", CompletionKind.keyword, s); s.parts ~= sizeof_; s.parts ~= stringof_; - s.sortParts(); } ireal_.parts ~= new ACSymbol("im", CompletionKind.keyword, real_); @@ -331,8 +320,8 @@ static this() float_, idouble_, ifloat_, ireal_, real_, ucent_, void_]; } -ACSymbol*[] builtinSymbols; -ACSymbol*[] arraySymbols; -ACSymbol*[] assocArraySymbols; -ACSymbol*[] classSymbols; -ACSymbol*[] structSymbols; +const(ACSymbol)*[] builtinSymbols; +const(ACSymbol)*[] arraySymbols; +const(ACSymbol)*[] assocArraySymbols; +const(ACSymbol)*[] classSymbols; +const(ACSymbol)*[] structSymbols; diff --git a/astconverter.d b/astconverter.d index 1a833d9..feff926 100644 --- a/astconverter.d +++ b/astconverter.d @@ -25,6 +25,7 @@ import std.array; import std.conv; import std.path; import std.range; +import std.typecons; import stdx.d.ast; import stdx.d.lexer; @@ -35,6 +36,7 @@ import constants; import messages; import semantic; import stupidlog; +import modulecache; // circular import /** * First Pass handles the following: @@ -256,9 +258,7 @@ final class FirstPass : ASTVisitor override void visit(ImportDeclaration importDeclaration) { - Log.trace(__FUNCTION__, " ImportDeclaration"); - - +// Log.trace(__FUNCTION__, " ImportDeclaration"); foreach (single; importDeclaration.singleImports.filter!( a => a !is null && a.identifierChain !is null)) { @@ -273,10 +273,10 @@ final class FirstPass : ASTVisitor importDeclaration.importBindings.singleImport.identifierChain); foreach (bind; importDeclaration.importBindings.importBinds) { - if (bind.right == TokenType.invalid) - info.importedSymbols[bind.left.value] = null; - else - info.importedSymbols[bind.left.value] = bind.right.value; + Tuple!(string, string) bindTuple; + bindTuple[0] = bind.left.value.dup; + bindTuple[1] = bind.right == TokenType.invalid ? null : bind.right.value.dup; + info.importedSymbols ~= bindTuple; } currentScope.importInformation ~= info; } @@ -462,7 +462,7 @@ public: private: - void assignToScopes(ACSymbol* currentSymbol) + void assignToScopes(const(ACSymbol)* currentSymbol) { moduleScope.getScopeByCursor(currentSymbol.location).symbols ~= currentSymbol; @@ -474,7 +474,35 @@ private: { foreach (importInfo; currentScope.importInformation) { - // TODO: actually process imports here + auto symbols = ModuleCache.getSymbolsInModule(importInfo.modulePath); + if (importInfo.importedSymbols.length == 0) + { + currentScope.symbols ~= symbols; + continue; + } + symbolLoop: foreach (symbol; symbols) + { + foreach (tup; importInfo.importedSymbols) + { + if (tup[0] != symbol.name) + continue symbolLoop; + if (tup[1] !is null) + { + ACSymbol* s = new ACSymbol(tup[1], + symbol.kind, symbol.type); + // TODO: Compiler gets confused here, so cast the types. + s.parts = cast(typeof(s.parts)) symbol.parts; + // TODO: Re-format callTip with new name? + s.callTip = symbol.callTip; + s.qualifier = symbol.qualifier; + s.location = symbol.location; + s.symbolFile = symbol.symbolFile; + currentScope.symbols ~= s; + } + else + currentScope.symbols ~= symbol; + } + } } } @@ -509,22 +537,26 @@ private: void thirdPass(SemanticSymbol* currentSymbol) { +// Log.trace("third pass on ", currentSymbol.acSymbol.name); with (CompletionKind) final switch (currentSymbol.acSymbol.kind) { case className: case interfaceName: - // resolve inheritance + resolveInheritance(currentSymbol); goto case structName; case structName: case unionName: + resolveAliasThis(currentSymbol); + resolveMixinTemplates(currentSymbol); break; case variableName: case memberVariableName: - break; case functionName: +// Log.trace("Resolving type of ", currentSymbol.acSymbol.name); + currentSymbol.acSymbol.type = resolveType(currentSymbol.type, + currentSymbol.acSymbol.location); break; case enumName: - break; case keyword: case enumMember: case packageName: @@ -535,33 +567,78 @@ private: case aliasName: case templateName: case mixinTemplateName: + break; + } + + foreach (child; currentSymbol.children) + thirdPass(child); + } + + void resolveInheritance(SemanticSymbol* currentSymbol) + { +// Log.trace("Resolving inheritance for ", currentSymbol.acSymbol.name); + outer: foreach (string[] base; currentSymbol.baseClasses) + { + const(ACSymbol)* baseClass; + if (base.length == 0) + continue; + auto symbols = moduleScope.getSymbolsByNameAndCursor( + base[0], currentSymbol.acSymbol.location); + if (symbols.length == 0) + continue; + baseClass = symbols[0]; + foreach (part; base[1..$]) + { + symbols = baseClass.getPartsByName(part); + if (symbols.length == 0) + continue outer; + baseClass = symbols[0]; + } + currentSymbol.acSymbol.parts ~= baseClass.parts; } } - ACSymbol* resolveType(const Type t) - in + void resolveAliasThis(SemanticSymbol* currentSymbol) { - assert (t !is null); - assert (t.type2 !is null); + // TODO: } - body + + void resolveMixinTemplates(SemanticSymbol* currentSymbol) { - ACSymbol* s; + // TODO: + } + + const(ACSymbol)* resolveType(Type t, size_t location) + { + if (t is null) return null; + if (t.type2 is null) return null; + const(ACSymbol)* s; if (t.type2.builtinType != TokenType.invalid) s = convertBuiltinType(t.type2); else if (t.type2.typeConstructor != TokenType.invalid) - s = resolveType(t.type2.type); + s = resolveType(t.type2.type, location); else if (t.type2.symbol !is null) { - if (t.type2.symbol.dot) - Log.error("TODO: global scoped symbol handling"); + // TODO: global scoped symbol handling string[] symbolParts = expandSymbol( t.type2.symbol.identifierOrTemplateChain); - + auto symbols = moduleScope.getSymbolsByNameAndCursor( + symbolParts[0], location); + if (symbols.length == 0) + goto resolveSuffixes; + s = symbols[0]; + foreach (symbolPart; symbolParts[1..$]) + { + auto parts = s.getPartsByName(symbolPart); + if (parts.length == 0) + goto resolveSuffixes; + s = parts[0]; + } } + resolveSuffixes: foreach (suffix; t.typeSuffixes) s = processSuffix(s, suffix); - return null; + return s; } static string[] expandSymbol(const IdentifierOrTemplateChain chain) @@ -570,6 +647,8 @@ private: for (size_t i = 0; i != chain.identifiersOrTemplateInstances.length; ++i) { auto identOrTemplate = chain.identifiersOrTemplateInstances[i]; + if (identOrTemplate is null) + continue; strings[i] = identOrTemplate.templateInstance is null ? identOrTemplate.identifier.value.dup : identOrTemplate.identifier.value.dup; @@ -577,14 +656,14 @@ private: return strings; } - static ACSymbol* processSuffix(ACSymbol* symbol, const TypeSuffix suffix) + static const(ACSymbol)* processSuffix(const(ACSymbol)* symbol, const TypeSuffix suffix) { if (suffix.star) return symbol; if (suffix.array || suffix.type) { ACSymbol* s = new ACSymbol; - s.parts = arraySymbols; + s.parts = suffix.array ? arraySymbols : assocArraySymbols; s.type = symbol; s.qualifier = suffix.array ? SymbolQualifier.array : SymbolQualifier.assocArray; return s; @@ -597,7 +676,7 @@ private: return null; } - static ACSymbol* convertBuiltinType(const Type2 type2) + static const(ACSymbol)* convertBuiltinType(const Type2 type2) { string stringRepresentation = getTokenValue(type2.builtinType); if (stringRepresentation is null) return null; @@ -619,6 +698,7 @@ const(ACSymbol)*[] convertAstToSymbols(Module m, string symbolFile) SecondPass second = SecondPass(first.rootSymbol, first.moduleScope); second.run(); ThirdPass third = ThirdPass(second.rootSymbol, second.moduleScope); + third.run(); return cast(typeof(return)) third.rootSymbol.acSymbol.parts; } @@ -634,19 +714,6 @@ const(Scope)* generateAutocompleteTrees(const(Token)[] tokens, string symbolFile return cast(typeof(return)) third.moduleScope; } -version(unittest) Module parseTestCode(string code) -{ - LexerConfig config; - const(Token)[] tokens = byToken(cast(ubyte[]) code, config); - Parser p = new Parser; - p.fileName = "unittest"; - p.tokens = tokens; - Module m = p.parseModule(); - assert (p.errorCount == 0); - assert (p.warningCount == 0); - return m; -} - private: string[] iotcToStringArray(const IdentifierOrTemplateChain iotc) @@ -664,5 +731,18 @@ string[] iotcToStringArray(const IdentifierOrTemplateChain iotc) private static string convertChainToImportPath(IdentifierChain chain) { - return to!string(chain.identifiers.map!(a => a.value).join(dirSeparator).array) ~ ".d"; + return to!string(chain.identifiers.map!(a => a.value.dup).join(dirSeparator).array) ~ ".d"; +} + +version(unittest) Module parseTestCode(string code) +{ + LexerConfig config; + const(Token)[] tokens = byToken(cast(ubyte[]) code, config); + Parser p = new Parser; + p.fileName = "unittest"; + p.tokens = tokens; + Module m = p.parseModule(); + assert (p.errorCount == 0); + assert (p.warningCount == 0); + return m; } diff --git a/modulecache.d b/modulecache.d index 31587e4..aa4ca88 100644 --- a/modulecache.d +++ b/modulecache.d @@ -63,18 +63,24 @@ struct ModuleCache /** * Adds the given path to the list of directories checked for imports */ - static void addImportPath(string path) + static void addImportPaths(string[] paths) { - if (!exists(path)) - return; - importPaths ~= path; - foreach (fileName; dirEntries(path, "*.{d,di}", SpanMode.depth)) + foreach (path; paths) { - Log.info("Loading and caching completions for ", fileName); - getSymbolsInModule(fileName); - import core.memory; - GC.collect(); - GC.minimize(); + if (!exists(path)) + { + Log.error("Cannot cache modules in ", path, " because it does not exist"); + continue; + } + importPaths ~= path; + } + foreach (path; paths) + { + foreach (fileName; dirEntries(path, "*.{d,di}", SpanMode.depth)) + { + Log.info("Loading and caching completions for ", fileName); + getSymbolsInModule(fileName); + } } } @@ -90,8 +96,15 @@ struct ModuleCache string location = resolveImportLoctation(moduleName); if (location is null) return []; + if (!needsReparsing(location)) - return cache[location].symbols; + { + if (location in cache) + return cache[location].symbols; + return []; + } + + recursionGuard[location] = true; const(ACSymbol)*[] symbols; try @@ -117,6 +130,10 @@ struct ModuleCache getTimes(location, access, modification); CacheEntry c = CacheEntry(symbols, modification); cache[location] = c; + recursionGuard[location] = false; + import core.memory; + GC.collect(); + GC.minimize(); return symbols; } @@ -142,7 +159,7 @@ struct ModuleCache if (filePath.exists()) return filePath; } - writeln("Could not find ", moduleName); + Log.error("Could not find ", moduleName); return null; } @@ -161,6 +178,10 @@ private: */ static bool needsReparsing(string mod) { + if (mod !in recursionGuard) + return true; + if (recursionGuard[mod]) + return false; if (!exists(mod) || mod !in cache) return true; SysTime access; @@ -172,6 +193,8 @@ private: // Mapping of file paths to their cached symbols. static CacheEntry[string] cache; + static bool[string] recursionGuard; + // Listing of paths to check for imports static string[] importPaths; } diff --git a/semantic.d b/semantic.d index 36596e5..85c1d2d 100644 --- a/semantic.d +++ b/semantic.d @@ -22,6 +22,7 @@ import messages; import actypes; import stdx.d.ast; import stdx.d.lexer; +import stupidlog; /** * Intermediate form between ACSymbol and the AST classes. Stores enough @@ -41,6 +42,11 @@ public: acSymbol.symbolFile = symbolFile; } + ~this() + { + Log.trace(acSymbol.name, " destructor"); + } + void addChild(SemanticSymbol* child) { children ~= child; diff --git a/server.d b/server.d index 4bf0ca1..0e9853f 100644 --- a/server.d +++ b/server.d @@ -91,8 +91,7 @@ int main(string[] args) Log.info("Sockets shut down."); } - foreach (path; importPaths) - ModuleCache.addImportPath(path); + ModuleCache.addImportPaths(importPaths); Log.info("Import directories: ", ModuleCache.getImportPaths()); ubyte[] buffer = new ubyte[1024 * 1024 * 4]; // 4 megabytes should be enough for anybody... @@ -150,11 +149,7 @@ int main(string[] args) msgpack.unpack(buffer[size_t.sizeof .. bytesReceived], request); if (request.kind == RequestKind.addImport) { - foreach (path; request.importPaths) - { - ModuleCache.addImportPath(path); - } - + ModuleCache.addImportPaths(request.importPaths); } else if (request.kind == RequestKind.clearCache) { From cc49cef4c709de37f1b1f2c944e637f973e540c3 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Sun, 13 Oct 2013 23:23:31 +0000 Subject: [PATCH 06/27] Made some effort at reducing memory use. Implemented #54 --- actypes.d | 73 ++++++++++++++++++++++++++++++++++++-------------- astconverter.d | 36 ++++++++++++++++++++++++- build.sh | 4 +++ modulecache.d | 20 ++++++++++---- semantic.d | 5 ---- server.d | 3 +-- 6 files changed, 108 insertions(+), 33 deletions(-) diff --git a/actypes.d b/actypes.d index f788250..435e892 100644 --- a/actypes.d +++ b/actypes.d @@ -98,6 +98,23 @@ public: return this.opCmp(other.name); } + /** + * Gets all parts whose name matches the given string. + */ + const(ACSymbol)*[] getPartsByName(string name) const + { + return cast(typeof(return)) parts.filter!(a => a.name == name).array; + } + + size_t estimateMemory(size_t runningTotal) const + { + runningTotal = runningTotal + name.length + callTip.length + + ACSymbol.sizeof; + foreach (part; parts) + runningTotal = part.estimateMemory(runningTotal); + return runningTotal; + } + /** * Symbols that compose this symbol, such as enum members, class variables, * methods, etc. @@ -109,15 +126,25 @@ public: */ string name; + /** + * Calltip to display if this is a function + */ + string callTip; + + /** + * Module containing the symbol. + */ + string symbolFile; + /** * The symbol that represents the type. */ const(ACSymbol)* type; /** - * Calltip to display if this is a function + * Symbol location */ - string callTip; + size_t location; /** * The kind of symbol @@ -128,24 +155,6 @@ public: * Symbol qualifier */ SymbolQualifier qualifier; - - /** - * Symbol location - */ - size_t location; - - /** - * Module containing the symbol. - */ - string symbolFile; - - /** - * Gets all parts whose name matches the given string. - */ - const(ACSymbol)*[] getPartsByName(string name) const - { - return cast(typeof(return)) parts.filter!(a => a.name == name).array; - } } struct Scope @@ -318,6 +327,28 @@ static this() builtinSymbols = [bool_, int_, long_, byte_, char_, dchar_, short_, ubyte_, uint_, ulong_, ushort_, wchar_, cdouble_, cent_, cfloat_, creal_, double_, float_, idouble_, ifloat_, ireal_, real_, ucent_, void_]; + + // _argptr has type void* + argptrType = new Type; + argptrType.type2 = new Type2; + argptrType.type2.builtinType = TokenType.void_; + TypeSuffix argptrTypeSuffix = new TypeSuffix; + argptrTypeSuffix.star = true; + argptrType.typeSuffixes ~= argptrTypeSuffix; + + // _arguments has type TypeInfo[] + argumentsType = new Type; + argumentsType = new Type; + argumentsType.type2 = new Type2; + argumentsType.type2.symbol = new Symbol; + argumentsType.type2.symbol.identifierOrTemplateChain = new IdentifierOrTemplateChain; + IdentifierOrTemplateInstance i = new IdentifierOrTemplateInstance; + i.identifier.value = "TypeInfo"; + i.identifier.type = TokenType.identifier; + argumentsType.type2.symbol.identifierOrTemplateChain.identifiersOrTemplateInstances ~= i; + TypeSuffix argumentsTypeSuffix = new TypeSuffix; + argumentsTypeSuffix.array = true; + argumentsType.typeSuffixes ~= argptrTypeSuffix; } const(ACSymbol)*[] builtinSymbols; @@ -325,3 +356,5 @@ const(ACSymbol)*[] arraySymbols; const(ACSymbol)*[] assocArraySymbols; const(ACSymbol)*[] classSymbols; const(ACSymbol)*[] structSymbols; +Type argptrType; +Type argumentsType; diff --git a/astconverter.d b/astconverter.d index feff926..dfec652 100644 --- a/astconverter.d +++ b/astconverter.d @@ -62,6 +62,19 @@ final class FirstPass : ASTVisitor void run() { visit(mod); + mod = null; + } + + override void visit(Unittest u) + { + // Create a dummy symbol because we don't want unit test symbols leaking + // into the symbol they're declared in. + SemanticSymbol* s = new SemanticSymbol("*unittest*", + CompletionKind.dummy, null, 0); + s.parent = currentSymbol; + currentSymbol = s; + u.accept(this); + currentSymbol = s.parent; } override void visit(Constructor con) @@ -388,6 +401,20 @@ private: symbol.addChild(parameter); parameter.parent = symbol; } + if (parameters.hasVarargs) + { + SemanticSymbol* argptr = new SemanticSymbol("_argptr", + CompletionKind.variableName, null, 0); + argptr.type = argptrType; + argptr.parent = symbol; + symbol.addChild(argptr); + + SemanticSymbol* arguments = new SemanticSymbol("_arguments", + CompletionKind.variableName, null, 0); + arguments.type = argumentsType; + arguments.parent = symbol; + symbol.addChild(arguments); + } } symbol.acSymbol.callTip = formatCallTip(returnType, functionName, parameters); @@ -695,22 +722,29 @@ const(ACSymbol)*[] convertAstToSymbols(Module m, string symbolFile) { FirstPass first = new FirstPass(m, symbolFile); first.run(); + SecondPass second = SecondPass(first.rootSymbol, first.moduleScope); second.run(); + ThirdPass third = ThirdPass(second.rootSymbol, second.moduleScope); third.run(); + return cast(typeof(return)) third.rootSymbol.acSymbol.parts; } const(Scope)* generateAutocompleteTrees(const(Token)[] tokens, string symbolFile) { Module m = parseModule(tokens, null); + FirstPass first = new FirstPass(m, symbolFile); first.run(); + SecondPass second = SecondPass(first.rootSymbol, first.currentScope); second.run(); + ThirdPass third = ThirdPass(second.rootSymbol, second.moduleScope); third.run(); + return cast(typeof(return)) third.moduleScope; } @@ -731,7 +765,7 @@ string[] iotcToStringArray(const IdentifierOrTemplateChain iotc) private static string convertChainToImportPath(IdentifierChain chain) { - return to!string(chain.identifiers.map!(a => a.value.dup).join(dirSeparator).array) ~ ".d"; + return to!string(chain.identifiers.map!(a => a.value).join(dirSeparator).array) ~ ".d"; } version(unittest) Module parseTestCode(string code) diff --git a/build.sh b/build.sh index 1c0d5f6..a047116 100755 --- a/build.sh +++ b/build.sh @@ -2,6 +2,10 @@ dmd -wi client.d\ messages.d\ msgpack-d/src/msgpack.d\ -Imsgpack-d/src\ + -release\ + -inline\ + -noboundscheck\ + -O\ -ofdcd-client dmd \ diff --git a/modulecache.d b/modulecache.d index aa4ca88..280033d 100644 --- a/modulecache.d +++ b/modulecache.d @@ -60,6 +60,18 @@ struct ModuleCache cache = cache.init; } + static void estimateMemory() + { + size_t estimate = 0; + foreach (c; cache) + { + foreach (symbol; c.symbols) + estimate = symbol.estimateMemory(estimate); + } + double megabytes = estimate / (1024.0F * 1024.0F); + Log.info("Memory use estimated at ", megabytes, " megabytes"); + } + /** * Adds the given path to the list of directories checked for imports */ @@ -78,7 +90,6 @@ struct ModuleCache { foreach (fileName; dirEntries(path, "*.{d,di}", SpanMode.depth)) { - Log.info("Loading and caching completions for ", fileName); getSymbolsInModule(fileName); } } @@ -92,7 +103,7 @@ struct ModuleCache */ static const(ACSymbol)*[] getSymbolsInModule(string moduleName) { -// Log.info("Getting symbols for module ", moduleName); + string location = resolveImportLoctation(moduleName); if (location is null) return []; @@ -104,6 +115,8 @@ struct ModuleCache return []; } + Log.info("Getting symbols for module ", moduleName); + recursionGuard[location] = true; const(ACSymbol)*[] symbols; @@ -131,9 +144,6 @@ struct ModuleCache CacheEntry c = CacheEntry(symbols, modification); cache[location] = c; recursionGuard[location] = false; - import core.memory; - GC.collect(); - GC.minimize(); return symbols; } diff --git a/semantic.d b/semantic.d index 85c1d2d..b9796e5 100644 --- a/semantic.d +++ b/semantic.d @@ -42,11 +42,6 @@ public: acSymbol.symbolFile = symbolFile; } - ~this() - { - Log.trace(acSymbol.name, " destructor"); - } - void addChild(SemanticSymbol* child) { children ~= child; diff --git a/server.d b/server.d index 0e9853f..d6c0402 100644 --- a/server.d +++ b/server.d @@ -98,6 +98,7 @@ int main(string[] args) sw.stop(); Log.info("Startup completed in ", sw.peek().to!("msecs", float), " milliseconds"); + ModuleCache.estimateMemory(); while (true) { @@ -114,8 +115,6 @@ int main(string[] args) continue; } - // TODO: Only process connections from localhost - scope (exit) { s.shutdown(SocketShutdown.BOTH); From f97add145d62b90e170bd4d611e66fa9cd0476f0 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Mon, 14 Oct 2013 00:50:27 +0000 Subject: [PATCH 07/27] Fixed issue with 'this' not being present in struct bodies --- astconverter.d | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/astconverter.d b/astconverter.d index dfec652..9f7e7ef 100644 --- a/astconverter.d +++ b/astconverter.d @@ -116,7 +116,7 @@ final class FirstPass : ASTVisitor override void visit(FunctionDeclaration dec) { // Log.trace(__FUNCTION__, " ", typeof(dec).stringof); - SemanticSymbol* symbol = new SemanticSymbol(dec.name.value.dup, + SemanticSymbol* symbol = new SemanticSymbol(dec.name.value.dup, CompletionKind.functionName, symbolFile, dec.name.startIndex); processParameters(symbol, dec.returnType, symbol.acSymbol.name, dec.parameters); @@ -261,11 +261,18 @@ final class FirstPass : ASTVisitor s.startLocation = structBody.startLocation; s.endLocation = structBody.endLocation; // Log.trace("Added scope ", s.startLocation, " ", s.endLocation); + + ACSymbol* thisSymbol = new ACSymbol("this", CompletionKind.variableName, + currentSymbol.acSymbol); + thisSymbol.location = s.startLocation; + thisSymbol.symbolFile = symbolFile; + currentSymbol.acSymbol.parts ~= thisSymbol; + s.parent = currentScope; - currentScope = s; - foreach (dec; structBody.declarations) - visit(dec); - currentScope = s.parent; + currentScope = s; + foreach (dec; structBody.declarations) + visit(dec); + currentScope = s.parent; currentScope.children ~= s; } @@ -549,7 +556,7 @@ private: struct ThirdPass { public: - this(SemanticSymbol* rootSymbol, Scope* moduleScope) + this(SemanticSymbol* rootSymbol, Scope* moduleScope) pure { this.rootSymbol = rootSymbol; this.moduleScope = moduleScope; @@ -668,7 +675,7 @@ private: return s; } - static string[] expandSymbol(const IdentifierOrTemplateChain chain) + static string[] expandSymbol(const IdentifierOrTemplateChain chain) pure { string[] strings = new string[chain.identifiersOrTemplateInstances.length]; for (size_t i = 0; i != chain.identifiersOrTemplateInstances.length; ++i) From 2635ec5c6020297831eb0ea5929a014f39b8882d Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Mon, 14 Oct 2013 01:03:20 +0000 Subject: [PATCH 08/27] Fixed partial completion --- actypes.d | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/actypes.d b/actypes.d index 435e892..5e85e6e 100644 --- a/actypes.d +++ b/actypes.d @@ -172,12 +172,19 @@ struct Scope return cast(typeof(return)) &this; } - ACSymbol*[] getSymbolsInCursorScope(size_t cursorPosition) const + const(ACSymbol)*[] getSymbolsInCursorScope(size_t cursorPosition) const { auto s = getScopeByCursor(cursorPosition); if (s is null) return []; - return cast(typeof(return)) s.symbols; + const(ACSymbol)*[] symbols = cast(typeof(return)) s.symbols; + Scope* sc = s.parent; + while (sc !is null) + { + symbols ~= sc.symbols; + sc = sc.parent; + } + return symbols; } const(ACSymbol)*[] getSymbolsByName(string name) const From 7fd5fca74d444f0752634af6f385fb80c1e97eb5 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Mon, 14 Oct 2013 01:09:35 +0000 Subject: [PATCH 09/27] Initial work on #26 --- client.d | 7 ++++++- messages.d | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/client.d b/client.d index 38b4233..05fa0b1 100644 --- a/client.d +++ b/client.d @@ -40,12 +40,13 @@ int main(string[] args) bool help; bool shutdown; bool clearCache; + bool symbolLocation; try { getopt(args, "cursorPos|c", &cursorPos, "I", &importPaths, "port|p", &port, "help|h", &help, "shutdown", &shutdown, - "clearCache", &clearCache); + "clearCache", &clearCache, "symbolLocation", &symbolLocation); } catch (Exception e) { @@ -176,6 +177,10 @@ Options: --shutdown Instructs the server to shut down. + --symbolLocation + Get the file name and position that the symbol at the cursor location + was defined. + -IPATH Instructs the server to add PATH to its list of paths searced for imported modules. diff --git a/messages.d b/messages.d index 121adaa..8fc6638 100644 --- a/messages.d +++ b/messages.d @@ -108,7 +108,9 @@ enum RequestKind : ubyte /// Add import directory to server addImport, /// Shut down the server - shutdown + shutdown, + /// Get declaration location of given symbol + symbolLocation } /** From c8d755f74398d37d5643eeee14595cca1c5efb46 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Mon, 14 Oct 2013 01:28:42 +0000 Subject: [PATCH 10/27] Added struct and class properties to completions --- actypes.d | 18 ++++++++++++++---- astconverter.d | 12 +----------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/actypes.d b/actypes.d index 5e85e6e..3164dfe 100644 --- a/actypes.d +++ b/actypes.d @@ -229,7 +229,6 @@ struct ImportInformation */ static this() { - // TODO: make sure all parts are sorted. auto bool_ = new ACSymbol("bool", CompletionKind.keyword); auto int_ = new ACSymbol("int", CompletionKind.keyword); auto long_ = new ACSymbol("long", CompletionKind.keyword); @@ -246,12 +245,13 @@ static this() auto alignof_ = new ACSymbol("alignof", CompletionKind.keyword, ulong_); auto mangleof_ = new ACSymbol("mangleof", CompletionKind.keyword); auto sizeof_ = new ACSymbol("sizeof", CompletionKind.keyword, ulong_); - auto stringof_ = new ACSymbol("stringof", CompletionKind.keyword); + auto stringof_ = new ACSymbol("init", CompletionKind.keyword); + auto init = new ACSymbol("stringof", CompletionKind.keyword); arraySymbols ~= alignof_; arraySymbols ~= new ACSymbol("dup", CompletionKind.keyword); arraySymbols ~= new ACSymbol("idup", CompletionKind.keyword); - arraySymbols ~= new ACSymbol("init", CompletionKind.keyword); + arraySymbols ~= init; arraySymbols ~= new ACSymbol("length", CompletionKind.keyword, ulong_); arraySymbols ~= mangleof_; arraySymbols ~= new ACSymbol("ptr", CompletionKind.keyword); @@ -273,6 +273,7 @@ static this() assocArraySymbols ~= new ACSymbol("rehash", CompletionKind.keyword); assocArraySymbols ~= sizeof_; assocArraySymbols ~= stringof_; + assocArraySymbols ~= init; assocArraySymbols ~= new ACSymbol("values", CompletionKind.keyword); assocArraySymbols.sort(); @@ -286,6 +287,7 @@ static this() s.parts ~= sizeof_; s.parts ~= stringof_; s.parts ~= mangleof_; + s.parts ~= init; } auto cdouble_ = new ACSymbol("cdouble", CompletionKind.keyword); @@ -322,6 +324,15 @@ static this() s.parts ~= stringof_; } + classSymbols ~= new ACSymbol("classInfo", CompletionKind.variableName); + classSymbols ~= new ACSymbol("tupleof", CompletionKind.variableName); + classSymbols ~= new ACSymbol("__vptr", CompletionKind.variableName); + classSymbols ~= new ACSymbol("__monitor", CompletionKind.variableName); + classSymbols ~= mangleof_; + classSymbols ~= alignof_; + classSymbols ~= sizeof_; + classSymbols ~= init; + ireal_.parts ~= new ACSymbol("im", CompletionKind.keyword, real_); ifloat_.parts ~= new ACSymbol("im", CompletionKind.keyword, float_); idouble_.parts ~= new ACSymbol("im", CompletionKind.keyword, double_); @@ -362,6 +373,5 @@ const(ACSymbol)*[] builtinSymbols; const(ACSymbol)*[] arraySymbols; const(ACSymbol)*[] assocArraySymbols; const(ACSymbol)*[] classSymbols; -const(ACSymbol)*[] structSymbols; Type argptrType; Type argumentsType; diff --git a/astconverter.d b/astconverter.d index 9f7e7ef..d96c6b6 100644 --- a/astconverter.d +++ b/astconverter.d @@ -341,19 +341,9 @@ private: { // Log.trace("visiting aggregate declaration ", dec.name.value); - CompletionKind k; - static if (is (AggType == ClassDeclaration)) - k = CompletionKind.className; - else static if (is (AggType == InterfaceDeclaration)) - k = CompletionKind.interfaceName; - else static if (is (AggType == StructDeclaration)) - k = CompletionKind.structName; - else static if (is (AggType == UnionDeclaration)) - k = CompletionKind.unionName; - else static assert (false, "Unhandled aggregate type " ~ AggType.stringof); - SemanticSymbol* symbol = new SemanticSymbol(dec.name.value.dup, kind, symbolFile, dec.name.startIndex); + symbol.acSymbol.parts ~= classSymbols; symbol.parent = currentSymbol; symbol.protection = protection; currentSymbol = symbol; From e5784a85632b47f962d8d2742a7e0d12cf4a8563 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Fri, 18 Oct 2013 12:26:41 -0700 Subject: [PATCH 11/27] Fixes #30 --- astconverter.d | 66 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/astconverter.d b/astconverter.d index d96c6b6..805ebcc 100644 --- a/astconverter.d +++ b/astconverter.d @@ -715,8 +715,10 @@ private: Scope* moduleScope; } -const(ACSymbol)*[] convertAstToSymbols(Module m, string symbolFile) +const(ACSymbol)*[] convertAstToSymbols(const(Token)[] tokens, string symbolFile) { + Module m = parseModuleSimple(tokens, null); + FirstPass first = new FirstPass(m, symbolFile); first.run(); @@ -731,7 +733,7 @@ const(ACSymbol)*[] convertAstToSymbols(Module m, string symbolFile) const(Scope)* generateAutocompleteTrees(const(Token)[] tokens, string symbolFile) { - Module m = parseModule(tokens, null); + Module m = parseModule(tokens, "editor buffer", &doesNothing); FirstPass first = new FirstPass(m, symbolFile); first.run(); @@ -747,6 +749,64 @@ const(Scope)* generateAutocompleteTrees(const(Token)[] tokens, string symbolFile private: +Module parseModuleSimple(const(Token)[] tokens, string fileName) +{ + auto parser = new SimpleParser(); + parser.fileName = fileName; + parser.tokens = tokens; + parser.messageFunction = &doesNothing; + auto mod = parser.parseModule(); + return mod; +} + +class SimpleParser : Parser +{ + override Unittest parseUnittest() + { + expect(TokenType.unittest_); + skipBraces(); + return null; + } + + override FunctionBody parseFunctionBody() + { + if (currentIs(TokenType.semicolon)) + advance(); + else if (currentIs(TokenType.lBrace)) + skipBraces(); + else + { + if (currentIs(TokenType.in_)) + { + advance(); + skipBraces(); + if (currentIs(TokenType.out_)) + { + advance(); + if (currentIs(TokenType.lParen)) + skipParens(); + skipBraces(); + } + } + else if (currentIs(TokenType.out_)) + { + advance(); + if (currentIs(TokenType.lParen)) + skipParens(); + skipBraces(); + if (currentIs(TokenType.in_)) + { + advance(); + skipBraces(); + } + } + expect(TokenType.body_); + skipBraces(); + } + return null; + } +} + string[] iotcToStringArray(const IdentifierOrTemplateChain iotc) { string[] parts; @@ -777,3 +837,5 @@ version(unittest) Module parseTestCode(string code) assert (p.warningCount == 0); return m; } + +private void doesNothing(string a, int b, int c, string d) {} From 2e6c978e78f57d9b0de38a7c6bbe0296173a26ce Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Fri, 18 Oct 2013 12:28:17 -0700 Subject: [PATCH 12/27] Fixes #30 --- modulecache.d | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/modulecache.d b/modulecache.d index 280033d..50d27cf 100644 --- a/modulecache.d +++ b/modulecache.d @@ -129,9 +129,14 @@ struct ModuleCache LexerConfig config; config.fileName = location; auto tokens = source.byToken(config).array(); - Module mod = parseModule(tokens, location, &doesNothing); + symbols = convertAstToSymbols(tokens, location); - symbols = convertAstToSymbols(mod, location); + // Parsing allocates a lot of AST nodes. We can greatly reduce the + // program's idle memory use by running the GC here. + // TODO: Re-visit this when D gets a precise GC. + import core.memory; + GC.collect(); + GC.minimize(); } catch (Exception ex) { @@ -208,5 +213,3 @@ private: // Listing of paths to check for imports static string[] importPaths; } - -private void doesNothing(string a, int b, int c, string d) {} From 93f3986d770c9e59d91c39cd85aef351e61f4c49 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Fri, 18 Oct 2013 12:28:55 -0700 Subject: [PATCH 13/27] Removed broken connection filtering --- server.d | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/server.d b/server.d index d6c0402..960a30b 100644 --- a/server.d +++ b/server.d @@ -105,15 +105,7 @@ int main(string[] args) auto s = socket.accept(); s.blocking = true; - if (s.remoteAddress.toHostNameString() != "localhost") - { - Log.error("Warning: Connection attempt from ", - s.remoteAddress.toHostNameString(), "ignored. DCD only accepts " - ~ " requests from localhost"); - s.shutdown(SocketShutdown.BOTH); - s.close(); - continue; - } + // TODO: Restrict connections to localhost scope (exit) { From ef85d2de5f3f2131f85541f859d532321ecf7cb2 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Sat, 19 Oct 2013 17:43:04 -0700 Subject: [PATCH 14/27] More work on #26 --- autocomplete.d | 10 +++++++++- dscanner | 2 +- server.d | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/autocomplete.d b/autocomplete.d index 4ecdeef..6a97f31 100644 --- a/autocomplete.d +++ b/autocomplete.d @@ -40,7 +40,15 @@ import modulecache; import astconverter; import stupidlog; -AutocompleteResponse complete(AutocompleteRequest request, string[] importPaths) +AutocompleteResponse findDeclaration(const AutocompleteRequest request) +{ + AutocompleteResponse response; + + + return response; +} + +AutocompleteResponse complete(const AutocompleteRequest request) { Log.info("Got a completion request"); AutocompleteResponse response; diff --git a/dscanner b/dscanner index 982510c..7768456 160000 --- a/dscanner +++ b/dscanner @@ -1 +1 @@ -Subproject commit 982510c50c9b59c70619a80c771b202c5db12be1 +Subproject commit 77684566ec6e131649e4953221e5d09c7e656c46 diff --git a/server.d b/server.d index 960a30b..00da9d9 100644 --- a/server.d +++ b/server.d @@ -154,7 +154,7 @@ int main(string[] args) } else { - AutocompleteResponse response = complete(request, importPaths); + AutocompleteResponse response = complete(request); ubyte[] responseBytes = msgpack.pack(response); assert(s.send(responseBytes) == responseBytes.length); } From bb3b33b4712a99053ee56a2eaeb5748b05ea2616 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Mon, 21 Oct 2013 23:30:58 -0700 Subject: [PATCH 15/27] More work on go-to-location support --- autocomplete.d | 336 ++++++++++++++------------ client.d | 59 +++-- editors/textadept/modules/dmd/dcd.lua | 19 ++ messages.d | 17 +- server.d | 7 +- 5 files changed, 263 insertions(+), 175 deletions(-) diff --git a/autocomplete.d b/autocomplete.d index 6a97f31..eb12d43 100644 --- a/autocomplete.d +++ b/autocomplete.d @@ -43,11 +43,193 @@ import stupidlog; AutocompleteResponse findDeclaration(const AutocompleteRequest request) { AutocompleteResponse response; + LexerConfig config; + config.fileName = "stdin"; + auto tokens = byToken(cast(ubyte[]) request.sourceCode, config); + const(Token)[] tokenArray = void; + try { + tokenArray = tokens.array(); + } catch (Exception e) { + Log.error("Could not provide autocomplete due to lexing exception: ", e.msg); + return response; + } + auto sortedTokens = assumeSorted(tokenArray); + string partial; + auto beforeTokens = sortedTokens.lowerBound(cast(size_t) request.cursorPosition); + + Log.info("Token at cursor: ", beforeTokens[$ - 1]); + + const(Scope)* completionScope = generateAutocompleteTrees(tokenArray, "stdin"); + auto expression = getExpression(beforeTokens); + + writeln(expression); + + const(ACSymbol)*[] symbols = getSymbolsByTokenChain(completionScope, expression, + request.cursorPosition, CompletionType.identifiers); + + if (symbols.length > 0) + { + response.symbolLocation = symbols[0].location; + response.symbolFilePath = symbols[0].symbolFile; + Log.info(beforeTokens[$ - 1].value, " declared in ", + response.symbolFilePath, " at ", response.symbolLocation); + } return response; } +const(ACSymbol)*[] getSymbolsByTokenChain(T)(const(Scope)* completionScope, + T tokens, size_t cursorPosition, CompletionType completionType) +{ + // Find the symbol corresponding to the beginning of the chain + const(ACSymbol)*[] symbols = completionScope.getSymbolsByNameAndCursor( + tokens[0].value, cursorPosition); + if (symbols.length == 0) + { + Log.trace("Could not find declaration of ", tokens[0].value); + return []; + } + + if (completionType == CompletionType.identifiers + && symbols[0].kind == CompletionKind.memberVariableName + || symbols[0].kind == CompletionKind.variableName + || symbols[0].kind == CompletionKind.aliasName + || symbols[0].kind == CompletionKind.enumMember) + { + symbols = symbols[0].type is null ? [] : [symbols[0].type]; + if (symbols.length == 0) + return symbols; + } + + loop: for (size_t i = 1; i < tokens.length; i++) + { + TokenType open; + TokenType close; + void skip() + { + i++; + for (int depth = 1; depth > 0 && i < tokens.length; i++) + { + if (tokens[i].type == open) + depth++; + else if (tokens[i].type == close) + { + depth--; + if (depth == 0) break; + } + } + } + with (TokenType) switch (tokens[i].type) + { + case int_: + case uint_: + case long_: + case ulong_: + case char_: + case wchar_: + case dchar_: + case bool_: + case byte_: + case ubyte_: + case short_: + case ushort_: + case cent_: + case ucent_: + case float_: + case ifloat_: + case cfloat_: + case idouble_: + case cdouble_: + case double_: + case real_: + case ireal_: + case creal_: + case this_: + symbols = symbols[0].getPartsByName(getTokenValue(tokens[i].type)); + if (symbols.length == 0) + break loop; + break; + case identifier: + Log.trace("looking for ", tokens[i].value, " in ", symbols[0].name); + symbols = symbols[0].getPartsByName(tokens[i].value); + if (symbols.length == 0) + { + Log.trace("Couldn't find it."); + break loop; + } + if (symbols[0].kind == CompletionKind.variableName + || symbols[0].kind == CompletionKind.memberVariableName + || symbols[0].kind == CompletionKind.enumMember + || (symbols[0].kind == CompletionKind.functionName + && (completionType == CompletionType.identifiers + || i + 1 < tokens.length))) + { + symbols = symbols[0].type is null ? [] : [symbols[0].type]; + } + if (symbols.length == 0) + break loop; + if (symbols[0].kind == CompletionKind.aliasName + && (completionType == CompletionType.identifiers + || i + 1 < tokens.length)) + { + symbols = symbols[0].type is null ? [] : [symbols[0].type]; + } + if (symbols.length == 0) + break loop; + break; + case lParen: + open = TokenType.lParen; + close = TokenType.rParen; + skip(); + break; + case lBracket: + open = TokenType.lBracket; + close = TokenType.rBracket; + if (symbols[0].qualifier == SymbolQualifier.array) + { + auto h = i; + skip(); + Parser p; + p.setTokens(tokens[h .. i].array()); + if (!p.isSliceExpression()) + { + symbols = symbols[0].type is null ? [] : [symbols[0].type]; + } + } + else if (symbols[0].qualifier == SymbolQualifier.assocArray) + { + symbols = symbols[0].type is null ? [] :[symbols[0].type]; + skip(); + } + else + { + auto h = i; + skip(); + Parser p; + p.setTokens(tokens[h .. i].array()); + const(ACSymbol)*[] overloads; + if (p.isSliceExpression()) + overloads = symbols[0].getPartsByName("opSlice"); + else + overloads = symbols[0].getPartsByName("opIndex"); + if (overloads.length > 0) + { + symbols = overloads[0].type is null ? [] : [overloads[0].type]; + } + else + return []; + } + break; + case dot: + break; + default: + break loop; + } + } + return symbols; +} + AutocompleteResponse complete(const AutocompleteRequest request) { Log.info("Got a completion request"); @@ -210,157 +392,12 @@ void setCompletions(T)(ref AutocompleteResponse response, if (tokens.length == 0) return; - // Find the symbol corresponding to the beginning of the chain - const(ACSymbol)*[] symbols = completionScope.getSymbolsByNameAndCursor( - tokens[0].value, cursorPosition); - if (symbols.length == 0) - { - Log.trace("Could not find declaration of ", tokens[0].value); - return; - } + const(ACSymbol)*[] symbols = getSymbolsByTokenChain(completionScope, tokens, + cursorPosition, completionType); - if (completionType == CompletionType.identifiers - && symbols[0].kind == CompletionKind.memberVariableName - || symbols[0].kind == CompletionKind.variableName - || symbols[0].kind == CompletionKind.aliasName - || symbols[0].kind == CompletionKind.enumMember) - { - symbols = symbols[0].type is null ? [] : [symbols[0].type]; - if (symbols.length == 0) - return; - } + if (symbols.length == 0) + return; - loop: for (size_t i = 1; i < tokens.length; i++) - { - TokenType open; - TokenType close; - void skip() - { - i++; - for (int depth = 1; depth > 0 && i < tokens.length; i++) - { - if (tokens[i].type == open) - depth++; - else if (tokens[i].type == close) - { - depth--; - if (depth == 0) break; - } - } - } - with (TokenType) switch (tokens[i].type) - { - case int_: - case uint_: - case long_: - case ulong_: - case char_: - case wchar_: - case dchar_: - case bool_: - case byte_: - case ubyte_: - case short_: - case ushort_: - case cent_: - case ucent_: - case float_: - case ifloat_: - case cfloat_: - case idouble_: - case cdouble_: - case double_: - case real_: - case ireal_: - case creal_: - case this_: - symbols = symbols[0].getPartsByName(getTokenValue(tokens[i].type)); - if (symbols.length == 0) - break loop; - break; - case identifier: - Log.trace("looking for ", tokens[i].value, " in ", symbols[0].name); - symbols = symbols[0].getPartsByName(tokens[i].value); - if (symbols.length == 0) - { - Log.trace("Couldn't find it."); - break loop; - } - if (symbols[0].kind == CompletionKind.variableName - || symbols[0].kind == CompletionKind.memberVariableName - || symbols[0].kind == CompletionKind.enumMember - || (symbols[0].kind == CompletionKind.functionName - && (completionType == CompletionType.identifiers - || i + 1 < tokens.length))) - { - symbols = symbols[0].type is null ? [] : [symbols[0].type]; - } - if (symbols.length == 0) - break loop; - if (symbols[0].kind == CompletionKind.aliasName - && (completionType == CompletionType.identifiers - || i + 1 < tokens.length)) - { - symbols = symbols[0].type is null ? [] : [symbols[0].type]; - } - if (symbols.length == 0) - break loop; - break; - case lParen: - open = TokenType.lParen; - close = TokenType.rParen; - skip(); - break; - case lBracket: - open = TokenType.lBracket; - close = TokenType.rBracket; - if (symbols[0].qualifier == SymbolQualifier.array) - { - auto h = i; - skip(); - Parser p; - p.setTokens(tokens[h .. i].array()); - if (!p.isSliceExpression()) - { - symbols = symbols[0].type is null ? [] : [symbols[0].type]; - } - } - else if (symbols[0].qualifier == SymbolQualifier.assocArray) - { - symbols = symbols[0].type is null ? [] :[symbols[0].type]; - skip(); - } - else - { - auto h = i; - skip(); - Parser p; - p.setTokens(tokens[h .. i].array()); - const(ACSymbol)*[] overloads; - if (p.isSliceExpression()) - overloads = symbols[0].getPartsByName("opSlice"); - else - overloads = symbols[0].getPartsByName("opIndex"); - if (overloads.length > 0) - { - symbols = overloads[0].type is null ? [] : [overloads[0].type]; - } - else - return; - } - break; - case dot: - break; - default: - break loop; - } - } - - if (symbols.length == 0) - { - Log.error("Could not get completions"); - return; - } if (completionType == CompletionType.identifiers) { foreach (s; symbols[0].parts.filter!(a => a.name !is null @@ -403,7 +440,6 @@ void setCompletions(T)(ref AutocompleteResponse response, response.completions ~= symbol.callTip; } } - } T getExpression(T)(T beforeTokens) diff --git a/client.d b/client.d index 05fa0b1..6b0dfbc 100644 --- a/client.d +++ b/client.d @@ -46,7 +46,7 @@ int main(string[] args) { getopt(args, "cursorPos|c", &cursorPos, "I", &importPaths, "port|p", &port, "help|h", &help, "shutdown", &shutdown, - "clearCache", &clearCache, "symbolLocation", &symbolLocation); + "clearCache", &clearCache, "symbolLocation|l", &symbolLocation); } catch (Exception e) { @@ -119,6 +119,7 @@ int main(string[] args) request.importPaths = importPaths; request.sourceCode = sourceCode; request.cursorPosition = cursorPos; + request.kind = symbolLocation ? RequestKind.symbolLocation : RequestKind.autocomplete; // Send message to server TcpSocket socket = createSocket(port); @@ -128,26 +129,11 @@ int main(string[] args) AutocompleteResponse response = getResponse(socket); - if (response.completions.length > 0) - { - writeln(response.completionType); - auto app = appender!(string[])(); - if (response.completionType == CompletionType.identifiers) - { - for (size_t i = 0; i < response.completions.length; i++) - app.put(format("%s\t%s", response.completions[i], response.completionKinds[i])); - } - else - { - foreach (completion; response.completions) - { - app.put(completion); - } - } - // Deduplicate overloaded methods - foreach (line; app.data.sort.uniq) - writeln(line); - } + if (symbolLocation) + printLocationResponse(response); + else + printCompletionResponse(response); + return 0; } @@ -177,7 +163,7 @@ Options: --shutdown Instructs the server to shut down. - --symbolLocation + --symbolLocation | -l Get the file name and position that the symbol at the cursor location was defined. @@ -225,3 +211,32 @@ AutocompleteResponse getResponse(TcpSocket socket) msgpack.unpack(buffer[0..bytesReceived], response); return response; } + +void printLocationResponse(AutocompleteResponse response) +{ + writefln("%s\t%d", response.symbolFilePath, response.symbolLocation); +} + +void printCompletionResponse(AutocompleteResponse response) +{ + if (response.completions.length > 0) + { + writeln(response.completionType); + auto app = appender!(string[])(); + if (response.completionType == CompletionType.identifiers) + { + for (size_t i = 0; i < response.completions.length; i++) + app.put(format("%s\t%s", response.completions[i], response.completionKinds[i])); + } + else + { + foreach (completion; response.completions) + { + app.put(completion); + } + } + // Deduplicate overloaded methods + foreach (line; app.data.sort.uniq) + writeln(line); + } +} diff --git a/editors/textadept/modules/dmd/dcd.lua b/editors/textadept/modules/dmd/dcd.lua index 1081b64..6c6326b 100644 --- a/editors/textadept/modules/dmd/dcd.lua +++ b/editors/textadept/modules/dmd/dcd.lua @@ -100,6 +100,25 @@ function M.cycleCalltips(delta) showCurrentCallTip() end +function M.gotoDeclaration() + local fileName = os.tmpname() + local command = M.PATH_TO_DCD_CLIENT .. " -l -c" .. buffer.current_pos .. " > " .. fileName + local mode = "w" + if _G.WIN32 then + mode = "wb" + end + local p = io.popen(command, mode) + p:write(buffer:get_text()) + p:flush() + p:close() + local tmpFile = io.open(fileName, "r") + local r = tmpFile:read("*a") + if r ~= "\n" then + -- TODO: Go to declaration + end + os.remove(fileName) +end + events.connect(events.CALL_TIP_CLICK, function(arrow) if buffer:get_lexer() ~= "dmd" then return end if arrow == 1 then diff --git a/messages.d b/messages.d index 8fc6638..7649e57 100644 --- a/messages.d +++ b/messages.d @@ -93,7 +93,12 @@ enum CompletionType : string * The auto-completion list consists of a listing of functions and their * parameters. */ - calltips = "calltips" + calltips = "calltips", + + /** + * The response contains the location of a symbol declaration. + */ + location = "location" } /** @@ -154,6 +159,16 @@ struct AutocompleteResponse */ string completionType; + /** + * The path to the file that contains the symbol. + */ + string symbolFilePath; + + /** + * The byte offset at which the symbol is located. + */ + size_t symbolLocation; + /** * The completions */ diff --git a/server.d b/server.d index 00da9d9..7613d0e 100644 --- a/server.d +++ b/server.d @@ -154,9 +154,12 @@ int main(string[] args) } else { - AutocompleteResponse response = complete(request); + AutocompleteResponse response = + request.kind == RequestKind.autocomplete + ? complete(request) + : findDeclaration(request); ubyte[] responseBytes = msgpack.pack(response); - assert(s.send(responseBytes) == responseBytes.length); + s.send(responseBytes); } Log.info("Request processed in ", requestWatch.peek().to!("msecs", float), " milliseconds"); } From 1d807075fdadc655a2a7a52c5b3e34eec58bdba7 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Thu, 24 Oct 2013 18:03:12 -0700 Subject: [PATCH 16/27] #26 --- autocomplete.d | 53 ++++++++++++++------------ client.d | 17 +++++---- editors/textadept/modules/dmd/dcd.lua | 11 +++++- editors/textadept/modules/dmd/init.lua | 1 + 4 files changed, 48 insertions(+), 34 deletions(-) diff --git a/autocomplete.d b/autocomplete.d index eb12d43..341311b 100644 --- a/autocomplete.d +++ b/autocomplete.d @@ -42,8 +42,9 @@ import stupidlog; AutocompleteResponse findDeclaration(const AutocompleteRequest request) { - AutocompleteResponse response; - LexerConfig config; + Log.info("Finding declaration"); + AutocompleteResponse response; + LexerConfig config; config.fileName = "stdin"; auto tokens = byToken(cast(ubyte[]) request.sourceCode, config); const(Token)[] tokenArray = void; @@ -58,31 +59,33 @@ AutocompleteResponse findDeclaration(const AutocompleteRequest request) auto beforeTokens = sortedTokens.lowerBound(cast(size_t) request.cursorPosition); - Log.info("Token at cursor: ", beforeTokens[$ - 1]); + Log.info("Token at cursor: ", beforeTokens[$ - 1]); - const(Scope)* completionScope = generateAutocompleteTrees(tokenArray, "stdin"); - auto expression = getExpression(beforeTokens); + const(Scope)* completionScope = generateAutocompleteTrees(tokenArray, "stdin"); + auto expression = getExpression(beforeTokens); - writeln(expression); + const(ACSymbol)*[] symbols = getSymbolsByTokenChain(completionScope, expression, + request.cursorPosition, CompletionType.identifiers); - const(ACSymbol)*[] symbols = getSymbolsByTokenChain(completionScope, expression, - request.cursorPosition, CompletionType.identifiers); + if (symbols.length > 0) + { + response.symbolLocation = symbols[0].location; + response.symbolFilePath = symbols[0].symbolFile; + Log.info(beforeTokens[$ - 1].value, " declared in ", + response.symbolFilePath, " at ", response.symbolLocation); + } + else + { + Log.error("Could not find symbol"); + } - if (symbols.length > 0) - { - response.symbolLocation = symbols[0].location; - response.symbolFilePath = symbols[0].symbolFile; - Log.info(beforeTokens[$ - 1].value, " declared in ", - response.symbolFilePath, " at ", response.symbolLocation); - } - - return response; + return response; } const(ACSymbol)*[] getSymbolsByTokenChain(T)(const(Scope)* completionScope, - T tokens, size_t cursorPosition, CompletionType completionType) + T tokens, size_t cursorPosition, CompletionType completionType) { - // Find the symbol corresponding to the beginning of the chain + // Find the symbol corresponding to the beginning of the chain const(ACSymbol)*[] symbols = completionScope.getSymbolsByNameAndCursor( tokens[0].value, cursorPosition); if (symbols.length == 0) @@ -227,7 +230,7 @@ const(ACSymbol)*[] getSymbolsByTokenChain(T)(const(Scope)* completionScope, break loop; } } - return symbols; + return symbols; } AutocompleteResponse complete(const AutocompleteRequest request) @@ -392,11 +395,11 @@ void setCompletions(T)(ref AutocompleteResponse response, if (tokens.length == 0) return; - const(ACSymbol)*[] symbols = getSymbolsByTokenChain(completionScope, tokens, - cursorPosition, completionType); + const(ACSymbol)*[] symbols = getSymbolsByTokenChain(completionScope, tokens, + cursorPosition, completionType); - if (symbols.length == 0) - return; + if (symbols.length == 0) + return; if (completionType == CompletionType.identifiers) { @@ -554,7 +557,7 @@ void setImportCompletions(T)(T tokens, ref AutocompleteResponse response) foreach (importDirectory; ModuleCache.getImportPaths()) { string p = format("%s%s%s", importDirectory, dirSeparator, path); - writeln("Checking for ", p); + Log.trace("Checking for ", p); if (!exists(p)) continue; foreach (string name; dirEntries(p, SpanMode.shallow)) diff --git a/client.d b/client.d index 6b0dfbc..8aab9ec 100644 --- a/client.d +++ b/client.d @@ -119,7 +119,7 @@ int main(string[] args) request.importPaths = importPaths; request.sourceCode = sourceCode; request.cursorPosition = cursorPos; - request.kind = symbolLocation ? RequestKind.symbolLocation : RequestKind.autocomplete; + request.kind = symbolLocation ? RequestKind.symbolLocation : RequestKind.autocomplete; // Send message to server TcpSocket socket = createSocket(port); @@ -129,10 +129,10 @@ int main(string[] args) AutocompleteResponse response = getResponse(socket); - if (symbolLocation) - printLocationResponse(response); - else - printCompletionResponse(response); + if (symbolLocation) + printLocationResponse(response); + else + printCompletionResponse(response); return 0; } @@ -214,12 +214,15 @@ AutocompleteResponse getResponse(TcpSocket socket) void printLocationResponse(AutocompleteResponse response) { - writefln("%s\t%d", response.symbolFilePath, response.symbolLocation); + if (response.symbolFilePath is null) + writeln("Not found"); + else + writefln("%s\t%d", response.symbolFilePath, response.symbolLocation); } void printCompletionResponse(AutocompleteResponse response) { - if (response.completions.length > 0) + if (response.completions.length > 0) { writeln(response.completionType); auto app = appender!(string[])(); diff --git a/editors/textadept/modules/dmd/dcd.lua b/editors/textadept/modules/dmd/dcd.lua index 6c6326b..d089ea6 100644 --- a/editors/textadept/modules/dmd/dcd.lua +++ b/editors/textadept/modules/dmd/dcd.lua @@ -113,8 +113,15 @@ function M.gotoDeclaration() p:close() local tmpFile = io.open(fileName, "r") local r = tmpFile:read("*a") - if r ~= "\n" then - -- TODO: Go to declaration + if r ~= "Not found\n" then + path, position = r:match("^(.-)\t(%d+)") + if (path ~= nil and position ~= nil) then + if (path ~= "stdin") then + io.open_file(path) + end + buffer:goto_pos(tonumber(position)) + buffer:word_right_end_extend() + end end os.remove(fileName) end diff --git a/editors/textadept/modules/dmd/init.lua b/editors/textadept/modules/dmd/init.lua index 3302e79..c696616 100644 --- a/editors/textadept/modules/dmd/init.lua +++ b/editors/textadept/modules/dmd/init.lua @@ -31,6 +31,7 @@ keys.dmd = { (_USERHOME..'/modules/dmd/init.lua'):iconv('UTF-8', _CHARSET) }, }, ['c\n'] = {autocomplete}, + ['cG'] = {_M.dcd.gotoDeclaration}, ['down'] = {_M.dcd.cycleCalltips, 1}, ['up'] = {_M.dcd.cycleCalltips, -1}, } From 5911c2409cad608b97f228a5986e07bf080d82da Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Tue, 29 Oct 2013 20:39:45 -0700 Subject: [PATCH 17/27] Dependency update --- dscanner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dscanner b/dscanner index 7768456..1c05726 160000 --- a/dscanner +++ b/dscanner @@ -1 +1 @@ -Subproject commit 77684566ec6e131649e4953221e5d09c7e656c46 +Subproject commit 1c057260d5541f2ebd4d5b1780bacf45e482df39 From 13ef990fa96fd32d65b93d4ce95afeb0fe550fff Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Tue, 29 Oct 2013 21:15:48 -0700 Subject: [PATCH 18/27] #62 --- server.d | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server.d b/server.d index 7613d0e..80c7289 100644 --- a/server.d +++ b/server.d @@ -49,8 +49,6 @@ int main(string[] args) Log.info("Starting up..."); StopWatch sw = StopWatch(AutoStart.yes); - // No relative paths - version (Posix) chdir("/"); Log.output = stdout; Log.level = LogLevel.trace; @@ -100,6 +98,9 @@ int main(string[] args) Log.info("Startup completed in ", sw.peek().to!("msecs", float), " milliseconds"); ModuleCache.estimateMemory(); + // No relative paths + version (Posix) chdir("/"); + while (true) { auto s = socket.accept(); From 506573b493dc5a66ddea4ba1cc4ad62021678fb3 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Tue, 29 Oct 2013 23:10:01 -0700 Subject: [PATCH 19/27] Fix #11 --- astconverter.d | 208 ++++++++++++++++++++++++++----------------------- dscanner | 2 +- modulecache.d | 2 +- 3 files changed, 112 insertions(+), 100 deletions(-) diff --git a/astconverter.d b/astconverter.d index 805ebcc..b3da675 100644 --- a/astconverter.d +++ b/astconverter.d @@ -77,41 +77,41 @@ final class FirstPass : ASTVisitor currentSymbol = s.parent; } - override void visit(Constructor con) - { + override void visit(Constructor con) + { // Log.trace(__FUNCTION__, " ", typeof(con).stringof); - visitConstructor(con.location, con.parameters, con.functionBody); - } + visitConstructor(con.location, con.parameters, con.functionBody); + } - override void visit(SharedStaticConstructor con) - { + override void visit(SharedStaticConstructor con) + { // Log.trace(__FUNCTION__, " ", typeof(con).stringof); visitConstructor(con.location, null, con.functionBody); - } + } - override void visit(StaticConstructor con) - { + override void visit(StaticConstructor con) + { // Log.trace(__FUNCTION__, " ", typeof(con).stringof); - visitConstructor(con.location, null, con.functionBody); - } + visitConstructor(con.location, null, con.functionBody); + } - override void visit(Destructor des) - { + override void visit(Destructor des) + { // Log.trace(__FUNCTION__, " ", typeof(des).stringof); - visitDestructor(des.location, des.functionBody); - } + visitDestructor(des.location, des.functionBody); + } - override void visit(SharedStaticDestructor des) - { + override void visit(SharedStaticDestructor des) + { // Log.trace(__FUNCTION__, " ", typeof(des).stringof); - visitDestructor(des.location, des.functionBody); - } + visitDestructor(des.location, des.functionBody); + } - override void visit(StaticDestructor des) - { + override void visit(StaticDestructor des) + { // Log.trace(__FUNCTION__, " ", typeof(des).stringof); - visitDestructor(des.location, des.functionBody); - } + visitDestructor(des.location, des.functionBody); + } override void visit(FunctionDeclaration dec) { @@ -199,7 +199,7 @@ final class FirstPass : ASTVisitor foreach (Attribute attr; dec.attributes) { if (isProtection(attr.attribute)) - p = attr.attribute; + protection = attr.attribute; } dec.accept(this); protection = p; @@ -221,30 +221,30 @@ final class FirstPass : ASTVisitor mod.accept(this); } - override void visit(EnumDeclaration dec) - { + override void visit(EnumDeclaration dec) + { assert (currentSymbol); // Log.trace(__FUNCTION__, " ", typeof(dec).stringof); - SemanticSymbol* symbol = new SemanticSymbol(dec.name.value.dup, + SemanticSymbol* symbol = new SemanticSymbol(dec.name.value.dup, CompletionKind.enumName, symbolFile, dec.name.startIndex); - symbol.type = dec.type; - symbol.parent = currentSymbol; - currentSymbol = symbol; - if (dec.enumBody !is null) - dec.enumBody.accept(this); - currentSymbol = symbol.parent; - currentSymbol.addChild(symbol); - } + symbol.type = dec.type; + symbol.parent = currentSymbol; + currentSymbol = symbol; + if (dec.enumBody !is null) + dec.enumBody.accept(this); + currentSymbol = symbol.parent; + currentSymbol.addChild(symbol); + } - override void visit(EnumMember member) - { + override void visit(EnumMember member) + { // Log.trace(__FUNCTION__, " ", typeof(member).stringof); - SemanticSymbol* symbol = new SemanticSymbol(member.name.value.dup, + SemanticSymbol* symbol = new SemanticSymbol(member.name.value.dup, CompletionKind.enumMember, symbolFile, member.name.startIndex); - symbol.type = member.type; - symbol.parent = currentSymbol; - currentSymbol.addChild(symbol); - } + symbol.type = member.type; + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol); + } override void visit(ModuleDeclaration dec) { @@ -284,6 +284,7 @@ final class FirstPass : ASTVisitor { ImportInformation info; info.modulePath = convertChainToImportPath(single.identifierChain); + info.isPublic = protection == TokenType.public_; currentScope.importInformation ~= info; } if (importDeclaration.importBindings is null) return; @@ -298,6 +299,7 @@ final class FirstPass : ASTVisitor bindTuple[1] = bind.right == TokenType.invalid ? null : bind.right.value.dup; info.importedSymbols ~= bindTuple; } + info.isPublic = protection == TokenType.public_; currentScope.importInformation ~= info; } @@ -339,7 +341,6 @@ private: void visitAggregateDeclaration(AggType)(AggType dec, CompletionKind kind) { - // Log.trace("visiting aggregate declaration ", dec.name.value); SemanticSymbol* symbol = new SemanticSymbol(dec.name.value.dup, kind, symbolFile, dec.name.startIndex); @@ -449,8 +450,8 @@ private: /// Package and module name string[] moduleName; - /// Current scope - Scope* currentScope; + /// Current scope + Scope* currentScope; /// Module scope Scope* moduleScope; @@ -502,6 +503,11 @@ private: if (importInfo.importedSymbols.length == 0) { currentScope.symbols ~= symbols; + if (importInfo.isPublic && currentScope.parent is null) + { + Log.trace("Public import"); + rootSymbol.acSymbol.parts ~= symbols; + } continue; } symbolLoop: foreach (symbol; symbols) @@ -522,9 +528,15 @@ private: s.location = symbol.location; s.symbolFile = symbol.symbolFile; currentScope.symbols ~= s; + if (importInfo.isPublic && currentScope.parent is null) + rootSymbol.acSymbol.parts ~= s; } else + { currentScope.symbols ~= symbol; + if (importInfo.isPublic && currentScope.parent is null) + rootSymbol.acSymbol.parts ~= symbol; + } } } } @@ -717,18 +729,18 @@ private: const(ACSymbol)*[] convertAstToSymbols(const(Token)[] tokens, string symbolFile) { - Module m = parseModuleSimple(tokens, null); + Module m = parseModuleSimple(tokens, symbolFile); - FirstPass first = new FirstPass(m, symbolFile); + FirstPass first = new FirstPass(m, symbolFile); first.run(); - SecondPass second = SecondPass(first.rootSymbol, first.moduleScope); + SecondPass second = SecondPass(first.rootSymbol, first.moduleScope); second.run(); ThirdPass third = ThirdPass(second.rootSymbol, second.moduleScope); third.run(); - return cast(typeof(return)) third.rootSymbol.acSymbol.parts; + return cast(typeof(return)) third.rootSymbol.acSymbol.parts; } const(Scope)* generateAutocompleteTrees(const(Token)[] tokens, string symbolFile) @@ -751,60 +763,60 @@ private: Module parseModuleSimple(const(Token)[] tokens, string fileName) { - auto parser = new SimpleParser(); - parser.fileName = fileName; - parser.tokens = tokens; - parser.messageFunction = &doesNothing; - auto mod = parser.parseModule(); - return mod; + auto parser = new SimpleParser(); + parser.fileName = fileName; + parser.tokens = tokens; + parser.messageFunction = &doesNothing; + auto mod = parser.parseModule(); + return mod; } class SimpleParser : Parser { - override Unittest parseUnittest() - { - expect(TokenType.unittest_); - skipBraces(); - return null; - } + override Unittest parseUnittest() + { + expect(TokenType.unittest_); + skipBraces(); + return null; + } - override FunctionBody parseFunctionBody() - { - if (currentIs(TokenType.semicolon)) - advance(); - else if (currentIs(TokenType.lBrace)) - skipBraces(); - else - { - if (currentIs(TokenType.in_)) - { - advance(); - skipBraces(); - if (currentIs(TokenType.out_)) - { - advance(); - if (currentIs(TokenType.lParen)) - skipParens(); - skipBraces(); - } - } - else if (currentIs(TokenType.out_)) - { - advance(); - if (currentIs(TokenType.lParen)) - skipParens(); - skipBraces(); - if (currentIs(TokenType.in_)) - { - advance(); - skipBraces(); - } - } - expect(TokenType.body_); - skipBraces(); - } - return null; - } + override FunctionBody parseFunctionBody() + { + if (currentIs(TokenType.semicolon)) + advance(); + else if (currentIs(TokenType.lBrace)) + skipBraces(); + else + { + if (currentIs(TokenType.in_)) + { + advance(); + skipBraces(); + if (currentIs(TokenType.out_)) + { + advance(); + if (currentIs(TokenType.lParen)) + skipParens(); + skipBraces(); + } + } + else if (currentIs(TokenType.out_)) + { + advance(); + if (currentIs(TokenType.lParen)) + skipParens(); + skipBraces(); + if (currentIs(TokenType.in_)) + { + advance(); + skipBraces(); + } + } + expect(TokenType.body_); + skipBraces(); + } + return null; + } } string[] iotcToStringArray(const IdentifierOrTemplateChain iotc) diff --git a/dscanner b/dscanner index 1c05726..e381964 160000 --- a/dscanner +++ b/dscanner @@ -1 +1 @@ -Subproject commit 1c057260d5541f2ebd4d5b1780bacf45e482df39 +Subproject commit e3819643bbec121e16abda6f980cc096d2e7f4f1 diff --git a/modulecache.d b/modulecache.d index 50d27cf..99d5954 100644 --- a/modulecache.d +++ b/modulecache.d @@ -69,7 +69,7 @@ struct ModuleCache estimate = symbol.estimateMemory(estimate); } double megabytes = estimate / (1024.0F * 1024.0F); - Log.info("Memory use estimated at ", megabytes, " megabytes"); + Log.trace("Memory use estimated at ", megabytes, " megabytes"); } /** From ff5c7a43e85486dfbe6f0b517686af44c1d31264 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Tue, 29 Oct 2013 23:16:13 -0700 Subject: [PATCH 20/27] Fix #58 --- msgpack-d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack-d b/msgpack-d index 40c797c..30a7d3f 160000 --- a/msgpack-d +++ b/msgpack-d @@ -1 +1 @@ -Subproject commit 40c797cb8ae3eb56cf88399ef3532fc29abd238a +Subproject commit 30a7d3fb38b43dccef3be7cea1f40b4dc61a3474 From ed7a667ae4f53c0e1c2e41103a1b0de59f4b8a3e Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Thu, 31 Oct 2013 14:51:02 -0700 Subject: [PATCH 21/27] Fix #64 --- astconverter.d | 2 ++ 1 file changed, 2 insertions(+) diff --git a/astconverter.d b/astconverter.d index b3da675..3526518 100644 --- a/astconverter.d +++ b/astconverter.d @@ -540,6 +540,8 @@ private: } } } + foreach (childScope; currentScope.children) + resolveImports(childScope); } SemanticSymbol* rootSymbol; From 52b919bd3af9e783762c3eab9d1c551919e2ba48 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Mon, 4 Nov 2013 16:21:33 -0800 Subject: [PATCH 22/27] Fix #67 and update DScanner --- astconverter.d | 33 ++++++++++++++++++--------------- autocomplete.d | 4 +++- dscanner | 2 +- modulecache.d | 39 +++++++++++++++++++++++---------------- 4 files changed, 45 insertions(+), 33 deletions(-) diff --git a/astconverter.d b/astconverter.d index 3526518..a59a306 100644 --- a/astconverter.d +++ b/astconverter.d @@ -428,16 +428,6 @@ private: return "%s %s%s".format(formatNode(returnType), name, parameterString); } - static string formatNode(T)(T node) - { - if (node is null) return ""; - import formatter; - auto app = appender!(char[])(); - auto f = new Formatter!(typeof(app))(app); - f.format(node); - return to!string(app.data); - } - /// Current protection type TokenType protection; @@ -499,13 +489,13 @@ private: { foreach (importInfo; currentScope.importInformation) { - auto symbols = ModuleCache.getSymbolsInModule(importInfo.modulePath); + auto symbols = ModuleCache.getSymbolsInModule( + ModuleCache.resolveImportLoctation(importInfo.modulePath)); if (importInfo.importedSymbols.length == 0) { currentScope.symbols ~= symbols; if (importInfo.isPublic && currentScope.parent is null) { - Log.trace("Public import"); rootSymbol.acSymbol.parts ~= symbols; } continue; @@ -708,8 +698,11 @@ private: } if (suffix.parameters) { - Log.error("TODO: Function type suffix"); - return null; + ACSymbol* s = new ACSymbol; + s.type = symbol; + s.qualifier = SymbolQualifier.func; + s.callTip = suffix.delegateOrFunction.value ~ formatNode(suffix.parameters); + return s; } return null; } @@ -836,7 +829,7 @@ string[] iotcToStringArray(const IdentifierOrTemplateChain iotc) private static string convertChainToImportPath(IdentifierChain chain) { - return to!string(chain.identifiers.map!(a => a.value).join(dirSeparator).array) ~ ".d"; + return to!string(chain.identifiers.map!(a => a.value).join(dirSeparator).array); } version(unittest) Module parseTestCode(string code) @@ -852,4 +845,14 @@ version(unittest) Module parseTestCode(string code) return m; } +string formatNode(T)(T node) +{ + if (node is null) return ""; + import formatter; + auto app = appender!(char[])(); + auto f = new Formatter!(typeof(app))(app); + f.format(node); + return to!string(app.data); +} + private void doesNothing(string a, int b, int c, string d) {} diff --git a/autocomplete.d b/autocomplete.d index 341311b..4525640 100644 --- a/autocomplete.d +++ b/autocomplete.d @@ -570,7 +570,9 @@ void setImportCompletions(T)(T tokens, ref AutocompleteResponse response) else if (isDir(name)) { response.completions ~= name.baseName(); - response.completionKinds ~= CompletionKind.packageName; + response.completionKinds ~= + exists(buildPath(name, "package.d")) || exists(buildPath(name, "package.di")) + ? CompletionKind.packageName : CompletionKind.moduleName; } } } diff --git a/dscanner b/dscanner index e381964..c522170 160000 --- a/dscanner +++ b/dscanner @@ -1 +1 @@ -Subproject commit e3819643bbec121e16abda6f980cc096d2e7f4f1 +Subproject commit c522170983576c533b8a378040648612aeee7953 diff --git a/modulecache.d b/modulecache.d index 99d5954..01f249d 100644 --- a/modulecache.d +++ b/modulecache.d @@ -97,14 +97,12 @@ struct ModuleCache /** * Params: - * moduleName = the name of the module in "a/b.d" form + * moduleName = the name of the module in "a/b/c" form * Returns: * The symbols defined in the given module */ - static const(ACSymbol)*[] getSymbolsInModule(string moduleName) + static const(ACSymbol)*[] getSymbolsInModule(string location) { - - string location = resolveImportLoctation(moduleName); if (location is null) return []; @@ -115,7 +113,7 @@ struct ModuleCache return []; } - Log.info("Getting symbols for module ", moduleName); + Log.info("Getting symbols for ", location); recursionGuard[location] = true; @@ -154,28 +152,37 @@ struct ModuleCache /** * Params: - * moduleName the name of the module being imported, in "a/b/c.d" style + * moduleName the name of the module being imported, in "a/b/c" style * Returns: * The absolute path to the file that contains the module, or null if * not found. */ static string resolveImportLoctation(string moduleName) { -// Log.trace("Resolving location of ", moduleName); if (isRooted(moduleName)) return moduleName; - + string[] alternatives; foreach (path; importPaths) { - string filePath = path ~ "/" ~ moduleName; - if (filePath.exists()) - return filePath; - filePath ~= "i"; // check for x.di if x.d isn't found - if (filePath.exists()) - return filePath; + string filePath = buildPath(path, moduleName); + if (exists(filePath ~ ".d") && isFile(filePath ~ ".d")) + alternatives = (filePath ~ ".d") ~ alternatives; + else if (exists(filePath ~ ".di") && isFile(filePath ~ ".di")) + alternatives = (filePath ~ ".di") ~ alternatives; + else if (exists(filePath) && isDir(filePath)) + { + string packagePath = buildPath(filePath, "package.d"); + if (exists(packagePath) && isFile(packagePath)) + { + alternatives ~= packagePath; + continue; + } + packagePath ~= "i"; + if (exists(packagePath) && isFile(packagePath)) + alternatives ~= packagePath; + } } - Log.error("Could not find ", moduleName); - return null; + return alternatives.length > 0 ? alternatives[0] : null; } static const(string[]) getImportPaths() From 4c78e06ded6b0254ecfe1d753793b6b70468e0ee Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Tue, 5 Nov 2013 00:23:17 +0000 Subject: [PATCH 23/27] DScanner update --- dscanner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dscanner b/dscanner index c522170..87ed0bd 160000 --- a/dscanner +++ b/dscanner @@ -1 +1 @@ -Subproject commit c522170983576c533b8a378040648612aeee7953 +Subproject commit 87ed0bd3b42758488ad59fb26f2180a01740dfa9 From 55e8577067de9fcad756ba4bee35f9b1cc898b3b Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Tue, 5 Nov 2013 14:38:37 +0000 Subject: [PATCH 24/27] #66 --- astconverter.d | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/astconverter.d b/astconverter.d index a59a306..4015d50 100644 --- a/astconverter.d +++ b/astconverter.d @@ -180,6 +180,37 @@ final class FirstPass : ASTVisitor } } + override void visit(AliasDeclaration aliasDeclaration) + { + if (aliasDeclaration.initializers.length == 0) + { + SemanticSymbol* symbol = new SemanticSymbol( + aliasDeclaration.name.value.dup, + CompletionKind.aliasName, + symbolFile, + aliasDeclaration.name.startIndex); + symbol.type = aliasDeclaration.type; + symbol.protection = protection; + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol); + } + else + { + foreach (initializer; aliasDeclaration.initializers) + { + SemanticSymbol* symbol = new SemanticSymbol( + initializer.name.value.dup, + CompletionKind.aliasName, + symbolFile, + initializer.name.startIndex); + symbol.type = initializer.type; + symbol.protection = protection; + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol); + } + } + } + override void visit(AliasThisDeclaration dec) { // Log.trace(__FUNCTION__, " ", typeof(dec).stringof); @@ -192,8 +223,8 @@ final class FirstPass : ASTVisitor if (dec.attributeDeclaration !is null && isProtection(dec.attributeDeclaration.attribute.attribute)) { - protection = dec.attributeDeclaration.attribute.attribute; - return; + protection = dec.attributeDeclaration.attribute.attribute; + return; } TokenType p = protection; foreach (Attribute attr; dec.attributes) @@ -545,6 +576,7 @@ private: * $(LI base classes) * $(LI mixin templates) * $(LI alias this) + * $(LI alias declarations) * ) */ struct ThirdPass @@ -584,6 +616,9 @@ private: currentSymbol.acSymbol.type = resolveType(currentSymbol.type, currentSymbol.acSymbol.location); break; + case aliasName: + // TODO + break; case enumName: case keyword: case enumMember: @@ -592,7 +627,6 @@ private: case dummy: case array: case assocArray: - case aliasName: case templateName: case mixinTemplateName: break; From ba64604a552988f814e4efe72a04d174272d7646 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Tue, 5 Nov 2013 17:56:22 -0800 Subject: [PATCH 25/27] #66 and partial resolution to constructor completion --- astconverter.d | 8 ++++++-- autocomplete.d | 10 ++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/astconverter.d b/astconverter.d index 4015d50..f14522a 100644 --- a/astconverter.d +++ b/astconverter.d @@ -387,7 +387,7 @@ private: void visitConstructor(size_t location, Parameters parameters, FunctionBody functionBody) { - SemanticSymbol* symbol = new SemanticSymbol("this", + SemanticSymbol* symbol = new SemanticSymbol("*constructor*", CompletionKind.functionName, symbolFile, location); processParameters(symbol, null, "this", parameters); symbol.protection = protection; @@ -617,7 +617,11 @@ private: currentSymbol.acSymbol.location); break; case aliasName: - // TODO + const(ACSymbol)* t = resolveType(currentSymbol.type, + currentSymbol.acSymbol.location); + while (t !is null && t.kind == CompletionKind.aliasName) + t = t.type; + currentSymbol.acSymbol.type = t; break; case enumName: case keyword: diff --git a/autocomplete.d b/autocomplete.d index 4525640..14df1fd 100644 --- a/autocomplete.d +++ b/autocomplete.d @@ -421,7 +421,7 @@ void setCompletions(T)(ref AutocompleteResponse response, && symbols[0].callTip is null) { auto call = symbols[0].getPartsByName("opCall"); - if (call.length == 0) + if (call.length > 0) { symbols = call; goto setCallTips; @@ -431,6 +431,7 @@ void setCompletions(T)(ref AutocompleteResponse response, return; else { + Log.trace("Not a function, but it has a constructor"); symbols = constructor; goto setCallTips; } @@ -440,7 +441,8 @@ void setCompletions(T)(ref AutocompleteResponse response, foreach (symbol; symbols) { Log.trace("Adding calltip ", symbol.callTip); - response.completions ~= symbol.callTip; + if (symbol.kind != CompletionKind.aliasName) + response.completions ~= symbol.callTip; } } } @@ -571,8 +573,8 @@ void setImportCompletions(T)(T tokens, ref AutocompleteResponse response) { response.completions ~= name.baseName(); response.completionKinds ~= - exists(buildPath(name, "package.d")) || exists(buildPath(name, "package.di")) - ? CompletionKind.packageName : CompletionKind.moduleName; + exists(buildPath(name, "package.d")) || exists(buildPath(name, "package.di")) + ? CompletionKind.packageName : CompletionKind.moduleName; } } } From 4b1bc4a2835aa7ad436691de43890f641fe2e194 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Wed, 6 Nov 2013 15:40:28 -0800 Subject: [PATCH 26/27] Fix #66 and #65 --- astconverter.d | 5 +---- autocomplete.d | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/astconverter.d b/astconverter.d index f14522a..9db1716 100644 --- a/astconverter.d +++ b/astconverter.d @@ -447,6 +447,7 @@ private: } symbol.acSymbol.callTip = formatCallTip(returnType, functionName, parameters); + symbol.type = returnType; } static string formatCallTip(Type returnType, string name, Parameters parameters, @@ -612,10 +613,6 @@ private: case variableName: case memberVariableName: case functionName: -// Log.trace("Resolving type of ", currentSymbol.acSymbol.name); - currentSymbol.acSymbol.type = resolveType(currentSymbol.type, - currentSymbol.acSymbol.location); - break; case aliasName: const(ACSymbol)* t = resolveType(currentSymbol.type, currentSymbol.acSymbol.location); diff --git a/autocomplete.d b/autocomplete.d index 14df1fd..f7488e1 100644 --- a/autocomplete.d +++ b/autocomplete.d @@ -154,6 +154,15 @@ const(ACSymbol)*[] getSymbolsByTokenChain(T)(const(Scope)* completionScope, break loop; break; case identifier: + // Use function return type instead of the function itself + if (symbols[0].qualifier == SymbolQualifier.func + || symbols[0].kind == CompletionKind.functionName) + { + symbols = symbols[0].type is null ? [] :[symbols[0].type]; + if (symbols.length == 0) + break loop; + } + Log.trace("looking for ", tokens[i].value, " in ", symbols[0].name); symbols = symbols[0].getPartsByName(tokens[i].value); if (symbols.length == 0) @@ -193,7 +202,7 @@ const(ACSymbol)*[] getSymbolsByTokenChain(T)(const(Scope)* completionScope, { auto h = i; skip(); - Parser p; + Parser p = new Parser(); p.setTokens(tokens[h .. i].array()); if (!p.isSliceExpression()) { @@ -209,7 +218,7 @@ const(ACSymbol)*[] getSymbolsByTokenChain(T)(const(Scope)* completionScope, { auto h = i; skip(); - Parser p; + Parser p = new Parser(); p.setTokens(tokens[h .. i].array()); const(ACSymbol)*[] overloads; if (p.isSliceExpression()) @@ -403,6 +412,14 @@ void setCompletions(T)(ref AutocompleteResponse response, if (completionType == CompletionType.identifiers) { + if (symbols[0].qualifier == SymbolQualifier.func + || symbols[0].kind == CompletionKind.functionName) + { + Log.trace("Completion list for return type of function ", symbols[0].name); + symbols = symbols[0].type is null ? [] : [symbols[0].type]; + if (symbols.length == 0) + return; + } foreach (s; symbols[0].parts.filter!(a => a.name !is null && a.name[0] != '*' && (partial is null ? true : a.name.toUpper().startsWith(partial.toUpper())) From 073bb639a9d0d42ae8ae58dd6e0236b86097fe11 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Wed, 6 Nov 2013 15:58:53 -0800 Subject: [PATCH 27/27] Fix #26 --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5fd34c1..1045ca6 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ back to the client. * Autocompletion of class, struct, and interface instances. * Display of call tips for functions, constructors, and variables of function type * alias declarations + * Public imports + * Finding the declaration location of a symbol at the cursor * *import* statement completions * Not working: * Automatic starting of the server by the client @@ -25,7 +27,6 @@ back to the client. * *auto* declarations * *alias this* * Determining the type of an enum member when no base type is specified, but the first member has an initialaizer - * Public imports * That one feature that you *REALLY* needed #Setup @@ -105,6 +106,18 @@ this, run the client with the -I option: dcd-client -Ipath/to/imports + +##Find declaration of symbol at cursor +```dcd-client --symbolLocation -c 123``` + +The "--symbolLocation" or "-l" flags cause the client to instruct the server +to return the path to the file and the byte offset of the declaration of the +symbol at the given cursor position. + +The output consists of the absolute path to the file followed by a tab character +followed by the byte offset, folled by a newline character. For example + /home/example/src/project/bar.d 3482 + #Server The server must be running for the DCD client to provide autocomplete information. In future versions the client may start the server if it is not running, but for