Basic CTags support based on the AST classes

build.sh

@@ -1,5 +1,5 @@
-dmd *.d std/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner-dmd
+#dmd *.d std/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner-dmd
-#dmd *.d std/d/*.d -g -m64 -w -wi -ofdscanner -unittest
+dmd *.d std/d/*.d -g -m64 -w -wi -ofdscanner 
-ldc2 -O3 *.d std/d/*.d -of=dscanner-ldc -release -m64
+#ldc2 -O3 *.d std/d/*.d -of=dscanner-ldc -release -m64
 #ldc2 *.d std/d/*.d -of=dscanner -unittest -m64 -g
-/opt/gdc/bin/gdc -O3 -odscanner-gdc -fno-bounds-check -frelease -m64 *.d std/d/*.d
+#/opt/gdc/bin/gdc -O3 -odscanner-gdc -fno-bounds-check -frelease -m64 *.d std/d/*.d

createTable.sql

@@ -1,4 +0,0 @@
-create table modules (path, mtime, id);
-create table publicImports (importerId, importedId);
-create table containers (name, protection, moduleId, id);
-create table symbols (name, type, kind, containerId, id);

ctags.d

@@ -5,10 +5,64 @@
 
 module ctags;
 
-void printCtags(Tokens)(File output, ref Tokens tokens)
+import std.d.parser;
-{
+import std.d.lexer;
-	output.write("!_TAG_FILE_FORMAT 2\n"
+import std.d.ast;
-		~ "!_TAG_FILE_SORTED 1\n"
+import std.algorithm;
-		~ "!_TAG_PROGRAM_URL https://github.com/Hackerpilot/Dscanner/\n");
+import std.stdio;
+import std.array;
 
+void doNothing(string, int, int, string) {}
+
+void printCtags(Tokens)(File output, ref Tokens tokens, string fileName)
+{
+	Module m = parseModule(tokens.array(), fileName, &doNothing);
+	auto printer = new CTagsPrinter;
+	printer.fileName = fileName;
+	printer.visit(m);
+	printer.print(output);
+}
+
+class CTagsPrinter : ASTVisitor
+{
+
+	alias ASTVisitor.visit visit;
+
+	override void visit(ClassDeclaration dec)
+	{
+		tagLines ~= "%s\t%s\t%d;\"\tc".format(dec.name.value, fileName, dec.name.line);
+		dec.structBody.accept(this);
+	}
+
+	override void visit(InterfaceDeclaration dec)
+	{
+		tagLines ~= "%s\t%s\t%d;\"\tc".format(dec.name.value, fileName, dec.name.line);
+		dec.structBody.accept(this);
+	}
+
+	override void visit(FunctionDeclaration dec)
+	{
+		tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d".format(dec.name.value, fileName,
+			dec.name.line, dec.parameters.parameters.length);
+	}
+
+	override void visit(EnumDeclaration dec)
+	{
+		tagLines ~= "%s\t%s\t%d;\"\tg".format(dec.name.value, fileName, dec.name.line);
+	}
+
+	void print(File output)
+	{
+		output.write("!_TAG_FILE_FORMAT 2\n"
+			~ "!_TAG_FILE_SORTED 1\n"
+			~ "!_TAG_FILE_AUTHOR Brian Schott\n"
+			~ "!_TAG_PROGRAM_URL https://github.com/Hackerpilot/Dscanner/\n");
+		foreach (str; sort(tagLines))
+		{
+			output.writeln(str);
+		}
+	}
+
+	string fileName;
+	string[] tagLines;
 }

editors/textadept/modules/dmd/init.lua

@@ -317,7 +317,7 @@ local keywords = {
 
 -- For this module to work the dscanner program must be installed. Configure the
 -- path to the executable here
-M.PATH_TO_DSCANNER = "/home/alaran/src/dscanner-master/dscanner"
+M.PATH_TO_DSCANNER = "/home/alaran/src/dscanner/dscanner"
 
 _M.textadept.editing.comment_string.dmd = '//'
 _M.textadept.run.compile_command.dmd = 'dmd -c -o- %(filename)'
@@ -368,32 +368,56 @@ local function showCompletionList(r)
 	buffer.auto_c_choose_single = setting
 end
 
-events.connect(events.CHAR_ADDED, function(ch)
+--events.connect(events.CHAR_ADDED, function(ch)
+--	if buffer:get_lexer() ~= "dmd" then return end
+--	if ch > 255 then return end
+--	local character = string.char(ch)
+--	if character == "." or character == "(" then
+--		local fileName = os.tmpname()
+--		local tmpFile = io.open(fileName, "w")
+--		tmpFile:write(buffer:get_text())
+--		local command = M.PATH_TO_DSCANNER
+--			.. (character == "." and " --dotComplete " or " --parenComplete ")
+--			.. fileName .. " " .. buffer.current_pos .. " -I" .. buffer.filename:match(".+[\\/]")
+--		local p = io.popen(command)
+--		local r = p:read("*a")
+--		if r ~= "\n" then
+--			if character == "." then
+--				showCompletionList(r)
+--			elseif character == "(" then
+--				if r:find("^completions\n") then
+--					showCompletionList(r)
+--				elseif r:find("^calltips\n.*") then
+--					r = r:gsub("^calltips\n", "")
+--					buffer:call_tip_show(buffer.current_pos, r:gsub("\\n", "\n"):gsub("\\t", "\t"):match("(.*)%s+$"))
+--				end
+--			end
+--		end
+--		os.remove(fileName)
+--	end
+--end)
+
+events.connect(events.FILE_AFTER_SAVE, function()
 	if buffer:get_lexer() ~= "dmd" then return end
-	if ch > 255 then return end
+	buffer:annotation_clear_all()
-	local character = string.char(ch)
+	--buffer.annotation_visible = _SCINTILLA.constants.ANNOTATION_STANDARD
-	if character == "." or character == "(" then
+	local command = M.PATH_TO_DSCANNER .. " --syntaxCheck " .. buffer.filename
-		local fileName = os.tmpname()
+	local p = io.popen(command)
-		local tmpFile = io.open(fileName, "w")
+	for line in p:lines() do
-		tmpFile:write(buffer:get_text())
+		lineNumber, column, level, message = string.match(line, "^.-%((%d+):(%d+)%)%[(%w+)%]: (.+)$")
-		local command = M.PATH_TO_DSCANNER
+		local l = tonumber(lineNumber) - 1
-			.. (character == "." and " --dotComplete " or " --parenComplete ")
+		local c = tonumber(column)
-			.. fileName .. " " .. buffer.current_pos .. " -I" .. buffer.filename:match(".+[\\/]")
+		if level == "error" then
-		local p = io.popen(command)
+			buffer.annotation_style[l] = 8
-		local r = p:read("*a")
+		else
-		if r ~= "\n" then
+			buffer.annotation_style[l] = 2
-			if character == "." then
+		end
-				showCompletionList(r)
+		local t = buffer.annotation_text[l]
-			elseif character == "(" then
+		if #t > 0 then
-				if r:find("^completions\n") then
+			buffer.annotation_text[l] = buffer.annotation_text[l] .. "\n" .. message
-					showCompletionList(r)
+		else
-				elseif r:find("^calltips\n.*") then
+			buffer.annotation_text[l] = message
-					r = r:gsub("^calltips\n", "")
-					buffer:call_tip_show(buffer.current_pos, r:gsub("\\n", "\n"):gsub("\\t", "\t"):match("(.*)%s+$"))
-				end
-			end
 		end
-		os.remove(fileName)
 	end
 end)
 

main.d

@@ -16,10 +16,12 @@ import std.regex;
 import std.stdio;
 import std.range;
 import std.d.lexer;
+import std.d.parser;
 
 import highlighter;
 import autocomplete;
 import stats;
+import ctags;
 
 /**
  * Loads any import directories specified in /etc/dmd.conf.
@@ -92,13 +94,14 @@ int main(string[] args)
 	bool format;
 	bool help;
 	bool tokenCount;
+	bool syntaxCheck;
 
 	try
 	{
 		getopt(args, "I", &importDirs, "dotComplete|d", &dotComplete, "sloc|l", &sloc,
 			"json|j", &json, "parenComplete|p", &parenComplete, "highlight", &highlight,
 			"ctags|c", &ctags, "recursive|r|R", &recursive, "help|h", &help,
-			"tokenCount", &tokenCount,
+			"tokenCount", &tokenCount, "syntaxCheck", &syntaxCheck,
             "declaration|e", &declaration, "symbolComplete|s", &symbolComplete);
 	}
 	catch (Exception e)
@@ -112,7 +115,8 @@ int main(string[] args)
         return 0;
     }
 
-    auto optionCount = count!"a"([sloc, highlight, ctags, json, tokenCount]);
+    auto optionCount = count!"a"([sloc, highlight, ctags, json, tokenCount,
+		syntaxCheck]);
     if (optionCount > 1)
     {
         stderr.writeln("Too many options specified");
@@ -172,6 +176,14 @@ int main(string[] args)
 			{
 			}
 		}
+		else if (ctags)
+		{
+			printCtags(stdout, tokens, args[1]);
+		}
+		else if (syntaxCheck)
+		{
+			parseModule(tokens.array(), args[1]);
+		}
 	}
 
 	return 0;

std/d/ast.d

@@ -26,6 +26,7 @@ import std.d.lexer;
  */
 abstract class ASTVisitor
 {
+public:
     /** */ void visit(AddExpression addExpression) { addExpression.accept(this); }
     /** */ void visit(AliasDeclaration aliasDeclaration) { aliasDeclaration.accept(this); }
     /** */ void visit(AliasInitializer aliasInitializer) { aliasInitializer.accept(this); }
@@ -846,7 +847,7 @@ class Declarator : ASTNode
 {
 public:
     mixin(DEFAULT_ACCEPT);
-    /** */ Token identifier;
+    /** */ Token name;
     /** */ Initializer initializer;
 }
 
@@ -912,7 +913,7 @@ class EnumDeclaration : ASTNode
 {
 public:
     mixin(DEFAULT_ACCEPT);
-    /** */ Token identifier;
+    /** */ Token name;
     /** */ Type type;
     /** */ EnumBody enumBody;
 }
@@ -1231,7 +1232,7 @@ class InterfaceDeclaration : ASTNode
 {
 public:
     mixin(DEFAULT_ACCEPT);
-    /** */ Token identifier;
+    /** */ Token name;
     /** */ TemplateParameters templateParameters;
     /** */ Constraint constraint;
     /** */ BaseClassList baseClassList;
@@ -1290,6 +1291,7 @@ class LambdaExpression : ExpressionNode
 {
 public:
     mixin(DEFAULT_ACCEPT);
+    /** */ TokenType functionType;
     /** */ Token identifier;
     /** */ Parameters parameters;
     /** */ FunctionAttribute[] functionAttributes;

std/d/lexer.d

@@ -128,12 +128,12 @@ public:
  */
 struct Token
 {
-	/**
+    /**
      * The representation of the token in the original source code.
      */
     string value;
 
-	/**
+    /**
      * The index of the start of the token in the original source.
      * $(LPAREN)measured in ASCII characters or UTF-8 code units$(RPAREN)
      */
@@ -144,13 +144,13 @@ struct Token
      */
     uint line;
 
-	/**
+    /**
      * The column number of the start of the token in the original source.
      * $(LPAREN)measured in ASCII characters or UTF-8 code units$(RPAREN)
      */
     ushort column;
 
-	/**
+    /**
      * The token type.
      */
     TokenType type;
@@ -159,7 +159,7 @@ struct Token
      * Check to see if the token is of the same type and has the same string
      * representation as the given token.
      */
-    bool opEquals(ref const(Token) other) const
+    bool opEquals(ref const(Token) other) const nothrow pure
     {
         return other.type == type && other.value == value;
     }
@@ -168,17 +168,23 @@ struct Token
      * Checks to see if the token's string representation is equal to the given
      * string.
      */
-    bool opEquals(string value) const { return this.value == value; }
+    bool opEquals(string value) const nothrow pure
+    {
+        return this.value == value;
+    }
 
     /**
      * Checks to see if the token is of the given type.
      */
-    bool opEquals(TokenType type) const { return this.type == type; }
+    bool opEquals(TokenType type) const nothrow pure
+    {
+        return this.type == type;
+    }
 
     /**
      * Comparison operator orders tokens by start index.
      */
-    int opCmp(ref const(Token) other) const
+    int opCmp(ref const(Token) other) const nothrow pure
     {
         if (startIndex < other.startIndex) return -1;
         if (startIndex > other.startIndex) return 1;
@@ -216,7 +222,7 @@ enum TokenStyle : ushort
      * Escape sequences will be replaced with their equivalent characters,
      * enclosing quote characters will not be included. Special tokens such as
      * $(D_KEYWORD ___VENDOR__) will be replaced with their equivalent strings.
-	 * Useful for creating a compiler or interpreter.
+     * Useful for creating a compiler or interpreter.
      */
     default_ = 0b0000,
 
@@ -237,7 +243,7 @@ enum TokenStyle : ushort
 
     /**
      * Do not replace the value field of the special tokens such as
-	 * $(D_KEYWORD ___DATE__) with their string equivalents.
+     * $(D_KEYWORD ___DATE__) with their string equivalents.
      */
     doNotReplaceSpecial = 0b0100,
 
@@ -449,7 +455,7 @@ L_advance:
             ">>>",             "TokenType.unsignedShiftRight",
             ">>>=",            "TokenType.unsignedShiftRightEqual",
             "^",               "TokenType.xor",
-            "^=",              "TokenType.xorEqual",
+            "^=",              "TokenType.xorEqual"
         ));
         case '/':
             nextCharNonLF();
@@ -2030,7 +2036,7 @@ pure nothrow bool isMisc(ref const Token t)
  */
 enum TokenType: ushort
 {
-	invalid, /// Not a valid token
+    invalid, /// Not a valid token
     assign, /// =
     at, /// @
     bitAnd, /// &
@@ -2528,7 +2534,7 @@ bool isRangeEoF(R)(ref R range)
 
 // Lookup table for token values
 package immutable(string[TokenType.max + 1]) tokenValues = [
-	null,
+    null,
     "=",
     "@",
     "&",
@@ -3109,7 +3115,7 @@ private:
         string value;
         Slot* next;
         uint hash;
-    };
+    }
 
     void printLoadFactor()
     {

std/d/parser.d

@@ -77,18 +77,20 @@ import std.string : format;
 
 // Uncomment this if you want ALL THE OUTPUT
 // Caution: generates 180 megabytes of logging for std.datetime
-// version = std_parser_verbose;
+//version = std_parser_verbose;
 
 /**
  * Params:
  *     tokens = the tokens parsed by std.d.lexer
  * Returns: the parsed module
  */
-Module parseModule(const(Token)[] tokens, string fileName)
+Module parseModule(const(Token)[] tokens, string fileName,
+	void function(string, int, int, string) messageFunction = null)
 {
     auto parser = new Parser();
     parser.fileName = fileName;
     parser.tokens = tokens;
+	parser.messageFunction = messageFunction;
     auto mod = parser.parseModule();
     // writefln("Parsing finished with %d errors and %d warnings.",
     //     parser.errorCount, parser.warningCount);
@@ -1832,7 +1834,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         auto node = new Declarator;
         auto id = expect(TokenType.identifier);
         if (id is null) return null;
-        node.identifier = *id;
+        node.name = *id;
         if (currentIsOneOf(TokenType.lBracket, TokenType.star))
         {
             error("C-style variable declarations are not supported.");
@@ -2005,7 +2007,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         auto node = new EnumDeclaration;
         if (expect(TokenType.enum_) is null) return null;
         if (currentIs(TokenType.identifier))
-            node.identifier = advance();
+            node.name = advance();
         if (currentIs(TokenType.colon))
         {
             advance();
@@ -2668,7 +2670,8 @@ body {} // six
         {
             auto b = setBookmark();
             auto t = parseType();
-            if (t is null || !currentIs(TokenType.identifier))
+            if (t is null || !currentIs(TokenType.identifier)
+				|| !peekIs(TokenType.assign))
             {
                 goToBookmark(b);
                 node.expression = parseExpression();
@@ -2959,7 +2962,7 @@ import core.stdc.stdio, std.string : KeepTerminator;
         if (expect(TokenType.interface_) is null) return null;
         auto ident = expect(TokenType.identifier);
         if (ident is null) return null;
-        node.identifier = *ident;
+        node.name = *ident;
         if (currentIs(TokenType.lParen))
         {
             node.templateParameters = parseTemplateParameters();
@@ -3170,10 +3173,16 @@ invariant() foo();
     {
         mixin(traceEnterAndExit!(__FUNCTION__));
         auto node = new LambdaExpression;
-        if (currentIs(TokenType.identifier))
+        if (currentIsOneOf(TokenType.function_, TokenType.delegate_))
+        {
+            node.functionType = advance().type;
+            goto lParen;
+        }
+        else if (currentIs(TokenType.identifier))
             node.identifier = advance();
         else if (currentIs(TokenType.lParen))
         {
+        lParen:
             node.parameters = parseParameters();
             do
             {
@@ -6431,9 +6440,9 @@ private:
             if (suppressMessages > 0)
                 return;
             if (index < tokens.length)
-                stderr.writeln(message, "(", current.line, ":", current.column + 1, ")");
+                writeln(message, "(", current.line, ":", current.column + 1, ")");
             else
-                stderr.writeln(message, "(EOF:0)");
+                writeln(message, "(EOF:0)");
         }
     }
     else

std/d/runtester.sh

@@ -1,12 +0,0 @@
-if [ ! -d runs ]; then
-    mkdir runs
-fi
-for file in /usr/include/d/std/*.d; do
-    shortFile=$(basename $file)
-    echo "Parsing" $shortFile "..."
-    outFile=runs/$shortFile.txt
-    ./tester $file > $outFile
-done
-echo
-grep -l "Parsing finished with 0 errors" runs/*.txt | sed -e "s/runs\//Pass /" -e "s/.txt//"
-grep -L "Parsing finished with 0 errors" runs/*.txt | sed -e "s/runs\//Fail /" -e "s/.txt//"

std/d/tester.d

@@ -1,69 +0,0 @@
-import std.d.lexer;
-import std.d.ast;
-import std.d.parser;
-import std.stdio;
-import std.file;
-import std.array;
-
-class TestVisitor : ASTVisitor
-{
-    override void visit(ClassDeclaration classDeclaration)
-    {
-        writeln("class ", classDeclaration.name.value, " on line ", classDeclaration.name.line);
-    }
-
-    override void visit(StructDeclaration structDeclaration)
-    {
-        writeln("struct ", structDeclaration.name.value, " on line ", structDeclaration.name.line);
-    }
-
-    override void visit(ModuleDeclaration md)
-    {
-        writeln("module declaration found");
-    }
-
-    override void visit(FunctionDeclaration funDec)
-    {
-        writeln("function ", funDec.name.value, " on line ", funDec.name.line);
-    }
-
-    override void visit(VariableDeclaration varDec)
-    {
-        foreach (decl; varDec.declarators)
-        {
-            writeln("variable ", decl.identifier.value,
-                " on line ", decl.identifier.line);
-        }
-    }
-
-    override void visit(ImportDeclaration impDec)
-    {
-        writeln("import declaration found");
-    }
-
-    override void visit(InterfaceDeclaration intDec)
-    {
-        writeln("Interface ", intDec.identifier.value,
-            " on line ", intDec.identifier.line);
-    }
-
-    override void visit(VersionSpecification verSpec)
-    {
-        writeln("Version specification");
-    }
-
-    alias ASTVisitor.visit visit;
-}
-
-void main(string[] args)
-{
-    auto de = dirEntry(args[1]);
-    ubyte[] sourceBuffer = new ubyte[de.size];
-    auto f = File(args[1]);
-    ubyte[] rawSource = f.rawRead(sourceBuffer);
-    LexerConfig config;
-    auto tokens = byToken(rawSource, config).array();
-    Module m = parseModule(tokens, args[1]);
-    //ASTVisitor visitor = new TestVisitor;
-    //visitor.visit(m);
-}