Added a README. Enhanced CTAGS

README.md

@@ -0,0 +1,149 @@
+# Overview
+DScanner is a tool for analyzing D source code
+
+# Usage
+The following examples assume that we are analyzing a simple file called helloworld.d
+
+	import std.stdio;
+	void main(string[] args)
+	{
+		writeln("Hello World");
+	}
+
+### Token Count
+The "--tokenCount" or "-t" option prints the number of tokens in the given file
+
+	$ dscanner --tokencount helloworld.d
+	20
+
+### Import Listing
+The "--imports" or "-i" option prints a listing of modules imported by the given
+source file.
+
+	$ dscanner --imports helloworld.d
+	std.stdio
+
+### Line of Code Count
+The "--sloc" or "-l" option prints the number of lines of code in the file.
+Instead of simply printing the number of line breaks, this counts the number of
+semicolon, while, if, do, else, switch, for, foreach, foreach\_reverse, default,
+and case tokens in the file.
+
+	$ ./dscanner --sloc helloworld.d
+	2
+
+### CTAGS output
+The "--ctags" or "-c" option generates CTAGS information and writes it to the
+standard output. When used with the "--recursive", "-R", or "-r" option, CTAGS
+information will be generated for a specified directory and all of its
+sub-directories.
+
+	$ dscanner --ctags helloworld.d
+	!_TAG_FILE_FORMAT	2
+	!_TAG_FILE_SORTED	1
+	!_TAG_FILE_AUTHOR	Brian Schott
+	!_TAG_PROGRAM_URL	https://github.com/Hackerpilot/Dscanner/
+	main	helloworld.d	3;"	f	arity:1
+
+CTAGS output uses the following tag kinds:
+
+* g -- enum declarataion
+* e -- enum member
+* v -- variable declaration
+* i -- interface declaration
+* c -- class declaration
+* s -- struct declaration
+* f -- function declaration
+* u -- union declaration
+* T -- template declaration
+
+More information on the CTAGS format can be found [here](http://ctags.sourceforge.net/FORMAT).
+
+### AST Dump
+The "--ast" or "--xml" options will dump the complete abstract syntax tree of
+the given source file to standard output in XML format. JSON output is planned
+but not yet implemented.
+
+	$ dscanner --ast helloworld.d
+	<module>
+	<declaration>
+	<importDeclaration>
+	<singleImport>
+	<identifierChain>
+	<identifier>std</identifier>
+	<identifier>stdio</identifier>
+	</identifierChain>
+	</singleImport>
+	</importDeclaration>
+	</declaration>
+	<declaration>
+	<functionDeclaration line="3">
+	<name>main</name>
+	<type pretty="void">
+	<type2>
+	void
+	</type2>
+	</type>
+	<parameters>
+	<parameter>
+	<name>args</name>
+	<type pretty="string[]">
+	<type2>
+	<symbol>
+	<identifierOrTemplateChain>
+	<identifierOrTemplateInstance>
+	<identifier>string</identifier>
+	</identifierOrTemplateInstance>
+	</identifierOrTemplateChain>
+	</symbol>
+	</type2>
+	<typeSuffix type="[]"/>
+	</type>
+	<identifier>args</identifier>
+	</parameter>
+	</parameters>
+	<functionBody>
+	<blockStatement>
+	<declarationsAndStatements>
+	<declarationOrStatement>
+	<statement>
+	<statementNoCaseNoDefault>
+	<expressionStatement>
+	<expression>
+	<assignExpression>
+	<functionCallExpression>
+	<unaryExpression>
+	<primaryExpression>
+	<identifierOrTemplateInstance>
+	<identifier>writeln</identifier>
+	</identifierOrTemplateInstance>
+	</primaryExpression>
+	</unaryExpression>
+	<arguments>
+	<argumentList>
+	<assignExpression>
+	<primaryExpression>
+	<stringLiteral>Hello World</stringLiteral>
+	</primaryExpression>
+	</assignExpression>
+	</argumentList>
+	</arguments>
+	</functionCallExpression>
+	</assignExpression>
+	</expression>
+	</expressionStatement>
+	</statementNoCaseNoDefault>
+	</statement>
+	</declarationOrStatement>
+	</declarationsAndStatements>
+	</blockStatement>
+	</functionBody>
+	</functionDeclaration>
+	</declaration>
+	</module>
+
+# Useful code
+The source code for DScanner has a complete lexer, parser, and abstact syntax
+tree library for D code under the std/d/ directory. It is intended that these
+modules eventually end up in Phobos, so feel free to use them for your own D
+tools.

astprinter.d

@@ -1166,8 +1166,8 @@ class XMLPrinter : ASTVisitor
 	override void visit(TemplateDeclaration templateDeclaration)
 	{
 		output.writeln("<templateDeclaration line=\"",
-			templateDeclaration.identifier.line, "\">");
+			templateDeclaration.name.line, "\">");
-		output.writeln("<name>", templateDeclaration.identifier, "</name>");
+		output.writeln("<name>", templateDeclaration.name.value, "</name>");
 		visit(templateDeclaration.templateParameters);
 		if (templateDeclaration.constraint !is null)
 			visit(templateDeclaration.constraint);
@@ -1447,7 +1447,7 @@ class XMLPrinter : ASTVisitor
 
 	private string xmlEscape(string s)
 	{
-		return s.translate(['<' : "&lt;", '>' : "&gt;", '&', "&amp;"]);
+		return s.translate(['<' : "&lt;", '>' : "&gt;", '&' : "&amp;"]);
 	}
 
 	File output;

ctags.d

@@ -54,13 +54,22 @@ class CTagsPrinter : ASTVisitor
 
 	override void visit(InterfaceDeclaration dec)
 	{
-		tagLines ~= "%s\t%s\t%d;\"\tc%s\n".format(dec.name.value, fileName, dec.name.line, context);
+		tagLines ~= "%s\t%s\t%d;\"\ti%s\n".format(dec.name.value, fileName, dec.name.line, context);
         auto c = context;
         context = "\tclass:" ~ dec.name.value;
 		dec.accept(this);
         context = c;
 	}
 
+	override void visit(TemplateDeclaration dec)
+	{
+		tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.value, fileName, dec.name.line, context);
+        auto c = context;
+        context = "\ttemplate:" ~ dec.name.value;
+		dec.accept(this);
+        context = c;
+	}
+
 	override void visit(FunctionDeclaration dec)
 	{
 		tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d%s\n".format(dec.name.value, fileName,
@@ -73,6 +82,11 @@ class CTagsPrinter : ASTVisitor
 
 	override void visit(EnumDeclaration dec)
 	{
+		if (dec.name == TokenType.invalid)
+		{
+			dec.accept(this);
+			return;
+		}
 		tagLines ~= "%s\t%s\t%d;\"\tg%s\n".format(dec.name.value, fileName,
             dec.name.line, context);
         auto c = context;
@@ -81,6 +95,21 @@ class CTagsPrinter : ASTVisitor
         context = c;
 	}
 
+	override void visit(UnionDeclaration dec)
+	{
+		if (dec.name == TokenType.invalid)
+		{
+			dec.accept(this);
+			return;
+		}
+		tagLines ~= "%s\t%s\t%d;\"\tu%s\n".format(dec.name.value, fileName,
+            dec.name.line, context);
+        auto c = context;
+        context = "\tunion:" ~ dec.name.value;
+		dec.accept(this);
+        context = c;
+	}
+
     override void visit(EnumMember mem)
     {
         tagLines ~= "%s\t%s\t%d;\"\te%s\n".format(mem.name.value, fileName,
@@ -97,13 +126,6 @@ class CTagsPrinter : ASTVisitor
 		dec.accept(this);
 	}
 
-    override void visit(FunctionBody fBody)
-    {
-        ++suppressDepth;
-        fBody.accept(this);
-        --suppressDepth;
-    }
-
 	string fileName;
 	string[] tagLines;
     int suppressDepth;

imports.d

@@ -0,0 +1,37 @@
+//          Copyright Brian Schott (Sir Alaran) 2012.
+// Distributed under the Boost Software License, Version 1.0.
+//    (See accompanying file LICENSE_1_0.txt or copy at
+//          http://www.boost.org/LICENSE_1_0.txt)
+
+module imports;
+
+import std.d.ast;
+import std.stdio;
+
+class ImportPrinter : ASTVisitor
+{
+	override void visit(SingleImport singleImport)
+	{
+		ignore = false;
+		singleImport.accept(this);
+		writeln();
+		ignore = true;
+	}
+
+	override void visit(IdentifierChain identifierChain)
+	{
+		if (ignore) return;
+		bool first = true;
+		foreach (ident; identifierChain.identifiers)
+		{
+			if (!first)
+				write(".");
+			write(ident.value);
+			first = false;
+		}
+	}
+
+	alias ASTVisitor.visit visit;
+
+	bool ignore = true;
+}

main.d

@@ -22,10 +22,10 @@ import highlighter;
 import stats;
 import ctags;
 import astprinter;
+import imports;
 
 int main(string[] args)
 {
-	string[] importDirs;
 	bool sloc;
 	bool highlight;
 	bool ctags;
@@ -35,14 +35,14 @@ int main(string[] args)
 	bool tokenCount;
 	bool syntaxCheck;
 	bool ast;
+	bool imports;
 
 	try
 	{
-		getopt(args, "I", &importDirs, "sloc|l", &sloc,
+		getopt(args, "sloc|l", &sloc, "highlight", &highlight,
-			"highlight", &highlight,
 			"ctags|c", &ctags, "recursive|r|R", &recursive, "help|h", &help,
-			"tokenCount", &tokenCount, "syntaxCheck|s", &syntaxCheck,
+			"tokenCount|t", &tokenCount, "syntaxCheck|s", &syntaxCheck,
-			"ast|xml", &ast);
+			"ast|xml", &ast, "imports|i", &imports);
 	}
 	catch (Exception e)
 	{
@@ -56,7 +56,7 @@ int main(string[] args)
 	}
 
 	auto optionCount = count!"a"([sloc, highlight, ctags, tokenCount,
-		syntaxCheck, ast]);
+		syntaxCheck, ast, imports]);
 	if (optionCount > 1)
 	{
 		stderr.writeln("Too many options specified");
@@ -79,17 +79,17 @@ int main(string[] args)
 			args.length == 1 ? "stdin" : args[1]);
 		return 0;
 	}
-    else if (ctags)
+	else if (ctags)
-    {
+	{
-        if (recursive)
+		if (recursive)
-        {
+		{
-            stdout.printCtags(dirEntries(args[1], SpanMode.depth)
+			stdout.printCtags(dirEntries(args[1], SpanMode.depth)
-                .filter!(a => a.name.endsWith(".d") || a.name.endsWith(".di"))()
+				.filter!(a => a.name.endsWith(".d") || a.name.endsWith(".di"))()
-                .map!(a => a.name)().array());
+				.map!(a => a.name)().array());
-        }
+		}
-        else
+		else
-            stdout.printCtags(args[1 .. $]);
+			stdout.printCtags(args[1 .. $]);
-    }
+	}
 	else
 	{
 		LexerConfig config;
@@ -111,11 +111,17 @@ int main(string[] args)
 		}
 		else if (syntaxCheck)
 		{
-			parseModule(tokens.array(), args[1]);
+			parseModule(tokens.array(), config.fileName);
+		}
+		else if (imports)
+		{
+			auto mod = parseModule(tokens.array(), config.fileName);
+			auto visitor = new ImportPrinter;
+			visitor.visit(mod);
 		}
 		else if (ast)
 		{
-			auto mod = parseModule(tokens.array(), args[1]);
+			auto mod = parseModule(tokens.array(), config.fileName);
 			auto printer = new XMLPrinter;
 			printer.output = stdout;
 			printer.visit(mod);
@@ -135,22 +141,19 @@ options:
     --help | -h
         Prints this help message
 
-    --sloc | -l [sourceFiles]
+    --sloc | -l [sourceFile]
-        count the number of logical lines of code in the given
+        Prints the number of logical lines of code in the given
-        source files. If no files are specified, a file is read from stdin.
+        source file. If no files are specified, a file is read from stdin.
+
+    --tokenCount | t [sourceFile]
+        Prints the number of tokens in the given source file.
 
     --highlight [sourceFile] - Syntax-highlight the given source file. The
         resulting HTML will be written to standard output.
 
-    --imports | -i [sourceFiles]
+    --imports | -i [sourceFile]
         Prints modules imported by the given source file.
 
-    -I includePath
-        Include _includePath_ in the list of paths used to search for imports.
-        By default dscanner will search in the current working directory as
-        well as any paths specified in /etc/dmd.conf. This is only used for the
-        --parenComplete and --dotComplete options.
-
     --syntaxCheck | -s [sourceFile]
         Lexes and parses sourceFile, printing the line and column number of any
         syntax errors to stdout. One error or warning is printed per line.
@@ -160,8 +163,8 @@ options:
         ctags information requires a filename, so stdin cannot be used in place
         of a filename.
 
-	--ast | --xml sourceFile
+    --ast | --xml sourceFile
-		Generates an XML representation of the source files abstract syntax tree
+        Generates an XML representation of the source files abstract syntax tree
 
     --recursive | -R | -r directory
         When used with --ctags, dscanner will produce ctags output for all .d

std/d/ast.d

@@ -2367,10 +2367,10 @@ class TemplateDeclaration : ASTNode
 public:
     override void accept(ASTVisitor visitor)
     {
-        mixin (visitIfNotNull!(identifier, templateParameters, constraint,
+        mixin (visitIfNotNull!(name, templateParameters, constraint,
             declarations));
     }
-    /** */ Token identifier;
+    /** */ Token name;
     /** */ TemplateParameters templateParameters;
     /** */ Constraint constraint;
     /** */ Declaration[] declarations;

std/d/parser.d

@@ -4838,7 +4838,7 @@ q{(int a, ...)
         expect(TokenType.template_);
         auto ident = expect(TokenType.identifier);
         if (ident is null) return null;
-        node.identifier = *ident;
+        node.name = *ident;
         node.templateParameters = parseTemplateParameters();
         if (currentIs(TokenType.if_))
             node.constraint = parseConstraint();