From e09051cf1d5439551650a125ab639162296b615f Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Sat, 27 Jul 2013 14:49:48 +0000 Subject: [PATCH] Added a README. Enhanced CTAGS --- README.md | 149 +++++++++++++++++++++++++++++++++++++++++++++++++ astprinter.d | 6 +- ctags.d | 38 ++++++++++--- imports.d | 37 ++++++++++++ main.d | 65 +++++++++++---------- std/d/ast.d | 4 +- std/d/parser.d | 2 +- 7 files changed, 256 insertions(+), 45 deletions(-) create mode 100644 README.md create mode 100644 imports.d diff --git a/README.md b/README.md new file mode 100644 index 0000000..e84998b --- /dev/null +++ b/README.md @@ -0,0 +1,149 @@ +# Overview +DScanner is a tool for analyzing D source code + +# Usage +The following examples assume that we are analyzing a simple file called helloworld.d + + import std.stdio; + void main(string[] args) + { + writeln("Hello World"); + } + +### Token Count +The "--tokenCount" or "-t" option prints the number of tokens in the given file + + $ dscanner --tokencount helloworld.d + 20 + +### Import Listing +The "--imports" or "-i" option prints a listing of modules imported by the given +source file. + + $ dscanner --imports helloworld.d + std.stdio + +### Line of Code Count +The "--sloc" or "-l" option prints the number of lines of code in the file. +Instead of simply printing the number of line breaks, this counts the number of +semicolon, while, if, do, else, switch, for, foreach, foreach\_reverse, default, +and case tokens in the file. + + $ ./dscanner --sloc helloworld.d + 2 + +### CTAGS output +The "--ctags" or "-c" option generates CTAGS information and writes it to the +standard output. When used with the "--recursive", "-R", or "-r" option, CTAGS +information will be generated for a specified directory and all of its +sub-directories. + + $ dscanner --ctags helloworld.d + !_TAG_FILE_FORMAT 2 + !_TAG_FILE_SORTED 1 + !_TAG_FILE_AUTHOR Brian Schott + !_TAG_PROGRAM_URL https://github.com/Hackerpilot/Dscanner/ + main helloworld.d 3;" f arity:1 + +CTAGS output uses the following tag kinds: + +* g -- enum declarataion +* e -- enum member +* v -- variable declaration +* i -- interface declaration +* c -- class declaration +* s -- struct declaration +* f -- function declaration +* u -- union declaration +* T -- template declaration + +More information on the CTAGS format can be found [here](http://ctags.sourceforge.net/FORMAT). + +### AST Dump +The "--ast" or "--xml" options will dump the complete abstract syntax tree of +the given source file to standard output in XML format. JSON output is planned +but not yet implemented. + + $ dscanner --ast helloworld.d + + + + + + std + stdio + + + + + + + main + + + void + + + + + args + + + + + + string + + + + + + + args + + + + + + + + + + + + + + + + writeln + + + + + + + + Hello World + + + + + + + + + + + + + + + + + + +# Useful code +The source code for DScanner has a complete lexer, parser, and abstact syntax +tree library for D code under the std/d/ directory. It is intended that these +modules eventually end up in Phobos, so feel free to use them for your own D +tools. diff --git a/astprinter.d b/astprinter.d index c90703f..d7a7b5d 100644 --- a/astprinter.d +++ b/astprinter.d @@ -1166,8 +1166,8 @@ class XMLPrinter : ASTVisitor override void visit(TemplateDeclaration templateDeclaration) { output.writeln(""); - output.writeln("", templateDeclaration.identifier, ""); + templateDeclaration.name.line, "\">"); + output.writeln("", templateDeclaration.name.value, ""); visit(templateDeclaration.templateParameters); if (templateDeclaration.constraint !is null) visit(templateDeclaration.constraint); @@ -1447,7 +1447,7 @@ class XMLPrinter : ASTVisitor private string xmlEscape(string s) { - return s.translate(['<' : "<", '>' : ">", '&', "&"]); + return s.translate(['<' : "<", '>' : ">", '&' : "&"]); } File output; diff --git a/ctags.d b/ctags.d index 255b86b..87ea9e3 100755 --- a/ctags.d +++ b/ctags.d @@ -54,13 +54,22 @@ class CTagsPrinter : ASTVisitor override void visit(InterfaceDeclaration dec) { - tagLines ~= "%s\t%s\t%d;\"\tc%s\n".format(dec.name.value, fileName, dec.name.line, context); + tagLines ~= "%s\t%s\t%d;\"\ti%s\n".format(dec.name.value, fileName, dec.name.line, context); auto c = context; context = "\tclass:" ~ dec.name.value; dec.accept(this); context = c; } + override void visit(TemplateDeclaration dec) + { + tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.value, fileName, dec.name.line, context); + auto c = context; + context = "\ttemplate:" ~ dec.name.value; + dec.accept(this); + context = c; + } + override void visit(FunctionDeclaration dec) { tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d%s\n".format(dec.name.value, fileName, @@ -73,6 +82,11 @@ class CTagsPrinter : ASTVisitor override void visit(EnumDeclaration dec) { + if (dec.name == TokenType.invalid) + { + dec.accept(this); + return; + } tagLines ~= "%s\t%s\t%d;\"\tg%s\n".format(dec.name.value, fileName, dec.name.line, context); auto c = context; @@ -81,6 +95,21 @@ class CTagsPrinter : ASTVisitor context = c; } + override void visit(UnionDeclaration dec) + { + if (dec.name == TokenType.invalid) + { + dec.accept(this); + return; + } + tagLines ~= "%s\t%s\t%d;\"\tu%s\n".format(dec.name.value, fileName, + dec.name.line, context); + auto c = context; + context = "\tunion:" ~ dec.name.value; + dec.accept(this); + context = c; + } + override void visit(EnumMember mem) { tagLines ~= "%s\t%s\t%d;\"\te%s\n".format(mem.name.value, fileName, @@ -97,13 +126,6 @@ class CTagsPrinter : ASTVisitor dec.accept(this); } - override void visit(FunctionBody fBody) - { - ++suppressDepth; - fBody.accept(this); - --suppressDepth; - } - string fileName; string[] tagLines; int suppressDepth; diff --git a/imports.d b/imports.d new file mode 100644 index 0000000..90bcab5 --- /dev/null +++ b/imports.d @@ -0,0 +1,37 @@ +// Copyright Brian Schott (Sir Alaran) 2012. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +module imports; + +import std.d.ast; +import std.stdio; + +class ImportPrinter : ASTVisitor +{ + override void visit(SingleImport singleImport) + { + ignore = false; + singleImport.accept(this); + writeln(); + ignore = true; + } + + override void visit(IdentifierChain identifierChain) + { + if (ignore) return; + bool first = true; + foreach (ident; identifierChain.identifiers) + { + if (!first) + write("."); + write(ident.value); + first = false; + } + } + + alias ASTVisitor.visit visit; + + bool ignore = true; +} diff --git a/main.d b/main.d index e913a9d..435756b 100755 --- a/main.d +++ b/main.d @@ -22,10 +22,10 @@ import highlighter; import stats; import ctags; import astprinter; +import imports; int main(string[] args) { - string[] importDirs; bool sloc; bool highlight; bool ctags; @@ -35,14 +35,14 @@ int main(string[] args) bool tokenCount; bool syntaxCheck; bool ast; + bool imports; try { - getopt(args, "I", &importDirs, "sloc|l", &sloc, - "highlight", &highlight, + getopt(args, "sloc|l", &sloc, "highlight", &highlight, "ctags|c", &ctags, "recursive|r|R", &recursive, "help|h", &help, - "tokenCount", &tokenCount, "syntaxCheck|s", &syntaxCheck, - "ast|xml", &ast); + "tokenCount|t", &tokenCount, "syntaxCheck|s", &syntaxCheck, + "ast|xml", &ast, "imports|i", &imports); } catch (Exception e) { @@ -56,7 +56,7 @@ int main(string[] args) } auto optionCount = count!"a"([sloc, highlight, ctags, tokenCount, - syntaxCheck, ast]); + syntaxCheck, ast, imports]); if (optionCount > 1) { stderr.writeln("Too many options specified"); @@ -79,17 +79,17 @@ int main(string[] args) args.length == 1 ? "stdin" : args[1]); return 0; } - else if (ctags) - { - if (recursive) - { - stdout.printCtags(dirEntries(args[1], SpanMode.depth) - .filter!(a => a.name.endsWith(".d") || a.name.endsWith(".di"))() - .map!(a => a.name)().array()); - } - else - stdout.printCtags(args[1 .. $]); - } + else if (ctags) + { + if (recursive) + { + stdout.printCtags(dirEntries(args[1], SpanMode.depth) + .filter!(a => a.name.endsWith(".d") || a.name.endsWith(".di"))() + .map!(a => a.name)().array()); + } + else + stdout.printCtags(args[1 .. $]); + } else { LexerConfig config; @@ -111,11 +111,17 @@ int main(string[] args) } else if (syntaxCheck) { - parseModule(tokens.array(), args[1]); + parseModule(tokens.array(), config.fileName); + } + else if (imports) + { + auto mod = parseModule(tokens.array(), config.fileName); + auto visitor = new ImportPrinter; + visitor.visit(mod); } else if (ast) { - auto mod = parseModule(tokens.array(), args[1]); + auto mod = parseModule(tokens.array(), config.fileName); auto printer = new XMLPrinter; printer.output = stdout; printer.visit(mod); @@ -135,22 +141,19 @@ options: --help | -h Prints this help message - --sloc | -l [sourceFiles] - count the number of logical lines of code in the given - source files. If no files are specified, a file is read from stdin. + --sloc | -l [sourceFile] + Prints the number of logical lines of code in the given + source file. If no files are specified, a file is read from stdin. + + --tokenCount | t [sourceFile] + Prints the number of tokens in the given source file. --highlight [sourceFile] - Syntax-highlight the given source file. The resulting HTML will be written to standard output. - --imports | -i [sourceFiles] + --imports | -i [sourceFile] Prints modules imported by the given source file. - -I includePath - Include _includePath_ in the list of paths used to search for imports. - By default dscanner will search in the current working directory as - well as any paths specified in /etc/dmd.conf. This is only used for the - --parenComplete and --dotComplete options. - --syntaxCheck | -s [sourceFile] Lexes and parses sourceFile, printing the line and column number of any syntax errors to stdout. One error or warning is printed per line. @@ -160,8 +163,8 @@ options: ctags information requires a filename, so stdin cannot be used in place of a filename. - --ast | --xml sourceFile - Generates an XML representation of the source files abstract syntax tree + --ast | --xml sourceFile + Generates an XML representation of the source files abstract syntax tree --recursive | -R | -r directory When used with --ctags, dscanner will produce ctags output for all .d diff --git a/std/d/ast.d b/std/d/ast.d index 141bfbb..d5334b3 100755 --- a/std/d/ast.d +++ b/std/d/ast.d @@ -2367,10 +2367,10 @@ class TemplateDeclaration : ASTNode public: override void accept(ASTVisitor visitor) { - mixin (visitIfNotNull!(identifier, templateParameters, constraint, + mixin (visitIfNotNull!(name, templateParameters, constraint, declarations)); } - /** */ Token identifier; + /** */ Token name; /** */ TemplateParameters templateParameters; /** */ Constraint constraint; /** */ Declaration[] declarations; diff --git a/std/d/parser.d b/std/d/parser.d index 2b0be49..8ceb484 100755 --- a/std/d/parser.d +++ b/std/d/parser.d @@ -4838,7 +4838,7 @@ q{(int a, ...) expect(TokenType.template_); auto ident = expect(TokenType.identifier); if (ident is null) return null; - node.identifier = *ident; + node.name = *ident; node.templateParameters = parseTemplateParameters(); if (currentIs(TokenType.if_)) node.constraint = parseConstraint();