From 81bc94e8730032eb6f2ef3e3f15751c129928700 Mon Sep 17 00:00:00 2001 From: Joakim Brannstrom Date: Fri, 8 May 2015 21:29:01 +0200 Subject: [PATCH 1/2] Extended ctags generation with Exuberant metadata used by Vim. --- src/ctags.d | 256 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 197 insertions(+), 59 deletions(-) diff --git a/src/ctags.d b/src/ctags.d index f40f570..c4b2fc1 100644 --- a/src/ctags.d +++ b/src/ctags.d @@ -13,11 +13,13 @@ import std.range; import std.stdio; import std.array; import std.conv; +import std.typecons; /** * Prints CTAGS information to the given file. + * Includes metadata according to exuberant format used by Vim. * Params: - * outpt = the file that CTAGS info is written to + * outpt = the file that Exuberant TAGS info is written to * fileNames = tags will be generated from these files */ void printCtags(File output, string[] fileNames) @@ -28,101 +30,143 @@ void printCtags(File output, string[] fileNames) foreach (fileName; fileNames) { File f = File(fileName); - if (f.size == 0) continue; + if (f.size == 0) + continue; auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size)); f.rawRead(bytes); auto tokens = getTokensForParser(bytes, config, &cache); Module m = parseModule(tokens.array, fileName, null, &doNothing); + auto printer = new CTagsPrinter; printer.fileName = fileName; printer.visit(m); tags ~= printer.tagLines; } - output.write("!_TAG_FILE_FORMAT\t2\n" - ~ "!_TAG_FILE_SORTED\t1\n" - ~ "!_TAG_FILE_AUTHOR\tBrian Schott\n" - ~ "!_TAG_PROGRAM_URL\thttps://github.com/Hackerpilot/Dscanner/\n"); + output.write( + "!_TAG_FILE_FORMAT\t2\n" ~ "!_TAG_FILE_SORTED\t1\n" ~ "!_TAG_FILE_AUTHOR\tBrian Schott\n" ~ "!_TAG_PROGRAM_URL\thttps://github.com/Hackerpilot/Dscanner/\n"); tags.sort().copy(output.lockingTextWriter); } private: -void doNothing(string, size_t, size_t, string, bool) {} +/// States determining how an access modifier affects the parsning. +/// Reset, when ascending the AST reset back to the previous access. +/// Keep, when ascending the AST keep the new access. +enum AccessState +{ + Reset, + Keep +} -class CTagsPrinter : ASTVisitor +alias ContextType = Tuple!(string, "c", string, "access"); + +void doNothing(string, size_t, size_t, string, bool) +{ +} + +string paramsToString(Dec)(const Dec dec) +{ + import std.d.formatter : Formatter; + + auto app = appender!string(); + auto formatter = new Formatter!(typeof(app))(app); + + static if (is(Dec == FunctionDeclaration)) + { + formatter.format(dec.parameters); + } + else static if (is(Dec == TemplateDeclaration)) + { + formatter.format(dec.templateParameters); + } + else static if (is(Dec == Constructor)) + { + formatter.format(dec.parameters); + } + + return app.data; +} + +final class CTagsPrinter + : ASTVisitor { override void visit(const ClassDeclaration dec) { - tagLines ~= "%s\t%s\t%d;\"\tc\tline:%d%s\n".format(dec.name.text, fileName, dec.name.line, dec.name.line, context); - const c = context; - context = "\tclass:" ~ dec.name.text; + tagLines ~= "%s\t%s\t%d;\"\tc\tline:%d%s%s\n".format(dec.name.text, + fileName, dec.name.line, dec.name.line, context.c, context.access); + auto c = context; + context = ContextType("\tclass:" ~ dec.name.text, "\taccess:public"); dec.accept(this); context = c; } override void visit(const StructDeclaration dec) { - tagLines ~= "%s\t%s\t%d;\"\ts\tline:%d%s\n".format(dec.name.text, fileName, dec.name.line, dec.name.line, context); - const c = context; - context = "\tstruct:" ~ dec.name.text; + if (dec.name == tok!"") + { + dec.accept(this); + return; + } + tagLines ~= "%s\t%s\t%d;\"\ts\tline:%d%s%s\n".format(dec.name.text, + fileName, dec.name.line, dec.name.line, context.c, context.access); + auto c = context; + context = ContextType("\tstruct:" ~ dec.name.text, "\taccess:public"); dec.accept(this); context = c; } override void visit(const InterfaceDeclaration dec) { - tagLines ~= "%s\t%s\t%d;\"\ti\tline:%d%s\n".format(dec.name.text, fileName, dec.name.line, dec.name.line, context); - const c = context; - context = "\tclass:" ~ dec.name.text; + tagLines ~= "%s\t%s\t%d;\"\ti\tline:%d%s%s\n".format(dec.name.text, + fileName, dec.name.line, dec.name.line, context.c, context.access); + auto c = context; + context = ContextType("\tclass:" ~ dec.name.text, context.access); dec.accept(this); context = c; } override void visit(const TemplateDeclaration dec) { - tagLines ~= "%s\t%s\t%d;\"\tT\tline:%d%s\n".format(dec.name.text, fileName, dec.name.line, dec.name.line, context); - const c = context; - context = "\ttemplate:" ~ dec.name.text; + auto params = paramsToString(dec); + + tagLines ~= "%s\t%s\t%d;\"\tT\tline:%d%s%s\tsignature:%s\n".format( + dec.name.text, fileName, dec.name.line, dec.name.line, context.c, + context.access, params); + auto c = context; + context = ContextType("\ttemplate:" ~ dec.name.text, context.access); dec.accept(this); context = c; } override void visit(const FunctionDeclaration dec) { - tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d\tline:%d%s\n".format(dec.name.text, fileName, - dec.name.line, dec.parameters.parameters.length, dec.name.line, context); - const c = context; - context = "\tfunction:" ~ dec.name.text; - dec.accept(this); - context = c; + auto params = paramsToString(dec); + + tagLines ~= "%s\t%s\t%d;\"\tf\tline:%d%s%s\tsignature:%s\n".format( + dec.name.text, fileName, dec.name.line, dec.name.line, context.c, + context.access, params); } override void visit(const Constructor dec) { - tagLines ~= "this\t%s\t%d;\"\tf\tarity:%d\tline:%d%s\n".format(fileName, - dec.line, dec.parameters.parameters.length, dec.line, context); - const c = context; - context = "\tfunction: this"; - dec.accept(this); - context = c; + auto params = paramsToString(dec); + + tagLines ~= "this\t%s\t%d;\"\tf\tline:%d%s\tsignature:%s%s\n".format(fileName, + dec.line, dec.line, context.c, params, context.access); } override void visit(const Destructor dec) { - tagLines ~= "~this\t%s\t%d;\"\tf\tline:%d%s\n".format(fileName, dec.line, - dec.line, context); - const c = context; - context = "\tfunction: this"; - dec.accept(this); - context = c; + tagLines ~= "~this\t%s\t%d;\"\tf\tline:%d%s\n".format(fileName, + dec.line, dec.line, context.c); } override void visit(const EnumDeclaration dec) { - tagLines ~= "%s\t%s\t%d;\"\tg\tline:%d%s\n".format(dec.name.text, fileName, - dec.name.line, dec.name.line, context); - const c = context; - context = "\tenum:" ~ dec.name.text; + tagLines ~= "%s\t%s\t%d;\"\tg\tline:%d%s%s\n".format(dec.name.text, + fileName, dec.name.line, dec.name.line, context.c, context.access); + auto c = context; + context = ContextType("\tenum:" ~ dec.name.text, context.access); dec.accept(this); context = c; } @@ -134,32 +178,32 @@ class CTagsPrinter : ASTVisitor dec.accept(this); return; } - tagLines ~= "%s\t%s\t%d;\"\tu\tline:%d%s\n".format(dec.name.text, fileName, - dec.name.line, dec.name.line, context); - const c = context; - context = "\tunion:" ~ dec.name.text; + tagLines ~= "%s\t%s\t%d;\"\tu\tline:%d%s\n".format(dec.name.text, + fileName, dec.name.line, dec.name.line, context.c); + auto c = context; + context = ContextType("\tunion:" ~ dec.name.text, context.access); dec.accept(this); context = c; } override void visit(const AnonymousEnumMember mem) { - tagLines ~= "%s\t%s\t%d;\"\te\tline:%d%s\n".format(mem.name.text, fileName, - mem.name.line, mem.name.line, context); + tagLines ~= "%s\t%s\t%d;\"\te\tline:%d%s\n".format(mem.name.text, + fileName, mem.name.line, mem.name.line, context.c); } override void visit(const EnumMember mem) { - tagLines ~= "%s\t%s\t%d;\"\te\tline:%d%s\n".format(mem.name.text, fileName, - mem.name.line, mem.name.line, context); + tagLines ~= "%s\t%s\t%d;\"\te\tline:%d%s\n".format(mem.name.text, + fileName, mem.name.line, mem.name.line, context.c); } override void visit(const VariableDeclaration dec) { foreach (d; dec.declarators) { - tagLines ~= "%s\t%s\t%d;\"\tv\tline:%d%s\n".format(d.name.text, fileName, - d.name.line, d.name.line, context); + tagLines ~= "%s\t%s\t%d;\"\tv\tline:%d%s%s\n".format(d.name.text, + fileName, d.name.line, d.name.line, context.c, context.access); } dec.accept(this); } @@ -168,22 +212,116 @@ class CTagsPrinter : ASTVisitor { foreach (i; dec.identifiers) { - tagLines ~= "%s\t%s\t%d;\"\tv\tline:%d%s\n".format(i.text, fileName, - i.line, i.line, context); + tagLines ~= "%s\t%s\t%d;\"\tv\tline:%d%s%s\n".format(i.text, + fileName, i.line, i.line, context.c, context.access); } dec.accept(this); } override void visit(const Invariant dec) { - tagLines ~= "invariant\t%s\t%d;\"\tv\tline:%d%s\n".format(fileName, dec.line, dec.line, context); + tagLines ~= "invariant\t%s\t%d;\"\tv\tline:%d%s%s\n".format(fileName, + dec.line, dec.line, context.c, context.access); + } + + override void visit(const ModuleDeclaration dec) + { + context = ContextType("", "\taccess:public"); + dec.accept(this); + } + + override void visit(const Attribute attribute) + { + if (attribute.attribute != tok!"") + { + switch (attribute.attribute.type) + { + case tok!"export": + context.access = "\taccess:public"; + break; + case tok!"public": + context.access = "\taccess:public"; + break; + case tok!"package": + context.access = "\taccess:protected"; + break; + case tok!"protected": + context.access = "\taccess:protected"; + break; + case tok!"private": + context.access = "\taccess:private"; + break; + default: + } + } + + attribute.accept(this); + } + + override void visit(const AttributeDeclaration dec) + { + accessSt = AccessState.Keep; + dec.accept(this); + } + + override void visit(const Declaration dec) + { + auto c = context; + dec.accept(this); + + final switch (accessSt) with (AccessState) + { + case Reset: + context = c; + break; + case Keep: + break; + } + accessSt = AccessState.Reset; + } + + override void visit(const Unittest dec) + { + // skipping symbols inside a unit test. + //TODO investigate if it would be useful to show the unittests that was + //found when std.experimental.unittest is released. Could be possible + //to then use the UDA to show the unittest with a "name". + } + + override void visit(const AliasDeclaration dec) + { + // Old style alias + if (dec.identifierList) + { + foreach (i; dec.identifierList.identifiers) + { + tagLines ~= "%s\t%s\t%d;\"\ta\tline:%d%s%s\n".format(i.text, + fileName, i.line, i.line, context.c, context.access); + } + } + dec.accept(this); + } + + override void visit(const AliasInitializer dec) + { + tagLines ~= "%s\t%s\t%d;\"\ta\tline:%d%s%s\n".format(dec.name.text, + fileName, dec.name.line, dec.name.line, context.c, context.access); + + dec.accept(this); + } + + override void visit(const AliasThisDeclaration dec) + { + auto name = dec.identifier; + tagLines ~= "%s\t%s\t%d;\"\ta\tline:%d%s%s\n".format(name.text, + fileName, name.line, name.line, context.c, context.access); + + dec.accept(this); } alias visit = ASTVisitor.visit; - string fileName; string[] tagLines; - int suppressDepth; - string context; + ContextType context; + AccessState accessSt; } - From 3025185a55b95dfeb37a8292221b455ff8e21571 Mon Sep 17 00:00:00 2001 From: Joakim Brannstrom Date: Fri, 17 Jul 2015 13:01:16 +0200 Subject: [PATCH 2/2] Update after code review. --- src/ctags.d | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/ctags.d b/src/ctags.d index c4b2fc1..616226b 100644 --- a/src/ctags.d +++ b/src/ctags.d @@ -19,7 +19,7 @@ import std.typecons; * Prints CTAGS information to the given file. * Includes metadata according to exuberant format used by Vim. * Params: - * outpt = the file that Exuberant TAGS info is written to + * output = the file that Exuberant TAGS info is written to * fileNames = tags will be generated from these files */ void printCtags(File output, string[] fileNames) @@ -49,13 +49,15 @@ void printCtags(File output, string[] fileNames) private: -/// States determining how an access modifier affects the parsning. -/// Reset, when ascending the AST reset back to the previous access. -/// Keep, when ascending the AST keep the new access. +/// States determining how an access modifier affects tags when traversing the +/// AST. +/// The assumption is that there are fewer AST nodes and patterns that affects +/// the whole scope. +/// Therefor the default was chosen to be Reset. enum AccessState { - Reset, - Keep + Reset, /// when ascending the AST reset back to the previous access. + Keep /// when ascending the AST keep the new access. } alias ContextType = Tuple!(string, "c", string, "access"); @@ -71,7 +73,7 @@ string paramsToString(Dec)(const Dec dec) auto app = appender!string(); auto formatter = new Formatter!(typeof(app))(app); - static if (is(Dec == FunctionDeclaration)) + static if (is(Dec == FunctionDeclaration) || is(Dec == Constructor)) { formatter.format(dec.parameters); } @@ -79,10 +81,6 @@ string paramsToString(Dec)(const Dec dec) { formatter.format(dec.templateParameters); } - else static if (is(Dec == Constructor)) - { - formatter.format(dec.parameters); - } return app.data; } @@ -282,10 +280,11 @@ final class CTagsPrinter override void visit(const Unittest dec) { - // skipping symbols inside a unit test. - //TODO investigate if it would be useful to show the unittests that was - //found when std.experimental.unittest is released. Could be possible - //to then use the UDA to show the unittest with a "name". + // skipping symbols inside a unit test to not clutter the ctags file + // with "temporary" symbols. + // TODO when phobos have a unittest library investigate how that could + // be used to describe the tests. + // Maybe with UDA's to give the unittest a "name". } override void visit(const AliasDeclaration dec)