From 7ceebb28a3748a0a655850974b4118e1ae9689ca Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Mon, 15 Jul 2013 02:52:50 +0000 Subject: [PATCH] Basic CTags support based on the AST classes --- build.sh | 8 +-- createTable.sql | 4 -- ctags.d | 64 ++++++++++++++++++++-- editors/textadept/modules/dmd/init.lua | 74 +++++++++++++++++--------- main.d | 16 +++++- std/d/ast.d | 8 +-- std/d/lexer.d | 34 +++++++----- std/d/parser.d | 27 ++++++---- std/d/runtester.sh | 12 ----- std/d/tester.d | 69 ------------------------ 10 files changed, 169 insertions(+), 147 deletions(-) delete mode 100755 createTable.sql delete mode 100755 std/d/runtester.sh delete mode 100644 std/d/tester.d diff --git a/build.sh b/build.sh index 9d3a865..98a5369 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,5 @@ -dmd *.d std/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner-dmd -#dmd *.d std/d/*.d -g -m64 -w -wi -ofdscanner -unittest -ldc2 -O3 *.d std/d/*.d -of=dscanner-ldc -release -m64 +#dmd *.d std/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner-dmd +dmd *.d std/d/*.d -g -m64 -w -wi -ofdscanner +#ldc2 -O3 *.d std/d/*.d -of=dscanner-ldc -release -m64 #ldc2 *.d std/d/*.d -of=dscanner -unittest -m64 -g -/opt/gdc/bin/gdc -O3 -odscanner-gdc -fno-bounds-check -frelease -m64 *.d std/d/*.d +#/opt/gdc/bin/gdc -O3 -odscanner-gdc -fno-bounds-check -frelease -m64 *.d std/d/*.d diff --git a/createTable.sql b/createTable.sql deleted file mode 100755 index 7b65c55..0000000 --- a/createTable.sql +++ /dev/null @@ -1,4 +0,0 @@ -create table modules (path, mtime, id); -create table publicImports (importerId, importedId); -create table containers (name, protection, moduleId, id); -create table symbols (name, type, kind, containerId, id); diff --git a/ctags.d b/ctags.d index 5fc346a..a608923 100755 --- a/ctags.d +++ b/ctags.d @@ -5,10 +5,64 @@ module ctags; -void printCtags(Tokens)(File output, ref Tokens tokens) -{ - output.write("!_TAG_FILE_FORMAT 2\n" - ~ "!_TAG_FILE_SORTED 1\n" - ~ "!_TAG_PROGRAM_URL https://github.com/Hackerpilot/Dscanner/\n"); +import std.d.parser; +import std.d.lexer; +import std.d.ast; +import std.algorithm; +import std.stdio; +import std.array; +void doNothing(string, int, int, string) {} + +void printCtags(Tokens)(File output, ref Tokens tokens, string fileName) +{ + Module m = parseModule(tokens.array(), fileName, &doNothing); + auto printer = new CTagsPrinter; + printer.fileName = fileName; + printer.visit(m); + printer.print(output); +} + +class CTagsPrinter : ASTVisitor +{ + + alias ASTVisitor.visit visit; + + override void visit(ClassDeclaration dec) + { + tagLines ~= "%s\t%s\t%d;\"\tc".format(dec.name.value, fileName, dec.name.line); + dec.structBody.accept(this); + } + + override void visit(InterfaceDeclaration dec) + { + tagLines ~= "%s\t%s\t%d;\"\tc".format(dec.name.value, fileName, dec.name.line); + dec.structBody.accept(this); + } + + override void visit(FunctionDeclaration dec) + { + tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d".format(dec.name.value, fileName, + dec.name.line, dec.parameters.parameters.length); + } + + override void visit(EnumDeclaration dec) + { + tagLines ~= "%s\t%s\t%d;\"\tg".format(dec.name.value, fileName, dec.name.line); + } + + void print(File output) + { + output.write("!_TAG_FILE_FORMAT 2\n" + ~ "!_TAG_FILE_SORTED 1\n" + ~ "!_TAG_FILE_AUTHOR Brian Schott\n" + ~ "!_TAG_PROGRAM_URL https://github.com/Hackerpilot/Dscanner/\n"); + foreach (str; sort(tagLines)) + { + output.writeln(str); + } + } + + string fileName; + string[] tagLines; } diff --git a/editors/textadept/modules/dmd/init.lua b/editors/textadept/modules/dmd/init.lua index f26178d..f74f0a4 100755 --- a/editors/textadept/modules/dmd/init.lua +++ b/editors/textadept/modules/dmd/init.lua @@ -317,7 +317,7 @@ local keywords = { -- For this module to work the dscanner program must be installed. Configure the -- path to the executable here -M.PATH_TO_DSCANNER = "/home/alaran/src/dscanner-master/dscanner" +M.PATH_TO_DSCANNER = "/home/alaran/src/dscanner/dscanner" _M.textadept.editing.comment_string.dmd = '//' _M.textadept.run.compile_command.dmd = 'dmd -c -o- %(filename)' @@ -368,32 +368,56 @@ local function showCompletionList(r) buffer.auto_c_choose_single = setting end -events.connect(events.CHAR_ADDED, function(ch) +--events.connect(events.CHAR_ADDED, function(ch) +-- if buffer:get_lexer() ~= "dmd" then return end +-- if ch > 255 then return end +-- local character = string.char(ch) +-- if character == "." or character == "(" then +-- local fileName = os.tmpname() +-- local tmpFile = io.open(fileName, "w") +-- tmpFile:write(buffer:get_text()) +-- local command = M.PATH_TO_DSCANNER +-- .. (character == "." and " --dotComplete " or " --parenComplete ") +-- .. fileName .. " " .. buffer.current_pos .. " -I" .. buffer.filename:match(".+[\\/]") +-- local p = io.popen(command) +-- local r = p:read("*a") +-- if r ~= "\n" then +-- if character == "." then +-- showCompletionList(r) +-- elseif character == "(" then +-- if r:find("^completions\n") then +-- showCompletionList(r) +-- elseif r:find("^calltips\n.*") then +-- r = r:gsub("^calltips\n", "") +-- buffer:call_tip_show(buffer.current_pos, r:gsub("\\n", "\n"):gsub("\\t", "\t"):match("(.*)%s+$")) +-- end +-- end +-- end +-- os.remove(fileName) +-- end +--end) + +events.connect(events.FILE_AFTER_SAVE, function() if buffer:get_lexer() ~= "dmd" then return end - if ch > 255 then return end - local character = string.char(ch) - if character == "." or character == "(" then - local fileName = os.tmpname() - local tmpFile = io.open(fileName, "w") - tmpFile:write(buffer:get_text()) - local command = M.PATH_TO_DSCANNER - .. (character == "." and " --dotComplete " or " --parenComplete ") - .. fileName .. " " .. buffer.current_pos .. " -I" .. buffer.filename:match(".+[\\/]") - local p = io.popen(command) - local r = p:read("*a") - if r ~= "\n" then - if character == "." then - showCompletionList(r) - elseif character == "(" then - if r:find("^completions\n") then - showCompletionList(r) - elseif r:find("^calltips\n.*") then - r = r:gsub("^calltips\n", "") - buffer:call_tip_show(buffer.current_pos, r:gsub("\\n", "\n"):gsub("\\t", "\t"):match("(.*)%s+$")) - end - end + buffer:annotation_clear_all() + --buffer.annotation_visible = _SCINTILLA.constants.ANNOTATION_STANDARD + local command = M.PATH_TO_DSCANNER .. " --syntaxCheck " .. buffer.filename + local p = io.popen(command) + for line in p:lines() do + lineNumber, column, level, message = string.match(line, "^.-%((%d+):(%d+)%)%[(%w+)%]: (.+)$") + local l = tonumber(lineNumber) - 1 + local c = tonumber(column) + if level == "error" then + buffer.annotation_style[l] = 8 + else + buffer.annotation_style[l] = 2 + end + local t = buffer.annotation_text[l] + if #t > 0 then + buffer.annotation_text[l] = buffer.annotation_text[l] .. "\n" .. message + else + buffer.annotation_text[l] = message end - os.remove(fileName) end end) diff --git a/main.d b/main.d index d353dfd..c9f457a 100755 --- a/main.d +++ b/main.d @@ -16,10 +16,12 @@ import std.regex; import std.stdio; import std.range; import std.d.lexer; +import std.d.parser; import highlighter; import autocomplete; import stats; +import ctags; /** * Loads any import directories specified in /etc/dmd.conf. @@ -92,13 +94,14 @@ int main(string[] args) bool format; bool help; bool tokenCount; + bool syntaxCheck; try { getopt(args, "I", &importDirs, "dotComplete|d", &dotComplete, "sloc|l", &sloc, "json|j", &json, "parenComplete|p", &parenComplete, "highlight", &highlight, "ctags|c", &ctags, "recursive|r|R", &recursive, "help|h", &help, - "tokenCount", &tokenCount, + "tokenCount", &tokenCount, "syntaxCheck", &syntaxCheck, "declaration|e", &declaration, "symbolComplete|s", &symbolComplete); } catch (Exception e) @@ -112,7 +115,8 @@ int main(string[] args) return 0; } - auto optionCount = count!"a"([sloc, highlight, ctags, json, tokenCount]); + auto optionCount = count!"a"([sloc, highlight, ctags, json, tokenCount, + syntaxCheck]); if (optionCount > 1) { stderr.writeln("Too many options specified"); @@ -172,6 +176,14 @@ int main(string[] args) { } } + else if (ctags) + { + printCtags(stdout, tokens, args[1]); + } + else if (syntaxCheck) + { + parseModule(tokens.array(), args[1]); + } } return 0; diff --git a/std/d/ast.d b/std/d/ast.d index 895f6e7..6059a9d 100755 --- a/std/d/ast.d +++ b/std/d/ast.d @@ -26,6 +26,7 @@ import std.d.lexer; */ abstract class ASTVisitor { +public: /** */ void visit(AddExpression addExpression) { addExpression.accept(this); } /** */ void visit(AliasDeclaration aliasDeclaration) { aliasDeclaration.accept(this); } /** */ void visit(AliasInitializer aliasInitializer) { aliasInitializer.accept(this); } @@ -846,7 +847,7 @@ class Declarator : ASTNode { public: mixin(DEFAULT_ACCEPT); - /** */ Token identifier; + /** */ Token name; /** */ Initializer initializer; } @@ -912,7 +913,7 @@ class EnumDeclaration : ASTNode { public: mixin(DEFAULT_ACCEPT); - /** */ Token identifier; + /** */ Token name; /** */ Type type; /** */ EnumBody enumBody; } @@ -1231,7 +1232,7 @@ class InterfaceDeclaration : ASTNode { public: mixin(DEFAULT_ACCEPT); - /** */ Token identifier; + /** */ Token name; /** */ TemplateParameters templateParameters; /** */ Constraint constraint; /** */ BaseClassList baseClassList; @@ -1290,6 +1291,7 @@ class LambdaExpression : ExpressionNode { public: mixin(DEFAULT_ACCEPT); + /** */ TokenType functionType; /** */ Token identifier; /** */ Parameters parameters; /** */ FunctionAttribute[] functionAttributes; diff --git a/std/d/lexer.d b/std/d/lexer.d index 7cf564f..6b07f57 100755 --- a/std/d/lexer.d +++ b/std/d/lexer.d @@ -128,12 +128,12 @@ public: */ struct Token { - /** + /** * The representation of the token in the original source code. */ string value; - /** + /** * The index of the start of the token in the original source. * $(LPAREN)measured in ASCII characters or UTF-8 code units$(RPAREN) */ @@ -144,13 +144,13 @@ struct Token */ uint line; - /** + /** * The column number of the start of the token in the original source. * $(LPAREN)measured in ASCII characters or UTF-8 code units$(RPAREN) */ ushort column; - /** + /** * The token type. */ TokenType type; @@ -159,7 +159,7 @@ struct Token * Check to see if the token is of the same type and has the same string * representation as the given token. */ - bool opEquals(ref const(Token) other) const + bool opEquals(ref const(Token) other) const nothrow pure { return other.type == type && other.value == value; } @@ -168,17 +168,23 @@ struct Token * Checks to see if the token's string representation is equal to the given * string. */ - bool opEquals(string value) const { return this.value == value; } + bool opEquals(string value) const nothrow pure + { + return this.value == value; + } /** * Checks to see if the token is of the given type. */ - bool opEquals(TokenType type) const { return this.type == type; } + bool opEquals(TokenType type) const nothrow pure + { + return this.type == type; + } /** * Comparison operator orders tokens by start index. */ - int opCmp(ref const(Token) other) const + int opCmp(ref const(Token) other) const nothrow pure { if (startIndex < other.startIndex) return -1; if (startIndex > other.startIndex) return 1; @@ -216,7 +222,7 @@ enum TokenStyle : ushort * Escape sequences will be replaced with their equivalent characters, * enclosing quote characters will not be included. Special tokens such as * $(D_KEYWORD ___VENDOR__) will be replaced with their equivalent strings. - * Useful for creating a compiler or interpreter. + * Useful for creating a compiler or interpreter. */ default_ = 0b0000, @@ -237,7 +243,7 @@ enum TokenStyle : ushort /** * Do not replace the value field of the special tokens such as - * $(D_KEYWORD ___DATE__) with their string equivalents. + * $(D_KEYWORD ___DATE__) with their string equivalents. */ doNotReplaceSpecial = 0b0100, @@ -449,7 +455,7 @@ L_advance: ">>>", "TokenType.unsignedShiftRight", ">>>=", "TokenType.unsignedShiftRightEqual", "^", "TokenType.xor", - "^=", "TokenType.xorEqual", + "^=", "TokenType.xorEqual" )); case '/': nextCharNonLF(); @@ -2030,7 +2036,7 @@ pure nothrow bool isMisc(ref const Token t) */ enum TokenType: ushort { - invalid, /// Not a valid token + invalid, /// Not a valid token assign, /// = at, /// @ bitAnd, /// & @@ -2528,7 +2534,7 @@ bool isRangeEoF(R)(ref R range) // Lookup table for token values package immutable(string[TokenType.max + 1]) tokenValues = [ - null, + null, "=", "@", "&", @@ -3109,7 +3115,7 @@ private: string value; Slot* next; uint hash; - }; + } void printLoadFactor() { diff --git a/std/d/parser.d b/std/d/parser.d index aee3125..c2df671 100755 --- a/std/d/parser.d +++ b/std/d/parser.d @@ -77,18 +77,20 @@ import std.string : format; // Uncomment this if you want ALL THE OUTPUT // Caution: generates 180 megabytes of logging for std.datetime -// version = std_parser_verbose; +//version = std_parser_verbose; /** * Params: * tokens = the tokens parsed by std.d.lexer * Returns: the parsed module */ -Module parseModule(const(Token)[] tokens, string fileName) +Module parseModule(const(Token)[] tokens, string fileName, + void function(string, int, int, string) messageFunction = null) { auto parser = new Parser(); parser.fileName = fileName; parser.tokens = tokens; + parser.messageFunction = messageFunction; auto mod = parser.parseModule(); // writefln("Parsing finished with %d errors and %d warnings.", // parser.errorCount, parser.warningCount); @@ -1832,7 +1834,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c; auto node = new Declarator; auto id = expect(TokenType.identifier); if (id is null) return null; - node.identifier = *id; + node.name = *id; if (currentIsOneOf(TokenType.lBracket, TokenType.star)) { error("C-style variable declarations are not supported."); @@ -2005,7 +2007,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c; auto node = new EnumDeclaration; if (expect(TokenType.enum_) is null) return null; if (currentIs(TokenType.identifier)) - node.identifier = advance(); + node.name = advance(); if (currentIs(TokenType.colon)) { advance(); @@ -2668,7 +2670,8 @@ body {} // six { auto b = setBookmark(); auto t = parseType(); - if (t is null || !currentIs(TokenType.identifier)) + if (t is null || !currentIs(TokenType.identifier) + || !peekIs(TokenType.assign)) { goToBookmark(b); node.expression = parseExpression(); @@ -2959,7 +2962,7 @@ import core.stdc.stdio, std.string : KeepTerminator; if (expect(TokenType.interface_) is null) return null; auto ident = expect(TokenType.identifier); if (ident is null) return null; - node.identifier = *ident; + node.name = *ident; if (currentIs(TokenType.lParen)) { node.templateParameters = parseTemplateParameters(); @@ -3170,10 +3173,16 @@ invariant() foo(); { mixin(traceEnterAndExit!(__FUNCTION__)); auto node = new LambdaExpression; - if (currentIs(TokenType.identifier)) + if (currentIsOneOf(TokenType.function_, TokenType.delegate_)) + { + node.functionType = advance().type; + goto lParen; + } + else if (currentIs(TokenType.identifier)) node.identifier = advance(); else if (currentIs(TokenType.lParen)) { + lParen: node.parameters = parseParameters(); do { @@ -6431,9 +6440,9 @@ private: if (suppressMessages > 0) return; if (index < tokens.length) - stderr.writeln(message, "(", current.line, ":", current.column + 1, ")"); + writeln(message, "(", current.line, ":", current.column + 1, ")"); else - stderr.writeln(message, "(EOF:0)"); + writeln(message, "(EOF:0)"); } } else diff --git a/std/d/runtester.sh b/std/d/runtester.sh deleted file mode 100755 index e2655d1..0000000 --- a/std/d/runtester.sh +++ /dev/null @@ -1,12 +0,0 @@ -if [ ! -d runs ]; then - mkdir runs -fi -for file in /usr/include/d/std/*.d; do - shortFile=$(basename $file) - echo "Parsing" $shortFile "..." - outFile=runs/$shortFile.txt - ./tester $file > $outFile -done -echo -grep -l "Parsing finished with 0 errors" runs/*.txt | sed -e "s/runs\//Pass /" -e "s/.txt//" -grep -L "Parsing finished with 0 errors" runs/*.txt | sed -e "s/runs\//Fail /" -e "s/.txt//" diff --git a/std/d/tester.d b/std/d/tester.d deleted file mode 100644 index 791f50a..0000000 --- a/std/d/tester.d +++ /dev/null @@ -1,69 +0,0 @@ -import std.d.lexer; -import std.d.ast; -import std.d.parser; -import std.stdio; -import std.file; -import std.array; - -class TestVisitor : ASTVisitor -{ - override void visit(ClassDeclaration classDeclaration) - { - writeln("class ", classDeclaration.name.value, " on line ", classDeclaration.name.line); - } - - override void visit(StructDeclaration structDeclaration) - { - writeln("struct ", structDeclaration.name.value, " on line ", structDeclaration.name.line); - } - - override void visit(ModuleDeclaration md) - { - writeln("module declaration found"); - } - - override void visit(FunctionDeclaration funDec) - { - writeln("function ", funDec.name.value, " on line ", funDec.name.line); - } - - override void visit(VariableDeclaration varDec) - { - foreach (decl; varDec.declarators) - { - writeln("variable ", decl.identifier.value, - " on line ", decl.identifier.line); - } - } - - override void visit(ImportDeclaration impDec) - { - writeln("import declaration found"); - } - - override void visit(InterfaceDeclaration intDec) - { - writeln("Interface ", intDec.identifier.value, - " on line ", intDec.identifier.line); - } - - override void visit(VersionSpecification verSpec) - { - writeln("Version specification"); - } - - alias ASTVisitor.visit visit; -} - -void main(string[] args) -{ - auto de = dirEntry(args[1]); - ubyte[] sourceBuffer = new ubyte[de.size]; - auto f = File(args[1]); - ubyte[] rawSource = f.rawRead(sourceBuffer); - LexerConfig config; - auto tokens = byToken(rawSource, config).array(); - Module m = parseModule(tokens, args[1]); - //ASTVisitor visitor = new TestVisitor; - //visitor.visit(m); -}