From 2efa5242753e75f88c2f559c5be8adb4584d5e5c Mon Sep 17 00:00:00 2001 From: Cristian Creteanu Date: Sun, 7 Jun 2020 16:32:10 +0300 Subject: [PATCH] Add dmd as a lib callbacks for statement semantic (#11092) Add dmd as a lib callbacks for statement semantic merged-on-behalf-of: Razvan Nitu --- dub.sdl | 7 +- src/dmd/compiler.d | 21 +++ src/dmd/mars.d | 220 ++++++++++++++------------ src/dmd/statementsem.d | 8 + test/dub_package/retrieveScope.d | 223 +++++++++++++++++++++++++++ test/dub_package/testfiles/correct.d | 61 ++++++++ 6 files changed, 443 insertions(+), 97 deletions(-) create mode 100755 test/dub_package/retrieveScope.d create mode 100644 test/dub_package/testfiles/correct.d diff --git a/dub.sdl b/dub.sdl index e0c9d98f75..e0c6850123 100644 --- a/dub.sdl +++ b/dub.sdl @@ -31,6 +31,8 @@ subPackage { "src/dmd/utf.d" \ "src/dmd/utils.d" + versions "CallbackAPI" + preGenerateCommands ` "$${DUB_EXE}" \ --arch=$${DUB_ARCH} \ @@ -60,6 +62,8 @@ subPackage { "src/dmd/permissivevisitor.d" \ "src/dmd/strictvisitor.d" + versions "CallbackAPI" + dependency "dmd:lexer" version="*" } @@ -73,7 +77,8 @@ subPackage { "NoBackend" \ "GC" \ "NoMain" \ - "MARS" + "MARS" \ + "CallbackAPI" excludedSourceFiles "src/dmd/backend/*" excludedSourceFiles "src/dmd/root/*" diff --git a/src/dmd/compiler.d b/src/dmd/compiler.d index 429ea37174..360a2935c8 100644 --- a/src/dmd/compiler.d +++ b/src/dmd/compiler.d @@ -28,6 +28,7 @@ import dmd.root.ctfloat; import dmd.semantic2; import dmd.semantic3; import dmd.tokens; +import dmd.statement; extern (C++) __gshared { @@ -152,6 +153,26 @@ extern (C++) struct Compiler } return false; // this import will not be compiled } + + version (CallbackAPI) + { + alias OnStatementSemanticStart = void function(Statement, Scope*); + alias OnStatementSemanticDone = void function(Statement, Scope*); + + /** + * Used to insert functionality before the start of the + * semantic analysis of a statement when importing DMD as a library + */ + __gshared OnStatementSemanticStart onStatementSemanticStart + = function void(Statement s, Scope *sc) {}; + + /** + * Used to insert functionality after the end of the + * semantic analysis of a statement when importing DMD as a library + */ + __gshared OnStatementSemanticDone onStatementSemanticDone + = function void(Statement s, Scope *sc) {}; + } } /****************************** diff --git a/src/dmd/mars.d b/src/dmd/mars.d index e9ed3bf087..a30c36e527 100644 --- a/src/dmd/mars.d +++ b/src/dmd/mars.d @@ -2617,6 +2617,125 @@ private void reconcileCommands(ref Param params, size_t numSrcFiles) params.useDIP25 = false; } +/** +Creates the module based on the file provided + +The file is dispatched in one of the various arrays +(global.params.{ddocfiles,dllfiles,jsonfiles,etc...}) +according to its extension. +If it is a binary file, it is added to libmodules. + +Params: + file = File name to dispatch + libmodules = Array to which binaries (shared/static libs and object files) + will be appended + +Returns: + A D module +*/ +Module createModule(const(char)* file, ref Strings libmodules) +{ + const(char)[] name; + version (Windows) + { + file = toWinPath(file); + } + const(char)[] p = file.toDString(); + p = FileName.name(p); // strip path + const(char)[] ext = FileName.ext(p); + if (!ext) + { + if (!p.length) + { + error(Loc.initial, "invalid file name '%s'", file); + fatal(); + } + auto id = Identifier.idPool(p); + return new Module(file.toDString, id, global.params.doDocComments, global.params.doHdrGeneration); + } + + /* Deduce what to do with a file based on its extension + */ + if (FileName.equals(ext, global.obj_ext)) + { + global.params.objfiles.push(file); + libmodules.push(file); + return null; + } + if (FileName.equals(ext, global.lib_ext)) + { + global.params.libfiles.push(file); + libmodules.push(file); + return null; + } + static if (TARGET.Linux || TARGET.OSX || TARGET.FreeBSD || TARGET.OpenBSD || TARGET.Solaris || TARGET.DragonFlyBSD) + { + if (FileName.equals(ext, global.dll_ext)) + { + global.params.dllfiles.push(file); + libmodules.push(file); + return null; + } + } + if (ext == global.ddoc_ext) + { + global.params.ddocfiles.push(file); + return null; + } + if (FileName.equals(ext, global.json_ext)) + { + global.params.doJsonGeneration = true; + global.params.jsonfilename = file.toDString; + return null; + } + if (FileName.equals(ext, global.map_ext)) + { + global.params.mapfile = file.toDString; + return null; + } + static if (TARGET.Windows) + { + if (FileName.equals(ext, "res")) + { + global.params.resfile = file.toDString; + return null; + } + if (FileName.equals(ext, "def")) + { + global.params.deffile = file.toDString; + return null; + } + if (FileName.equals(ext, "exe")) + { + assert(0); // should have already been handled + } + } + /* Examine extension to see if it is a valid + * D source file extension + */ + if (FileName.equals(ext, global.mars_ext) || FileName.equals(ext, global.hdr_ext) || FileName.equals(ext, "dd")) + { + name = FileName.removeExt(p); + if (!name.length || name == ".." || name == ".") + { + error(Loc.initial, "invalid file name '%s'", file); + fatal(); + } + } + else + { + error(Loc.initial, "unrecognized file extension %.*s", cast(int)ext.length, ext.ptr); + fatal(); + } + + /* At this point, name is the D source file name stripped of + * its path and extension. + */ + auto id = Identifier.idPool(name); + + return new Module(file.toDString, id, global.params.doDocComments, global.params.doHdrGeneration); +} + /** Creates the list of modules based on the files provided @@ -2640,102 +2759,11 @@ Modules createModules(ref Strings files, ref Strings libmodules) bool firstmodule = true; for (size_t i = 0; i < files.dim; i++) { - const(char)[] name; - version (Windows) - { - files[i] = toWinPath(files[i]); - } - const(char)[] p = files[i].toDString(); - p = FileName.name(p); // strip path - const(char)[] ext = FileName.ext(p); - if (ext) - { - /* Deduce what to do with a file based on its extension - */ - if (FileName.equals(ext, global.obj_ext)) - { - global.params.objfiles.push(files[i]); - libmodules.push(files[i]); - continue; - } - if (FileName.equals(ext, global.lib_ext)) - { - global.params.libfiles.push(files[i]); - libmodules.push(files[i]); - continue; - } - static if (TARGET.Linux || TARGET.OSX || TARGET.FreeBSD || TARGET.OpenBSD || TARGET.Solaris || TARGET.DragonFlyBSD) - { - if (FileName.equals(ext, global.dll_ext)) - { - global.params.dllfiles.push(files[i]); - libmodules.push(files[i]); - continue; - } - } - if (ext == global.ddoc_ext) - { - global.params.ddocfiles.push(files[i]); - continue; - } - if (FileName.equals(ext, global.json_ext)) - { - global.params.doJsonGeneration = true; - global.params.jsonfilename = files[i].toDString; - continue; - } - if (FileName.equals(ext, global.map_ext)) - { - global.params.mapfile = files[i].toDString; - continue; - } - static if (TARGET.Windows) - { - if (FileName.equals(ext, "res")) - { - global.params.resfile = files[i].toDString; - continue; - } - if (FileName.equals(ext, "def")) - { - global.params.deffile = files[i].toDString; - continue; - } - if (FileName.equals(ext, "exe")) - { - assert(0); // should have already been handled - } - } - /* Examine extension to see if it is a valid - * D source file extension - */ - if (FileName.equals(ext, global.mars_ext) || FileName.equals(ext, global.hdr_ext) || FileName.equals(ext, "dd")) - { - name = FileName.removeExt(p); - if (!name.length || name == ".." || name == ".") - { - Linvalid: - error(Loc.initial, "invalid file name '%s'", files[i]); - fatal(); - } - } - else - { - error(Loc.initial, "unrecognized file extension %.*s", cast(int)ext.length, ext.ptr); - fatal(); - } - } - else - { - name = p; - if (!name.length) - goto Linvalid; - } - /* At this point, name is the D source file name stripped of - * its path and extension. - */ - auto id = Identifier.idPool(name); - auto m = new Module(files[i].toDString, id, global.params.doDocComments, global.params.doHdrGeneration); + auto m = createModule(files[i], libmodules); + + if (m is null) + continue; + modules.push(m); if (firstmodule) { diff --git a/src/dmd/statementsem.d b/src/dmd/statementsem.d index 45e1872a6c..f625d6e892 100644 --- a/src/dmd/statementsem.d +++ b/src/dmd/statementsem.d @@ -57,6 +57,7 @@ import dmd.target; import dmd.tokens; import dmd.typesem; import dmd.visitor; +import dmd.compiler; /***************************************** * CTFE requires FuncDeclaration::labtab for the interpretation. @@ -123,8 +124,15 @@ private Expression checkAssignmentAsCondition(Expression e) // Performs semantic analysis in Statement AST nodes extern(C++) Statement statementSemantic(Statement s, Scope* sc) { + version (CallbackAPI) + Compiler.onStatementSemanticStart(s, sc); + scope v = new StatementSemanticVisitor(sc); s.accept(v); + + version (CallbackAPI) + Compiler.onStatementSemanticDone(s, sc); + return v.result; } diff --git a/test/dub_package/retrieveScope.d b/test/dub_package/retrieveScope.d new file mode 100755 index 0000000000..53749ebe03 --- /dev/null +++ b/test/dub_package/retrieveScope.d @@ -0,0 +1,223 @@ +#!/usr/bin/env dub +/+dub.sdl: +dependency "dmd" path="../.." +versions "CallbackAPI" ++/ +/* + * This file contains an example of how to retrieve the scope of a statement. + * First, the callback system is used. This, however, will not work for fields + * of structs or classes, which is why the visitor will cover this corner case + */ + +import core.stdc.stdarg; +import core.stdc.string; + +import std.conv; +import std.string; +import std.algorithm.sorting; +import std.algorithm.mutation : SwapStrategy; +import std.path : dirName; + +import dmd.errors; +import dmd.frontend; +import dmd.mars; +import dmd.console; +import dmd.arraytypes; +import dmd.compiler; +import dmd.dmodule; +import dmd.dsymbol; +import dmd.dsymbolsem; +import dmd.semantic2; +import dmd.semantic3; +import dmd.statement; +import dmd.visitor; +import dmd.dscope; +import dmd.denum; +import dmd.nspace; +import dmd.dstruct; +import dmd.dclass; +import dmd.globals; + +import std.stdio : writeln; + +private bool isBefore(Loc loc1, Loc loc2) +{ + return loc1.linnum != loc2.linnum? loc1.linnum < loc2.linnum + : loc1.charnum < loc2.charnum; +} + +private struct CallbackHelper { + static Loc cursorLoc; + static Scope *scp; + + static extern (C++) void statementSem(Statement s, Scope *sc) { + if (s.loc.linnum == cursorLoc.linnum + && strcmp(s.loc.filename, cursorLoc.filename) == 0) { + sc.setNoFree(); + scp = sc; + } + } +}; + +int main() +{ + auto dmdParentDir = dirName(dirName(dirName(__FILE_FULL_PATH__))); + global.path = new Strings(); + global.path.push((dmdParentDir ~ "/phobos").ptr); + global.path.push((dmdParentDir ~ "/druntime/import").ptr); + + /* comment for error output in parsing & semantic */ + diagnosticHandler = (const ref Loc location, + Color headerColor, + const(char)* header, + const(char)* messageFormat, + va_list args, + const(char)* prefix1, + const(char)* prefix2) => true; + global.gag = 1; + initDMD(diagnosticHandler); + + Strings libmodules; + Module m = createModule((dirName(__FILE_FULL_PATH__) ~ "/testfiles/correct.d").ptr, + libmodules); + m.importedFrom = m; // m.isRoot() == true + + m.read(Loc.initial); + m.parse(); + + CallbackHelper.cursorLoc = Loc(to!string(m.srcfile).ptr, 22, 10); + + Compiler.onStatementSemanticStart = &CallbackHelper.statementSem; + + m.importAll(null); + + // semantic + m.dsymbolSemantic(null); + + Module.dprogress = 1; + Module.runDeferredSemantic(); + + m.semantic2(null); + Module.runDeferredSemantic2(); + + m.semantic3(null); + Module.runDeferredSemantic3(); + + + Dsymbol[] symbols; + + // if scope could not be retrieved through the callback, then traverse AST + if (!CallbackHelper.scp) { + auto visitor = new DsymbolsScopeRetrievingVisitor(CallbackHelper.cursorLoc); + m.accept(visitor); + + symbols = visitor.symbols; + } + + while (CallbackHelper.scp) { + if (CallbackHelper.scp.scopesym && CallbackHelper.scp.scopesym.symtab) + foreach (x; CallbackHelper.scp.scopesym.symtab.tab.asRange()) { + symbols ~= x.value; + } + CallbackHelper.scp = CallbackHelper.scp.enclosing; + } + + sort!("to!string(a.ident) < to!string(b.ident)", SwapStrategy.stable)(symbols); + + foreach (sym; symbols) { + writeln(sym.ident); + } + + deinitializeDMD(); + + return 0; +} + +private extern (C++) final class DsymbolsScopeRetrievingVisitor : Visitor +{ + Loc loc; + Dsymbol[] symbols; + alias visit = Visitor.visit; + +public: + extern (D) this(Loc loc) + { + this.loc = loc; + } + + override void visit(Dsymbol s) + { + } + + override void visit(ScopeDsymbol s) + { + visitScopeDsymbol(s); + } + + override void visit(EnumDeclaration d) + { + visitScopeDsymbol(d); + } + + override void visit(Nspace d) + { + visitScopeDsymbol(d); + } + + override void visit(StructDeclaration d) + { + visitScopeDsymbol(d); + } + + override void visit(ClassDeclaration d) + { + visitScopeDsymbol(d); + } + + void visitBaseClasses(ClassDeclaration d) + { + visitScopeDsymbol(d); + } + + override void visit(Module m) + { + visitScopeDsymbol(m); + } + + private void visitScopeDsymbol(ScopeDsymbol scopeDsym) + { + if (!scopeDsym.members) + return; + + Dsymbol dsym; + foreach (i, s; *scopeDsym.members) + { + if (s is null || s.ident is null) + continue; + + // if the current symbol is from another module + if (auto m = scopeDsym.isModule()) + if (!(to!string(s.loc.filename).endsWith(m.ident.toString() ~ ".d"))) + continue; + + if (!s.isImport()) + symbols ~= s; + + if (!i || dsym is null) { + dsym = s; + continue; + } + + // only visit a symbol which contains the cursor + // choose the symbol which is before and the closest to the cursor + if (isBefore(dsym.loc, loc) + && isBefore(dsym.loc, s.loc) + && isBefore(s.loc, loc)) { + dsym = s; + } + } + + dsym.accept(this); + } +} + diff --git a/test/dub_package/testfiles/correct.d b/test/dub_package/testfiles/correct.d new file mode 100644 index 0000000000..f96c038a7c --- /dev/null +++ b/test/dub_package/testfiles/correct.d @@ -0,0 +1,61 @@ +module correct; +dafaenfai; +int[] g; + +struct Foo { + int a; + + void b() { + + } + + struct Bar { + int c; + } +} + +/** + * Params: + * x = something cool + */ +int foo(int x) { + int a = 0; + + if (a == 1) { + int b, c, d; + } + + for (int i = 0; i < 10; ++i) { + int e; + int j; + + e = 7; + } + + while (0) { + int f = 5; + } + + foreach(k; 0..3) { + int g; + + g = 2; + } + + int z = a; + + return 0; +} + +int main(string[] args) { + int y = 0; + int x = 1; + int xx = 2; + int yy = 0; + int k = xx; + + return 0; +} + +int bar() {return 0;} +