Add dmd as a lib callbacks for statement semantic (#11092)

Add dmd as a lib callbacks for statement semantic
merged-on-behalf-of: Razvan Nitu <RazvanN7@users.noreply.github.com>

dub.sdl

@@ -31,6 +31,8 @@ subPackage {
     "src/dmd/utf.d" \
     "src/dmd/utils.d"
 
+  versions "CallbackAPI"
+
   preGenerateCommands `
     "$${DUB_EXE}" \
     --arch=$${DUB_ARCH} \
@@ -60,6 +62,8 @@ subPackage {
     "src/dmd/permissivevisitor.d" \
     "src/dmd/strictvisitor.d"
 
+  versions "CallbackAPI"
+
   dependency "dmd:lexer" version="*"
 }
 
@@ -73,7 +77,8 @@ subPackage {
     "NoBackend" \
     "GC" \
     "NoMain" \
-    "MARS"
+    "MARS" \
+    "CallbackAPI"
 
   excludedSourceFiles "src/dmd/backend/*"
   excludedSourceFiles "src/dmd/root/*"

src/dmd/compiler.d

@@ -28,6 +28,7 @@ import dmd.root.ctfloat;
 import dmd.semantic2;
 import dmd.semantic3;
 import dmd.tokens;
+import dmd.statement;
 
 extern (C++) __gshared
 {
@@ -152,6 +153,26 @@ extern (C++) struct Compiler
         }
         return false; // this import will not be compiled
     }
+
+    version (CallbackAPI)
+    {
+        alias OnStatementSemanticStart = void function(Statement, Scope*);
+        alias OnStatementSemanticDone = void function(Statement, Scope*);
+
+        /**
+         * Used to insert functionality before the start of the
+         * semantic analysis of a statement when importing DMD as a library
+         */
+        __gshared OnStatementSemanticStart onStatementSemanticStart
+                    = function void(Statement s, Scope *sc) {};
+
+        /**
+         * Used to insert functionality after the end of the
+         * semantic analysis of a statement when importing DMD as a library
+         */
+        __gshared OnStatementSemanticDone onStatementSemanticDone
+                    = function void(Statement s, Scope *sc) {};
+    }
 }
 
 /******************************

src/dmd/mars.d

@@ -2617,6 +2617,125 @@ private void reconcileCommands(ref Param params, size_t numSrcFiles)
         params.useDIP25 = false;
 }
 
+/**
+Creates the module based on the file provided
+
+The file is dispatched in one of the various arrays
+(global.params.{ddocfiles,dllfiles,jsonfiles,etc...})
+according to its extension.
+If it is a binary file, it is added to libmodules.
+
+Params:
+  file = File name to dispatch
+  libmodules = Array to which binaries (shared/static libs and object files)
+               will be appended
+
+Returns:
+  A D module
+*/
+Module createModule(const(char)* file, ref Strings libmodules)
+{
+    const(char)[] name;
+    version (Windows)
+    {
+        file = toWinPath(file);
+    }
+    const(char)[] p = file.toDString();
+    p = FileName.name(p); // strip path
+    const(char)[] ext = FileName.ext(p);
+    if (!ext)
+    {
+        if (!p.length)
+        {
+            error(Loc.initial, "invalid file name '%s'", file);
+            fatal();
+        }
+        auto id = Identifier.idPool(p);
+        return new Module(file.toDString, id, global.params.doDocComments, global.params.doHdrGeneration);
+    }
+
+    /* Deduce what to do with a file based on its extension
+        */
+    if (FileName.equals(ext, global.obj_ext))
+    {
+        global.params.objfiles.push(file);
+        libmodules.push(file);
+        return null;
+    }
+    if (FileName.equals(ext, global.lib_ext))
+    {
+        global.params.libfiles.push(file);
+        libmodules.push(file);
+        return null;
+    }
+    static if (TARGET.Linux || TARGET.OSX || TARGET.FreeBSD || TARGET.OpenBSD || TARGET.Solaris || TARGET.DragonFlyBSD)
+    {
+        if (FileName.equals(ext, global.dll_ext))
+        {
+            global.params.dllfiles.push(file);
+            libmodules.push(file);
+            return null;
+        }
+    }
+    if (ext == global.ddoc_ext)
+    {
+        global.params.ddocfiles.push(file);
+        return null;
+    }
+    if (FileName.equals(ext, global.json_ext))
+    {
+        global.params.doJsonGeneration = true;
+        global.params.jsonfilename = file.toDString;
+        return null;
+    }
+    if (FileName.equals(ext, global.map_ext))
+    {
+        global.params.mapfile = file.toDString;
+        return null;
+    }
+    static if (TARGET.Windows)
+    {
+        if (FileName.equals(ext, "res"))
+        {
+            global.params.resfile = file.toDString;
+            return null;
+        }
+        if (FileName.equals(ext, "def"))
+        {
+            global.params.deffile = file.toDString;
+            return null;
+        }
+        if (FileName.equals(ext, "exe"))
+        {
+            assert(0); // should have already been handled
+        }
+    }
+    /* Examine extension to see if it is a valid
+        * D source file extension
+        */
+    if (FileName.equals(ext, global.mars_ext) || FileName.equals(ext, global.hdr_ext) || FileName.equals(ext, "dd"))
+    {
+        name = FileName.removeExt(p);
+        if (!name.length || name == ".." || name == ".")
+        {
+            error(Loc.initial, "invalid file name '%s'", file);
+            fatal();
+        }
+    }
+    else
+    {
+        error(Loc.initial, "unrecognized file extension %.*s", cast(int)ext.length, ext.ptr);
+        fatal();
+    }
+
+    /* At this point, name is the D source file name stripped of
+     * its path and extension.
+     */
+    auto id = Identifier.idPool(name);
+
+    return new Module(file.toDString, id, global.params.doDocComments, global.params.doHdrGeneration);
+}
+
 /**
 Creates the list of modules based on the files provided
 
@@ -2640,102 +2759,11 @@ Modules createModules(ref Strings files, ref Strings libmodules)
     bool firstmodule = true;
     for (size_t i = 0; i < files.dim; i++)
     {
-        const(char)[] name;
+        auto m = createModule(files[i], libmodules);
-        version (Windows)
+
-        {
+        if (m is null)
-            files[i] = toWinPath(files[i]);
+            continue;
-        }
+
-        const(char)[] p = files[i].toDString();
-        p = FileName.name(p); // strip path
-        const(char)[] ext = FileName.ext(p);
-        if (ext)
-        {
-            /* Deduce what to do with a file based on its extension
-             */
-            if (FileName.equals(ext, global.obj_ext))
-            {
-                global.params.objfiles.push(files[i]);
-                libmodules.push(files[i]);
-                continue;
-            }
-            if (FileName.equals(ext, global.lib_ext))
-            {
-                global.params.libfiles.push(files[i]);
-                libmodules.push(files[i]);
-                continue;
-            }
-            static if (TARGET.Linux || TARGET.OSX || TARGET.FreeBSD || TARGET.OpenBSD || TARGET.Solaris || TARGET.DragonFlyBSD)
-            {
-                if (FileName.equals(ext, global.dll_ext))
-                {
-                    global.params.dllfiles.push(files[i]);
-                    libmodules.push(files[i]);
-                    continue;
-                }
-            }
-            if (ext == global.ddoc_ext)
-            {
-                global.params.ddocfiles.push(files[i]);
-                continue;
-            }
-            if (FileName.equals(ext, global.json_ext))
-            {
-                global.params.doJsonGeneration = true;
-                global.params.jsonfilename = files[i].toDString;
-                continue;
-            }
-            if (FileName.equals(ext, global.map_ext))
-            {
-                global.params.mapfile = files[i].toDString;
-                continue;
-            }
-            static if (TARGET.Windows)
-            {
-                if (FileName.equals(ext, "res"))
-                {
-                    global.params.resfile = files[i].toDString;
-                    continue;
-                }
-                if (FileName.equals(ext, "def"))
-                {
-                    global.params.deffile = files[i].toDString;
-                    continue;
-                }
-                if (FileName.equals(ext, "exe"))
-                {
-                    assert(0); // should have already been handled
-                }
-            }
-            /* Examine extension to see if it is a valid
-             * D source file extension
-             */
-            if (FileName.equals(ext, global.mars_ext) || FileName.equals(ext, global.hdr_ext) || FileName.equals(ext, "dd"))
-            {
-                name = FileName.removeExt(p);
-                if (!name.length || name == ".." || name == ".")
-                {
-                Linvalid:
-                    error(Loc.initial, "invalid file name '%s'", files[i]);
-                    fatal();
-                }
-            }
-            else
-            {
-                error(Loc.initial, "unrecognized file extension %.*s", cast(int)ext.length, ext.ptr);
-                fatal();
-            }
-        }
-        else
-        {
-            name = p;
-            if (!name.length)
-                goto Linvalid;
-        }
-        /* At this point, name is the D source file name stripped of
-         * its path and extension.
-         */
-        auto id = Identifier.idPool(name);
-        auto m = new Module(files[i].toDString, id, global.params.doDocComments, global.params.doHdrGeneration);
         modules.push(m);
         if (firstmodule)
         {

src/dmd/statementsem.d

@@ -57,6 +57,7 @@ import dmd.target;
 import dmd.tokens;
 import dmd.typesem;
 import dmd.visitor;
+import dmd.compiler;
 
 /*****************************************
  * CTFE requires FuncDeclaration::labtab for the interpretation.
@@ -123,8 +124,15 @@ private Expression checkAssignmentAsCondition(Expression e)
 // Performs semantic analysis in Statement AST nodes
 extern(C++) Statement statementSemantic(Statement s, Scope* sc)
 {
+    version (CallbackAPI)
+        Compiler.onStatementSemanticStart(s, sc);
+
     scope v = new StatementSemanticVisitor(sc);
     s.accept(v);
+
+    version (CallbackAPI)
+        Compiler.onStatementSemanticDone(s, sc);
+
     return v.result;
 }
 

test/dub_package/retrieveScope.d

@@ -0,0 +1,223 @@
+#!/usr/bin/env dub
+/+dub.sdl:
+dependency "dmd" path="../.."
+versions "CallbackAPI"
++/
+/*
+ * This file contains an example of how to retrieve the scope of a statement.
+ * First, the callback system is used. This, however, will not work for fields
+ * of structs or classes, which is why the visitor will cover this corner case
+ */
+
+import core.stdc.stdarg;
+import core.stdc.string;
+
+import std.conv;
+import std.string;
+import std.algorithm.sorting;
+import std.algorithm.mutation : SwapStrategy;
+import std.path : dirName;
+
+import dmd.errors;
+import dmd.frontend;
+import dmd.mars;
+import dmd.console;
+import dmd.arraytypes;
+import dmd.compiler;
+import dmd.dmodule;
+import dmd.dsymbol;
+import dmd.dsymbolsem;
+import dmd.semantic2;
+import dmd.semantic3;
+import dmd.statement;
+import dmd.visitor;
+import dmd.dscope;
+import dmd.denum;
+import dmd.nspace;
+import dmd.dstruct;
+import dmd.dclass;
+import dmd.globals;
+
+import std.stdio : writeln;
+
+private bool isBefore(Loc loc1, Loc loc2)
+{
+    return loc1.linnum != loc2.linnum? loc1.linnum < loc2.linnum
+                                    : loc1.charnum < loc2.charnum;
+}
+
+private struct CallbackHelper {
+    static Loc cursorLoc;
+    static Scope *scp;
+
+    static extern (C++) void statementSem(Statement s, Scope *sc) {
+        if (s.loc.linnum == cursorLoc.linnum
+                && strcmp(s.loc.filename, cursorLoc.filename) == 0) {
+            sc.setNoFree();
+            scp = sc;
+        }
+    }
+};
+
+int main()
+{
+    auto dmdParentDir = dirName(dirName(dirName(__FILE_FULL_PATH__)));
+    global.path = new Strings();
+    global.path.push((dmdParentDir ~ "/phobos").ptr);
+    global.path.push((dmdParentDir ~ "/druntime/import").ptr);
+
+    /* comment for error output in parsing & semantic */
+    diagnosticHandler = (const ref Loc location,
+                            Color headerColor,
+                            const(char)* header,
+                            const(char)* messageFormat,
+                            va_list args,
+                            const(char)* prefix1,
+                            const(char)* prefix2) => true;
+    global.gag = 1;
+    initDMD(diagnosticHandler);
+
+    Strings libmodules;
+    Module m = createModule((dirName(__FILE_FULL_PATH__) ~ "/testfiles/correct.d").ptr,
+                                libmodules);
+    m.importedFrom = m; // m.isRoot() == true
+
+    m.read(Loc.initial);
+    m.parse();
+
+    CallbackHelper.cursorLoc = Loc(to!string(m.srcfile).ptr, 22, 10);
+
+    Compiler.onStatementSemanticStart = &CallbackHelper.statementSem;
+
+    m.importAll(null);
+
+    // semantic
+    m.dsymbolSemantic(null);
+
+    Module.dprogress = 1;
+    Module.runDeferredSemantic();
+
+    m.semantic2(null);
+    Module.runDeferredSemantic2();
+
+    m.semantic3(null);
+    Module.runDeferredSemantic3();
+
+
+    Dsymbol[] symbols;
+
+    // if scope could not be retrieved through the callback, then traverse AST
+    if (!CallbackHelper.scp) {
+        auto visitor = new DsymbolsScopeRetrievingVisitor(CallbackHelper.cursorLoc);
+        m.accept(visitor);
+
+        symbols = visitor.symbols;
+    }
+
+    while (CallbackHelper.scp) {
+        if (CallbackHelper.scp.scopesym && CallbackHelper.scp.scopesym.symtab)
+            foreach (x; CallbackHelper.scp.scopesym.symtab.tab.asRange()) {
+                symbols ~= x.value;
+            }
+        CallbackHelper.scp = CallbackHelper.scp.enclosing;
+    }
+
+    sort!("to!string(a.ident) < to!string(b.ident)", SwapStrategy.stable)(symbols);
+
+    foreach (sym; symbols) {
+        writeln(sym.ident);
+    }
+
+    deinitializeDMD();
+
+    return 0;
+}
+
+private extern (C++) final class DsymbolsScopeRetrievingVisitor : Visitor
+{
+    Loc loc;
+    Dsymbol[] symbols;
+    alias visit = Visitor.visit;
+
+public:
+    extern (D) this(Loc loc)
+    {
+        this.loc = loc;
+    }
+
+    override void visit(Dsymbol s)
+    {
+    }
+
+    override void visit(ScopeDsymbol s)
+    {
+        visitScopeDsymbol(s);
+    }
+
+    override void visit(EnumDeclaration d)
+    {
+        visitScopeDsymbol(d);
+    }
+
+    override void visit(Nspace d)
+    {
+        visitScopeDsymbol(d);
+    }
+
+    override void visit(StructDeclaration d)
+    {
+        visitScopeDsymbol(d);
+    }
+
+    override void visit(ClassDeclaration d)
+    {
+        visitScopeDsymbol(d);
+    }
+
+    void visitBaseClasses(ClassDeclaration d)
+    {
+        visitScopeDsymbol(d);
+    }
+
+    override void visit(Module m)
+    {
+        visitScopeDsymbol(m);
+    }
+
+    private void visitScopeDsymbol(ScopeDsymbol scopeDsym)
+    {
+        if (!scopeDsym.members)
+            return;
+
+        Dsymbol dsym;
+        foreach (i, s; *scopeDsym.members)
+        {
+            if (s is null || s.ident is null)
+                continue;
+
+            // if the current symbol is from another module
+            if (auto m = scopeDsym.isModule())
+                if (!(to!string(s.loc.filename).endsWith(m.ident.toString() ~ ".d")))
+                    continue;
+
+            if (!s.isImport())
+                symbols ~= s;
+
+            if (!i || dsym is null) {
+                dsym = s;
+                continue;
+            }
+
+            // only visit a symbol which contains the cursor
+            // choose the symbol which is before and the closest to the cursor
+            if (isBefore(dsym.loc, loc)
+                && isBefore(dsym.loc, s.loc)
+                && isBefore(s.loc, loc)) {
+                dsym = s;
+            }
+        }
+
+        dsym.accept(this);
+    }
+}
+

test/dub_package/testfiles/correct.d

@@ -0,0 +1,61 @@
+module correct;
+dafaenfai;
+int[] g;
+
+struct Foo {
+    int a;
+
+    void b() {
+
+    }
+
+    struct Bar {
+        int c;
+    }
+}
+
+/**
+ * Params:
+ *      x = something cool
+ */
+int foo(int x) { 
+    int a = 0;
+
+    if (a == 1) {
+        int b, c, d;
+    }
+
+    for (int i = 0; i < 10; ++i) {
+        int e;
+        int j;
+
+        e = 7;
+    }
+
+    while (0) {
+        int f = 5;
+    }
+
+    foreach(k; 0..3) {
+        int g;
+
+        g = 2;
+    }
+
+    int z = a;
+
+    return 0;
+}
+
+int main(string[] args) {
+    int y = 0;
+    int x = 1;
+    int xx = 2;
+    int yy = 0;
+    int k = xx;
+
+    return 0;
+}
+
+int bar() {return 0;}
+