symbol list for #39

cedast/build/cedast.coedit

@@ -1,7 +1,9 @@
 object CurrentProject: TCENativeProject
   OptionsCollection = <  
     item
-      name = 'default'
+      name = 'win32-dbg'
+      debugingOptions.debug = True
+      debugingOptions.codeviewDexts = True
       messagesOptions.additionalWarnings = True
       outputOptions.binaryKind = sharedlib
       pathsOptions.outputFilename = '..\..\lazproj\cedast.dll'

cedast/src/ast.d

@@ -1,13 +1,296 @@
 module ast;
 
 import std.d.lexer, std.d.parser, std.d.ast;
+import std.json, std.array, std.conv;
+import iz.enumset, iz.memory;
+
 import common;
-import iz.enumset;
 
 private
 {
-    enum AstInfos {moduleName, symbolList, todoList, WarnAndErr}
+    enum AstInfos {ModuleName, ErrorsJson, ErrorsPas, SymsJson, SymsPas}
     alias CachedInfos = EnumSet!(AstInfos, Set8);
+
+    enum SymbolType
+    {
+        _alias,
+        _class,
+        _enum,
+        _error,
+        _function,
+        _interface,
+        _import,
+        _mixin,   // (template decl)
+        _struct,
+        _template,
+        _union,
+        _variable,
+        _warning
+    }
+
+    struct Symbol
+    {
+        size_t line;
+        size_t col;
+        string name;
+        SymbolType type;
+        Symbol * [] subs;
+
+        ~this()
+        {
+            foreach_reverse(i; 0 .. subs.length)
+                subs[i].destruct;
+        }
+
+        void serializePas(ref Appender!string lfmApp)
+        {
+            lfmApp.put("\ritem\r");
+
+            lfmApp.put(format("line = %d\r", line));
+            lfmApp.put(format("col = %d\r", col));
+            lfmApp.put(format("name = '%s'\r", name));
+            lfmApp.put(format("symType = %s\r", type));
+
+            lfmApp.put("subs = <");
+            if (subs.length) foreach(Symbol * sub; subs)
+                sub.serializePas(lfmApp);
+            lfmApp.put(">\r");
+            lfmApp.put("end");
+        }
+
+        void serializeJson(ref JSONValue json)
+        {
+            auto vobj = parseJSON("{}");
+            vobj["line"]= JSONValue(line);
+            vobj["col"] = JSONValue(col);
+            vobj["name"]= JSONValue(name);
+            vobj["type"]= JSONValue(to!string(type));
+            if (subs.length)
+            {
+                auto vsubs = parseJSON("[]");
+                foreach(Symbol * sub; subs)
+                    sub.serializeJson(vsubs);
+                vobj["items"] = vsubs;
+            }
+            json.array ~= vobj;
+        }
+    }
+
+    struct AstError
+    {
+        size_t line, col;
+        string msg;
+        bool isErr;
+    }
+
+    class SymbolListBuilder : ASTVisitor
+    {
+        Symbol * root;
+        Symbol * parent;
+
+        size_t count;
+
+        alias visit = ASTVisitor.visit;
+
+        this(Module mod)
+        {
+            root = construct!Symbol;
+            resetRoot;
+            foreach(Declaration d; mod.declarations)
+                visit(d);
+        }
+
+        ~this()
+        {
+            root.destruct;
+        }
+
+        final void resetRoot(){parent = root;}
+
+        final string serializePas()
+        {
+            Appender!string lfmApp;
+            lfmApp.reserve(count * 64);
+
+            lfmApp.put("object TSymbolList\rsymbols = <");
+            foreach(sym; root.subs) sym.serializePas(lfmApp);
+            lfmApp.put(">\rend\r\n");
+
+            return lfmApp.data;
+        }
+
+        final JSONValue serializeJson()
+        {
+            JSONValue result = parseJSON("{}");
+            JSONValue vsubs = parseJSON("[]");
+            foreach(sym; root.subs) sym.serializeJson(vsubs);
+            result["items"] = vsubs;
+            return result;
+        }
+
+        /// returns a new symbol if the declarator is based on a Token named "name".
+        final Symbol * addDeclaration(DT)(DT adt)
+        {
+            static if
+            (
+                is(DT == const(EponymousTemplateDeclaration)) ||
+                is(DT == const(AnonymousEnumMember))    ||
+                is(DT == const(AliasInitializer))       ||
+                is(DT == const(ClassDeclaration))       ||
+                is(DT == const(Declarator))             ||
+                is(DT == const(EnumDeclaration))        ||
+                is(DT == const(FunctionDeclaration))    ||
+                is(DT == const(InterfaceDeclaration))   ||
+                is(DT == const(StructDeclaration))      ||
+                is(DT == const(TemplateDeclaration))    ||
+                is(DT == const(UnionDeclaration))
+
+            )
+            {
+                count++;
+                auto result = construct!Symbol;
+                result.name = adt.name.text;
+                result.line = adt.name.line;
+                result.col  = adt.name.column;
+                parent.subs ~= result;
+                return result;
+            }
+
+            version(none) assert(0, "addDeclaration no implemented for " ~ DT.stringof);
+        }
+
+        /// visitor implementation if the declarator is based on a Token named "name".
+        final void namedVisitorImpl(DT, SymbolType st, bool dig = true)(const(DT) dt)
+        {
+            auto newSymbol = addDeclaration(dt);
+            newSymbol.type = st;
+            //
+            static if (dig)
+            {
+                auto previousParent = parent;
+                scope(exit) parent = previousParent;
+                parent = newSymbol;
+                dt.accept(this);
+            }
+        }
+
+        /// visitor implementation for special cases.
+        final void otherVisitorImpl(SymbolType st, string name, size_t line, size_t col)
+        {
+            count++;
+            auto result = construct!Symbol;
+            result.name = name;
+            result.line = line;
+            result.col  = col;
+            result.type = st;
+            parent.subs ~= result;
+        }
+
+        final override void visit(const AliasDeclaration decl)
+        {
+            // why is initializers an array ?
+            if (decl.initializers.length > 0)
+                namedVisitorImpl!(AliasInitializer, SymbolType._alias)(decl.initializers[0]);
+        }
+
+        final override void visit(const AnonymousEnumDeclaration decl)
+        {
+            if (decl.members.length > 0)
+                namedVisitorImpl!(AnonymousEnumMember, SymbolType._enum)(decl.members[0]);
+        }
+
+        final override void visit(const ClassDeclaration decl)
+        {
+            namedVisitorImpl!(ClassDeclaration, SymbolType._class)(decl);
+        }
+
+        final override void visit(const Constructor decl)
+        {
+            otherVisitorImpl(SymbolType._function, "this", decl.line, decl.column);
+        }
+
+        final override void visit(const Destructor decl)
+        {
+            otherVisitorImpl(SymbolType._function, "~this", decl.line, decl.column);
+        }
+
+        final override void visit(const EnumDeclaration decl)
+        {
+            namedVisitorImpl!(EnumDeclaration, SymbolType._enum)(decl);
+        }
+
+        final override void visit(const EponymousTemplateDeclaration decl)
+        {
+            namedVisitorImpl!(EponymousTemplateDeclaration, SymbolType._template)(decl);
+        }
+
+        final override void visit(const FunctionDeclaration decl)
+        {
+            namedVisitorImpl!(FunctionDeclaration, SymbolType._function)(decl);
+        }
+
+        final override void visit(const InterfaceDeclaration decl)
+        {
+            namedVisitorImpl!(InterfaceDeclaration, SymbolType._interface)(decl);
+        }
+
+        final override void visit(const ImportDeclaration decl)
+        {
+            foreach(const(SingleImport) si; decl.singleImports)
+            {
+                if (!si.identifierChain.identifiers.length)
+                    continue;
+                //
+                string[] modules;
+                foreach(ident; si.identifierChain.identifiers)
+                {
+                    modules ~= ident.text;
+                    modules ~= ".";
+                }
+                //
+                otherVisitorImpl(SymbolType._import, modules[0..$-1].join,
+                    si.identifierChain.identifiers[0].line,
+                    si.identifierChain.identifiers[0].column
+                );
+            }
+        }
+
+        final override void visit(const MixinTemplateDeclaration decl)
+        {
+            namedVisitorImpl!(TemplateDeclaration, SymbolType._mixin)(decl.templateDeclaration);
+        }
+
+        final override void visit(const StructDeclaration decl)
+        {
+            namedVisitorImpl!(StructDeclaration, SymbolType._struct)(decl);
+        }
+
+        final override void visit(const TemplateDeclaration decl)
+        {
+            namedVisitorImpl!(TemplateDeclaration, SymbolType._template)(decl);
+        }
+
+        final override void visit(const UnionDeclaration decl)
+        {
+            namedVisitorImpl!(UnionDeclaration, SymbolType._union)(decl);
+        }
+
+        final override void visit(const VariableDeclaration decl)
+        {
+            foreach(elem; decl.declarators)
+                namedVisitorImpl!(Declarator, SymbolType._variable, false)(elem);
+        }
+
+        final override void visit(const StaticConstructor decl)
+        {
+            otherVisitorImpl(SymbolType._function, "static this", decl.line, decl.column);
+        }
+
+        final override void visit(const StaticDestructor decl)
+        {
+            otherVisitorImpl(SymbolType._function, "static ~this", decl.line, decl.column);
+        }
+    }
 }
 
 
@@ -21,18 +304,28 @@ private:
     LexerConfig config;
     StringCache strcache;
     Module mod;
-    CachedInfos cachedInfos;
     bool scanned;
+
+    CachedInfos cachedInfos;
     string modName;
+    ubyte[] jsonErrors;
+    ubyte[] pasErrors;
+    ubyte[] todosPas;
+    ubyte[] todosJson;
+    ubyte[] symsPas;
+    ubyte[] symsJson;
+    static AstError*[] errors;
 
     final static void parserError(string fname, size_t line, size_t col, string msg, bool isErr)
     {
+        errors ~= new AstError(line, col, msg, isErr);
     }
 
     final void resetCachedInfo()
     {
         cachedInfos = 0;
-        modName = "";
+        modName = modName.init;
+        errors = errors.init;
     }
 
     final void scan()
@@ -49,27 +342,30 @@ public:
 
     this(string filename)
     {
-        mixin(logcall);
-
-        import std.file;
         fname = filename;
-        src = cast(ubyte[]) read(fname, size_t.max);
         strcache = StringCache(StringCache.defaultBucketCount);
-        scan;
+        rescanFile();
     }
 
     this(ubyte[] buffer)
     {
-        mixin(logcall);
-        src = buffer.dup;
         strcache = StringCache(StringCache.defaultBucketCount);
+        rescanBuffer(buffer);
+    }
+
+    final void rescanFile()
+    {
+        resetCachedInfo;
+        import std.file;
+        src = cast(ubyte[]) read(fname, size_t.max);
         scan;
     }
 
-    final void rescan()
+    final void rescanBuffer(ubyte[] buffer)
     {
-        mixin(logcall);
         resetCachedInfo;
+        src = buffer.dup;
+        scan;
     }
 
     final string moduleName()
@@ -78,10 +374,10 @@ public:
 
         if (!scanned)
             return result;
-        if (AstInfos.moduleName in cachedInfos)
+        if (AstInfos.ModuleName in cachedInfos)
             return modName;
 
-        cachedInfos += AstInfos.moduleName;
+        cachedInfos += AstInfos.ModuleName;
         if (mod.moduleDeclaration)
         foreach(Token t; mod.moduleDeclaration.moduleName.identifiers)
             result ~= t.text ~ ".";
@@ -90,4 +386,40 @@ public:
             modName = result[0 .. $-1];
         return modName;
     }
+
+    final ubyte[] todoListPas()
+    {
+        return null;
+    }
+
+    final ubyte[] todoListJson()
+    {
+        return null;
+    }
+
+    final ubyte[] symbolListPas()
+    {
+        if (AstInfos.SymsPas !in cachedInfos)
+        {
+            cachedInfos += AstInfos.SymsPas;
+            SymbolListBuilder slb = construct!SymbolListBuilder(mod);
+            scope(exit) destruct(slb);
+            symsPas = cast(ubyte[]) slb.serializePas();
+        }
+        return symsPas;
+    }
+
+    final ubyte[] symbolListJson()
+    {
+        if (AstInfos.SymsJson !in cachedInfos)
+        {
+            cachedInfos += AstInfos.SymsJson;
+            SymbolListBuilder slb = construct!SymbolListBuilder(mod);
+            scope(exit) destruct(slb);
+            JSONValue v = slb.serializeJson();
+            symsJson = cast(ubyte[]) v.toPrettyString;
+        }
+        return symsJson;
+    }
+
 }

cedast/src/cedast.d

@@ -1,6 +1,7 @@
 module cedast;
 
 import core.runtime, common, ast;
+import iz.memory;
 
 __gshared Ast*[] modules;
 
@@ -12,7 +13,7 @@ AstToken scanFile(char* filename)
     {
         import std.string: fromStringz;
         import std.algorithm: countUntil;
-        Ast* ast = new Ast(filename.fromStringz.idup);
+        Ast* ast = construct!Ast(filename.fromStringz.idup);
         result = countUntil(modules, null);
         if (result == -1)
         {
@@ -40,7 +41,7 @@ AstToken scanBuffer(ubyte* buffer, size_t len)
     try
     {
         import std.algorithm: countUntil;
-        Ast* ast = new Ast(buffer[0 .. len]);
+        Ast* ast = construct!Ast(buffer[0 .. len]);
         result = countUntil(modules, null);
         if (result == -1)
         {
@@ -62,11 +63,19 @@ AstToken scanBuffer(ubyte* buffer, size_t len)
 }
 
 extern(C) export
-void rescan(AstToken tok)
+void rescanFile(AstToken tok)
 {
     if (tok < 1 || tok > modules.length)
         return;
-    modules[tok - 1].rescan;
+    modules[tok - 1].rescanFile;
+}
+
+extern(C) export
+void rescanBuffer(AstToken tok, ubyte* buffer, size_t len)
+{
+    if (tok < 1 || tok > modules.length)
+        return;
+    modules[tok - 1].rescanBuffer(buffer[0 .. len]);
 }
 
 extern(C) export
@@ -77,7 +86,7 @@ void unleash(AstToken tok)
         return;
     if (modules[tok - 1] == null)
         return;
-    delete modules[tok - 1];
+    destruct(modules[tok - 1]);
     modules[tok - 1] = null;
     if (tok == modules.length)
         modules.length -= 1;
@@ -98,15 +107,39 @@ immutable(char*) moduleName(AstToken tok)
 extern(C) export
 ubyte* symbolList(AstToken tok, ref size_t len, SerializationFormat fmt)
 {
-    ubyte* result = null;
+    if (tok < 1 || tok > modules.length)
-    return result;
+        return null;
+    Ast* mod = modules[tok - 1];
+    if (mod == null)
+        return null;
+
+    ubyte[] result;
+    if (fmt == SerializationFormat.json)
+        result = mod.symbolListJson;
+    else
+        result = mod.symbolListPas;
+
+    len = result.length;
+    return result.ptr;
 }
 
 extern(C) export
 ubyte* todoList(AstToken tok, ref size_t len, SerializationFormat fmt)
 {
-    ubyte* result = null;
+    if (tok < 1 || tok > modules.length)
-    return result;
+        return null;
+    Ast* mod = modules[tok - 1];
+    if (mod == null)
+        return null;
+
+    ubyte[] result;
+    if (fmt == SerializationFormat.json)
+        result = mod.todoListJson;
+    else
+        result = mod.todoListPas;
+
+    len = result.length;
+    return result.ptr;
 }
 
 version(Windows)
@@ -119,7 +152,7 @@ version(Windows)
     extern (Windows)
     BOOL DllMain(HINSTANCE hInstance, ULONG ulReason, LPVOID pvReserved)
     {
-        switch (ulReason)
+        final switch (ulReason)
         {
             case DLL_PROCESS_ATTACH:
                 Runtime.initialize;
@@ -139,11 +172,7 @@ version(Windows)
             case DLL_THREAD_DETACH:
                 dll_thread_detach( true, true );
                 break;
-
-            default:
-                assert(0);
         }
-
         return true;
     }
 }

cedast/src/common.d

@@ -4,7 +4,11 @@ alias AstToken = ptrdiff_t;
 
 __gshared immutable AstToken invalidAstToken = 0;
 
-enum SerializationFormat {json, pascal}
+enum SerializationFormat : byte
+{
+    json,
+    pascal
+}
 
 enum logcall = q{
     import std.file;

lazproj/cedast_loader.pas

@@ -6,23 +6,35 @@ type
 
   TAstToken = NativeInt;
 
+  {$Z1}
+  TSerializationFormat = (json, pas);
+
   TScanFile     = function(filename: PChar): TAstToken; cdecl;
-  TScanBuffer = function (buffer: PByte; len: NativeUint): TAstToken; cdecl;
+  TScanBuffer   = function(buffer: PByte; len: NativeUint): TAstToken; cdecl;
-  TRescan     = procedure (tok: TAstToken); cdecl;
+  TRescanFile   = procedure(tok: TAstToken); cdecl;
-  TUnleash    = procedure (tok: TAstToken); cdecl;
+  TRescanBuffer = procedure(tok: TAstToken; buffer: PByte; len: NativeUint); cdecl;
-  TModuleName = function (tok: TAstToken): PChar; cdecl;
+  TUnleash      = procedure(tok: TAstToken); cdecl;
+
+  TModuleName   = function(tok: TAstToken): PChar; cdecl;
+  TSymbolList   = function(tok: TAstToken; var len: NativeUint ; fmt: TSerializationFormat): PByte; cdecl;
+
+
 
 var
   dast: TLibHandle;
   scanfile: TScanFile;
   scanbuffer: TScanBuffer;
-  rescan: TRescan;
+  rescanfile: TRescanFile;
+  rescanbuffer: TRescanBuffer;
   unleash: TUnleash;
   moduleName: TModuleName;
+  symlist: TSymbolList;
   tok: TAstToken;
+  len: NativeUint = 0;
+  ptr: PByte;
 
 const
-  testModule = 'module a.b.c.d.e.f.g.h; import std.stdio;';
+  testModule = 'module a.b.c.d.e.f.g.h; import std.stdio; uint a; struct F{long c;}';
 
 
 begin
@@ -35,18 +47,32 @@ begin
     if scanFile = nil then writeln('invalid scanfile proc ptr')
     else tok := scanfile(PChar('exception in call so ticket value is 0'));
 
-    rescan := TRescan(GetProcAddress(dast, 'rescan'));
+    rescanfile := TRescanFile(GetProcAddress(dast, 'rescanFile'));
-    if rescan = nil then writeln('invalid rescan proc ptr')
+    if rescanfile = nil then writeln('invalid rescanFile proc ptr')
-    else rescan(tok);
+    else rescanfile(tok);
 
     scanbuffer := TScanBuffer(GetProcAddress(dast, 'scanBuffer'));
     if scanbuffer = nil then writeln('invalid scanBuffer proc ptr')
     else tok := scanbuffer(@testModule[1], length(testModule));
 
+    rescanbuffer := TRescanBuffer(GetProcAddress(dast, 'rescanBuffer'));
+    if rescanbuffer = nil then writeln('invalid rescanBuffer proc ptr')
+    else rescanbuffer(tok, @testmodule[1], length(testModule));
+
     moduleName := TModuleName(GetProcAddress(dast, 'moduleName'));
     if moduleName = nil then writeln('invalid moduleName proc ptr')
     else if tok <> 0 then writeln(moduleName(tok));
 
+    symlist := TSymbolList(GetProcAddress(dast, 'symbolList'));
+    if symlist = nil then writeln('invalid symbolList proc ptr')
+    else if tok <> 0 then with TMemoryStream.Create do try
+      ptr := symlist(tok, len, TSerializationFormat.json);
+      write(ptr^, len);
+      SaveToFile('testsymlist.txt');
+    finally
+      free;
+    end;
+
     unleash := TUnleash(GetProcAddress(dast, 'unleash'));
     if unleash = nil then writeln('invalid unleash proc ptr')
     else unleash(tok);