more for #39

- simplified API
- renamed token to handle to avoid confusion
- scan in a thread
- notification

cedast/src/ast.d

@@ -1,7 +1,7 @@
 module ast;
 
 import std.d.lexer, std.d.parser, std.d.ast;
-import std.json, std.array, std.conv;
+import std.json, std.array, std.conv, std.parallelism;
 import iz.enumset, iz.memory;
 
 import common;
@@ -294,7 +294,7 @@ private
 }
 
 
-struct Ast
+class Ast
 {
 
 private:
@@ -304,6 +304,8 @@ private:
     LexerConfig config;
     StringCache strcache;
     Module mod;
+    AstNotification notif;
+    void* notifparam;
     bool scanned;
 
     CachedInfos cachedInfos;
@@ -325,49 +327,55 @@ private:
     {
         cachedInfos = 0;
         modName = modName.init;
+        jsonErrors = jsonErrors.init;
+        pasErrors = pasErrors.init;
+        todosPas = todosPas.init;
+        todosJson = todosJson.init;
+        symsPas = symsPas.init;
+        symsJson = symsJson.init;
         errors = errors.init;
     }
 
-    final void scan()
+    final void taskScan()
     {
-        resetCachedInfo;
-        scanned = false;
-        scope(success) scanned = true;
-
         config = LexerConfig(fname, StringBehavior.source, WhitespaceBehavior.skip);
         mod = parseModule(getTokensForParser(src, config, &strcache), fname, null, &parserError);
+        if (notif) notif(notifparam);
+        scanned = true;
     }
 
 public:
 
-    this(string filename)
+    this()
-    {
-        fname = filename;
-        strcache = StringCache(StringCache.defaultBucketCount);
-        rescanFile();
-    }
-
-    this(ubyte[] buffer)
     {
         strcache = StringCache(StringCache.defaultBucketCount);
-        rescanBuffer(buffer);
     }
 
-    final void rescanFile()
+    final void scanFile(string filename)
     {
         resetCachedInfo;
+        fname = filename;
         import std.file;
-        src = cast(ubyte[]) read(fname, size_t.max);
+        try src = cast(ubyte[]) read(fname, size_t.max);
-        scan;
+        catch(Exception e){}
+        scanned = false;
+        task(&taskScan).executeInNewThread;
     }
 
-    final void rescanBuffer(ubyte[] buffer)
+    final void scanBuffer(ubyte[] buffer)
     {
         resetCachedInfo;
         src = buffer.dup;
-        scan;
+        scanned = false;
+        task(&taskScan).executeInNewThread;
     }
 
+    @property AstNotification notification(){return notif;}
+    @property void notification(AstNotification value){notif = value;}
+
+    @property void* notificationParameter(){return notifparam;}
+    @property void notificationParameter(void* value){notifparam = value;}
+
     final string moduleName()
     {
         string result;
@@ -423,3 +431,4 @@ public:
     }
 
 }
+

cedast/src/cedast.d

@@ -3,17 +3,19 @@ module cedast;
 import core.runtime, common, ast;
 import iz.memory;
 
-__gshared Ast*[] modules;
+__gshared Ast[] modules;
+
 
 extern(C) export
-AstToken scanFile(char* filename)
+AstHandle newAst(void* param, AstNotification clbck)
 {
-    AstToken result;
+    AstHandle result;
     try
     {
-        import std.string: fromStringz;
         import std.algorithm: countUntil;
-        Ast* ast = construct!Ast(filename.fromStringz.idup);
+        Ast ast = construct!Ast;
+        ast.notification = clbck;
+        ast.notificationParameter = param;
         result = countUntil(modules, null);
         if (result == -1)
         {
@@ -28,115 +30,92 @@ AstToken scanFile(char* filename)
     }
     catch(Exception e)
     {
-        if (result != 0) unleash(result);
+        if (result != 0) deleteAst(result);
-        result = invalidAstToken;
+        result = invalidAstHandle;
     }
     return result;
 }
 
 extern(C) export
-AstToken scanBuffer(ubyte* buffer, size_t len)
+void deleteAst(AstHandle hdl)
 {
-    AstToken result;
+    if (hdl < 1 || hdl > modules.length)
-    try
+        return;
-    {
+    if (modules[hdl - 1] is null)
-        import std.algorithm: countUntil;
+        return;
-        Ast* ast = construct!Ast(buffer[0 .. len]);
-        result = countUntil(modules, null);
-        if (result == -1)
-        {
-            modules ~= ast;
-            result = modules.length;
-        }
-        else
-        {
-            modules[result] = ast;
-            ++result;
-        }
-    }
-    catch(Exception e)
-    {
-        if (result != 0) unleash(result);
-        result = invalidAstToken;
-    }
-    return result;
-}
 
-extern(C) export
+    destruct(modules[hdl - 1]);
-void rescanFile(AstToken tok)
+    modules[hdl - 1] = null;
-{
+    if (hdl == modules.length)
-    if (tok < 1 || tok > modules.length)
-        return;
-    modules[tok - 1].rescanFile;
-}
-
-extern(C) export
-void rescanBuffer(AstToken tok, ubyte* buffer, size_t len)
-{
-    if (tok < 1 || tok > modules.length)
-        return;
-    modules[tok - 1].rescanBuffer(buffer[0 .. len]);
-}
-
-extern(C) export
-void unleash(AstToken tok)
-{
-    mixin(logcall);
-    if (tok < 1 || tok > modules.length)
-        return;
-    if (modules[tok - 1] == null)
-        return;
-    destruct(modules[tok - 1]);
-    modules[tok - 1] = null;
-    if (tok == modules.length)
         modules.length -= 1;
 }
 
 extern(C) export
-immutable(char*) moduleName(AstToken tok)
+void scanFile(AstHandle hdl, char* filename)
 {
-    if (tok < 1 || tok > modules.length)
+    if (hdl < 1 || hdl > modules.length)
-        return null;
+        return;
-    Ast* mod = modules[tok - 1];
+    if (modules[hdl - 1] is null)
-    if (mod == null)
+        return;
-        return null;
+
-    import std.string: toStringz;
+    import std.string: fromStringz;
-    return toStringz(mod.moduleName);
+    modules[hdl - 1].scanFile(fromStringz(filename).idup);
 }
 
 extern(C) export
-ubyte* symbolList(AstToken tok, ref size_t len, SerializationFormat fmt)
+void scanBuffer(AstHandle hdl, ubyte* buffer, size_t len)
 {
-    if (tok < 1 || tok > modules.length)
+    if (hdl < 1 || hdl > modules.length)
+        return;
+    if (modules[hdl - 1] is null)
+        return;
+
+    modules[hdl - 1].scanBuffer(buffer[0 .. len]);
+}
+
+extern(C) export
+immutable(char*) moduleName(AstHandle hdl)
+{
+    if (hdl < 1 || hdl > modules.length)
         return null;
-    Ast* mod = modules[tok - 1];
+    if (modules[hdl - 1] is null)
-    if (mod == null)
+        return null;
+
+    import std.string: toStringz;
+    return toStringz(modules[hdl - 1].moduleName);
+}
+
+extern(C) export
+ubyte* symbolList(AstHandle hdl, ref size_t len, SerializationFormat fmt)
+{
+    if (hdl < 1 || hdl > modules.length)
+        return null;
+    if (modules[hdl - 1] is null)
         return null;
 
     ubyte[] result;
     if (fmt == SerializationFormat.json)
-        result = mod.symbolListJson;
+        result = modules[hdl - 1].symbolListJson;
     else
-        result = mod.symbolListPas;
+        result = modules[hdl - 1].symbolListPas;
 
     len = result.length;
     return result.ptr;
 }
 
 extern(C) export
-ubyte* todoList(AstToken tok, ref size_t len, SerializationFormat fmt)
+ubyte* todoList(AstHandle hdl, ref size_t len, SerializationFormat fmt)
 {
-    if (tok < 1 || tok > modules.length)
+    if (hdl < 1 || hdl > modules.length)
         return null;
-    Ast* mod = modules[tok - 1];
+    if (modules[hdl - 1] is null)
-    if (mod == null)
         return null;
 
     ubyte[] result;
     if (fmt == SerializationFormat.json)
-        result = mod.todoListJson;
+        result = modules[hdl - 1].todoListJson;
     else
-        result = mod.todoListPas;
+        result = modules[hdl - 1].todoListPas;
 
     len = result.length;
     return result.ptr;
@@ -177,4 +156,3 @@ version(Windows)
     }
 }
 
-

cedast/src/common.d

@@ -1,8 +1,10 @@
 module common;
 
-alias AstToken = ptrdiff_t;
+alias AstHandle = ptrdiff_t;
 
-__gshared immutable AstToken invalidAstToken = 0;
+alias AstNotification = extern(C) void function(void* param);
+
+__gshared immutable AstHandle invalidAstHandle = 0;
 
 enum SerializationFormat : byte
 {

lazproj/cedast_loader.pas

@@ -1,41 +1,52 @@
 program cedast_loader;
 
+{$MODE OBJFPC}
+
 uses dynlibs, classes, sysutils;
 
 type
 
-  TAstToken = NativeInt;
+  TAstHandle = NativeInt;
+
+  TAstNotification = procedure(param: pointer); cdecl;
 
   {$Z1}
+
   TSerializationFormat = (json, pas);
 
-  TScanFile     = function(filename: PChar): TAstToken; cdecl;
+  TNewAst     = function(param: Pointer; clbck: TAstNotification): TAstHandle; cdecl;
-  TScanBuffer   = function(buffer: PByte; len: NativeUint): TAstToken; cdecl;
+  TDeleteAst  = procedure(tok: TAstHandle); cdecl;
-  TRescanFile   = procedure(tok: TAstToken); cdecl;
-  TRescanBuffer = procedure(tok: TAstToken; buffer: PByte; len: NativeUint); cdecl;
-  TUnleash      = procedure(tok: TAstToken); cdecl;
 
-  TModuleName   = function(tok: TAstToken): PChar; cdecl;
+  TScanFile   = procedure(tok: TAstHandle; filename: PChar); cdecl;
-  TSymbolList   = function(tok: TAstToken; var len: NativeUint ; fmt: TSerializationFormat): PByte; cdecl;
+  TScanBuffer = procedure(tok: TAstHandle; buffer: PByte; len: NativeUint); cdecl;
+
+
+  TModuleName = function(tok: TAstHandle): PChar; cdecl;
+  TSymbolList = function(tok: TAstHandle; var len: NativeUint ; fmt: TSerializationFormat): PByte; cdecl;
 
 
 
 var
   dast: TLibHandle;
-  scanfile: TScanFile;
+  newAst: TNewAst;
-  scanbuffer: TScanBuffer;
+  deleteast: TDeleteAst;
-  rescanfile: TRescanFile;
+  scanFile: TScanFile;
-  rescanbuffer: TRescanBuffer;
+  scanBuffer: TScanBuffer;
-  unleash: TUnleash;
   moduleName: TModuleName;
-  symlist: TSymbolList;
+  symbolList: TSymbolList;
-  tok: TAstToken;
+  hdl: TAstHandle;
   len: NativeUint = 0;
   ptr: PByte;
+  done: boolean;
 
 const
   testModule = 'module a.b.c.d.e.f.g.h; import std.stdio; uint a; struct F{long c;}';
 
+procedure notif(param: Pointer); cdecl;
+begin
+  done := true;
+end;
+
 
 begin
 
@@ -43,39 +54,44 @@ begin
   if dast = NilHandle then
     writeln('dast invalid handle')
   else begin
-    scanfile := TScanFile(GetProcAddress(dast, 'scanFile'));
+
+    newAst := TNewAst(GetProcAddress(dast, 'newAst'));
+    if newAst = nil then writeln('invalid newAst proc ptr')
+    else hdl := newAst(nil, @notif);
+
+    scanFile := TScanFile(GetProcAddress(dast, 'scanFile'));
     if scanFile = nil then writeln('invalid scanfile proc ptr')
-    else tok := scanfile(PChar('exception in call so ticket value is 0'));
+    else begin
+      done := false;
+      scanFile(hdl, PChar('exception in call'));
+      while not done do sleep(20);
+    end;
 
-    rescanfile := TRescanFile(GetProcAddress(dast, 'rescanFile'));
+    scanBuffer := TScanBuffer(GetProcAddress(dast, 'scanBuffer'));
-    if rescanfile = nil then writeln('invalid rescanFile proc ptr')
+    if scanBuffer = nil then writeln('invalid scanBuffer proc ptr')
-    else rescanfile(tok);
+    else  begin
-
+      done := false;
-    scanbuffer := TScanBuffer(GetProcAddress(dast, 'scanBuffer'));
+      scanBuffer(hdl, @testModule[1], length(testModule));
-    if scanbuffer = nil then writeln('invalid scanBuffer proc ptr')
+      while not done do sleep(20);
-    else tok := scanbuffer(@testModule[1], length(testModule));
+    end;
-
-    rescanbuffer := TRescanBuffer(GetProcAddress(dast, 'rescanBuffer'));
-    if rescanbuffer = nil then writeln('invalid rescanBuffer proc ptr')
-    else rescanbuffer(tok, @testmodule[1], length(testModule));
 
     moduleName := TModuleName(GetProcAddress(dast, 'moduleName'));
     if moduleName = nil then writeln('invalid moduleName proc ptr')
-    else if tok <> 0 then writeln(moduleName(tok));
+    else if hdl <> 0 then writeln(moduleName(hdl));
 
-    symlist := TSymbolList(GetProcAddress(dast, 'symbolList'));
+    symbolList := TSymbolList(GetProcAddress(dast, 'symbolList'));
-    if symlist = nil then writeln('invalid symbolList proc ptr')
+    if symbolList = nil then writeln('invalid symbolList proc ptr')
-    else if tok <> 0 then with TMemoryStream.Create do try
+    else if hdl <> 0 then with TMemoryStream.Create do try
-      ptr := symlist(tok, len, TSerializationFormat.json);
+      ptr := symbolList(hdl, len, TSerializationFormat.json);
       write(ptr^, len);
       SaveToFile('testsymlist.txt');
     finally
       free;
     end;
 
-    unleash := TUnleash(GetProcAddress(dast, 'unleash'));
+    deleteAst := TDeleteAst(GetProcAddress(dast, 'deleteAst'));
-    if unleash = nil then writeln('invalid unleash proc ptr')
+    if deleteAst = nil then writeln('invalid deleteAst proc ptr')
-    else unleash(tok);
+    else deleteAst(hdl);
 
   end;