From d0da5ebb4e109ceeaf5ef28bb25b0e02bc453de9 Mon Sep 17 00:00:00 2001 From: Basile Burg Date: Tue, 15 Sep 2015 08:13:40 +0200 Subject: [PATCH] symbol list for #39 --- cedast/build/cedast.coedit | 4 +- cedast/src/ast.d | 362 +++++++++++++++++++++++++++++++++++-- cedast/src/cedast.d | 57 ++++-- cedast/src/common.d | 6 +- lazproj/cedast_loader.pas | 46 ++++- 5 files changed, 434 insertions(+), 41 deletions(-) diff --git a/cedast/build/cedast.coedit b/cedast/build/cedast.coedit index 5c678244..1516a19b 100644 --- a/cedast/build/cedast.coedit +++ b/cedast/build/cedast.coedit @@ -1,7 +1,9 @@ object CurrentProject: TCENativeProject OptionsCollection = < item - name = 'default' + name = 'win32-dbg' + debugingOptions.debug = True + debugingOptions.codeviewDexts = True messagesOptions.additionalWarnings = True outputOptions.binaryKind = sharedlib pathsOptions.outputFilename = '..\..\lazproj\cedast.dll' diff --git a/cedast/src/ast.d b/cedast/src/ast.d index b4814feb..a590b771 100644 --- a/cedast/src/ast.d +++ b/cedast/src/ast.d @@ -1,13 +1,296 @@ module ast; import std.d.lexer, std.d.parser, std.d.ast; +import std.json, std.array, std.conv; +import iz.enumset, iz.memory; + import common; -import iz.enumset; private { - enum AstInfos {moduleName, symbolList, todoList, WarnAndErr} + enum AstInfos {ModuleName, ErrorsJson, ErrorsPas, SymsJson, SymsPas} alias CachedInfos = EnumSet!(AstInfos, Set8); + + enum SymbolType + { + _alias, + _class, + _enum, + _error, + _function, + _interface, + _import, + _mixin, // (template decl) + _struct, + _template, + _union, + _variable, + _warning + } + + struct Symbol + { + size_t line; + size_t col; + string name; + SymbolType type; + Symbol * [] subs; + + ~this() + { + foreach_reverse(i; 0 .. subs.length) + subs[i].destruct; + } + + void serializePas(ref Appender!string lfmApp) + { + lfmApp.put("\ritem\r"); + + lfmApp.put(format("line = %d\r", line)); + lfmApp.put(format("col = %d\r", col)); + lfmApp.put(format("name = '%s'\r", name)); + lfmApp.put(format("symType = %s\r", type)); + + lfmApp.put("subs = <"); + if (subs.length) foreach(Symbol * sub; subs) + sub.serializePas(lfmApp); + lfmApp.put(">\r"); + lfmApp.put("end"); + } + + void serializeJson(ref JSONValue json) + { + auto vobj = parseJSON("{}"); + vobj["line"]= JSONValue(line); + vobj["col"] = JSONValue(col); + vobj["name"]= JSONValue(name); + vobj["type"]= JSONValue(to!string(type)); + if (subs.length) + { + auto vsubs = parseJSON("[]"); + foreach(Symbol * sub; subs) + sub.serializeJson(vsubs); + vobj["items"] = vsubs; + } + json.array ~= vobj; + } + } + + struct AstError + { + size_t line, col; + string msg; + bool isErr; + } + + class SymbolListBuilder : ASTVisitor + { + Symbol * root; + Symbol * parent; + + size_t count; + + alias visit = ASTVisitor.visit; + + this(Module mod) + { + root = construct!Symbol; + resetRoot; + foreach(Declaration d; mod.declarations) + visit(d); + } + + ~this() + { + root.destruct; + } + + final void resetRoot(){parent = root;} + + final string serializePas() + { + Appender!string lfmApp; + lfmApp.reserve(count * 64); + + lfmApp.put("object TSymbolList\rsymbols = <"); + foreach(sym; root.subs) sym.serializePas(lfmApp); + lfmApp.put(">\rend\r\n"); + + return lfmApp.data; + } + + final JSONValue serializeJson() + { + JSONValue result = parseJSON("{}"); + JSONValue vsubs = parseJSON("[]"); + foreach(sym; root.subs) sym.serializeJson(vsubs); + result["items"] = vsubs; + return result; + } + + /// returns a new symbol if the declarator is based on a Token named "name". + final Symbol * addDeclaration(DT)(DT adt) + { + static if + ( + is(DT == const(EponymousTemplateDeclaration)) || + is(DT == const(AnonymousEnumMember)) || + is(DT == const(AliasInitializer)) || + is(DT == const(ClassDeclaration)) || + is(DT == const(Declarator)) || + is(DT == const(EnumDeclaration)) || + is(DT == const(FunctionDeclaration)) || + is(DT == const(InterfaceDeclaration)) || + is(DT == const(StructDeclaration)) || + is(DT == const(TemplateDeclaration)) || + is(DT == const(UnionDeclaration)) + + ) + { + count++; + auto result = construct!Symbol; + result.name = adt.name.text; + result.line = adt.name.line; + result.col = adt.name.column; + parent.subs ~= result; + return result; + } + + version(none) assert(0, "addDeclaration no implemented for " ~ DT.stringof); + } + + /// visitor implementation if the declarator is based on a Token named "name". + final void namedVisitorImpl(DT, SymbolType st, bool dig = true)(const(DT) dt) + { + auto newSymbol = addDeclaration(dt); + newSymbol.type = st; + // + static if (dig) + { + auto previousParent = parent; + scope(exit) parent = previousParent; + parent = newSymbol; + dt.accept(this); + } + } + + /// visitor implementation for special cases. + final void otherVisitorImpl(SymbolType st, string name, size_t line, size_t col) + { + count++; + auto result = construct!Symbol; + result.name = name; + result.line = line; + result.col = col; + result.type = st; + parent.subs ~= result; + } + + final override void visit(const AliasDeclaration decl) + { + // why is initializers an array ? + if (decl.initializers.length > 0) + namedVisitorImpl!(AliasInitializer, SymbolType._alias)(decl.initializers[0]); + } + + final override void visit(const AnonymousEnumDeclaration decl) + { + if (decl.members.length > 0) + namedVisitorImpl!(AnonymousEnumMember, SymbolType._enum)(decl.members[0]); + } + + final override void visit(const ClassDeclaration decl) + { + namedVisitorImpl!(ClassDeclaration, SymbolType._class)(decl); + } + + final override void visit(const Constructor decl) + { + otherVisitorImpl(SymbolType._function, "this", decl.line, decl.column); + } + + final override void visit(const Destructor decl) + { + otherVisitorImpl(SymbolType._function, "~this", decl.line, decl.column); + } + + final override void visit(const EnumDeclaration decl) + { + namedVisitorImpl!(EnumDeclaration, SymbolType._enum)(decl); + } + + final override void visit(const EponymousTemplateDeclaration decl) + { + namedVisitorImpl!(EponymousTemplateDeclaration, SymbolType._template)(decl); + } + + final override void visit(const FunctionDeclaration decl) + { + namedVisitorImpl!(FunctionDeclaration, SymbolType._function)(decl); + } + + final override void visit(const InterfaceDeclaration decl) + { + namedVisitorImpl!(InterfaceDeclaration, SymbolType._interface)(decl); + } + + final override void visit(const ImportDeclaration decl) + { + foreach(const(SingleImport) si; decl.singleImports) + { + if (!si.identifierChain.identifiers.length) + continue; + // + string[] modules; + foreach(ident; si.identifierChain.identifiers) + { + modules ~= ident.text; + modules ~= "."; + } + // + otherVisitorImpl(SymbolType._import, modules[0..$-1].join, + si.identifierChain.identifiers[0].line, + si.identifierChain.identifiers[0].column + ); + } + } + + final override void visit(const MixinTemplateDeclaration decl) + { + namedVisitorImpl!(TemplateDeclaration, SymbolType._mixin)(decl.templateDeclaration); + } + + final override void visit(const StructDeclaration decl) + { + namedVisitorImpl!(StructDeclaration, SymbolType._struct)(decl); + } + + final override void visit(const TemplateDeclaration decl) + { + namedVisitorImpl!(TemplateDeclaration, SymbolType._template)(decl); + } + + final override void visit(const UnionDeclaration decl) + { + namedVisitorImpl!(UnionDeclaration, SymbolType._union)(decl); + } + + final override void visit(const VariableDeclaration decl) + { + foreach(elem; decl.declarators) + namedVisitorImpl!(Declarator, SymbolType._variable, false)(elem); + } + + final override void visit(const StaticConstructor decl) + { + otherVisitorImpl(SymbolType._function, "static this", decl.line, decl.column); + } + + final override void visit(const StaticDestructor decl) + { + otherVisitorImpl(SymbolType._function, "static ~this", decl.line, decl.column); + } + } } @@ -21,18 +304,28 @@ private: LexerConfig config; StringCache strcache; Module mod; - CachedInfos cachedInfos; bool scanned; + + CachedInfos cachedInfos; string modName; + ubyte[] jsonErrors; + ubyte[] pasErrors; + ubyte[] todosPas; + ubyte[] todosJson; + ubyte[] symsPas; + ubyte[] symsJson; + static AstError*[] errors; final static void parserError(string fname, size_t line, size_t col, string msg, bool isErr) { + errors ~= new AstError(line, col, msg, isErr); } final void resetCachedInfo() { cachedInfos = 0; - modName = ""; + modName = modName.init; + errors = errors.init; } final void scan() @@ -49,27 +342,30 @@ public: this(string filename) { - mixin(logcall); - - import std.file; fname = filename; - src = cast(ubyte[]) read(fname, size_t.max); strcache = StringCache(StringCache.defaultBucketCount); - scan; + rescanFile(); } this(ubyte[] buffer) { - mixin(logcall); - src = buffer.dup; strcache = StringCache(StringCache.defaultBucketCount); + rescanBuffer(buffer); + } + + final void rescanFile() + { + resetCachedInfo; + import std.file; + src = cast(ubyte[]) read(fname, size_t.max); scan; } - final void rescan() + final void rescanBuffer(ubyte[] buffer) { - mixin(logcall); resetCachedInfo; + src = buffer.dup; + scan; } final string moduleName() @@ -78,10 +374,10 @@ public: if (!scanned) return result; - if (AstInfos.moduleName in cachedInfos) + if (AstInfos.ModuleName in cachedInfos) return modName; - cachedInfos += AstInfos.moduleName; + cachedInfos += AstInfos.ModuleName; if (mod.moduleDeclaration) foreach(Token t; mod.moduleDeclaration.moduleName.identifiers) result ~= t.text ~ "."; @@ -90,4 +386,40 @@ public: modName = result[0 .. $-1]; return modName; } + + final ubyte[] todoListPas() + { + return null; + } + + final ubyte[] todoListJson() + { + return null; + } + + final ubyte[] symbolListPas() + { + if (AstInfos.SymsPas !in cachedInfos) + { + cachedInfos += AstInfos.SymsPas; + SymbolListBuilder slb = construct!SymbolListBuilder(mod); + scope(exit) destruct(slb); + symsPas = cast(ubyte[]) slb.serializePas(); + } + return symsPas; + } + + final ubyte[] symbolListJson() + { + if (AstInfos.SymsJson !in cachedInfos) + { + cachedInfos += AstInfos.SymsJson; + SymbolListBuilder slb = construct!SymbolListBuilder(mod); + scope(exit) destruct(slb); + JSONValue v = slb.serializeJson(); + symsJson = cast(ubyte[]) v.toPrettyString; + } + return symsJson; + } + } diff --git a/cedast/src/cedast.d b/cedast/src/cedast.d index 8ebba4dc..f8649ef3 100644 --- a/cedast/src/cedast.d +++ b/cedast/src/cedast.d @@ -1,6 +1,7 @@ module cedast; import core.runtime, common, ast; +import iz.memory; __gshared Ast*[] modules; @@ -12,7 +13,7 @@ AstToken scanFile(char* filename) { import std.string: fromStringz; import std.algorithm: countUntil; - Ast* ast = new Ast(filename.fromStringz.idup); + Ast* ast = construct!Ast(filename.fromStringz.idup); result = countUntil(modules, null); if (result == -1) { @@ -40,7 +41,7 @@ AstToken scanBuffer(ubyte* buffer, size_t len) try { import std.algorithm: countUntil; - Ast* ast = new Ast(buffer[0 .. len]); + Ast* ast = construct!Ast(buffer[0 .. len]); result = countUntil(modules, null); if (result == -1) { @@ -62,11 +63,19 @@ AstToken scanBuffer(ubyte* buffer, size_t len) } extern(C) export -void rescan(AstToken tok) +void rescanFile(AstToken tok) { if (tok < 1 || tok > modules.length) return; - modules[tok - 1].rescan; + modules[tok - 1].rescanFile; +} + +extern(C) export +void rescanBuffer(AstToken tok, ubyte* buffer, size_t len) +{ + if (tok < 1 || tok > modules.length) + return; + modules[tok - 1].rescanBuffer(buffer[0 .. len]); } extern(C) export @@ -77,7 +86,7 @@ void unleash(AstToken tok) return; if (modules[tok - 1] == null) return; - delete modules[tok - 1]; + destruct(modules[tok - 1]); modules[tok - 1] = null; if (tok == modules.length) modules.length -= 1; @@ -98,15 +107,39 @@ immutable(char*) moduleName(AstToken tok) extern(C) export ubyte* symbolList(AstToken tok, ref size_t len, SerializationFormat fmt) { - ubyte* result = null; - return result; + if (tok < 1 || tok > modules.length) + return null; + Ast* mod = modules[tok - 1]; + if (mod == null) + return null; + + ubyte[] result; + if (fmt == SerializationFormat.json) + result = mod.symbolListJson; + else + result = mod.symbolListPas; + + len = result.length; + return result.ptr; } extern(C) export ubyte* todoList(AstToken tok, ref size_t len, SerializationFormat fmt) { - ubyte* result = null; - return result; + if (tok < 1 || tok > modules.length) + return null; + Ast* mod = modules[tok - 1]; + if (mod == null) + return null; + + ubyte[] result; + if (fmt == SerializationFormat.json) + result = mod.todoListJson; + else + result = mod.todoListPas; + + len = result.length; + return result.ptr; } version(Windows) @@ -119,7 +152,7 @@ version(Windows) extern (Windows) BOOL DllMain(HINSTANCE hInstance, ULONG ulReason, LPVOID pvReserved) { - switch (ulReason) + final switch (ulReason) { case DLL_PROCESS_ATTACH: Runtime.initialize; @@ -139,11 +172,7 @@ version(Windows) case DLL_THREAD_DETACH: dll_thread_detach( true, true ); break; - - default: - assert(0); } - return true; } } diff --git a/cedast/src/common.d b/cedast/src/common.d index c4c2cfb8..405a4804 100644 --- a/cedast/src/common.d +++ b/cedast/src/common.d @@ -4,7 +4,11 @@ alias AstToken = ptrdiff_t; __gshared immutable AstToken invalidAstToken = 0; -enum SerializationFormat {json, pascal} +enum SerializationFormat : byte +{ + json, + pascal +} enum logcall = q{ import std.file; diff --git a/lazproj/cedast_loader.pas b/lazproj/cedast_loader.pas index cab26af7..de0b0496 100644 --- a/lazproj/cedast_loader.pas +++ b/lazproj/cedast_loader.pas @@ -6,23 +6,35 @@ type TAstToken = NativeInt; - TScanFile = function(filename: PChar): TAstToken; cdecl; - TScanBuffer = function (buffer: PByte; len: NativeUint): TAstToken; cdecl; - TRescan = procedure (tok: TAstToken); cdecl; - TUnleash = procedure (tok: TAstToken); cdecl; - TModuleName = function (tok: TAstToken): PChar; cdecl; + {$Z1} + TSerializationFormat = (json, pas); + + TScanFile = function(filename: PChar): TAstToken; cdecl; + TScanBuffer = function(buffer: PByte; len: NativeUint): TAstToken; cdecl; + TRescanFile = procedure(tok: TAstToken); cdecl; + TRescanBuffer = procedure(tok: TAstToken; buffer: PByte; len: NativeUint); cdecl; + TUnleash = procedure(tok: TAstToken); cdecl; + + TModuleName = function(tok: TAstToken): PChar; cdecl; + TSymbolList = function(tok: TAstToken; var len: NativeUint ; fmt: TSerializationFormat): PByte; cdecl; + + var dast: TLibHandle; scanfile: TScanFile; scanbuffer: TScanBuffer; - rescan: TRescan; + rescanfile: TRescanFile; + rescanbuffer: TRescanBuffer; unleash: TUnleash; moduleName: TModuleName; + symlist: TSymbolList; tok: TAstToken; + len: NativeUint = 0; + ptr: PByte; const - testModule = 'module a.b.c.d.e.f.g.h; import std.stdio;'; + testModule = 'module a.b.c.d.e.f.g.h; import std.stdio; uint a; struct F{long c;}'; begin @@ -35,18 +47,32 @@ begin if scanFile = nil then writeln('invalid scanfile proc ptr') else tok := scanfile(PChar('exception in call so ticket value is 0')); - rescan := TRescan(GetProcAddress(dast, 'rescan')); - if rescan = nil then writeln('invalid rescan proc ptr') - else rescan(tok); + rescanfile := TRescanFile(GetProcAddress(dast, 'rescanFile')); + if rescanfile = nil then writeln('invalid rescanFile proc ptr') + else rescanfile(tok); scanbuffer := TScanBuffer(GetProcAddress(dast, 'scanBuffer')); if scanbuffer = nil then writeln('invalid scanBuffer proc ptr') else tok := scanbuffer(@testModule[1], length(testModule)); + rescanbuffer := TRescanBuffer(GetProcAddress(dast, 'rescanBuffer')); + if rescanbuffer = nil then writeln('invalid rescanBuffer proc ptr') + else rescanbuffer(tok, @testmodule[1], length(testModule)); + moduleName := TModuleName(GetProcAddress(dast, 'moduleName')); if moduleName = nil then writeln('invalid moduleName proc ptr') else if tok <> 0 then writeln(moduleName(tok)); + symlist := TSymbolList(GetProcAddress(dast, 'symbolList')); + if symlist = nil then writeln('invalid symbolList proc ptr') + else if tok <> 0 then with TMemoryStream.Create do try + ptr := symlist(tok, len, TSerializationFormat.json); + write(ptr^, len); + SaveToFile('testsymlist.txt'); + finally + free; + end; + unleash := TUnleash(GetProcAddress(dast, 'unleash')); if unleash = nil then writeln('invalid unleash proc ptr') else unleash(tok);