diff --git a/cedast/build/cedast.coedit b/cedast/build/cedast.coedit deleted file mode 100644 index c029f46c..00000000 --- a/cedast/build/cedast.coedit +++ /dev/null @@ -1,36 +0,0 @@ -object CurrentProject: TCENativeProject - OptionsCollection = < - item - name = 'win32-dbg' - debugingOptions.debug = True - debugingOptions.codeviewDexts = True - messagesOptions.additionalWarnings = True - outputOptions.binaryKind = sharedlib - pathsOptions.outputFilename = '../bin/cedast.so' - end - item - name = 'linux-dbg' - debugingOptions.debug = True - debugingOptions.codeviewDexts = True - messagesOptions.additionalWarnings = True - messagesOptions.tlsInformations = True - outputOptions.binaryKind = obj - pathsOptions.outputFilename = '../bin/cedast.o' - otherOptions.customOptions.Strings = ( - '-fPIC' - ) - postBuildProcess.executable = 'sh' - postBuildProcess.parameters.Strings = ( - '/nux-postbuild.sh' - ) - end> - Sources.Strings = ( - '../src/cedast.d' - '../src/common.d' - '../src/ast.d' - ) - ConfigurationIndex = 1 - LibraryAliases.Strings = ( - '*' - ) -end diff --git a/cedast/build/nux-postbuild.sh b/cedast/build/nux-postbuild.sh deleted file mode 100644 index ee838ad9..00000000 --- a/cedast/build/nux-postbuild.sh +++ /dev/null @@ -1 +0,0 @@ -dmd ../bin/cedast.o /home/basile/Dev/dproj/Iz/lib/iz.a /home/basile/Dev/metad/libs/libdparse.a -of../bin/cedast.so -shared -defaultlib=libphobos2.so diff --git a/cedast/src/ast.d b/cedast/src/ast.d deleted file mode 100644 index 704a970a..00000000 --- a/cedast/src/ast.d +++ /dev/null @@ -1,446 +0,0 @@ -module ast; - -import dparse.lexer, dparse.parser, dparse.ast; -import std.json, std.array, std.conv, std.parallelism, std.concurrency; -import iz.enumset, iz.memory; - -import common; - -private -{ - enum AstInfos - { - ModuleName, - ErrorsJson, ErrorsPas, - SymsJson, SymsPas, - TodosJson, TodosPas - } - - alias CachedInfos = EnumSet!(AstInfos, Set8); - - enum SymbolType - { - _alias, - _class, - _enum, - _error, - _function, - _interface, - _import, - _mixin, // (template decl) - _struct, - _template, - _union, - _variable, - _warning - } - - struct Symbol - { - size_t line; - size_t col; - string name; - SymbolType type; - Symbol * [] subs; - - ~this() - { - foreach_reverse(i; 0 .. subs.length) - subs[i].destruct; - } - - void serializePas(ref Appender!string lfmApp) - { - lfmApp.put("\ritem\r"); - - lfmApp.put(format("line = %d\r", line)); - lfmApp.put(format("col = %d\r", col)); - lfmApp.put(format("name = '%s'\r", name)); - lfmApp.put(format("symType = %s\r", type)); - - lfmApp.put("subs = <"); - if (subs.length) foreach(Symbol * sub; subs) - sub.serializePas(lfmApp); - lfmApp.put(">\r"); - lfmApp.put("end"); - } - - void serializeJson(ref JSONValue json) - { - auto vobj = parseJSON("{}"); - vobj["line"]= JSONValue(line); - vobj["col"] = JSONValue(col); - vobj["name"]= JSONValue(name); - vobj["type"]= JSONValue(to!string(type)); - if (subs.length) - { - auto vsubs = parseJSON("[]"); - foreach(Symbol * sub; subs) - sub.serializeJson(vsubs); - vobj["items"] = vsubs; - } - json.array ~= vobj; - } - } - - struct AstError - { - size_t line, col; - string msg; - bool isErr; - } - - class SymbolListBuilder : ASTVisitor - { - Symbol * root; - Symbol * parent; - - size_t count; - - alias visit = ASTVisitor.visit; - - this(Module mod) - { - root = construct!Symbol; - resetRoot; - foreach(Declaration d; mod.declarations) - visit(d); - } - - ~this() - { - root.destruct; - } - - final void resetRoot(){parent = root;} - - final string serializePas() - { - Appender!string lfmApp; - lfmApp.reserve(count * 64); - - lfmApp.put("object TSymbolList\rsymbols = <"); - foreach(sym; root.subs) sym.serializePas(lfmApp); - lfmApp.put(">\rend\r\n"); - - return lfmApp.data; - } - - final JSONValue serializeJson() - { - JSONValue result = parseJSON("{}"); - JSONValue vsubs = parseJSON("[]"); - foreach(sym; root.subs) sym.serializeJson(vsubs); - result["items"] = vsubs; - return result; - } - - /// returns a new symbol if the declarator is based on a Token named "name". - final Symbol * addDeclaration(DT)(DT adt) - { - static if - ( - is(DT == const(EponymousTemplateDeclaration)) || - is(DT == const(AnonymousEnumMember)) || - is(DT == const(AliasInitializer)) || - is(DT == const(ClassDeclaration)) || - is(DT == const(Declarator)) || - is(DT == const(EnumDeclaration)) || - is(DT == const(FunctionDeclaration)) || - is(DT == const(InterfaceDeclaration)) || - is(DT == const(StructDeclaration)) || - is(DT == const(TemplateDeclaration)) || - is(DT == const(UnionDeclaration)) - - ) - { - count++; - auto result = construct!Symbol; - result.name = adt.name.text; - result.line = adt.name.line; - result.col = adt.name.column; - parent.subs ~= result; - return result; - } - - version(none) assert(0, "addDeclaration no implemented for " ~ DT.stringof); - } - - /// visitor implementation if the declarator is based on a Token named "name". - final void namedVisitorImpl(DT, SymbolType st, bool dig = true)(const(DT) dt) - { - auto newSymbol = addDeclaration(dt); - newSymbol.type = st; - // - static if (dig) - { - auto previousParent = parent; - scope(exit) parent = previousParent; - parent = newSymbol; - dt.accept(this); - } - } - - /// visitor implementation for special cases. - final void otherVisitorImpl(SymbolType st, string name, size_t line, size_t col) - { - count++; - auto result = construct!Symbol; - result.name = name; - result.line = line; - result.col = col; - result.type = st; - parent.subs ~= result; - } - - final override void visit(const AliasDeclaration decl) - { - // why is initializers an array ? - if (decl.initializers.length > 0) - namedVisitorImpl!(AliasInitializer, SymbolType._alias)(decl.initializers[0]); - } - - final override void visit(const AnonymousEnumDeclaration decl) - { - if (decl.members.length > 0) - namedVisitorImpl!(AnonymousEnumMember, SymbolType._enum)(decl.members[0]); - } - - final override void visit(const ClassDeclaration decl) - { - namedVisitorImpl!(ClassDeclaration, SymbolType._class)(decl); - } - - final override void visit(const Constructor decl) - { - otherVisitorImpl(SymbolType._function, "this", decl.line, decl.column); - } - - final override void visit(const Destructor decl) - { - otherVisitorImpl(SymbolType._function, "~this", decl.line, decl.column); - } - - final override void visit(const EnumDeclaration decl) - { - namedVisitorImpl!(EnumDeclaration, SymbolType._enum)(decl); - } - - final override void visit(const EponymousTemplateDeclaration decl) - { - namedVisitorImpl!(EponymousTemplateDeclaration, SymbolType._template)(decl); - } - - final override void visit(const FunctionDeclaration decl) - { - namedVisitorImpl!(FunctionDeclaration, SymbolType._function)(decl); - } - - final override void visit(const InterfaceDeclaration decl) - { - namedVisitorImpl!(InterfaceDeclaration, SymbolType._interface)(decl); - } - - final override void visit(const ImportDeclaration decl) - { - foreach(const(SingleImport) si; decl.singleImports) - { - if (!si.identifierChain.identifiers.length) - continue; - // - string[] modules; - foreach(ident; si.identifierChain.identifiers) - { - modules ~= ident.text; - modules ~= "."; - } - // - otherVisitorImpl(SymbolType._import, modules[0..$-1].join, - si.identifierChain.identifiers[0].line, - si.identifierChain.identifiers[0].column - ); - } - } - - final override void visit(const MixinTemplateDeclaration decl) - { - namedVisitorImpl!(TemplateDeclaration, SymbolType._mixin)(decl.templateDeclaration); - } - - final override void visit(const StructDeclaration decl) - { - namedVisitorImpl!(StructDeclaration, SymbolType._struct)(decl); - } - - final override void visit(const TemplateDeclaration decl) - { - namedVisitorImpl!(TemplateDeclaration, SymbolType._template)(decl); - } - - final override void visit(const UnionDeclaration decl) - { - namedVisitorImpl!(UnionDeclaration, SymbolType._union)(decl); - } - - final override void visit(const VariableDeclaration decl) - { - foreach(elem; decl.declarators) - namedVisitorImpl!(Declarator, SymbolType._variable, false)(elem); - } - - final override void visit(const StaticConstructor decl) - { - otherVisitorImpl(SymbolType._function, "static this", decl.line, decl.column); - } - - final override void visit(const StaticDestructor decl) - { - otherVisitorImpl(SymbolType._function, "static ~this", decl.line, decl.column); - } - } -} - - -class Ast -{ - -private: - - ubyte[] src; - string fname; - LexerConfig config; - StringCache strcache; - Module mod; - AstNotification notif; - void* notifparam; - bool scanned; - - CachedInfos cachedInfos; - string modName; - ubyte[] jsonErrors; - ubyte[] pasErrors; - ubyte[] todosPas; - ubyte[] todosJson; - ubyte[] symsPas; - ubyte[] symsJson; - __gshared static AstError*[] errors; - - final static void parserError(string fname, size_t line, size_t col, string msg, bool isErr) - { - errors ~= new AstError(line, col, msg, isErr); - } - - final void resetCachedInfo() - { - cachedInfos = 0; - modName = modName.init; - jsonErrors = jsonErrors.init; - pasErrors = pasErrors.init; - todosPas = todosPas.init; - todosJson = todosJson.init; - symsPas = symsPas.init; - symsJson = symsJson.init; - errors = errors.init; - } - - final void taskScan() - { - config = LexerConfig(fname, StringBehavior.source, WhitespaceBehavior.skip); - mod = parseModule(getTokensForParser(src, config, &strcache), fname, null, &parserError); - if (notif) notif(notifparam); - scanned = true; - - } - -public: - - this() - { - strcache = StringCache(StringCache.defaultBucketCount); - } - - final void scanFile(string filename) - { - resetCachedInfo; - fname = filename; - import std.file; - try src = cast(ubyte[]) read(fname, size_t.max); - catch(Exception e){} - scanned = false; - version(Windows)task(&taskScan).executeInNewThread; - else taskScan; - } - - final void scanBuffer(ubyte[] buffer) - { - resetCachedInfo; - src = buffer.dup; - scanned = false; - version(Windows) task(&taskScan).executeInNewThread; - else taskScan; - } - - @property AstNotification notification(){return notif;} - @property void notification(AstNotification value){notif = value;} - - @property void* notificationParameter(){return notifparam;} - @property void notificationParameter(void* value){notifparam = value;} - - final string moduleName() - { - if (scanned && AstInfos.ModuleName !in cachedInfos) - { - string result; - cachedInfos += AstInfos.ModuleName; - if (mod.moduleDeclaration) - foreach(Token t; mod.moduleDeclaration.moduleName.identifiers) - result ~= t.text ~ "."; - if (result.length) - modName = result[0 .. $-1].idup; - } - return modName; - } - - final ubyte[] todoListPas() - { - if (scanned && AstInfos.TodosPas !in cachedInfos) - { - } - return todosPas; - } - - final ubyte[] todoListJson() - { - if (scanned && AstInfos.TodosJson !in cachedInfos) - { - } - return todosJson; - } - - final ubyte[] symbolListPas() - { - if (scanned && AstInfos.SymsPas !in cachedInfos) - { - cachedInfos += AstInfos.SymsPas; - SymbolListBuilder slb = construct!SymbolListBuilder(mod); - scope(exit) destruct(slb); - symsPas = cast(ubyte[]) slb.serializePas().dup; - } - return symsPas; - } - - final ubyte[] symbolListJson() - { - if (scanned && AstInfos.SymsJson !in cachedInfos) - { - cachedInfos += AstInfos.SymsJson; - SymbolListBuilder slb = construct!SymbolListBuilder(mod); - scope(exit) destruct(slb); - JSONValue v = slb.serializeJson(); - symsJson = cast(ubyte[]) v.toPrettyString; - } - return symsJson; - } - -} - diff --git a/cedast/src/cedast.d b/cedast/src/cedast.d deleted file mode 100644 index 562a8dc6..00000000 --- a/cedast/src/cedast.d +++ /dev/null @@ -1,165 +0,0 @@ -module cedast; - -import core.runtime, common, ast; -import iz.memory; - -__gshared Ast[] modules; -__gshared bool init; - -void tryInit() -{ - if (init) return; - Runtime.initialize; - init = true; -} - -extern(C) export -AstHandle newAst(void* param, AstNotification clbck) -{ - version(linux) tryInit; - AstHandle result; - try - { - import std.algorithm: countUntil; - Ast ast = construct!Ast; - ast.notification = clbck; - ast.notificationParameter = param; - result = countUntil(modules, null); - if (result == -1) - { - modules ~= ast; - result = modules.length; - } - else - { - modules[result] = ast; - ++result; - } - } - catch(Exception e) - { - if (result != 0) deleteAst(result); - result = invalidAstHandle; - } - return result; -} - -extern(C) export -void deleteAst(AstHandle hdl) -{ - if (hdl < 1 || hdl > modules.length) - return; - if (modules[hdl - 1] is null) - return; - - destruct(modules[hdl - 1]); - modules[hdl - 1] = null; - if (hdl == modules.length) - modules.length -= 1; -} - -extern(C) export -void scanFile(AstHandle hdl, char* filename) -{ - if (hdl < 1 || hdl > modules.length) - return; - if (modules[hdl - 1] is null) - return; - - import std.string: fromStringz; - modules[hdl - 1].scanFile(fromStringz(filename).idup); -} - -extern(C) export -void scanBuffer(AstHandle hdl, ubyte* buffer, size_t len) -{ - if (hdl < 1 || hdl > modules.length) - return; - if (modules[hdl - 1] is null) - return; - - modules[hdl - 1].scanBuffer(buffer[0 .. len]); -} - -extern(C) export -immutable(char*) moduleName(AstHandle hdl) -{ - if (hdl < 1 || hdl > modules.length) - return null; - if (modules[hdl - 1] is null) - return null; - - import std.string: toStringz; - return toStringz(modules[hdl - 1].moduleName); -} - -extern(C) export -ubyte* symbolList(AstHandle hdl, ref size_t len, SerializationFormat fmt) -{ - if (hdl < 1 || hdl > modules.length) - return null; - if (modules[hdl - 1] is null) - return null; - - ubyte[] result; - if (fmt == SerializationFormat.json) - result = modules[hdl - 1].symbolListJson; - else - result = modules[hdl - 1].symbolListPas; - - len = result.length; - return result.ptr; -} - -extern(C) export -ubyte* todoList(AstHandle hdl, ref size_t len, SerializationFormat fmt) -{ - if (hdl < 1 || hdl > modules.length) - return null; - if (modules[hdl - 1] is null) - return null; - - ubyte[] result; - if (fmt == SerializationFormat.json) - result = modules[hdl - 1].todoListJson; - else - result = modules[hdl - 1].todoListPas; - - len = result.length; - return result.ptr; -} - -version(Windows) -{ - import core.sys.windows.windows; - import core.sys.windows.dll; - - __gshared HINSTANCE g_hInst; - - extern (Windows) - BOOL DllMain(HINSTANCE hInstance, ULONG ulReason, LPVOID pvReserved) - { - final switch (ulReason) - { - case DLL_PROCESS_ATTACH: - tryInit; - g_hInst = hInstance; - dll_process_attach( hInstance, true ); - break; - - case DLL_PROCESS_DETACH: - Runtime.terminate; - dll_process_detach( hInstance, true ); - break; - - case DLL_THREAD_ATTACH: - dll_thread_attach( true, true ); - break; - - case DLL_THREAD_DETACH: - dll_thread_detach( true, true ); - break; - } - return true; - } -} diff --git a/cedast/src/common.d b/cedast/src/common.d deleted file mode 100644 index cdb171e3..00000000 --- a/cedast/src/common.d +++ /dev/null @@ -1,15 +0,0 @@ -module common; - -extern(C): - -alias AstHandle = ptrdiff_t; - -alias AstNotification = void function(void* param); - -__gshared immutable AstHandle invalidAstHandle = 0; - -enum SerializationFormat : byte -{ - json, - pascal -} diff --git a/dastworx/dastworx.ce b/dastworx/dastworx.ce new file mode 100644 index 00000000..9dabf8fa --- /dev/null +++ b/dastworx/dastworx.ce @@ -0,0 +1,29 @@ +object CurrentProject: TCENativeProject + OptionsCollection = < + item + name = 'devel' + outputOptions.inlining = True + outputOptions.boundsCheck = offAlways + outputOptions.optimizations = True + outputOptions.release = True + outputOptions.versionIdentifiers.Strings = ( + 'devel' + ) + pathsOptions.outputFilename = '../bin/dastworx' + runOptions.options = [poUsePipes, poStderrToOutPut] + end> + Sources.Strings = ( + 'src/main.d' + 'src/todos.d' + 'src/symlist.d' + 'src/imports.d' + 'src/mainfun.d' + 'src/common.d' + 'src/runnableflags.d' + ) + ConfigurationIndex = 0 + LibraryAliases.Strings = ( + 'iz' + 'libdparse' + ) +end diff --git a/dastworx/dastworx.cegrp b/dastworx/dastworx.cegrp new file mode 100644 index 00000000..ae71c63d --- /dev/null +++ b/dastworx/dastworx.cegrp @@ -0,0 +1,13 @@ +object _1: TProjectGroup + items = < + item + filename = '../../../dproj/iz/iz.coedit' + end + item + filename = '../../../metad/projects/libdparse.coedit' + end + item + filename = 'dastworx.ce' + end> + index = 0 +end diff --git a/dastworx/readme.md b/dastworx/readme.md new file mode 100644 index 00000000..acb72023 --- /dev/null +++ b/dastworx/readme.md @@ -0,0 +1,5 @@ +dastworx +======== + +_D AST works_ is a tool designed to simple tasks on the AST of a module. +In development, will replace _cesyms_ and _cetodo_ from version 3 beta 1. \ No newline at end of file diff --git a/dastworx/src/common.d b/dastworx/src/common.d new file mode 100644 index 00000000..e6fea690 --- /dev/null +++ b/dastworx/src/common.d @@ -0,0 +1,231 @@ +module common; + +import + std.array, std.traits, std.meta, std.conv; +import + dparse.lexer, dparse.ast, dparse.parser, dparse.rollback_allocator; +import + iz.memory; + +enum ErrorType: ubyte +{ + warning, + error +} + +@NoInit @NoGc struct AstError +{ + ErrorType type; + @NoGc string message; + size_t line, column; + + @disable this(); + + this(ErrorType type, string message, size_t line, size_t column) @nogc @safe + { + this.type = type; + this.message = message; + this.line = line; + this.column = column; + } +} + +alias AstErrors = AstError*[]; + +@nogc @safe unittest +{ + AstError* err = construct!(AstError)(ErrorType.warning, "warning", 0, 0); + assert(err); + assert(err.type == ErrorType.warning); + assert(err.message == "warning"); + assert(err.column == 0); + assert(err.line == 0); + destruct(err); +} + +enum logCall = +q{ + import std.experimental.logger: log; + version(devel) log(); + else version(unittest) log(); +}; + +// TODO: define accurately the list of bad versions +version(linux) + enum badVersions = ["none", "Windows", "Win32", "Win64", "OSX"]; +else version(Windows) + enum badVersions = ["none", "linux", "OSX", "Posix", + "FreeBSD", "Solaris"]; +else version(OSX) + enum badVersions = ["none", "linux", "Windows", "Win32", "Win64"]; +else static assert(0); + +/** + * Make a D string compatible with an Object Pascal string literal. + */ +string patchPascalString(string value) +{ + Appender!string app; + app.reserve(value.length); + bool skip; + foreach (immutable i; 0..value.length) + { + char c = value[i]; + if (c > 0x7F) + { + app ~= value[i]; + skip = true; + } + else if (c == '\'') + { + if (skip) + app ~= value[i]; + else + app ~= "'#39'"; + skip = false; + } + else + { + app ~= value[i]; + skip = false; + } + } + return app.data; +} + +/// Used to annotate a public field that is written in `pascalStreaming()`. +enum Pascal; + +/** + * Streams a class or a struct using the Object Pascal streaming format, as defined + * in FPC's FCL or Delphi's RTL. + */ +void pascalStreaming(T, string[][] enumLuts = [[""]], bool bin = false)(auto ref T t, + ref Appender!string stream) +if (is(T == struct) || is(T == class)) +{ + + // TODO: find speification of the Pascal binary format + static if (bin) {} + else + { + stream ~= "object "; + stream ~= T.stringof; + stream ~= "\r"; + } + + foreach(member; __traits(allMembers, T)) + { + static if (is(typeof(__traits(getMember, t, member))) && + hasUDA!(__traits(getMember, t, member), Pascal)) + { + alias MT = typeof(__traits(getMember, t, member)); + alias TestBasicTypes = templateOr!(isSomeString, isIntegral, + isFloatingPoint, isBoolean,); + + static if (is(MT == class) || is(MT == struct)) + { + pascalStreaming!(MT, enumLuts, bin)(__traits(getMember, t, member), stream); + } + else static if (is(MT == enum)) + { + import std.range: iota; + bool done; + static if (isIntegral!(OriginalType!MT)) + foreach (i; aliasSeqOf!(iota(0,enumLuts.length))) + { + static if (enumLuts[i].length == MT.max+1) + { + static if (bin) {} + else + { + stream ~= member; + stream ~= " = "; + stream ~= enumLuts[i][__traits(getMember, t, member)]; + stream ~= "\r"; + } + done = true; + break; + } + } + if (!done) + { + static if (bin) {} + else + { + stream ~= member; + stream ~= " = "; + static if (isSomeString!MT) + stream ~= "'"; + stream ~= to!string(__traits(getMember, t, member)); + static if (isSomeString!MT) + stream ~= "'"; + stream ~= "\r"; + } + } + } + else static if (TestBasicTypes!MT) + { + static if (bin) {} + else + { + stream ~= member; + stream ~= " = "; + stream ~= to!string(__traits(getMember, t, member)); + stream ~= "\r"; + } + } + else static assert(0); + } + } + + static if (bin) {} + else + { + stream ~= "end\r"; + } +} + +unittest +{ + enum Bar{bar} + + static struct TRat + { + int notaprop1 = 0; + @Pascal ubyte subProperty1 = 1; + @Pascal string subProperty2 = "pascal"; + } + + static class TFoo + { + int notaprop1 = 0; + @Pascal ubyte property1 = 1; + @Pascal string property2 = "pascal"; + @Pascal Bar property3 = Bar.bar; + @Pascal TRat rat; + } + + Appender!string stream; + pascalStreaming!(TFoo, [["bar"]])(new TFoo, stream); + assert(stream.data != ""); +} + +/** + * Produces and visits the AST for a source code. + * + * This function is used to handle the content of a MixinExpression in an + * ASTVisitor. + */ +T parseAndVisit(T : ASTVisitor)(const(char)[] source) +{ + RollbackAllocator allocator; + LexerConfig config = LexerConfig("", StringBehavior.source, WhitespaceBehavior.skip); + StringCache cache = StringCache(StringCache.defaultBucketCount); + const(Token)[] tokens = getTokensForParser(cast(ubyte[]) source, config, &cache); + Module mod = parseModule(tokens, "", &allocator); + T result = construct!(T); + result.visit(mod); + return result; +} + diff --git a/dastworx/src/imports.d b/dastworx/src/imports.d new file mode 100644 index 00000000..a757dbf7 --- /dev/null +++ b/dastworx/src/imports.d @@ -0,0 +1,77 @@ +module imports; + +import + std.stdio, std.algorithm, std.array; +import + iz.memory; +import + dparse.lexer, dparse.ast, dparse.parser; +import + common; + +/** + * Lists the modules imported b y a module + * + * Each import is written in a new line. Import detection is not accurate, + * the imports injected by a mixin template or by a string variable are not detected, + * the imports deactivated by a static condition neither. + * + * The results are used by to detect which are the static libraries used by a + * runnable module. + */ +void listImports(const(Module) mod) +in +{ + assert(mod); +} +body +{ + mixin(logCall); + construct!(ImportLister).visit(mod); +} + +private final class ImportLister: ASTVisitor +{ + alias visit = ASTVisitor.visit; + size_t mixinDepth; + + override void visit(const ConditionalDeclaration decl) + { + const VersionCondition ver = decl.compileCondition.versionCondition; + if (ver is null || !canFind(badVersions, ver.token.text)) + decl.accept(this); + } + + override void visit(const(ImportDeclaration) decl) + { + foreach (const(SingleImport) si; decl.singleImports) + { + if (!si.identifierChain.identifiers.length) + continue; + si.identifierChain.identifiers.map!(a => a.text).join(".").writeln; + } + if (decl.importBindings) with (decl.importBindings.singleImport) + identifierChain.identifiers.map!(a => a.text).join(".").writeln; + } + + override void visit(const(MixinExpression) mix) + { + ++mixinDepth; + mix.accept(this); + --mixinDepth; + } + + override void visit(const PrimaryExpression primary) + { + if (mixinDepth && primary.primary.type.isStringLiteral) + { + assert(primary.primary.text.length > 1); + + size_t startIndex = 1; + startIndex += primary.primary.text[0] == 'q'; + parseAndVisit!(ImportLister)(primary.primary.text[startIndex..$-1]); + } + primary.accept(this); + } +} + diff --git a/dastworx/src/main.d b/dastworx/src/main.d new file mode 100644 index 00000000..842d7f84 --- /dev/null +++ b/dastworx/src/main.d @@ -0,0 +1,149 @@ +module ceasttools; + +import + core.memory; +import + std.array, std.getopt, std.stdio, std.path, std.algorithm; +import + iz.memory; +import + dparse.lexer, dparse.parser, dparse.ast, dparse.rollback_allocator; +import + common, todos, symlist, imports, mainfun, runnableflags; + + +private __gshared bool storeAstErrors = void, deepSymList = true; +private __gshared const(Token)[] tokens; +private __gshared Module module_ = void; +private __gshared static Appender!(ubyte[]) source; +private __gshared RollbackAllocator allocator; +private __gshared LexerConfig config; +private __gshared static Appender!(AstErrors) errors; +private __gshared string[] files; + + +static this() +{ + GC.disable; + source.reserve(1024^^2); + errors.reserve(32); +} + +void main(string[] args) +{ + version(devel) + { + mixin(logCall); + File f = File(__FILE__, "r"); + foreach(buffer; f.byChunk(4096)) + source.put(buffer); + f.close; + } + else + { + foreach(buffer; stdin.byChunk(4096)) + source.put(buffer); + } + + if (args.length > 2) + files = args[1].splitter(pathSeparator).array; + + config = LexerConfig("", StringBehavior.source, WhitespaceBehavior.skip); + StringCache cache = StringCache(StringCache.defaultBucketCount); + tokens = getTokensForParser(source.data, config, &cache); + + getopt(args, std.getopt.config.passThrough, + "d", &deepSymList + ); + + getopt(args, std.getopt.config.passThrough, + "i", &handleImportsOption, + "m", &handleMainfunOption, + "r", &handleRunnableFlags, + "s", &handleSymListOption, + "t", &handleTodosOption, + ); +} + +void handleSymListOption() +{ + mixin(logCall); + bool deep; + storeAstErrors = true; + parseTokens; + listSymbols(module_, errors.data, deepSymList); +} + +void handleTodosOption() +{ + mixin(logCall); + const(Token)[]*[] tokensArray; + if (tokens.length) + tokensArray ~= &tokens; + + import std.file: exists; + if (files.length) + { + StringCache cache = StringCache(StringCache.defaultBucketCount); + foreach(fname; files) + if (fname.exists) + { + try + { + File f = File(fname, "r"); + ubyte[] src; + foreach(buffer; f.byChunk(4096)) + src ~= buffer; + //tokensArray ~= getTokensForParser(src, config, &cache); + f.close; + } + catch (Exception e) continue; + } + } + getTodos(tokensArray); +} + +void handleRunnableFlags() +{ + mixin(logCall); + getRunnableFlags(tokens); +} + +void handleImportsOption() +{ + mixin(logCall); + storeAstErrors = false; + parseTokens; + listImports(module_); +} + +void handleMainfunOption() +{ + mixin(logCall); + storeAstErrors = false; + parseTokens; + detectMainFun(module_); +} + +void handleErrors(string fname, size_t line, size_t col, string message, bool err) +{ + if (storeAstErrors) + errors ~= construct!(AstError)(cast(ErrorType) err, message, line, col); +} + +void parseTokens() +{ + mixin(logCall); + if (!module_) + module_ = parseModule(tokens, "", &allocator, &handleErrors); +} + +version(devel) +{ + version(none) import std.compiler; + version(all) import std.uri; + mixin(q{import std.c.time;}); + + //TODO: something +} + diff --git a/dastworx/src/mainfun.d b/dastworx/src/mainfun.d new file mode 100644 index 00000000..87b4fc5b --- /dev/null +++ b/dastworx/src/mainfun.d @@ -0,0 +1,54 @@ +module mainfun; + +import + std.stdio, std.algorithm; +import + iz.memory; +import + dparse.lexer, dparse.ast, dparse.parser; +import + common; + +/** + * Detects wether a main function is declared in a module. + * + * Writes "1" if a main is found otherwise "0". The detection is not accurate, + * if the main is injected by a mixin template or by a string it is not detected, + * if the main is deactivated by a static condition neither. + * + * The result is used by to detect if the "-main" switch has to be passed to + * the compiler. + */ +void detectMainFun(const(Module) mod) +{ + mixin(logCall); + MainFunctionDetector mfd = construct!(MainFunctionDetector); + mfd.visit(mod); + write(mfd.hasMain); +} + +private final class MainFunctionDetector: ASTVisitor +{ + alias visit = ASTVisitor.visit; + + ubyte hasMain; + + this() + { + hasMain = false; + } + + override void visit(const ConditionalDeclaration decl) + { + const VersionCondition ver = decl.compileCondition.versionCondition; + if (ver is null || !canFind(badVersions, ver.token.text)) + decl.accept(this); + } + + override void visit(const(FunctionDeclaration) decl) + { + if (decl.name.text == "main") + hasMain = true; + } +} + diff --git a/dastworx/src/runnableflags.d b/dastworx/src/runnableflags.d new file mode 100644 index 00000000..b1fb08ad --- /dev/null +++ b/dastworx/src/runnableflags.d @@ -0,0 +1,20 @@ +module runnableflags; + +import + std.stdio; +import + dparse.lexer; +import + common; + +/** + * Parse the compiler switch defined in the comments located before a + * ModuleDeclaration and that are passed to the compiler when a runnable is + * launched. + * + * each line of the soutput contains an option. + */ +void getRunnableFlags(const(Token)[] tokens) +{ + mixin(logCall); +} diff --git a/dastworx/src/symlist.d b/dastworx/src/symlist.d new file mode 100644 index 00000000..e867f8a0 --- /dev/null +++ b/dastworx/src/symlist.d @@ -0,0 +1,351 @@ +module symlist; + +import + std.stdio, std.array, std.traits, std.conv, std.json, std.format, + std.algorithm; +import + iz.memory; +import + dparse.lexer, dparse.ast, dparse.parser; +import + common; + +private __gshared bool deep = void; + +/** + * Serializes the symbols in the standard output + */ +void listSymbols(const(Module) mod, AstErrors errors, bool deep = true) +{ + mixin(logCall); + symlist.deep = deep; + alias SL = SymbolListBuilder!(ListFmt.Pas); + SL.addAstErrors(errors); + SL sl = construct!(SL); + sl.visit(mod); + sl.serialize.writeln; +} + +enum ListFmt +{ + Pas, + Json +} + +enum SymbolType +{ + _alias, + _class, + _enum, + _error, + _function, + _interface, + _import, + _mixin, // (template decl) + _struct, + _template, + _union, + _unittest, + _variable, + _warning +} + +string makeSymbolTypeArray() +{ + string result = "string[SymbolType.max + 1] symbolTypeStrings = ["; + foreach(st; EnumMembers!SymbolType) + result ~= `"` ~ to!string(st) ~ `",`; + result ~= "];"; + return result; +} + +mixin(makeSymbolTypeArray); + +class SymbolListBuilder(ListFmt Fmt): ASTVisitor +{ + static if (Fmt == ListFmt.Pas) + { + static Appender!string pasStream; + } + else + { + static JSONValue json; + static JSONValue* jarray; + } + + static uint utc; + + alias visit = ASTVisitor.visit; + + static this() + { + static if (Fmt == ListFmt.Pas) + { + pasStream.put("object TSymbolList\rsymbols = <"); + } + else + { + json = parseJSON("[]"); + jarray = &json; + } + } + + static void addAstErrors(AstErrors errors) + { + foreach(error; errors) + { + string type = (error.type == ErrorType.error) ? + symbolTypeStrings[SymbolType._error] : + symbolTypeStrings[SymbolType._warning]; + static if (Fmt == ListFmt.Pas) + { + pasStream.put("\ritem\r"); + pasStream.put(format("line = %d\r", error.line)); + pasStream.put(format("col = %d\r", error.column)); + pasStream.put(format("name = '%s'\r", patchPascalString(error.message))); + pasStream.put(format("symType = %s\r", type)); + pasStream.put("end"); + } + else + { + JSONValue item = parseJSON("{}"); + item["line"] = JSONValue(error.line); + item["col"] = JSONValue(error.column); + item["name"] = JSONValue(error.message); + item["type"] = JSONValue(type); + jarray.array ~= item; + } + } + } + + final string serialize() + { + static if (Fmt == ListFmt.Pas) + { + pasStream.put(">\rend\r\n"); + return pasStream.data; + } + else + { + JSONValue result = parseJSON("{}"); + result["items"] = json; + version (assert) + return result.toPrettyString; + else + return result.toString; + } + } + + /// visitor implementation if the declaration has a "name". + final void namedVisitorImpl(DT, SymbolType st, bool dig = true)(const(DT) dt) + if (__traits(hasMember, DT, "name")) + { + static if (Fmt == ListFmt.Pas) + { + pasStream.put("\ritem\r"); + pasStream.put(format("line = %d\r", dt.name.line)); + pasStream.put(format("col = %d\r", dt.name.column)); + pasStream.put(format("name = '%s'\r", dt.name.text)); + pasStream.put("symType = " ~ symbolTypeStrings[st] ~ "\r"); + static if (dig) if (deep) + { + pasStream.put("subs = <"); + dt.accept(this); + pasStream.put(">\r"); + } + pasStream.put("end"); + } + else + { + JSONValue item = parseJSON("{}"); + item["line"] = JSONValue(dt.name.line); + item["col"] = JSONValue(dt.name.column); + item["name"] = JSONValue(dt.name.text); + item["type"] = JSONValue(symbolTypeStrings[st]); + static if (dig) if (deep) + { + JSONValue subs = parseJSON("[]"); + JSONValue* old = jarray; + jarray = &subs; + dt.accept(this); + item["items"] = subs; + jarray = old; + } + json.array ~= item; + } + } + + /// visitor implementation for special cases. + final void otherVisitorImpl(DT, bool dig = true) + (const(DT) dt, SymbolType st, string name, size_t line, size_t col) + { + static if (Fmt == ListFmt.Pas) + { + pasStream.put("\ritem\r"); + pasStream.put(format("line = %d\r", line)); + pasStream.put(format("col = %d\r", col)); + pasStream.put(format("name = '%s'\r", name)); + pasStream.put("symType = " ~ symbolTypeStrings[st] ~ "\r"); + static if (dig) + { + pasStream.put("subs = <"); + dt.accept(this); + pasStream.put(">\r"); + } + pasStream.put("end"); + } + else + { + JSONValue item = parseJSON("{}"); + item["line"] = JSONValue(line); + item["col"] = JSONValue(col); + item["name"] = JSONValue(name); + item["type"] = JSONValue(symbolTypeStrings[st]); + static if (dig) + { + JSONValue subs = parseJSON("[]"); + JSONValue* old = jarray; + jarray = &subs; + dt.accept(this); + item["items"] = subs; + jarray = old; + } + json.array ~= item; + } + } + + final override void visit(const AliasDeclaration decl) + { + if (decl.initializers.length) + namedVisitorImpl!(AliasInitializer, SymbolType._alias)(decl.initializers[0]); + } + + final override void visit(const AnonymousEnumMember decl) + { + namedVisitorImpl!(AnonymousEnumMember, SymbolType._enum)(decl); + } + + final override void visit(const AnonymousEnumDeclaration decl) + { + decl.accept(this); + } + + final override void visit(const AutoDeclaration decl) + { + if (decl.identifiers.length) + { + otherVisitorImpl(decl, SymbolType._variable, decl.identifiers[0].text, + decl.identifiers[0].line, decl.identifiers[0].column); + } + } + + final override void visit(const ClassDeclaration decl) + { + namedVisitorImpl!(ClassDeclaration, SymbolType._class)(decl); + } + + final override void visit(const Constructor decl) + { + otherVisitorImpl(decl, SymbolType._function, "ctor", decl.line, decl.column); + } + + final override void visit(const Destructor decl) + { + otherVisitorImpl(decl, SymbolType._function, "dtor", decl.line, decl.column); + } + + final override void visit(const EnumDeclaration decl) + { + namedVisitorImpl!(EnumDeclaration, SymbolType._enum)(decl); + } + + final override void visit(const EponymousTemplateDeclaration decl) + { + namedVisitorImpl!(EponymousTemplateDeclaration, SymbolType._template)(decl); + } + + final override void visit(const FunctionDeclaration decl) + { + namedVisitorImpl!(FunctionDeclaration, SymbolType._function)(decl); + } + + final override void visit(const InterfaceDeclaration decl) + { + namedVisitorImpl!(InterfaceDeclaration, SymbolType._interface)(decl); + } + + final override void visit(const ImportDeclaration decl) + { + foreach (const(SingleImport) si; decl.singleImports) + { + if (!si.identifierChain.identifiers.length) + continue; + + otherVisitorImpl(decl, SymbolType._import, + si.identifierChain.identifiers.map!(a => a.text).join("."), + si.identifierChain.identifiers[0].line, + si.identifierChain.identifiers[0].column); + } + if (decl.importBindings) with (decl.importBindings.singleImport) + otherVisitorImpl(decl, SymbolType._import, + identifierChain.identifiers.map!(a => a.text).join("."), + identifierChain.identifiers[0].line, + identifierChain.identifiers[0].column); + } + + final override void visit(const MixinTemplateDeclaration decl) + { + namedVisitorImpl!(TemplateDeclaration, SymbolType._mixin)(decl.templateDeclaration); + } + + final override void visit(const StructDeclaration decl) + { + namedVisitorImpl!(StructDeclaration, SymbolType._struct)(decl); + } + + final override void visit(const TemplateDeclaration decl) + { + namedVisitorImpl!(TemplateDeclaration, SymbolType._template)(decl); + } + + final override void visit(const UnionDeclaration decl) + { + namedVisitorImpl!(UnionDeclaration, SymbolType._union)(decl); + } + + final override void visit(const Unittest decl) + { + otherVisitorImpl(decl, SymbolType._unittest, format("test%.4d",utc++), + decl.line, decl.column); + } + + final override void visit(const VariableDeclaration decl) + { + if (decl.declarators) + foreach (elem; decl.declarators) + namedVisitorImpl!(Declarator, SymbolType._variable, false)(elem); + else if (decl.autoDeclaration) + visit(decl.autoDeclaration); + } + + final override void visit(const StaticConstructor decl) + { + otherVisitorImpl(decl, SymbolType._function, "static ctor", decl.line, decl.column); + } + + final override void visit(const StaticDestructor decl) + { + otherVisitorImpl(decl, SymbolType._function, "static dtor", decl.line, decl.column); + } + + final override void visit(const SharedStaticConstructor decl) + { + otherVisitorImpl(decl, SymbolType._function, "shared static ctor", decl.line, decl.column); + } + + final override void visit(const SharedStaticDestructor decl) + { + otherVisitorImpl(decl, SymbolType._function, "shared static dtor", decl.line, decl.column); + } +} + diff --git a/dastworx/src/todos.d b/dastworx/src/todos.d new file mode 100644 index 00000000..19f69a5d --- /dev/null +++ b/dastworx/src/todos.d @@ -0,0 +1,127 @@ +module todos; + +import + std.stdio, std.string, std.algorithm, std.array, std.conv, std.traits, + std.ascii, std.range; +import + dparse.lexer; +import + common; + +private __gshared Appender!string stream; + +void getTodos(const(Token)[]*[] tokensArray) +{ + mixin(logCall); + stream.reserve(2^^16); + stream.put("object TTodoItems\r items = <"); + assert(tokensArray.length); + foreach(tokens; tokensArray) + //foreach(token; (*tokens).filter!((a) => a.type == tok!"comment")()) + foreach(token; *tokens) + if (token.type == tok!"comment") + token.analyze; + stream.put(">\rend\r\n"); + writeln(stream.data); +} + +private void analyze(const(Token) token) +{ + string text = token.text.strip.patchPascalString; + string identifier; + + mixin(logCall); + + // always comment + text.popFrontN(2); + if (text.empty) + return; + // ddoc suffix + if (text.front.among('/', '*', '+')) + { + text.popFront; + if (text.empty) + return; + } + // leading whites + while (text.front.isWhite) + { + text.popFront; + if (text.empty) + return; + } + + // "TODO|FIXME|NOTE" + bool isTodoComment; + while (!text.empty) + { + identifier ~= std.ascii.toUpper(text.front); + text.popFront; + if (identifier.among("TODO","FIXME, NOTE")) + { + isTodoComment = true; + break; + } + } + if (!isTodoComment) return; + identifier = ""; + + // splits "fields" and "description" + bool isWellFormed; + string fields; + while (!text.empty) + { + auto front = text.front; + identifier ~= front; + text.popFront; + if (front == ':') + { + if (identifier.length) fields = identifier; + isWellFormed = text.length > 0; + break; + } + } + if (!isWellFormed) return; + identifier = ""; + + // parses "fields" + string a, c, p, s; + while (!fields.empty) + { + const dchar front = fields.front; + fields.popFront; + if ((front == '-' || fields.empty) && identifier.length > 2) + { + string fieldContent = identifier[2..$].strip; + switch(identifier[0..2].toUpper) + { + default: break; + case "-A": a = fieldContent; break; + case "-C": c = fieldContent; break; + case "-P": p = fieldContent; break; + case "-S": s = fieldContent; break; + } + identifier = ""; + } + identifier ~= front; + } + + if (text.length > 1 && text[$-2..$].among("*/", "+/")) + text.length -=2; + + + + stream.put("item\r"); + //stream.put(format("filename = '%s'\r", fname)); + stream.put(format("line = '%d'\r", token.line)); + stream.put(format("text = '%s'\r", text)); + if (c.length) + stream.put(format("category = '%s'\r", c)); + if (a.length) + stream.put(format("assignee = '%s'\r", a)); + if (p.length) + stream.put(format("priority = '%s'\r", p)); + if (s.length) + stream.put(format("status = '%s'\r", s)); + stream.put("end\r"); +} diff --git a/src/ce_projgroup.pas b/src/ce_projgroup.pas index ad9358fc..fad7ed68 100644 --- a/src/ce_projgroup.pas +++ b/src/ce_projgroup.pas @@ -13,6 +13,9 @@ type TProjectGroup = class; + //TODO-projectgroups: bug, load a free standing project, load a group that contains a link to the FSP. + //=> the FSP should be either closed or the lazy loader should trap the FSP + (** * Represents a project in a project group *) @@ -129,6 +132,7 @@ var constructor TProjectGroup.create(aOwner: TComponent); begin inherited; + Name := 'projectGroup'; fItems := TCollection.Create(TProjectGroupItem); fItems.FPOAttachObserver(self); EntitiesConnector.addSingleService(self);