diff --git a/.gitignore b/.gitignore index 50fdfc56..8ddbc4ac 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ public dcd d-scanner extract_last_changelog_part +dexed-d/.dub +dub.selections.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ad3e4f8..7c64eb8d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,14 +21,13 @@ release: GIT_SUBMODULE_STRATEGY: normal before_script: - apt-get update -y - - apt-get install -y dpkg - apt-get install -y rpm - apt-get install -y git - apt-get install -y zip - apt-get install -y libcurl4-openssl-dev - - curl -JLO https://sourceforge.net/projects/lazarus/files/Lazarus%20Linux%20amd64%20DEB/Lazarus%202.0.6/fpc-laz_3.0.4-1_amd64.deb/download && apt install -y ./fpc-laz_3.0.4-1_amd64.deb - - curl -JLO https://sourceforge.net/projects/lazarus/files/Lazarus%20Linux%20amd64%20DEB/Lazarus%202.0.6/fpc-src_3.0.4-2_amd64.deb/download && apt install -y ./fpc-src_3.0.4-2_amd64.deb - - curl -JLO https://sourceforge.net/projects/lazarus/files/Lazarus%20Linux%20amd64%20DEB/Lazarus%202.0.6/lazarus-project_2.0.6-0_amd64.deb/download && apt install -y ./lazarus-project_2.0.6-0_amd64.deb + - curl -JLO https://sourceforge.net/projects/lazarus/files/Lazarus%20Linux%20amd64%20DEB/Lazarus%202.0.8/fpc-laz_3.0.4-1_amd64.deb/download && apt install -y ./fpc-laz_3.0.4-1_amd64.deb + - curl -JLO https://sourceforge.net/projects/lazarus/files/Lazarus%20Linux%20amd64%20DEB/Lazarus%202.0.8/fpc-src_3.0.4-2_amd64.deb/download && apt install -y ./fpc-src_3.0.4-2_amd64.deb + - curl -JLO https://sourceforge.net/projects/lazarus/files/Lazarus%20Linux%20amd64%20DEB/Lazarus%202.0.8/lazarus-project_2.0.8-0_amd64.deb/download && apt install -y ./lazarus-project_2.0.8-0_amd64.deb script: - bash setup/build-release.sh artifacts: diff --git a/CHANGELOG.md b/CHANGELOG.md index 16de4fa0..0ad39077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# v3.9.0-dev + +## Other + +- Toolchain: removed he background tool _dastworx_ and replaced it with a library called _libdexed-d_. Although this will not change the user experience: + - Thousands of system calls to create the process and read its streams are saved. + - ddemangle not required anymore. + - crash in the new library will be fatal, i.e the IDE will have to be relaunched, while previously the tool was launched again, without significant impact on the IDE. + # v3.8.4 ## Bugs fixed diff --git a/dastworx/build.bat b/dastworx/build.bat deleted file mode 100644 index 1ea9dd23..00000000 --- a/dastworx/build.bat +++ /dev/null @@ -1,32 +0,0 @@ -:: D compiler and arch -if "%dc%"=="" set dc=dmd -if "%mflags%"=="" set mflags=-m32 - -::iz sources -set iz= -for /r "../etc/iz/import/" %%F in (*.d) do call set iz=%%iz%% "%%F" - -::dparse sources -set dparse= -for /r "../etc/libdparse/src/" %%F in (*.d) do call set dparse=%%dparse%% "%%F" - -::stdxalloc sources -set stdxalloc= -for /r "../etc/stdx-allocator/source/" %%F in (*.d) do call set stdxalloc=%%stdxalloc%% "%%F" - -::dast sources -set dast= -for /r "src/" %%F in (*.d) do call set dast=%%dast%% "%%F" - -echo building... - -::build -%dc% %dast% %dparse% %iz% %stdxalloc% ^ --O -release -inline -boundscheck=off %mflags% ^ --Isrc -I"..\etc\iz\import" -I"..\etc\libdparse\src" ^ -I"..\etc\stdx-allocator\source" ^ --of"..\bin\dastworx.exe" - -::cleanup -del ..\bin\dastworx.obj - -echo ...done diff --git a/dastworx/build.sh b/dastworx/build.sh deleted file mode 100755 index f6279929..00000000 --- a/dastworx/build.sh +++ /dev/null @@ -1,38 +0,0 @@ -if [[ -z "$DC" ]]; then DC=dmd; fi -if [[ "$DC" == "ldc" ]]; then DC=ldmd2; fi -if [[ "$DC" == "ldc2" ]]; then DC=ldmd2; fi -if [[ "$DC" == "gdc" ]]; then DC=gdmd; fi -if [[ -z "$MFLAGS" ]]; then MFLAGS=-m64; fi - -#iz sources -cd ../etc/iz/import/ -iz=$(find `pwd` -type f -name \*.d) -cd ../../../dastworx - -#dparse sources -cd ../etc/libdparse/src/ -dparse=$(find `pwd` -type f -name \*.d) -cd ../../../dastworx - -#stdx-alloc sources -cd ../etc/stdx-allocator/source/ -stdxalloc=$(find `pwd` -type f -name \*.d) -cd ../../../dastworx - -#dast sources -cd src/ -dast=$(find `pwd` -type f -name \*.d) -cd ../ - -echo building dastworx using $DC... - -#build -$DC ${dast[@]} ${dparse[@]} ${iz[@]} ${stdxalloc[@]} \ --O -release -inline -boundscheck=off $MFLAGS \ --Isrc -I../etc/iz/import -I../etc/libdparse/src -I../etc/stdx-allocator/source \ --of../bin/dastworx - -#cleanup -rm ../bin/dastworx.o - -echo ...done diff --git a/dastworx/dastworx.dprj b/dastworx/dastworx.dprj deleted file mode 100644 index b761a672..00000000 --- a/dastworx/dastworx.dprj +++ /dev/null @@ -1,52 +0,0 @@ -object CurrentProject: TCENativeProject - OptionsCollection = < - item - name = 'devel' - debugingOptions.generateInfos = True - outputOptions.boundsCheck = onAlways - outputOptions.versionIdentifiers.Strings = ( - 'devel' - ) - pathsOptions.outputFilename = '../bin/dastworx' - pathsOptions.extraSources.Strings = ( - '../etc/iz/import/*' - '../etc/libdparse/src/*' - '../etc/stdx-allocator/source/*' - ) - pathsOptions.importModulePaths.Strings = ( - '../etc/iz/import' - '../etc/libdparse/src' - '../etc/stdx-allocator/source' - ) - runOptions.options = [poUsePipes, poStderrToOutPut] - end - item - name = 'release' - outputOptions.boundsCheck = offAlways - outputOptions.optimizations = True - outputOptions.release = True - pathsOptions.outputFilename = '../bin/dastworx' - pathsOptions.extraSources.Strings = ( - '../etc/iz/import/*' - '../etc/libdparse/src/*' - '../etc/stdx-allocator/source/*' - ) - pathsOptions.importModulePaths.Strings = ( - '../etc/iz/import' - '../etc/libdparse/src' - '../etc/stdx-allocator/source' - ) - runOptions.options = [poUsePipes, poStderrToOutPut] - end> - Sources.Strings = ( - 'src/main.d' - 'src/todos.d' - 'src/symlist.d' - 'src/imports.d' - 'src/mainfun.d' - 'src/common.d' - 'src/halstead.d' - 'src/ddoc_template.d' - ) - ConfigurationIndex = 1 -end diff --git a/dastworx/readme.md b/dastworx/readme.md deleted file mode 100644 index b1ff3605..00000000 --- a/dastworx/readme.md +++ /dev/null @@ -1,22 +0,0 @@ -## Dastworx - -_D AST works_ is a tool that processes the AST of a D module to extract several information used by dexed. - -It's notably used by the _symbol list_ and the _todo list_ widgets. - -## Build - -If dexed is build manually you certainly have to build _dastworx_ too. -Two options exist. - -#### Using dexed & the submodules - -- If you've cloned this repository, make sure that the submodule are also here with `git submodule update --init`. -- In Dexed open the project `dastworx.dxp`. -- Select the `release` configuration. -- Click `Compile project` - -#### Using the scripts - -- Windows: `build.bat` -- Linux: `sh ./build.sh` diff --git a/dastworx/src/imports.d b/dastworx/src/imports.d deleted file mode 100644 index a8a2af1d..00000000 --- a/dastworx/src/imports.d +++ /dev/null @@ -1,126 +0,0 @@ -module imports; - -import - std.stdio, std.algorithm, std.array, std.file, std.functional; -import - iz.memory; -import - dparse.lexer, dparse.ast, dparse.parser, dparse.rollback_allocator; -import - common; - -/** - * Lists the modules imported by a module - * - * On the first line writes the module name or # between double quotes then - * each import is written in a new line. Import detection is not accurate, - * the imports injected by a mixin template or by a string variable are not detected, - * the imports deactivated by a static condition neither. - * - * The results are used by to detect which are the static libraries used by a - * runnable module. - */ -void listImports(const(Module) mod) -in -{ - assert(mod); -} -body -{ - mixin(logCall); - if (mod.moduleDeclaration) - writeln('"', mod.moduleDeclaration.moduleName.identifiers - .map!(a => a.text).join("."), '"'); - else - writeln("\"#\""); - construct!(ImportLister).visit(mod); -} - -/** - * Lists the modules imported by several modules - * - * The output consists of several consecutive lists, as formated for - * listImports. When no moduleDeclaration is available, the first line of - * a list matches the filename. - * - * The results are used by to detect which are the static libraries used by a - * runnable module. - */ -void listFilesImports(string[] files) -{ - mixin(logCall); - RollbackAllocator allocator; - StringCache cache = StringCache(StringCache.defaultBucketCount); - LexerConfig config = LexerConfig("", StringBehavior.source); - ImportLister il = construct!(ImportLister); - foreach(fname; files) - { - ubyte[] source = cast(ubyte[]) std.file.read(fname); - Module mod = parseModule(getTokensForParser(source, config, &cache), - fname, &allocator, toDelegate(&ignoreErrors)); - if (mod.moduleDeclaration) - writeln('"', mod.moduleDeclaration.moduleName.identifiers - .map!(a => a.text).join("."), '"'); - else - writeln('"', fname, '"'); - il.visit(mod); - } -} - -private final class ImportLister: ASTVisitor -{ - alias visit = ASTVisitor.visit; - size_t mixinDepth; - - override void visit(const(Module) mod) - { - mixinDepth = 0; - mod.accept(this); - } - - override void visit(const ConditionalDeclaration decl) - { - bool acc = true; - if (decl.compileCondition) - { - const ver = decl.compileCondition.versionCondition; - if (ver && ver.token.text in badVersions) - acc = false; - } - if (acc) - decl.accept(this); - } - - override void visit(const(ImportDeclaration) decl) - { - foreach (const(SingleImport) si; decl.singleImports) - { - if (!si.identifierChain.identifiers.length) - continue; - si.identifierChain.identifiers.map!(a => a.text).join(".").writeln; - } - if (decl.importBindings) with (decl.importBindings.singleImport) - identifierChain.identifiers.map!(a => a.text).join(".").writeln; - } - - override void visit(const(MixinExpression) mix) - { - ++mixinDepth; - mix.accept(this); - --mixinDepth; - } - - override void visit(const PrimaryExpression primary) - { - if (mixinDepth && primary.primary.type.isStringLiteral) - { - assert(primary.primary.text.length > 1); - - size_t startIndex = 1; - startIndex += primary.primary.text[0] == 'q'; - parseAndVisit!(ImportLister)(primary.primary.text[startIndex..$-1]); - } - primary.accept(this); - } -} - diff --git a/dastworx/src/main.d b/dastworx/src/main.d deleted file mode 100644 index 32ad14f2..00000000 --- a/dastworx/src/main.d +++ /dev/null @@ -1,156 +0,0 @@ -module dastworx; - -import - core.memory; -import - std.array, std.getopt, std.stdio, std.path, std.algorithm, std.functional, - std.file; -import - iz.memory: construct; -import - iz.options: Argument, ArgFlags, ArgFlag, handleArguments, CantThrow; -import - dparse.lexer, dparse.parser, dparse.ast, dparse.rollback_allocator; -import - common, todos, symlist, imports, mainfun, halstead, ddoc_template; - - -void main(string[] args) -{ - foreach(ref buffer; stdin.byChunk(4096)) - Launcher.source.put(buffer); - handleArguments!(CantThrow, Launcher)(args[1..$]); -} - -struct Launcher -{ - static this() - { - GC.disable; - source.reserve(1024^^2); - } - - __gshared @Argument("-l") int caretLine; - __gshared @Argument("-o") bool option1; - - __gshared Appender!(ubyte[]) source; - __gshared string[] files; - - // -o : deep visit the symbols - // alias deepSymList = option1; - // -o : outputs /++ +/ ddoc instead of /** */ - // alias plusComment = option1; - - /// Writes the list of files to process - @Argument("-f") - static void setFiles(string value) - { - files = value - .splitter(pathSeparator) - .filter!exists - .array; - } - - /// Writes the symbol list - @Argument("-s", "", ArgFlags(ArgFlag.stopper)) - static void handleSymListOption() - { - mixin(logCall); - - static Appender!(AstErrors) errors; - - static void handleErrors(string fname, size_t line, size_t col, string message, bool err) - { - errors ~= construct!(AstError)(cast(ErrorType) err, message, line, col); - } - - RollbackAllocator alloc; - StringCache cache = StringCache(StringCache.defaultBucketCount); - LexerConfig config = LexerConfig("", StringBehavior.source); - - source.data - .getTokensForParser(config, &cache) - .parseModule("", &alloc, &handleErrors) - .listSymbols(errors.data, option1); - } - - /// Writes the list of todo comments - @Argument("-t", "", ArgFlags(ArgFlag.stopper)) - static void handleTodosOption() - { - mixin(logCall); - if (files.length) - getTodos(files); - } - - /// Writes the import list - @Argument("-i", "", ArgFlags(ArgFlag.stopper)) - static void handleImportsOption() - { - mixin(logCall); - if (files.length) - { - listFilesImports(files); - } - else - { - RollbackAllocator alloc; - StringCache cache = StringCache(StringCache.defaultBucketCount); - LexerConfig config = LexerConfig("", StringBehavior.source); - - source.data - .getTokensForParser(config, &cache) - .parseModule("", &alloc, toDelegate(&ignoreErrors)) - .listImports(); - } - } - - /// Writes if a main() is present in the module - @Argument("-m", "", ArgFlags(ArgFlag.stopper)) - static void handleMainfunOption() - { - mixin(logCall); - - RollbackAllocator alloc; - StringCache cache = StringCache(StringCache.defaultBucketCount); - LexerConfig config = LexerConfig("", StringBehavior.source); - - source.data - .getTokensForParser(config, &cache) - .parseModule("", &alloc, toDelegate(&ignoreErrors)) - .detectMainFun(); - } - - /// Writes the halstead metrics - @Argument("-H", "", ArgFlags(ArgFlag.stopper)) - static void handleHalsteadOption() - { - mixin(logCall); - - RollbackAllocator alloc; - StringCache cache = StringCache(StringCache.defaultBucketCount); - LexerConfig config = LexerConfig("", StringBehavior.source); - - source.data - .getTokensForParser(config, &cache) - .parseModule("", &alloc, toDelegate(&ignoreErrors)) - .performHalsteadMetrics; - } - - /// Writes the ddoc template for a given declaration - @Argument("-K", "", ArgFlags(ArgFlag.stopper)) - static void handleDdocTemplateOption() - { - mixin(logCall); - - RollbackAllocator alloc; - StringCache cache = StringCache(StringCache.defaultBucketCount); - LexerConfig config = LexerConfig("", StringBehavior.source); - - source.data - .getTokensForParser(config, &cache) - .parseModule("", &alloc, toDelegate(&ignoreErrors)) - .getDdocTemplate(caretLine, option1); - } -} - diff --git a/dexed-d/dub.json b/dexed-d/dub.json new file mode 100644 index 00000000..1c28966e --- /dev/null +++ b/dexed-d/dub.json @@ -0,0 +1,17 @@ +{ + "name" : "dexed-d", + "targetType" : "dynamicLibrary", + "targetPath" : "../bin", + "targetName" : "dexed-d", + "dependencies" : { + "libdparse" : { + "path" : "../etc/libdparse" + }, + "iz" : { + "path" : "../etc/iz" + } + }, + "dflags" : [ + "-link-defaultlib-shared=false" + ] +} diff --git a/dastworx/src/common.d b/dexed-d/src/common.d similarity index 94% rename from dastworx/src/common.d rename to dexed-d/src/common.d index 47966e4f..ea3c3278 100644 --- a/dastworx/src/common.d +++ b/dexed-d/src/common.d @@ -1,7 +1,9 @@ module common; import - std.array, std.traits, std.meta, std.conv; + core.stdc.string; +import + std.array, std.traits, std.meta, std.conv, std.algorithm, std.file, std.path; import dparse.lexer, dparse.ast, dparse.parser, dparse.rollback_allocator; import @@ -385,7 +387,7 @@ unittest * This function is used to handle the content of a MixinExpression in an * ASTVisitor. */ -T parseAndVisit(T : ASTVisitor)(const(char)[] source) +T parseAndVisit(T : ASTVisitor, A...)(const(char)[] source, A a) { import std.functional; @@ -394,7 +396,7 @@ T parseAndVisit(T : ASTVisitor)(const(char)[] source) StringCache cache = StringCache(StringCache.defaultBucketCount); const(Token)[] tokens = getTokensForParser(cast(ubyte[]) source, config, &cache); Module mod = parseModule(tokens, "", &allocator, toDelegate(&ignoreErrors)); - T result = construct!(T); + T result = construct!(T)(a); result.visit(mod); return result; } @@ -403,6 +405,14 @@ T parseAndVisit(T : ASTVisitor)(const(char)[] source) * By default libdparse outputs errors and warnings to the standard streams. * This function prevents that. */ -void ignoreErrors(string, size_t, size_t, string, bool) @system -{} +void ignoreErrors(string, size_t, size_t, string, bool) @system {} +/** + * Split a C string representing a list of filenames into an array of strings. + * The filenames are separated with the system path separator. + */ +alias joinedFilesToFiles = (const char* a) => a[0 .. a.strlen] + .splitter(pathSeparator) + .filter!exists + .filter!(b => b != "") + .array; diff --git a/dexed-d/src/ddemangle.d b/dexed-d/src/ddemangle.d new file mode 100644 index 00000000..baee5885 --- /dev/null +++ b/dexed-d/src/ddemangle.d @@ -0,0 +1,37 @@ +module ddemangle; + +import core.demangle : demangle; +import std.regex : replaceAll, Captures, regex, Regex; +import core.stdc.string : strlen; +import std.string : toStringz; + +extern(C): + +const(char)* ddemangle(const(char)* line) +{ + __gshared Regex!char reDemangle; + __gshared bool reInit; + if (!reInit) + { + reInit = true; + reDemangle = regex(r"\b_?_D[0-9a-zA-Z_]+\b"); + } + return replaceAll!(demangleMatch)(line[0 .. line.strlen], reDemangle).toStringz; +} + +extern(D): private: + +const(char)[] demangleMatch(Captures!(const(char)[]) m) +{ + // If the second character is an underscore, it may be a D symbol with double leading underscore; + if (m.hit.length > 0 && m.hit[1] != '_') + { + return demangle(m.hit); + } + else + { + auto result = demangle(m.hit[1..$]); + return result == m.hit[1..$] ? m.hit : result; + } +} + diff --git a/dastworx/src/ddoc_template.d b/dexed-d/src/ddoc_template.d similarity index 58% rename from dastworx/src/ddoc_template.d rename to dexed-d/src/ddoc_template.d index 36103f23..5321fc9c 100644 --- a/dastworx/src/ddoc_template.d +++ b/dexed-d/src/ddoc_template.d @@ -1,20 +1,47 @@ module ddoc_template; import - std.stdio; + core.stdc.string; +import + std.array, std.conv, std.string; import iz.memory, iz.sugar; import dparse.ast, dparse.lexer, dparse.parser, dparse.rollback_allocator; +import + common; /** * Finds the declaration at caretLine and write its ddoc template - * in the standard output. + * and returns its DDOC template as a C string. + * + * Params: + * src = the module source code, as a C string. + * caretLine = the line where the declaration is located. + * plusComment = indicates if the template use the "*" or the "+" decoration. */ -void getDdocTemplate(const(Module) mod, int caretLine, bool plusComment) +extern(C) const(char)* ddocTemplate(const(char)* src, int caretLine, bool plusComment) { + LexerConfig config; + RollbackAllocator rba; + StringCache sCache = StringCache(StringCache.defaultBucketCount); + + scope mod = src[0 .. src.strlen] + .getTokensForParser(config, &sCache) + .parseModule("", &rba, &ignoreErrors); + DDocTemplateGenerator dtg = construct!DDocTemplateGenerator(caretLine, plusComment); + scope(exit) destruct(dtg); dtg.visit(mod); + + return dtg.result; +} + +private void putLine(T...)(ref Appender!string a, T t) +{ + static foreach (i; 0 .. T.length) + a.put(t[i].to!string); + a.put("\n"); } final class DDocTemplateGenerator: ASTVisitor @@ -26,6 +53,7 @@ private: immutable int _caretline; immutable char c1; immutable char[2] c2; + Appender!string app; bool _throws; public: @@ -37,6 +65,8 @@ public: c2 = plusComment ? "++" : "**"; } + const(char)* result() { return app.data.toStringz(); } + override void visit(const(ThrowStatement) ts) { _throws = true; @@ -53,26 +83,26 @@ public: if (decl.name.line == _caretline) { decl.accept(this); - writeln("/", c2, "\n ", c1, " \n ", c1, " \n ", c1, " ", c1); + app.putLine("/", c2, "\n ", c1, " \n ", c1, " \n ", c1, " ", c1); const TemplateParameterList tpl = safeAccess(decl).templateParameters.templateParameterList; if ((tpl && tpl.items.length) || (decl.parameters && decl.parameters.parameters.length)) { - writeln(" ", c1, " \n ", c1, " Params:"); + app.putLine(" ", c1, " \n ", c1, " Params:"); if (tpl) { foreach(const TemplateParameter p; tpl.items) { if (p.templateAliasParameter) - writeln(" ", c1, " ", p.templateAliasParameter.identifier.text, " = "); + app.putLine(" ", c1, " ", p.templateAliasParameter.identifier.text, " = "); else if (p.templateTupleParameter) - writeln(" ", c1, " ", p.templateTupleParameter.identifier.text, " = "); + app.putLine(" ", c1, " ", p.templateTupleParameter.identifier.text, " = "); else if (p.templateTypeParameter) - writeln(" ", c1, " ", p.templateTypeParameter.identifier.text, " = "); + app.putLine(" ", c1, " ", p.templateTypeParameter.identifier.text, " = "); else if (p.templateValueParameter) - writeln(" ", c1, " ", p.templateValueParameter.identifier.text, " = "); + app.putLine(" ", c1, " ", p.templateValueParameter.identifier.text, " = "); } } if (decl.parameters) @@ -80,9 +110,9 @@ public: foreach(i, const Parameter p; decl.parameters.parameters) { if (p.name.text != "") - writeln(" ", c1, " ", p.name.text, " = "); + app.putLine(" ", c1, " ", p.name.text, " = "); else - writeln(" ", c1, " __param", i, " = "); + app.putLine(" ", c1, " __param", i, " = "); } } } @@ -90,15 +120,15 @@ public: if (const Type2 tp2 = safeAccess(decl).returnType.type2) { if (tp2.builtinType != tok!"void") - writeln(" ", c1, " \n ", c1, " Returns: "); + app.putLine(" ", c1, " \n ", c1, " Returns: "); } if (_throws) { - writeln(" ", c1, " \n ", c1, " Throws: "); + app.putLine(" ", c1, " \n ", c1, " Throws: "); } - writeln(" ", c1, "/"); + app.putLine(" ", c1, "/"); } else if (decl.name.line > _caretline) @@ -141,26 +171,26 @@ public: if (_caretline == line) { - writeln("/", c2, "\n ", c1, " \n ", c1, " \n ", c1, " ", c1); + app.putLine("/", c2, "\n ", c1, " \n ", c1, " \n ", c1, " ", c1); const TemplateParameterList tpl = safeAccess(decl).templateParameters.templateParameterList; if (tpl && tpl.items.length) { - writeln(" ", c1, " \n ", c1, " Params:"); + app.putLine(" ", c1, " \n ", c1, " Params:"); foreach(const TemplateParameter p; tpl.items) { if (p.templateAliasParameter) - writeln(" ", c1, " ", p.templateAliasParameter.identifier.text, " = "); + app.putLine(" ", c1, " ", p.templateAliasParameter.identifier.text, " = "); else if (p.templateTupleParameter) - writeln(" ", c1, " ", p.templateTupleParameter.identifier.text, " = "); + app.putLine(" ", c1, " ", p.templateTupleParameter.identifier.text, " = "); else if (p.templateTypeParameter) - writeln(" ", c1, " ", p.templateTypeParameter.identifier.text, " = "); + app.putLine(" ", c1, " ", p.templateTypeParameter.identifier.text, " = "); else if (p.templateValueParameter) - writeln(" ", c1, " ", p.templateValueParameter.identifier.text, " = "); + app.putLine(" ", c1, " ", p.templateValueParameter.identifier.text, " = "); } } - writeln(" ", c1, "/"); + app.putLine(" ", c1, "/"); } else if (line > _caretline) @@ -169,61 +199,3 @@ public: } } -version(unittest) -{ - DDocTemplateGenerator parseAndVisit(const(char)[] source, int caretLine, bool p = false) - { - writeln; - RollbackAllocator allocator; - LexerConfig config = LexerConfig("", StringBehavior.source, WhitespaceBehavior.skip); - StringCache cache = StringCache(StringCache.defaultBucketCount); - const(Token)[] tokens = getTokensForParser(cast(ubyte[]) source, config, &cache); - Module mod = parseModule(tokens, "", &allocator); - DDocTemplateGenerator result = construct!(DDocTemplateGenerator)(caretLine, p); - result.visit(mod); - return result; - } -} - -unittest -{ - q{ module a; - void foo(A...)(A a){} - }.parseAndVisit(2, true); -} - -unittest -{ - q{ module a; - void foo()(){} - }.parseAndVisit(2); -} - -unittest -{ - q{ module a; - int foo(int){} - }.parseAndVisit(2, true); -} - -unittest -{ - q{ module a; - class Foo(T, A...){} - }.parseAndVisit(2); -} - -unittest -{ - q{ module a; - struct Foo(alias Fun, A...){} - }.parseAndVisit(2); -} - -unittest -{ - q{ module a; - enum trait(alias Variable) = whatever; - }.parseAndVisit(2); -} - diff --git a/dastworx/src/halstead.d b/dexed-d/src/halstead.d similarity index 95% rename from dastworx/src/halstead.d rename to dexed-d/src/halstead.d index 47ab789f..97dbdedb 100644 --- a/dastworx/src/halstead.d +++ b/dexed-d/src/halstead.d @@ -1,7 +1,9 @@ module halstead; import - std.algorithm, std.conv, std.json, std.meta; + core.stdc.string; +import + std.algorithm, std.conv, std.json, std.meta, std.string; import std.stdio, std.ascii, std.digest.crc, std.range: iota; import @@ -13,14 +15,36 @@ version(unittest){} else import /** * Retrieves the count and unique count of the operands and operators of - * each function (inc. methods) of a module. After the call the results are - * serialized as JSON in te standard output. + * each function (inc. member functions) of a module, allowing the compute + * the halstead complexity of the functions. + * + * Params: + * src = The source code of the module to analyze, as a C string. + * + * Returns: a string representing a JSON array named "functions". + * Each array item is a JSON object containing + * - a function name, named "name" (as JSONString) + * - a line number, named "line" (as JSONNumber) + * - the count of operators, named "n1count" (as JSONNumber) + * - the sum of operators, named "n1sum" (as JSONNumber) + * - the count of operands, named "n2count" (as JSONNumber) + * - the sum of operands, named "n2sum" (as JSONNumber) */ -void performHalsteadMetrics(const(Module) mod) +extern(C) const(char)* halsteadMetrics(const(char)* src) { + LexerConfig config; + RollbackAllocator rba; + StringCache sCache = StringCache(StringCache.defaultBucketCount); + + scope mod = src[0 .. src.strlen] + .getTokensForParser(config, &sCache) + .parseModule("", &rba, &ignoreErrors); + HalsteadMetric hm = construct!(HalsteadMetric); + scope (exit) destruct(hm); + hm.visit(mod); - hm.serialize; + return hm.serialize(); } private struct Function @@ -57,11 +81,6 @@ private final class HalsteadMetric: ASTVisitor void processCallChain() { - version(none) - { - import std.array : join; - writeln("chain: ", chain.map!(a => a.identifier.text).join(".")); - } if (chain.length) { static Token getIdent(const(IdentifierOrTemplateInstance) i) @@ -115,11 +134,11 @@ private final class HalsteadMetric: ASTVisitor inFunctionCallChain.length++; } - void serialize() + const(char)* serialize() { JSONValue js; js["functions"] = fs; - js.toString.write; + return js.toString.toStringz(); } override void visit(const(PragmaExpression)){} diff --git a/dexed-d/src/imports.d b/dexed-d/src/imports.d new file mode 100644 index 00000000..5c4a55e4 --- /dev/null +++ b/dexed-d/src/imports.d @@ -0,0 +1,152 @@ +module imports; + +import + core.stdc.string; +import + std.algorithm, std.array, std.file, std.functional; +import + iz.memory; +import + dparse.lexer, dparse.ast, dparse.parser, dparse.rollback_allocator; +import + common; + +private alias moduleDeclarationToText = (ModuleDeclaration md) => md.moduleName + .identifiers + .map!(a => a.text) + .join("."); + +/** + * Lists the modules imported by a module + * + * On the first line writes the module name or # between double quotes then + * each import is written on a new line. Import detection is not accurate, + * the imports injected by a mixin template or by a string variable are not detected, + * the imports deactivated by a static condition neither. + * + * The results are used by to automatically detect the static libraries used by a + * dexed runnable module. + */ +extern(C) string[] listImports(const(char)* src) +{ + string[] result; + LexerConfig config; + RollbackAllocator rba; + StringCache sCache = StringCache(StringCache.defaultBucketCount); + + scope mod = src[0 .. src.strlen] + .getTokensForParser(config, &sCache) + .parseModule("", &rba, &ignoreErrors); + if (auto md = mod.moduleDeclaration) + result ~= '"' ~ moduleDeclarationToText(md) ~ '"'; + else + result ~= "\"#\""; + + ImportLister il = construct!(ImportLister)(&result); + scope (exit) destruct(il); + + il.visit(mod); + return result; +} + +/** + * Lists the modules imported by several modules + * + * The output consists of several consecutive lists, as formated for + * listImports. When no moduleDeclaration is available, the first line of + * a list matches the filename. + * + * The results are used by to build a key value store linking libraries to other + * libraries, which is part of dexed "libman". + */ +extern(C) string[] listFilesImports(const(char)* joinedFiles) +{ + string[] result; + RollbackAllocator rba; + StringCache sCache = StringCache(StringCache.defaultBucketCount); + LexerConfig config = LexerConfig("", StringBehavior.source); + ImportLister il = construct!(ImportLister)(&result); + + scope(exit) destruct(il); + + foreach(fname; joinedFilesToFiles(joinedFiles)) + { + scope mod = readText(fname) + .getTokensForParser(config, &sCache) + .parseModule("", &rba, &ignoreErrors); + if (auto md = mod.moduleDeclaration) + result ~= '"' ~ moduleDeclarationToText(md) ~ '"'; + else + result ~= '"' ~ cast(string)fname ~ '"'; + + il.visit(mod); + } + + return result; +} + +private final class ImportLister: ASTVisitor +{ + alias visit = ASTVisitor.visit; + size_t mixinDepth; + string[]* results; + + this(string[]* results) + { + assert(results); + this.results = results; + } + + override void visit(const(Module) mod) + { + mixinDepth = 0; + mod.accept(this); + } + + override void visit(const ConditionalDeclaration decl) + { + bool acc = true; + if (decl.compileCondition) + { + const ver = decl.compileCondition.versionCondition; + if (ver && ver.token.text in badVersions) + acc = false; + } + if (acc) + decl.accept(this); + } + + override void visit(const(ImportDeclaration) decl) + { + foreach (const(SingleImport) si; decl.singleImports) + { + if (!si.identifierChain.identifiers.length) + continue; + *results ~= si.identifierChain.identifiers.map!(a => a.text).join("."); + } + if (decl.importBindings) with (decl.importBindings.singleImport) + *results ~= identifierChain.identifiers.map!(a => a.text).join("."); + } + + override void visit(const(MixinExpression) mix) + { + ++mixinDepth; + mix.accept(this); + --mixinDepth; + } + + override void visit(const PrimaryExpression primary) + { + if (mixinDepth && primary.primary.type.isStringLiteral) + { + assert(primary.primary.text.length > 1); + + size_t startIndex = 1; + startIndex += primary.primary.text[0] == 'q'; + auto il = parseAndVisit!(ImportLister)(primary.primary.text[startIndex..$-1], results); + destruct(il); + } + primary.accept(this); + } +} + diff --git a/dastworx/src/mainfun.d b/dexed-d/src/mainfun.d similarity index 61% rename from dastworx/src/mainfun.d rename to dexed-d/src/mainfun.d index c837fc47..bac17098 100644 --- a/dastworx/src/mainfun.d +++ b/dexed-d/src/mainfun.d @@ -5,33 +5,40 @@ import import iz.memory, iz.sugar; import - dparse.lexer, dparse.ast, dparse.parser; + core.stdc.string; +import + dparse.lexer, dparse.parser, dparse.ast, dparse.rollback_allocator; import common; /** - * Detects wether a main function is declared in a module. - * - * Writes "1" if a main is found otherwise "0". The detection is not accurate, - * if the main is injected by a mixin template or by a string it is not detected, - * if the main is deactivated by a static condition neither. - * - * The result is used to determine if the "-main" switch has to be passed to - * the compiler when a runnable module is executed or a module tested. + * Params: + * src = The source code for the module, as a null terminated string. + * Returns: + * wether a module contains the main function. */ -void detectMainFun(const(Module) mod) +extern(C) bool hasMainFun(const(char)* src) { - mixin(logCall); + LexerConfig config; + RollbackAllocator rba; + StringCache sCache = StringCache(StringCache.defaultBucketCount); + + scope mod = src[0 .. src.strlen] + .getTokensForParser(config, &sCache) + .parseModule("", &rba, &ignoreErrors); + MainFunctionDetector mfd = construct!(MainFunctionDetector); + scope (exit) destruct(mfd); + mfd.visit(mod); - write(mfd.hasMain); + return mfd.hasMain; } private final class MainFunctionDetector: ASTVisitor { alias visit = ASTVisitor.visit; - ubyte hasMain; + bool hasMain; override void visit(const ConditionalDeclaration decl) { diff --git a/dastworx/src/symlist.d b/dexed-d/src/symlist.d similarity index 87% rename from dastworx/src/symlist.d rename to dexed-d/src/symlist.d index eb780a71..d24efaa9 100644 --- a/dastworx/src/symlist.d +++ b/dexed-d/src/symlist.d @@ -1,28 +1,53 @@ module symlist; import - std.stdio, std.array, std.traits, std.conv, std.json, std.format, - std.algorithm; + core.stdc.string; import - iz.memory: construct; + std.array, std.traits, std.conv, std.json, std.format, + std.algorithm, std.string; +import + iz.memory: construct, destruct; import iz.containers : Array; import dparse.lexer, dparse.ast, dparse.parser, dparse.formatter : Formatter; +import + dparse.rollback_allocator; import common; /** - * Serializes the symbols in the standard output + * Visit and enumerate all the declaration of a module. + * + * Params: + * src = The module to visit, as source code. + * deep = Defines if the nested declarations are visited. + * Returns: + * The serialized symbols, as a C string. */ -void listSymbols(const(Module) mod, AstErrors errors, bool deep = true) +extern(C) const(char)* listSymbols(const(char)* src, bool deep) { - mixin(logCall); + Appender!(AstErrors) errors; + + void handleErrors(string fname, size_t line, size_t col, string message, bool err) + { + errors ~= construct!(AstError)(cast(ErrorType) err, message, line, col); + } + + LexerConfig config; + RollbackAllocator rba; + StringCache sCache = StringCache(StringCache.defaultBucketCount); + + scope mod = src[0 .. src.strlen] + .getTokensForParser(config, &sCache) + .parseModule("", &rba, &handleErrors); + alias SL = SymbolListBuilder!(ListFmt.Pas); - SL.addAstErrors(errors); - SL sl = construct!(SL)(deep); + SL sl = construct!(SL)(errors, deep); + scope(exit) destruct(sl); + sl.visit(mod); - sl.serialize.writeln; + return sl.serialize(); } private: @@ -68,27 +93,22 @@ final class SymbolListBuilder(ListFmt Fmt): ASTVisitor static if (Fmt == ListFmt.Pas) { - static Appender!string pasStream; + Appender!(char[]) pasStream; } else { - static JSONValue json; - static JSONValue* jarray; + JSONValue json; + JSONValue* jarray; } - static Array!(char) funcNameApp; - static Formatter!(typeof(&funcNameApp)) fmtVisitor; - static uint utc; + Array!char funcNameApp; + Formatter!(Array!char*) fmtVisitor; + uint utc; - this(bool deep) + this(Appender!(AstErrors) errors, bool deep) { _deep = deep; - } - - alias visit = ASTVisitor.visit; - - static this() - { + funcNameApp.length = 0; static if (Fmt == ListFmt.Pas) { pasStream.put("object TSymbolList\rsymbols=<"); @@ -99,9 +119,12 @@ final class SymbolListBuilder(ListFmt Fmt): ASTVisitor jarray = &json; } fmtVisitor = construct!(typeof(fmtVisitor))(&funcNameApp); + addAstErrors(errors.data); } - static void addAstErrors(AstErrors errors) + alias visit = ASTVisitor.visit; + + void addAstErrors(AstErrors errors) { foreach(error; errors) { @@ -129,21 +152,21 @@ final class SymbolListBuilder(ListFmt Fmt): ASTVisitor } } - string serialize() + const(char)* serialize() { static if (Fmt == ListFmt.Pas) { pasStream.put(">\rend"); - return pasStream.data; + return pasStream.data.toStringz; } else { JSONValue result = parseJSON("{}"); result["items"] = json; version (assert) - return result.toPrettyString; + return result.toPrettyString.toStringz; else - return result.toString; + return result.toString.toStringz; } } diff --git a/dastworx/src/todos.d b/dexed-d/src/todos.d similarity index 82% rename from dastworx/src/todos.d rename to dexed-d/src/todos.d index 727690b3..abb48e6b 100644 --- a/dastworx/src/todos.d +++ b/dexed-d/src/todos.d @@ -8,27 +8,27 @@ import import common; -private __gshared Appender!string stream; - -void getTodos(string[] files) +extern(C) const(char)* todoItems(const(char)* joinedFiles) { - mixin(logCall); - stream.reserve(32 + 256 * files.length); + scope Appender!string stream; + stream.reserve(32); stream.put("object TTodoItems\ritems=<"); - foreach(fname; files) + foreach(fname; joinedFilesToFiles(joinedFiles)) { - StringCache cache = StringCache(StringCache.defaultBucketCount); - LexerConfig config = LexerConfig(fname, StringBehavior.source); + stream.reserve(256); + scope StringCache cache = StringCache(StringCache.defaultBucketCount); + scope LexerConfig config = LexerConfig("", StringBehavior.source); ubyte[] source = cast(ubyte[]) std.file.read(fname); - foreach(ref token; DLexer(source, config, &cache).array + foreach(ref token; DLexer(source, config, &cache) + .array .filter!((a) => a.type == tok!"comment")) - analyze(token, fname); + analyze(token, fname, stream); } stream.put(">end"); - writeln(stream.data); + return stream.data.toStringz(); } -private void analyze(const(Token) token, string fname) +private void analyze(const(Token) token, const(char)[] fname, ref Appender!string stream) { string text = token.text.strip.patchPascalString; string identifier; diff --git a/docs/build.md b/docs/build.md index b76c55d2..4abcab9b 100644 --- a/docs/build.md +++ b/docs/build.md @@ -3,16 +3,15 @@ title: Build Dexed header-includes: --- -## Dexed - Dexed is mostly programmed in Object Pascal, using the the [Lazarus development platform](http://www.lazarus-ide.org/). -* [Download](http://lazarus.freepascal.org/index.php?page=downloads) and setup the latest Lazarus version (2.0.0) and FPC + FPC sources (3.0.4) for your platform. +* [Download](http://lazarus.freepascal.org/index.php?page=downloads) and setup the latest Lazarus version (2.0.5) and FPC + FPC sources (3.0.4) for your platform. * Windows: the three packages are bundled in an installer. * Linux: the three packages must be downloaded and setup individually. It's recommended to download the packages from _SourceForge_ and not from the official repository of the distribution because they don't always propose the latest version. +* [Download](https://github.com/ldc-developers/ldc/releases) and setup LDC2, the LLVM-based D compiler. It is used to compile the part of IDE written in D, a library called _libdexed-d_. the binaries must be visible in the system PATH variable. Note that building _libdexed-d_ is automatic. * `cd ` -* `git clone https://github.com/Basile-z/dexed.git` -* `git submodule update --init`, to clone the dependencies used by the background tool. +* `git clone https://gitlab.com/basile.b/dexed.git` +* `git submodule update --init`, to clone the dependencies used by libdexed-d. The Lazarus LCL and the FreePascal FCL may require patches that fix bugs or regressions present in the latest Lazarus release and for which Dexed cannot include workarounds. Any `.patch` file located in the `patches/` folder should be applied. On linux you'll have to set the write permissions to `/usr/lib64/fpc` and `/usr/lib64/lazarus`. @@ -31,19 +30,8 @@ You're now ready to build Dexed. This can be done in the IDE or using the _lazbu * in the **project** menu, click *open...* and select the file **dexed.lpi**, which is located in the sub-folder **lazproj**. * in the menu **Execute** click **Create**. -After what Dexed should be build. The executable is output to the _bin_ folder. - -## Dastworx - -The background tool used by the IDE is a D program. - -* [Download](https://dlang.org/download.html#dmd) and setup latest DMD version. Other D compiler are also supported. For example to compile with LDC, set the encironment variable DC: `DC=ldc2`. -* In the repository, browse to the `dastworx` folder. - * Windows: double click `build.bat` - * Linux: `bash ./build.sh` - -You can also build it in dexed using the project file _dastworx.dprj_. - +After what Dexed should be build. The executable and the library are output to the _bin_ folder. +The library might have to be copied to a specific path, e.g _/lib64/_ under linux. ## Third party tools Additionally you'll have to build [the completion daemon **DCD**](https://github.com/dlang-community/DCD#setup) and the [D linter **Dscanner**](https://github.com/dlang-community/Dscanner#building-and-installing). diff --git a/docs/widgets_symbol_list.md b/docs/widgets_symbol_list.md index 851f7671..8bac0090 100644 --- a/docs/widgets_symbol_list.md +++ b/docs/widgets_symbol_list.md @@ -4,7 +4,6 @@ header-includes: