diff --git a/.gitignore b/.gitignore index 09c7e79..10b0e81 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ /generated GNUmakefile +/.dub +/tests_extractor diff --git a/README.md b/README.md index 4b03060..e63b575 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ dget | Internal | D source code downloader. dman | Public | D documentation lookup tool. dustmite | Public | [Test case minimization tool](https://github.com/CyberShadow/DustMite/wiki). get_dlibcurl32 | Internal | Win32 libcurl downloader/converter. -has_public_example | Internal | Checks public functions for public examples (requires DUB) rdmd | Public | [D build tool](http://dlang.org/rdmd.html). rdmd_test | Internal | rdmd test suite. tests_extractor | Internal | Extracts public unittests (requires DUB) @@ -35,14 +34,16 @@ Running DUB tools ----------------- Some tools require D's package manager DUB. -By default DUB builds a binary and executes it: +By default, DUB builds a binary and executes it. On a Posix system, +the source files can directly be executed with DUB (e.g. `./tests_extractor.d`). +Alternatively, the full single file execution command can be used: ``` -dub --root styles -c has_public_example +dub --single tests_extractor.d ``` Remember that when programs are run via DUB, you need to pass in `--` before -the program's arguments, e.g `dub --root styles -c has_public_example -- -i ../phobos/std/algorithm`. +the program's arguments, e.g `dub --single tests_extractor.d -- -i ../phobos/std/algorithm`. For more information, please see [DUB's documentation][dub-doc]. diff --git a/styles/.gitignore b/styles/.gitignore deleted file mode 100644 index 367bd43..0000000 --- a/styles/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.dub -has_public_example -test_extractor -out diff --git a/styles/dub.sdl b/styles/dub.sdl deleted file mode 100644 index 66b9080..0000000 --- a/styles/dub.sdl +++ /dev/null @@ -1,16 +0,0 @@ -dependency "libdparse" version="~>0.7.0-beta.2" -name "styles" -targetType "executable" -sourceFiles "utils.d" - -configuration "has_public_example" { - name "has_public_example" - targetName "has_public_example" - sourceFiles "has_public_example.d" -} - -configuration "tests_extractor" { - name "test_extractor" - targetName "test_extractor" - sourceFiles "tests_extractor.d" -} diff --git a/styles/dub.selections.json b/styles/dub.selections.json deleted file mode 100644 index 5758f0b..0000000 --- a/styles/dub.selections.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "fileVersion": 1, - "versions": { - "libdparse": "0.7.0-beta.2" - } -} diff --git a/styles/has_public_example.d b/styles/has_public_example.d deleted file mode 100644 index 9b729e0..0000000 --- a/styles/has_public_example.d +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Checks that all functions have a public example - * - * Copyright (C) 2016 by D Language Foundation - * - * Author: Sebastian Wilzbach - * - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) -*/ -// Written in the D programming language. - -import dparse.ast; -import std.algorithm; -import std.experimental.logger; -import std.range; -import std.stdio; -import utils; - -bool hadError; - -class TestVisitor : ASTVisitor -{ - this(string fileName, ubyte[] sourceCode) - { - this.fileName = fileName; - this.sourceCode = sourceCode; - } - - alias visit = ASTVisitor.visit; - - override void visit(const Module mod) - { - Declaration lastDecl; - bool hasPublicUnittest; - - foreach (decl; mod.declarations) - { - if (!isPublic(decl.attributes)) - continue; - - if (decl.functionDeclaration !is null || decl.templateDeclaration !is null) - { - if (lastDecl !is null && - (hasDitto(decl.functionDeclaration) || hasDitto(decl.templateDeclaration))) - continue; - - if (lastDecl !is null && !hasPublicUnittest) - triggerError(lastDecl); - - if (hasDdocHeader(sourceCode, decl)) - lastDecl = cast(Declaration) decl; - else - lastDecl = null; - - hasPublicUnittest = false; - continue; - } - - if (decl.unittest_ !is null) - { - // ignore module header unittest blocks or already validated functions - hasPublicUnittest |= lastDecl is null || hasDdocHeader(sourceCode, decl); - continue; - } - - // ignore dittoed template declarations - if (decl.classDeclaration !is null && hasDitto(decl.classDeclaration) - || decl.structDeclaration !is null && hasDitto(decl.structDeclaration)) - continue; - - // ran into struct or something else -> reset - if (lastDecl !is null && !hasPublicUnittest) - triggerError(lastDecl); - - lastDecl = null; - } - - if (lastDecl !is null && !hasPublicUnittest) - triggerError(lastDecl); - } - -private: - string fileName; - ubyte[] sourceCode; - - void triggerError(const Declaration decl) - { - if (auto fn = decl.functionDeclaration) - stderr.writefln("function %s in %s:%d has no public unittest", fn.name.text, fileName, fn.name.line); - if (auto tpl = decl.templateDeclaration) - stderr.writefln("template %s in %s:%d has no public unittest", tpl.name.text, fileName, tpl.name.line); - hadError = true; - } - - bool hasDitto(Decl)(const Decl decl) - { - if (decl is null) - return false; - - if (decl.comment is null) - return false; - - if (decl.comment.among!("ditto", "Ditto")) - return true; - - return false; - } - - bool isPublic(const Attribute[] attrs) - { - import dparse.lexer : tok; - import std.algorithm.searching : any; - import std.algorithm.iteration : map; - - enum tokPrivate = tok!"private", tokProtected = tok!"protected", tokPackage = tok!"package"; - - if (attrs.map!`a.attribute`.any!(x => x == tokPrivate || x == tokProtected || x == tokPackage)) - return false; - - return true; - } -} - -void parseFile(string fileName) -{ - import dparse.lexer; - import dparse.parser : parseModule; - import dparse.rollback_allocator : RollbackAllocator; - import std.array : uninitializedArray; - - auto inFile = File(fileName); - if (inFile.size == 0) - warningf("%s is empty", inFile.name); - - ubyte[] sourceCode = uninitializedArray!(ubyte[])(to!size_t(inFile.size)); - inFile.rawRead(sourceCode); - LexerConfig config; - auto cache = StringCache(StringCache.defaultBucketCount); - auto tokens = getTokensForParser(sourceCode, config, &cache); - - RollbackAllocator rba; - auto m = parseModule(tokens.array, fileName, &rba); - auto visitor = new TestVisitor(fileName, sourceCode); - visitor.visit(m); -} - -void main(string[] args) -{ - import std.file; - import std.getopt; - import std.path : asNormalizedPath; - - string inputDir; - string ignoredFilesStr; - - auto helpInfo = getopt(args, config.required, - "inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir, - "ignore", "Comma-separated list of files to exclude (partial matching is supported)", &ignoredFilesStr); - - if (helpInfo.helpWanted) - { - return defaultGetoptPrinter(`has_public_example -Searches the input directory recursively to ensure that all public, ddoced functions -have at least one public, ddoced unittest blocks. -`, helpInfo.options); - } - - inputDir = inputDir.asNormalizedPath.array; - - DirEntry[] files; - - if (inputDir.isFile) - { - files = [DirEntry(inputDir)]; - inputDir = "."; - } - else - { - files = dirEntries(inputDir, SpanMode.depth).filter!( - a => a.name.endsWith(".d") && !a.name.canFind(".git")).array; - } - - auto ignoringFiles = ignoredFilesStr.split(","); - - foreach (file; files) - if (!ignoringFiles.any!(x => file.name.canFind(x))) - file.name.parseFile; - - import core.stdc.stdlib : exit; - if (hadError) - exit(1); -} diff --git a/styles/utils.d b/styles/utils.d deleted file mode 100644 index 8470cdf..0000000 --- a/styles/utils.d +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Shared methods between style checkers - * - * Copyright (C) 2016 by D Language Foundation - * - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) -*/ -// Written in the D programming language. - -import dparse.ast; -import std.algorithm; -import std.ascii : whitespace; -import std.conv : to; -import std.experimental.logger; -import std.range; -import std.stdio : File; - -bool hasDdocHeader(const(ubyte)[] sourceCode, const Declaration decl) -{ - import std.algorithm.comparison : min; - - bool hasComment; - size_t firstPos = size_t.max; - - if (decl.unittest_ !is null) - { - firstPos = decl.unittest_.location; - hasComment = decl.unittest_.comment.length > 0; - } - else if (decl.functionDeclaration !is null) - { - // skip the return type - firstPos = sourceCode.skipPreviousWord(decl.functionDeclaration.name.index); - if (auto stClasses = decl.functionDeclaration.storageClasses) - firstPos = min(firstPos, stClasses[0].token.index); - hasComment = decl.functionDeclaration.comment.length > 0; - } - else if (decl.templateDeclaration !is null) - { - // skip the word `template` - firstPos = sourceCode.skipPreviousWord(decl.templateDeclaration.name.index); - hasComment = decl.templateDeclaration.comment.length > 0; - } - - // libdparse will put any ddoc comment with at least one character in the comment field - if (hasComment) - return true; - - firstPos = min(firstPos, getAttributesStartLocation(decl.attributes)); - - // scan the previous line for ddoc header -> skip to last real character - auto prevLine = sourceCode[0 .. firstPos].retro.find!(c => whitespace.countUntil(c) < 0); - - // if there is no comment annotation, only three possible cases remain. - // one line ddoc: ///, multi-line comments: /** */ or /++ +/ - return prevLine.filter!(c => !whitespace.canFind(c)).startsWith("///", "/+++/", "/***/") > 0; -} - -/** -The location of unittest token is known, but there might be attributes preceding it. -*/ -private size_t getAttributesStartLocation(const Attribute[] attrs) -{ - import dparse.lexer : tok; - - if (attrs.length == 0) - return size_t.max; - - if (attrs[0].atAttribute !is null) - return attrs[0].atAttribute.startLocation; - - if (attrs[0].attribute != tok!"") - return attrs[0].attribute.index; - - return size_t.max; -} - -private size_t skipPreviousWord(const(ubyte)[] sourceCode, size_t index) -{ - return index - sourceCode[0 .. index] - .retro - .enumerate - .find!(c => !whitespace.canFind(c.value)) - .find!(c => whitespace.canFind(c.value)) - .front.index; -} diff --git a/styles/tests_extractor.d b/tests_extractor.d similarity index 70% rename from styles/tests_extractor.d rename to tests_extractor.d index 0bcfdc4..b7a1807 100644 --- a/styles/tests_extractor.d +++ b/tests_extractor.d @@ -1,8 +1,13 @@ +#!/usr/bin/env dub +/++dub.sdl: +name "tests_extractor" +dependency "libdparse" version="~>0.7.0-beta.4" ++/ /* * Parses all public unittests that are visible on dlang.org * (= annotated with three slashes) * - * Copyright (C) 2016 by D Language Foundation + * Copyright (C) 2017 by D Language Foundation * * Author: Sebastian Wilzbach * @@ -14,6 +19,7 @@ import dparse.ast; import std.algorithm; +import std.ascii : whitespace; import std.conv; import std.exception; import std.experimental.logger; @@ -22,8 +28,6 @@ import std.path; import std.range; import std.stdio; -import utils; - class TestVisitor : ASTVisitor { File outFile; @@ -198,3 +202,73 @@ to in the output directory. } } } + +bool hasDdocHeader(const(ubyte)[] sourceCode, const Declaration decl) +{ + import std.algorithm.comparison : min; + + bool hasComment; + size_t firstPos = size_t.max; + + if (decl.unittest_ !is null) + { + firstPos = decl.unittest_.location; + hasComment = decl.unittest_.comment.length > 0; + } + else if (decl.functionDeclaration !is null) + { + // skip the return type + firstPos = sourceCode.skipPreviousWord(decl.functionDeclaration.name.index); + if (auto stClasses = decl.functionDeclaration.storageClasses) + firstPos = min(firstPos, stClasses[0].token.index); + hasComment = decl.functionDeclaration.comment.length > 0; + } + else if (decl.templateDeclaration !is null) + { + // skip the word `template` + firstPos = sourceCode.skipPreviousWord(decl.templateDeclaration.name.index); + hasComment = decl.templateDeclaration.comment.length > 0; + } + + // libdparse will put any ddoc comment with at least one character in the comment field + if (hasComment) + return true; + + firstPos = min(firstPos, getAttributesStartLocation(decl.attributes)); + + // scan the previous line for ddoc header -> skip to last real character + auto prevLine = sourceCode[0 .. firstPos].retro.find!(c => whitespace.countUntil(c) < 0); + + // if there is no comment annotation, only three possible cases remain. + // one line ddoc: ///, multi-line comments: /** */ or /++ +/ + return prevLine.filter!(c => !whitespace.canFind(c)).startsWith("///", "/+++/", "/***/") > 0; +} + +/** +The location of unittest token is known, but there might be attributes preceding it. +*/ +private size_t getAttributesStartLocation(const Attribute[] attrs) +{ + import dparse.lexer : tok; + + if (attrs.length == 0) + return size_t.max; + + if (attrs[0].atAttribute !is null) + return attrs[0].atAttribute.startLocation; + + if (attrs[0].attribute != tok!"") + return attrs[0].attribute.index; + + return size_t.max; +} + +private size_t skipPreviousWord(const(ubyte)[] sourceCode, size_t index) +{ + return index - sourceCode[0 .. index] + .retro + .enumerate + .find!(c => !whitespace.canFind(c.value)) + .find!(c => whitespace.canFind(c.value)) + .front.index; +}