Cleanup up styles directory

- has_public_example has been moved to Dscanner
- Reduce to a DUB single file setup for simplicity

.gitignore

@@ -6,3 +6,5 @@
 
 /generated
 GNUmakefile
+/.dub
+/tests_extractor

README.md

@@ -19,7 +19,6 @@ dget                   | Internal | D source code downloader.
 dman                   | Public   | D documentation lookup tool.
 dustmite               | Public   | [Test case minimization tool](https://github.com/CyberShadow/DustMite/wiki).
 get_dlibcurl32         | Internal | Win32 libcurl downloader/converter.
-has_public_example 	   | Internal | Checks public functions for public examples (requires DUB)
 rdmd                   | Public   | [D build tool](http://dlang.org/rdmd.html).
 rdmd_test              | Internal | rdmd test suite.
 tests_extractor 	   | Internal | Extracts public unittests (requires DUB)
@@ -35,14 +34,16 @@ Running DUB tools
 -----------------
 
 Some tools require D's package manager DUB.
-By default DUB builds a binary and executes it:
+By default, DUB builds a binary and executes it. On a Posix system,
+the source files can directly be executed with DUB (e.g. `./tests_extractor.d`).
+Alternatively, the full single file execution command can be used:
 
 ```
-dub --root styles -c has_public_example
+dub --single tests_extractor.d
 ```
 
 Remember that when programs are run via DUB, you need to pass in `--` before
-the program's arguments, e.g `dub --root styles -c has_public_example -- -i ../phobos/std/algorithm`.
+the program's arguments, e.g `dub --single tests_extractor.d -- -i ../phobos/std/algorithm`.
 
 For more information, please see [DUB's documentation][dub-doc].
 

styles/.gitignore

@@ -1,4 +0,0 @@
-.dub
-has_public_example
-test_extractor
-out

styles/dub.sdl

@@ -1,16 +0,0 @@
-dependency "libdparse" version="~>0.7.0-beta.2"
-name "styles"
-targetType "executable"
-sourceFiles "utils.d"
-
-configuration "has_public_example" {
-	name "has_public_example"
-	targetName "has_public_example"
-	sourceFiles "has_public_example.d"
-}
-
-configuration "tests_extractor" {
-	name "test_extractor"
-	targetName "test_extractor"
-	sourceFiles "tests_extractor.d"
-}

styles/dub.selections.json

@@ -1,6 +0,0 @@
-{
-	"fileVersion": 1,
-	"versions": {
-		"libdparse": "0.7.0-beta.2"
-	}
-}

styles/has_public_example.d

@@ -1,194 +0,0 @@
-/*
- * Checks that all functions have a public example
- *
- * Copyright (C) 2016 by D Language Foundation
- *
- * Author: Sebastian Wilzbach
- *
- * Distributed under the Boost Software License, Version 1.0.
- *    (See accompanying file LICENSE_1_0.txt or copy at
- *          http://www.boost.org/LICENSE_1_0.txt)
-*/
-// Written in the D programming language.
-
-import dparse.ast;
-import std.algorithm;
-import std.experimental.logger;
-import std.range;
-import std.stdio;
-import utils;
-
-bool hadError;
-
-class TestVisitor : ASTVisitor
-{
-    this(string fileName, ubyte[] sourceCode)
-    {
-        this.fileName = fileName;
-        this.sourceCode = sourceCode;
-    }
-
-    alias visit = ASTVisitor.visit;
-
-    override void visit(const Module mod)
-    {
-        Declaration lastDecl;
-        bool hasPublicUnittest;
-
-        foreach (decl; mod.declarations)
-        {
-            if (!isPublic(decl.attributes))
-                continue;
-
-            if (decl.functionDeclaration !is null || decl.templateDeclaration !is null)
-            {
-                if (lastDecl !is null &&
-                    (hasDitto(decl.functionDeclaration) || hasDitto(decl.templateDeclaration)))
-                    continue;
-
-                if (lastDecl !is null && !hasPublicUnittest)
-                    triggerError(lastDecl);
-
-                if (hasDdocHeader(sourceCode, decl))
-                    lastDecl = cast(Declaration) decl;
-                else
-                    lastDecl = null;
-
-                hasPublicUnittest = false;
-                continue;
-            }
-
-            if (decl.unittest_ !is null)
-            {
-                // ignore module header unittest blocks or already validated functions
-                hasPublicUnittest |= lastDecl is null || hasDdocHeader(sourceCode, decl);
-                continue;
-            }
-
-            // ignore dittoed template declarations
-            if (decl.classDeclaration !is null && hasDitto(decl.classDeclaration)
-                || decl.structDeclaration !is null && hasDitto(decl.structDeclaration))
-                    continue;
-
-            // ran into struct or something else -> reset
-            if (lastDecl !is null && !hasPublicUnittest)
-                triggerError(lastDecl);
-
-            lastDecl = null;
-        }
-
-        if (lastDecl !is null && !hasPublicUnittest)
-            triggerError(lastDecl);
-    }
-
-private:
-    string fileName;
-    ubyte[] sourceCode;
-
-    void triggerError(const Declaration decl)
-    {
-        if (auto fn = decl.functionDeclaration)
-            stderr.writefln("function %s in %s:%d has no public unittest", fn.name.text, fileName, fn.name.line);
-        if (auto tpl = decl.templateDeclaration)
-            stderr.writefln("template %s in %s:%d has no public unittest", tpl.name.text, fileName, tpl.name.line);
-        hadError = true;
-    }
-
-    bool hasDitto(Decl)(const Decl decl)
-    {
-        if (decl is null)
-            return false;
-
-        if (decl.comment is null)
-            return false;
-
-        if (decl.comment.among!("ditto", "Ditto"))
-            return true;
-
-        return false;
-    }
-
-    bool isPublic(const Attribute[] attrs)
-    {
-        import dparse.lexer : tok;
-        import std.algorithm.searching : any;
-        import std.algorithm.iteration : map;
-
-        enum tokPrivate = tok!"private", tokProtected = tok!"protected", tokPackage = tok!"package";
-
-        if (attrs.map!`a.attribute`.any!(x => x == tokPrivate || x == tokProtected || x == tokPackage))
-            return false;
-
-        return true;
-    }
-}
-
-void parseFile(string fileName)
-{
-    import dparse.lexer;
-    import dparse.parser : parseModule;
-    import dparse.rollback_allocator : RollbackAllocator;
-    import std.array : uninitializedArray;
-
-    auto inFile = File(fileName);
-    if (inFile.size == 0)
-        warningf("%s is empty", inFile.name);
-
-    ubyte[] sourceCode = uninitializedArray!(ubyte[])(to!size_t(inFile.size));
-    inFile.rawRead(sourceCode);
-    LexerConfig config;
-    auto cache = StringCache(StringCache.defaultBucketCount);
-    auto tokens = getTokensForParser(sourceCode, config, &cache);
-
-    RollbackAllocator rba;
-    auto m = parseModule(tokens.array, fileName, &rba);
-    auto visitor = new TestVisitor(fileName, sourceCode);
-    visitor.visit(m);
-}
-
-void main(string[] args)
-{
-    import std.file;
-    import std.getopt;
-    import std.path : asNormalizedPath;
-
-    string inputDir;
-    string ignoredFilesStr;
-
-    auto helpInfo = getopt(args, config.required,
-            "inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir,
-            "ignore", "Comma-separated list of files to exclude (partial matching is supported)", &ignoredFilesStr);
-
-    if (helpInfo.helpWanted)
-    {
-        return defaultGetoptPrinter(`has_public_example
-Searches the input directory recursively to ensure that all public, ddoced functions
-have at least one public, ddoced unittest blocks.
-`, helpInfo.options);
-    }
-
-    inputDir = inputDir.asNormalizedPath.array;
-
-    DirEntry[] files;
-
-    if (inputDir.isFile)
-    {
-        files = [DirEntry(inputDir)];
-        inputDir = ".";
-    }
-    else
-    {
-        files = dirEntries(inputDir, SpanMode.depth).filter!(
-                a => a.name.endsWith(".d") && !a.name.canFind(".git")).array;
-    }
-
-    auto ignoringFiles = ignoredFilesStr.split(",");
-
-    foreach (file; files)
-        if (!ignoringFiles.any!(x => file.name.canFind(x)))
-            file.name.parseFile;
-
-    import core.stdc.stdlib : exit;
-    if (hadError)
-        exit(1);
-}

styles/utils.d

@@ -1,88 +0,0 @@
-/*
- * Shared methods between style checkers
- *
- * Copyright (C) 2016 by D Language Foundation
- *
- * Distributed under the Boost Software License, Version 1.0.
- *    (See accompanying file LICENSE_1_0.txt or copy at
- *          http://www.boost.org/LICENSE_1_0.txt)
-*/
-// Written in the D programming language.
-
-import dparse.ast;
-import std.algorithm;
-import std.ascii : whitespace;
-import std.conv : to;
-import std.experimental.logger;
-import std.range;
-import std.stdio : File;
-
-bool hasDdocHeader(const(ubyte)[] sourceCode, const Declaration decl)
-{
-    import std.algorithm.comparison : min;
-
-    bool hasComment;
-    size_t firstPos = size_t.max;
-
-    if (decl.unittest_ !is null) 
-    {
-        firstPos = decl.unittest_.location;
-        hasComment = decl.unittest_.comment.length > 0;
-    }
-    else if (decl.functionDeclaration !is null) 
-    {
-        // skip the return type
-        firstPos = sourceCode.skipPreviousWord(decl.functionDeclaration.name.index);
-        if (auto stClasses = decl.functionDeclaration.storageClasses)
-            firstPos = min(firstPos, stClasses[0].token.index);
-        hasComment = decl.functionDeclaration.comment.length > 0;
-    }
-    else if (decl.templateDeclaration !is null) 
-    {
-        // skip the word `template`
-        firstPos = sourceCode.skipPreviousWord(decl.templateDeclaration.name.index);
-        hasComment = decl.templateDeclaration.comment.length > 0;
-    }
-
-    // libdparse will put any ddoc comment with at least one character in the comment field
-    if (hasComment)
-        return true;
-
-    firstPos = min(firstPos, getAttributesStartLocation(decl.attributes));
-
-    // scan the previous line for ddoc header -> skip to last real character
-    auto prevLine = sourceCode[0 .. firstPos].retro.find!(c => whitespace.countUntil(c) < 0);
-
-    // if there is no comment annotation, only three possible cases remain.
-    // one line ddoc: ///, multi-line comments: /** */ or /++ +/
-    return prevLine.filter!(c => !whitespace.canFind(c)).startsWith("///", "/+++/", "/***/") > 0;
-}
-
-/**
-The location of unittest token is known, but there might be attributes preceding it.
-*/
-private size_t getAttributesStartLocation(const Attribute[] attrs)
-{
-    import dparse.lexer : tok;
-
-    if (attrs.length == 0)
-        return size_t.max;
-
-    if (attrs[0].atAttribute !is null)
-        return attrs[0].atAttribute.startLocation;
-
-    if (attrs[0].attribute != tok!"")
-        return attrs[0].attribute.index;
-
-    return size_t.max;
-}
-
-private size_t skipPreviousWord(const(ubyte)[] sourceCode, size_t index)
-{
-    return index - sourceCode[0 .. index]
-                  .retro
-                  .enumerate
-                  .find!(c => !whitespace.canFind(c.value))
-                  .find!(c => whitespace.canFind(c.value))
-                  .front.index;
-}

tests_extractor.d

@@ -1,8 +1,13 @@
+#!/usr/bin/env dub
+/++dub.sdl:
+name "tests_extractor"
+dependency "libdparse" version="~>0.7.0-beta.4"
++/
 /*
  * Parses all public unittests that are visible on dlang.org
  * (= annotated with three slashes)
  *
- * Copyright (C) 2016 by D Language Foundation
+ * Copyright (C) 2017 by D Language Foundation
  *
  * Author: Sebastian Wilzbach
  *
@@ -14,6 +19,7 @@
 
 import dparse.ast;
 import std.algorithm;
+import std.ascii : whitespace;
 import std.conv;
 import std.exception;
 import std.experimental.logger;
@@ -22,8 +28,6 @@ import std.path;
 import std.range;
 import std.stdio;
 
-import utils;
-
 class TestVisitor : ASTVisitor
 {
     File outFile;
@@ -198,3 +202,73 @@ to in the output directory.
         }
     }
 }
+
+bool hasDdocHeader(const(ubyte)[] sourceCode, const Declaration decl)
+{
+    import std.algorithm.comparison : min;
+
+    bool hasComment;
+    size_t firstPos = size_t.max;
+
+    if (decl.unittest_ !is null)
+    {
+        firstPos = decl.unittest_.location;
+        hasComment = decl.unittest_.comment.length > 0;
+    }
+    else if (decl.functionDeclaration !is null)
+    {
+        // skip the return type
+        firstPos = sourceCode.skipPreviousWord(decl.functionDeclaration.name.index);
+        if (auto stClasses = decl.functionDeclaration.storageClasses)
+            firstPos = min(firstPos, stClasses[0].token.index);
+        hasComment = decl.functionDeclaration.comment.length > 0;
+    }
+    else if (decl.templateDeclaration !is null)
+    {
+        // skip the word `template`
+        firstPos = sourceCode.skipPreviousWord(decl.templateDeclaration.name.index);
+        hasComment = decl.templateDeclaration.comment.length > 0;
+    }
+
+    // libdparse will put any ddoc comment with at least one character in the comment field
+    if (hasComment)
+        return true;
+
+    firstPos = min(firstPos, getAttributesStartLocation(decl.attributes));
+
+    // scan the previous line for ddoc header -> skip to last real character
+    auto prevLine = sourceCode[0 .. firstPos].retro.find!(c => whitespace.countUntil(c) < 0);
+
+    // if there is no comment annotation, only three possible cases remain.
+    // one line ddoc: ///, multi-line comments: /** */ or /++ +/
+    return prevLine.filter!(c => !whitespace.canFind(c)).startsWith("///", "/+++/", "/***/") > 0;
+}
+
+/**
+The location of unittest token is known, but there might be attributes preceding it.
+*/
+private size_t getAttributesStartLocation(const Attribute[] attrs)
+{
+    import dparse.lexer : tok;
+
+    if (attrs.length == 0)
+        return size_t.max;
+
+    if (attrs[0].atAttribute !is null)
+        return attrs[0].atAttribute.startLocation;
+
+    if (attrs[0].attribute != tok!"")
+        return attrs[0].attribute.index;
+
+    return size_t.max;
+}
+
+private size_t skipPreviousWord(const(ubyte)[] sourceCode, size_t index)
+{
+    return index - sourceCode[0 .. index]
+                  .retro
+                  .enumerate
+                  .find!(c => !whitespace.canFind(c.value))
+                  .find!(c => whitespace.canFind(c.value))
+                  .front.index;
+}