Merge pull request #357 from wilzbach/attributes

Support parsing only specific unittests (e.g. @betterc-test)

posix.mak

@@ -107,8 +107,11 @@ $(ROOT)/tests_extractor: tests_extractor.d
 ################################################################################
 
 test_tests_extractor: $(ROOT)/tests_extractor
-	$< -i ./test/tests_extractor/ascii.d | diff - ./test/tests_extractor/ascii.d.ext
-	$< -i ./test/tests_extractor/iteration.d | diff - ./test/tests_extractor/iteration.d.ext
+	for file in ascii iteration ; do \
+		$< -i "./test/tests_extractor/$${file}.d" | diff -p - "./test/tests_extractor/$${file}.d.ext"; \
+	done
+	$< -a betterc -i "./test/tests_extractor/attributes.d" | diff -p - "./test/tests_extractor/attributes.d.ext";
+	$< --betterC -i "./test/tests_extractor/betterc.d" | diff -p - "./test/tests_extractor/betterc.d.ext";
 
 RDMD_TEST_COMPILERS = $(DMD)
 RDMD_TEST_EXECUTABLE = $(ROOT)/rdmd

test/tests_extractor/attributes.d

@@ -0,0 +1,44 @@
+module attributes;
+
+enum betterc;
+
+@betterc @safe @("foo") unittest
+{
+    assert(1 == 1);
+}
+
+@safe @("foo") unittest
+{
+    assert(2 == 2);
+}
+
+///
+@("foo") unittest
+{
+    assert(3 == 3);
+}
+
+@("foo") @betterc unittest
+{
+    assert(4 == 4);
+}
+
+@("betterc") @([1, 2, 3]) unittest
+{
+    assert(5 == 5);
+}
+
+@nogc @("foo", "betterc", "bar") @safe unittest
+{
+    assert(6 == 6);
+}
+
+@nogc @("foo", "better", "bar") @safe unittest
+{
+    assert(7 == 7);
+}
+
+@("betterd") unittest
+{
+    assert(8 == 8);
+}

test/tests_extractor/attributes.d.ext

@@ -0,0 +1,32 @@
+# line 3
+unittest
+{
+    import attributes;
+
+    assert(1 == 1);
+}
+
+# line 19
+unittest
+{
+    import attributes;
+
+    assert(4 == 4);
+}
+
+# line 24
+unittest
+{
+    import attributes;
+
+    assert(5 == 5);
+}
+
+# line 29
+unittest
+{
+    import attributes;
+
+    assert(6 == 6);
+}
+

test/tests_extractor/betterc.d

@@ -0,0 +1,16 @@
+module betterc;
+
+///
+unittest
+{
+    int a = 1;
+    assert(a == 2);
+}
+
+///
+unittest
+{
+    int b = 2;
+    assert(b == 2);
+    assert(b == 3);
+}

test/tests_extractor/betterc.d.ext

@@ -0,0 +1,24 @@
+# line 2
+unittest
+{
+    import betterc;
+
+    int a = 1;
+    assert(a == 2);
+}
+
+# line 9
+unittest
+{
+    import betterc;
+
+    int b = 2;
+    assert(b == 2);
+    assert(b == 3);
+}
+
+extern(C) void main()
+{
+    static foreach(u; __traits(getUnitTests, __traits(parent, main)))
+        u();
+}

tests_extractor.d

@@ -7,7 +7,7 @@ dependency "libdparse" version="~>0.8.0"
  * Parses all public unittests that are visible on dlang.org
  * (= annotated with three slashes)
  *
- * Copyright (C) 2017 by D Language Foundation
+ * Copyright (C) 2018 by D Language Foundation
  *
  * Author: Sebastian Wilzbach
  *
@@ -33,11 +33,13 @@ class TestVisitor : ASTVisitor
     File outFile;
     ubyte[] sourceCode;
     string moduleName;
+    VisitorConfig config;
 
-    this(File outFile, ubyte[] sourceCode)
+    this(File outFile, ubyte[] sourceCode, VisitorConfig config)
     {
         this.outFile = outFile;
         this.sourceCode = sourceCode;
+        this.config = config;
     }
 
     alias visit = ASTVisitor.visit;
@@ -54,17 +56,67 @@ class TestVisitor : ASTVisitor
             moduleName = outFile.name.replace(".d", "").replace(dirSeparator, ".").replace(".package", "");
         }
         m.accept(this);
+        // -betterC doesn't run unittests out of the box
+        if (config.betterCOutput)
+        {
+            outFile.writeln(q{extern(C) void main()
+{
+    static foreach(u; __traits(getUnitTests, __traits(parent, main)))
+        u();
+}});
+        }
     }
 
     override void visit(const Declaration decl)
     {
-        if (decl.unittest_ !is null && hasDdocHeader(sourceCode, decl))
+        if (decl.unittest_ !is null && shouldIncludeUnittest(decl))
             print(decl.unittest_);
 
         decl.accept(this);
     }
 
 private:
+
+    bool shouldIncludeUnittest(const Declaration decl)
+    {
+        if (!config.attributes.empty)
+            return filterForUDAs(decl);
+        else
+            return hasDdocHeader(sourceCode, decl);
+    }
+
+    bool filterForUDAs(const Declaration decl)
+    {
+        foreach (attr; decl.attributes)
+        {
+            if (attr.atAttribute is null)
+                continue;
+
+            // check for @myArg
+            if (config.attributes.canFind(attr.atAttribute.identifier.text))
+                return true;
+
+            // support @("myArg") too
+            if (auto argList = attr.atAttribute.argumentList)
+            {
+                foreach (arg; argList.items)
+                {
+                    if (auto unaryExp = cast(UnaryExpression) arg)
+                    if (auto primaryExp = unaryExp.primaryExpression)
+                    {
+                        auto attribute = primaryExp.primary.text;
+                        if (attribute.length >= 2)
+                        {
+                            attribute = attribute[1 .. $ - 1];
+                            if (config.attributes.canFind(attribute))
+                                return true;
+                        }
+                    }
+                }
+            }
+        }
+        return false;
+    }
     void print(const Unittest u)
     {
         /*
@@ -93,7 +145,7 @@ private:
     }
 }
 
-void parseFile(File inFile, File outFile)
+void parseFile(File inFile, File outFile, VisitorConfig visitorConfig)
 {
     import dparse.lexer;
     import dparse.parser : parseModule;
@@ -111,11 +163,11 @@ void parseFile(File inFile, File outFile)
 
     RollbackAllocator rba;
     auto m = parseModule(tokens.array, inFile.name, &rba);
-    auto visitor = new TestVisitor(outFile, sourceCode);
+    auto visitor = new TestVisitor(outFile, sourceCode, visitorConfig);
     visitor.visit(m);
 }
 
-void parseFileDir(string inputDir, string fileName, string outputDir)
+void parseFileDir(string inputDir, string fileName, string outputDir, VisitorConfig visitorConfig)
 {
     import std.path : buildPath, dirSeparator, buildNormalizedPath;
 
@@ -132,7 +184,13 @@ void parseFileDir(string inputDir, string fileName, string outputDir)
     // convert the file path to a nice output file, e.g. std/uni.d -> std_uni.d
     string outName = fileNameNormalized.replace(dirSeparator, "_");
 
-    parseFile(File(fileName), File(buildPath(outputDir, outName), "w"));
+    parseFile(File(fileName), File(buildPath(outputDir, outName), "w"), visitorConfig);
+}
+
+struct VisitorConfig
+{
+    string[] attributes; /// List of attributes to extract;
+    bool betterCOutput; /// Add custom extern(C) main method for running D's unittests
 }
 
 void main(string[] args)
@@ -143,12 +201,17 @@ void main(string[] args)
     string inputDir;
     string outputDir = "./out";
     string ignoredFilesStr;
-    string modulePrefix = "";
+    string modulePrefix;
+    string attributesStr;
+    VisitorConfig visitorConfig;
 
     auto helpInfo = getopt(args, config.required,
             "inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir,
             "outputdir|o", "Folder to which the extracted test files should be saved (stdout for a single file)", &outputDir,
-            "ignore", "Comma-separated list of files to exclude (partial matching is supported)", &ignoredFilesStr);
+            "ignore", "Comma-separated list of files to exclude (partial matching is supported)", &ignoredFilesStr,
+            "attributes|a", "Comma-separated list of UDAs that the unittest should have", &attributesStr,
+            "betterC", "Add custom extern(C) main method for running D's unittests", &visitorConfig.betterCOutput,
+    );
 
     if (helpInfo.helpWanted)
     {
@@ -162,6 +225,7 @@ to in the output directory.
 
     inputDir = inputDir.asNormalizedPath.array;
     Algebraic!(string, File) outputLocation = cast(string) outputDir.asNormalizedPath.array;
+    visitorConfig.attributes = attributesStr.split(",");
 
     if (!exists(outputDir))
         mkdir(outputDir);
@@ -196,8 +260,8 @@ to in the output directory.
         {
             stderr.writeln("parsing ", file);
             outputLocation.visit!(
-                (string outputFolder) => parseFileDir(inputDir, file, outputFolder),
-                (File outputFile) => parseFile(File(file.name, "r"), outputFile),
+                (string outputFolder) => parseFileDir(inputDir, file, outputFolder, visitorConfig),
+                (File outputFile) => parseFile(File(file.name, "r"), outputFile, visitorConfig),
             );
         }
         else