From c5d4547f26f194f5c965d6413a0d47891a984690 Mon Sep 17 00:00:00 2001 From: Vladimir Panteleev Date: Fri, 16 Aug 2019 11:57:17 +0000 Subject: [PATCH 1/4] Update DustMite * 234d1ee dustmite: Fix condition for printing "empty set" wiki link * 66f2083 dustmite: Add links to wiki documentation * 263d658 dustmite: Be more verbose why test commands fail * ad0877d dustmite: Add --version * de78d64 splitter: Avoid nondeterminism due to undefined AA iteration order * b918e2d splitter: Whitespace cleanup --- DustMite/dustmite.d | 118 ++++++++++++++++++++++++++++++++++---------- DustMite/splitter.d | 10 ++-- 2 files changed, 96 insertions(+), 32 deletions(-) diff --git a/DustMite/dustmite.d b/DustMite/dustmite.d index 344ba23..36742d3 100644 --- a/DustMite/dustmite.d +++ b/DustMite/dustmite.d @@ -98,7 +98,7 @@ auto nullReduction = Reduction(Reduction.Type.None); int main(string[] args) { - bool force, dump, dumpHtml, showTimes, stripComments, obfuscate, keepLength, showHelp, noOptimize; + bool force, dump, dumpHtml, showTimes, stripComments, obfuscate, keepLength, showHelp, showVersion, noOptimize; string coverageDir; string[] reduceOnly, noRemoveStr, splitRules; uint lookaheadCount; @@ -134,8 +134,23 @@ int main(string[] args) "nosave|no-save", &noSave, // for research "nooptimize|no-optimize", &noOptimize, // for research "h|help", &showHelp, + "V|version", &showVersion, ); + if (showVersion) + { + version (Dlang_Tools) + enum source = "dlang/tools"; + else + version (Dustmite_CustomSource) // Packaging Dustmite separately for a distribution? + enum source = import("source"); + else + enum source = "upstream"; + writeln("DustMite build ", __DATE__, " (", source, "), built with ", __VENDOR__, " ", __VERSION__); + if (args.length == 1) + return 0; + } + if (showHelp || args.length == 1 || args.length>3) { stderr.writef(q"EOS @@ -165,13 +180,14 @@ EOS", args[0], splitterNames, totalCPUs); if (!showHelp) { stderr.write(q"EOS - --help Show this message and some less interesting options + -h, --help Show this message and some less interesting options EOS"); } else { stderr.write(q"EOS - --help Show this message + -h, --help Show this message + -V, --version Show program version Less interesting options: --strategy STRAT Set strategy (careful/lookback/pingpong/indepth/inbreadth) --dump Dump parsed tree to DIR.dump file @@ -254,9 +270,14 @@ EOS"); } resultDir = dirSuffix("reduced"); - enforce(!exists(resultDir), "Result directory already exists"); + if (resultDir.exists) + { + writeln("Hint: read https://github.com/CyberShadow/DustMite/wiki#result-directory-already-exists"); + throw new Exception("Result directory already exists"); + } - if (!test(nullReduction)) + auto nullResult = test(nullReduction); + if (!nullResult.success) { auto testerFile = dir.buildNormalizedPath(tester); version (Posix) @@ -268,7 +289,10 @@ EOS"); writeln("Hint: test program path should be relative to the source directory, try " ~ tester.absolutePath.relativePath(dir.absolutePath).escapeShellFileName() ~ " instead of " ~ tester.escapeShellFileName()); - throw new Exception("Initial test fails" ~ (noRedirect ? "" : " (try using --no-redirect for details)")); + if (!noRedirect) + writeln("Hint: use --no-redirect to see test script output"); + writeln("Hint: read https://github.com/CyberShadow/DustMite/wiki#initial-test-fails"); + throw new Exception("Initial test fails: " ~ nullResult.reason); } lookaheadProcesses = new Lookahead[lookaheadCount]; @@ -290,7 +314,10 @@ EOS"); writefln("Done in %s tests and %s; reduced version is in %s", tests, duration, resultDir); } else + { + writeln("Hint: read https://github.com/CyberShadow/DustMite/wiki#reduced-to-empty-set"); writefln("Done in %s tests and %s; reduced to empty set", tests, duration); + } } else writefln("Done in %s tests and %s; no reductions found", tests, duration); @@ -1090,7 +1117,7 @@ Entity entityAt(size_t[] address) /// Try specified reduction. If it succeeds, apply it permanently and save intermediate result. bool tryReduction(Reduction r) { - if (test(r)) + if (test(r).success) { foundAnything = true; debug @@ -1331,7 +1358,7 @@ struct Lookahead } Lookahead[] lookaheadProcesses; -bool[HASH] lookaheadResults; +TestResult[HASH] lookaheadResults; bool lookaheadPredict() { return false; } @@ -1342,27 +1369,63 @@ else bool[HASH] cache; -bool test(Reduction reduction) +struct TestResult +{ + bool success; + + enum Source : ubyte + { + none, + tester, + lookahead, + diskCache, + ramCache, + } + Source source; + + int status; + string reason() + { + final switch (source) + { + case Source.none: + assert(false); + case Source.tester: + return format("Test script %(%s%) exited with exit code %d (%s)", + [tester], status, (success ? "success" : "failure")); + case Source.lookahead: + return format("Test script %(%s%) (in lookahead) exited with exit code %d (%s)", + [tester], status, (success ? "success" : "failure")); + case Source.diskCache: + return "Test result was cached on disk as " ~ (success ? "success" : "failure"); + case Source.ramCache: + return "Test result was cached in memory as " ~ (success ? "success" : "failure"); + } + } +} + +TestResult test(Reduction reduction) { write(reduction, " => "); stdout.flush(); HASH digest; measure!"cacheHash"({ digest = hash(reduction); }); - bool ramCached(lazy bool fallback) + TestResult ramCached(lazy TestResult fallback) { auto cacheResult = digest in cache; if (cacheResult) { // Note: as far as I can see, a cache hit for a positive reduction is not possible (except, perhaps, for a no-op reduction) writeln(*cacheResult ? "Yes" : "No", " (cached)"); - return *cacheResult; + return TestResult(*cacheResult, TestResult.Source.ramCache); } auto result = fallback; - return cache[digest] = result; + cache[digest] = result.success; + return result; } - bool diskCached(lazy bool fallback) + TestResult diskCached(lazy TestResult fallback) { tests++; @@ -1376,33 +1439,33 @@ bool test(Reduction reduction) if (found) { writeln("No (disk cache)"); - return false; + return TestResult(false, TestResult.Source.diskCache); } measure!"globalCache"({ found = exists(cacheBase~"1"); }); if (found) { writeln("Yes (disk cache)"); - return true; + return TestResult(true, TestResult.Source.diskCache); } auto result = fallback; - measure!"globalCache"({ autoRetry({ std.file.write(cacheBase ~ (result ? "1" : "0"), ""); }, "save result to disk cache"); }); + measure!"globalCache"({ autoRetry({ std.file.write(cacheBase ~ (result.success ? "1" : "0"), ""); }, "save result to disk cache"); }); return result; } else return fallback; } - bool lookahead(lazy bool fallback) + TestResult lookahead(lazy TestResult fallback) { if (iter.strategy) { // Handle existing lookahead jobs - bool reap(ref Lookahead process, int status) + TestResult reap(ref Lookahead process, int status) { safeDelete(process.testdir); process.pid = null; - return lookaheadResults[process.digest] = status == 0; + return lookaheadResults[process.digest] = TestResult(status == 0, TestResult.Source.lookahead, status); } foreach (ref process; lookaheadProcesses) @@ -1445,7 +1508,7 @@ bool test(Reduction reduction) prediction = cache[digest]; else if (digest in lookaheadResults) - prediction = lookaheadResults[digest]; + prediction = lookaheadResults[digest].success; else prediction = lookaheadPredict(); lookaheadIter.next(prediction); @@ -1474,7 +1537,7 @@ bool test(Reduction reduction) auto plookaheadResult = digest in lookaheadResults; if (plookaheadResult) { - writeln(*plookaheadResult ? "Yes" : "No", " (lookahead)"); + writeln(plookaheadResult.success ? "Yes" : "No", " (lookahead)"); return *plookaheadResult; } @@ -1487,7 +1550,7 @@ bool test(Reduction reduction) auto exitCode = process.pid.wait(); auto result = reap(process, exitCode); - writeln(result ? "Yes" : "No", " (lookahead-wait)"); + writeln(result.success ? "Yes" : "No", " (lookahead-wait)"); return result; } } @@ -1496,7 +1559,7 @@ bool test(Reduction reduction) return fallback; } - bool doTest() + TestResult doTest() { string testdir = dirSuffix("test"); measure!"testSave"({save(reduction, testdir);}); scope(exit) measure!"clean"({safeDelete(testdir);}); @@ -1510,14 +1573,15 @@ bool test(Reduction reduction) pid = spawnShell(tester, nul, nul, nul, null, Config.none, testdir); } - bool result; - measure!"test"({result = pid.wait() == 0;}); - writeln(result ? "Yes" : "No"); + int status; + measure!"test"({status = pid.wait();}); + auto result = TestResult(status == 0, TestResult.Source.tester, status); + writeln(result.success ? "Yes" : "No"); return result; } auto result = ramCached(diskCached(lookahead(doTest()))); - if (trace) saveTrace(reduction, dirSuffix("trace"), result); + if (trace) saveTrace(reduction, dirSuffix("trace"), result.success); return result; } diff --git a/DustMite/splitter.d b/DustMite/splitter.d index a22be0d..121fca1 100644 --- a/DustMite/splitter.d +++ b/DustMite/splitter.d @@ -612,7 +612,7 @@ struct DSplitter static bool isWordChar(char c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; } void reset(string code) @@ -873,8 +873,8 @@ struct DSplitter && entities[j].children.length == 2 && firstToken(entities[j].children[0]) == t) { - j++; - return true; + j++; + return true; } return false; } @@ -1130,13 +1130,13 @@ struct DSplitter debug callRoot.comments ~= "Args root"; entities ~= callRoot; - foreach (id, params; calls) + foreach (id; calls.keys.sort()) { auto funRoot = new Entity(); debug funRoot.comments ~= "%s root".format(id); callRoot.children ~= funRoot; - foreach (i, args; params) + foreach (i, args; calls[id]) { auto e = new Entity(); debug e.comments ~= "%s param %d".format(id, i); From c386750a1f87c49200ce317b128e8275effc42ca Mon Sep 17 00:00:00 2001 From: Vladimir Panteleev Date: Fri, 16 Aug 2019 15:36:43 +0000 Subject: [PATCH 2/4] setup.sh: Fix ShellCheck warnings ShellCheck version is unpinned so CI update caused master to break. Fix the warnings reported by the current CI version. --- setup.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.sh b/setup.sh index 6cbe6d6..d6365b6 100755 --- a/setup.sh +++ b/setup.sh @@ -83,7 +83,7 @@ function handleCmdLine() { esac done - if [ ! -z "${tag+x}" ] ; then + if [ -n "${tag+x}" ] ; then wd+="/$tag" mkdir -p "$wd" fi @@ -193,16 +193,16 @@ function update() { function makeWorld() { local BOOTSTRAP="" - which dmd >/dev/null || BOOTSTRAP="AUTO_BOOTSTRAP=1" + command -v dmd >/dev/null || BOOTSTRAP="AUTO_BOOTSTRAP=1" for repo in dmd druntime phobos ; do "$makecmd" -C "$wd/$repo" -f posix.mak clean "$makecmd" -C "$wd/$repo" -f posix.mak "-j${parallel}" MODEL="$model" BUILD="$build" $BOOTSTRAP done # Update the running dmd version (only required once) - if [[ ! -z "${install+x}" ]]; then + if [[ -n "${install+x}" ]]; then local old dmdBinary - old=$(which dmd) + old=$(command -v dmd) dmdBinary=$(ls -1 $wd/dmd/generated/*/$build/$model/dmd) if [ -f "$old" ]; then echo "Linking '$dmdBinary' to $old" From 8094e2998e438638460ad4576b600f7e1b8d9515 Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Thu, 3 Oct 2019 03:18:38 +0200 Subject: [PATCH 3/4] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..aa91291 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,19 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- master + +pool: + vmImage: 'ubuntu-latest' + +steps: +- script: echo Hello, world! + displayName: 'Run a one-line script' + +- script: | + echo Add other tasks to build, test, and deploy your project. + echo See https://aka.ms/yaml + displayName: 'Run a multi-line script' From 28029b1a5a47b748c62cf24673c4c154b5f37e1f Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Thu, 3 Oct 2019 03:20:07 +0200 Subject: [PATCH 4/4] Revert "Set up CI with Azure Pipelines" This reverts commit 8094e2998e438638460ad4576b600f7e1b8d9515. --- azure-pipelines.yml | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index aa91291..0000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Starter pipeline -# Start with a minimal pipeline that you can customize to build and deploy your code. -# Add steps that build, run tests, deploy, and more: -# https://aka.ms/yaml - -trigger: -- master - -pool: - vmImage: 'ubuntu-latest' - -steps: -- script: echo Hello, world! - displayName: 'Run a one-line script' - -- script: | - echo Add other tasks to build, test, and deploy your project. - echo See https://aka.ms/yaml - displayName: 'Run a multi-line script'