From a18ebc08f6e599fe4f0a02f13d3bd095f2ab1f5e Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Mon, 25 Apr 2016 02:13:02 +0300 Subject: [PATCH 1/5] Add changelog builder --- changed.d | 254 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 217 insertions(+), 37 deletions(-) diff --git a/changed.d b/changed.d index 3cd9d55..8498479 100755 --- a/changed.d +++ b/changed.d @@ -1,24 +1,35 @@ #!/usr/bin/env rdmd + // Written in the D programming language /** Change log generator which fetches the list of bugfixes from the D Bugzilla between the given dates. +Moreover manual changes are accumulated from raw text files in the +Dlang repositories. It stores its result in DDoc form to a text file. -Copyright: Dmitry Olshansky 2013. +Copyright: D Language Foundation 2016. License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). Authors: Dmitry Olshansky, - Andrej Mitrovic + Andrej Mitrovic, + Sebastian Wilzbach Example usage: --- +rdmd changed.d "v2.071.2..upstream/stable" rdmd changed.d --start=2013-01-01 --end=2013-04-01 rdmd changed.d --start=2013-01-01 (end date implicitly set to current date) --- +It is also possible to directly preview the generated changelog file: + +--- +rdmd changed.d "v2.071.2..upstream/stable" && dmd ../dlang.org/macros.ddoc ../dlang.org/html.ddoc ../dlang.org/dlang.org.ddoc ../dlang.org/doc.ddoc ../dlang.org/changelog/changelog.ddoc changelog.dd -Df../dlang.org/changelog/pending.html +--- + $(B Note:) The script will cache the results of an invocation, to avoid re-querying bugzilla when invoked with the same arguments. Use the --nocache option to override this behavior. @@ -32,6 +43,21 @@ import std.net.curl, std.conv, std.exception, std.algorithm, std.csv, std.typeco std.stdio, std.datetime, std.array, std.string, std.file, std.format, std.getopt, std.path; +import std.range.primitives, std.traits; + +struct BugzillaEntry +{ + int id; + string summary; +} + +struct ChangelogEntry +{ + string title; // the first line (can't contain links) + string description; // a detailed description (separated by a new line) + string basename; // basename without extension (used for the anchor link to the description) +} + auto templateRequest = `https://issues.dlang.org/buglist.cgi?bug_id={buglist}&bug_status=RESOLVED&resolution=FIXED&`~ `ctype=csv&columnlist=component,bug_severity,short_desc`; @@ -49,12 +75,6 @@ auto dateFromStr(string sdate) return Date(year, month, day); } -struct Entry -{ - int id; - string summary; -} - string[dchar] parenToMacro; shared static this() { @@ -105,17 +125,14 @@ auto getIssues(string revRange) } /** Generate and return the change log as a string. */ -string getChangeLog(string revRange) +auto getBugzillaChanges(string revRange) { auto req = generateRequest(templateRequest, getIssues(revRange)); debug stderr.writeln(req); // write text auto data = req.get; // component (e.g. DMD) -> bug type (e.g. regression) -> list of bug entries - Entry[][string][string] entries; - - immutable bugtypes = ["regressions", "bugs", "enhancements"]; - immutable components = ["DMD Compiler", "Phobos", "Druntime", "dlang.org", "Optlink", "Tools", "Installer"]; + BugzillaEntry[][string][string] entries; foreach (fields; csvReader!(Tuple!(int, string, string, string))(data, null)) { @@ -142,56 +159,219 @@ string getChangeLog(string revRange) type = "bugs"; break; - case "enhancement": + case "enhancement": type = "enhancements"; break; default: assert(0, type); } - entries[comp][type] ~= Entry(fields[0], fields[3].idup); + entries[comp][type] ~= BugzillaEntry(fields[0], fields[3].idup); } + return entries; +} - Appender!string result; +/** +Reads a single changelog file. - result ~= "$(BUGSTITLE Language Changes,\n"; - result ~= "-- Insert major language changes here --\n)\n\n"; +An entry consists of a title line, a blank separator line and +the description - result ~= "$(BUGSTITLE Library Changes,\n"; - result ~= "-- Insert major library changes here --\n)\n\n"; +Params: + filename = changelog file to be parsed - result ~= "$(BR)$(BIG List of all bug fixes and enhancements:)\n\n"; +Returns: The parsed `ChangelogEntry` +*/ +ChangelogEntry readChangelog(string filename) +{ + import std.algorithm.searching : countUntil; + import std.file: read; + import std.path : baseName, stripExtension; + import std.string : strip; + + auto fileRead = cast(ubyte[]) filename.read; + auto firstLineBreak = fileRead.countUntil("\n"); + + // filter empty files + if (firstLineBreak < 0) + return ChangelogEntry.init; + + // filter ddoc files + if (fileRead.length < 4 || fileRead[0..4] == "Ddoc") + return ChangelogEntry.init; + + ChangelogEntry entry = { + title: (cast(string) fileRead[0..firstLineBreak]).strip, + description: (cast(string) fileRead[firstLineBreak..$]).strip, + basename: filename.baseName.stripExtension // let's be safe here + }; + return entry; +} + +/** +Looks for changelog files (ending with `.dd`) in a directory and parses them. + +Params: + changelogDir = directory to search for changelog files + +Returns: An InputRange of `ChangelogEntry`s +*/ +auto readTextChanges(string changelogDir) +{ + import std.algorithm.iteration : filter, map; + import std.file: dirEntries, SpanMode; + import std.string : endsWith; + + return dirEntries(changelogDir, SpanMode.shallow) + .filter!(a => a.name().endsWith(".dd")) + .map!readChangelog + .filter!(a => a.title.length > 0); +} + +/** +Writes the overview headline of the manually listed changes in the ddoc format as list. + +Params: + changes = parsed InputRange of changelog information + w = Output range to use +*/ +void writeTextChangesHeader(Entries, Writer)(Entries changes, Writer w, string headline) + if (isInputRange!Entries && isOutputRange!(Writer, string)) +{ + // write the overview titles + w.formattedWrite("$(BUGSTITLE %s,\n\n", headline); + scope(exit) w.put("\n)\n\n"); + foreach(change; changes) + { + w.formattedWrite("$(LI $(RELATIVE_LINK2 %s,%s))\n", change.basename, change.title.escapeParens); + } +} +/** +Writes the long description of the manually listed changes in the ddoc format as list. + +Params: + changes = parsed InputRange of changelog information + w = Output range to use +*/ +void writeTextChangesBody(Entries, Writer)(Entries changes, Writer w, string headline) + if (isInputRange!Entries && isOutputRange!(Writer, string)) +{ + w.formattedWrite("$(BUGSTITLE %s,\n\n", headline); + scope(exit) w.put("\n)\n\n"); + foreach(change; changes) + { + w.formattedWrite("$(LI $(LNAME2 %s,%s)\n", change.basename, change.title.escapeParens); + scope(exit) w.put(")\n"); + w.formattedWrite(" $(P %s )\n", change.description.escapeParens); + } +} + +/** +Writes the fixed issued from Bugzilla in the ddoc format as a single list. + +Params: + changes = parsed InputRange of changelog information + w = Output range to use +*/ +void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w) + if (isOutputRange!(Writer, string)) +{ + immutable components = ["DMD Compiler", "Phobos", "Druntime", "dlang.org", "Optlink", "Tools", "Installer"]; + immutable bugtypes = ["regressions", "bugs", "enhancements"]; foreach (component; components) - if (auto comp = component in entries) { - foreach (bugtype; bugtypes) - if (auto bugs = bugtype in *comp) + if (auto comp = component in entries) { - result ~= format("$(BUGSTITLE %s %s,\n\n", component, bugtype); - foreach (bug; sort!"a.id < b.id"(*bugs)) + foreach (bugtype; bugtypes) + if (auto bugs = bugtype in *comp) { - result ~= format("$(LI $(BUGZILLA %s): %s)\n", - bug.id, bug.summary.escapeParens()); + w.formattedWrite("$(BUGSTITLE %s %s,\n\n", component, bugtype); + foreach (bug; sort!"a.id < b.id"(*bugs)) + { + w.formattedWrite("$(LI $(BUGZILLA %s): %s)\n", + bug.id, bug.summary.escapeParens()); + } + w.put(")\n"); } - result ~= ")\n"; } } - - return result.data; } int main(string[] args) { - if (args.length != 2) + auto outputFile = "./changelog.dd"; + auto nextVersionString = "LATEST"; + auto nextVersionDate = "September 19, 2016"; + string previousVersion = "Previous version"; + string revRange; + + auto helpInformation = getopt( + args, + std.getopt.config.passThrough, + "output|o", &outputFile); + + if (args.length >= 2) { - stderr.writeln("Usage: ./changed , e.g. ./changed v2.067.1..upstream/stable"); - return 1; + revRange = args[1]; + + // extract the previous version + auto parts = revRange.split(".."); + if (parts.length > 1) + previousVersion = parts[0]; + } + else + { + writeln("Skipped querying Bugzilla for changes. Please define a revision range e.g ./changed v2.072.2..upstream/stable"); } - string logPath = "./changelog.txt".absolutePath.buildNormalizedPath; - std.file.write(logPath, getChangeLog(args[1])); - writefln("Change log generated to: '%s'", logPath); + auto f = File(outputFile, "w"); + auto w = f.lockingTextWriter(); + w.put("Ddoc\n\n"); + w.formattedWrite("$(CHANGELOG_NAV_LAST %s)\n\n", previousVersion); + { + w.formattedWrite("$(VERSION %s, =================================================,\n\n", nextVersionDate); + scope(exit) w.put(")\n"); + + // search for raw change files + alias Repo = Tuple!(string, "path", string, "headline"); + auto repos = [Repo("dmd", "Language changes"), + Repo("druntime", "Runtime changes"), + Repo("phobos", "Library changes")]; + + auto changedRepos = repos + .map!(repo => Repo(buildPath("..", repo.path, "changelog"), repo.headline)) + .filter!(x => x.path.exists) + .map!(x => tuple!("headline", "changes")(x.headline, x.path.readTextChanges.array)) + .filter!(x => !x.changes.empty); + + // print the overview headers + changedRepos.each!(x => x.changes.writeTextChangesHeader(w, x.headline)); + + if (revRange.length) + w.put("$(BR)$(BIG $(RELATIVE_LINK2 bugfix-list, List of all bug fixes and enhancements in D $(VER).))\n\n"); + + w.put("$(HR)\n\n"); + + // print the detailed descriptions + changedRepos.each!(x => x.changes.writeTextChangesBody(w, x.headline)); + + // print the entire changelog history + if (revRange.length) + { + w.put("$(BR)$(BIG $(LNAME2 bugfix-list, List of all bug fixes and enhancements in D $(VER):))\n\n"); + revRange.getBugzillaChanges.writeBugzillaChanges(w); + } + } + + w.formattedWrite("$(CHANGELOG_NAV_LAST %s)\n", previousVersion); + + // write own macros + w.formattedWrite(`Macros: + VER=%s + TITLE=Change Log: $(VER)`, nextVersionString); + + writefln("Change log generated to: '%s'", outputFile); return 0; } From b0ed7fe61ea812884fb407b2d755863787b3b7b4 Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Fri, 7 Oct 2016 19:14:06 +0200 Subject: [PATCH 2/5] Prettify style + cleanup --- changed.d | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/changed.d b/changed.d index 8498479..cabaa87 100755 --- a/changed.d +++ b/changed.d @@ -18,10 +18,9 @@ Authors: Dmitry Olshansky, Sebastian Wilzbach Example usage: + --- rdmd changed.d "v2.071.2..upstream/stable" -rdmd changed.d --start=2013-01-01 --end=2013-04-01 -rdmd changed.d --start=2013-01-01 (end date implicitly set to current date) --- It is also possible to directly preview the generated changelog file: @@ -30,9 +29,8 @@ It is also possible to directly preview the generated changelog file: rdmd changed.d "v2.071.2..upstream/stable" && dmd ../dlang.org/macros.ddoc ../dlang.org/html.ddoc ../dlang.org/dlang.org.ddoc ../dlang.org/doc.ddoc ../dlang.org/changelog/changelog.ddoc changelog.dd -Df../dlang.org/changelog/pending.html --- -$(B Note:) The script will cache the results of an invocation, to avoid -re-querying bugzilla when invoked with the same arguments. -Use the --nocache option to override this behavior. +If no arguments are passed, only the manual changes will be accumulated and Bugzilla +won't be queried (faster). */ // NOTE: this script requires libcurl to be linked in (usually done by default). @@ -219,7 +217,7 @@ Returns: An InputRange of `ChangelogEntry`s auto readTextChanges(string changelogDir) { import std.algorithm.iteration : filter, map; - import std.file: dirEntries, SpanMode; + import std.file : dirEntries, SpanMode; import std.string : endsWith; return dirEntries(changelogDir, SpanMode.shallow) @@ -309,7 +307,10 @@ int main(string[] args) auto helpInformation = getopt( args, std.getopt.config.passThrough, - "output|o", &outputFile); + "output|o", &outputFile, + "date", &nextVersionDate, + "version", &nextVersionString, + "prev-version", &previousVersion); // this can automatically be detected if (args.length >= 2) { From 2e10f1b8454bac4136f1beb241f99e85667c4204 Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Fri, 7 Oct 2016 19:32:44 +0200 Subject: [PATCH 3/5] Added flag to hide manual text file changes --- changed.d | 51 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/changed.d b/changed.d index cabaa87..a525094 100755 --- a/changed.d +++ b/changed.d @@ -302,6 +302,7 @@ int main(string[] args) auto nextVersionString = "LATEST"; auto nextVersionDate = "September 19, 2016"; string previousVersion = "Previous version"; + bool hideTextChanges = false; string revRange; auto helpInformation = getopt( @@ -310,7 +311,8 @@ int main(string[] args) "output|o", &outputFile, "date", &nextVersionDate, "version", &nextVersionString, - "prev-version", &previousVersion); // this can automatically be detected + "prev-version", &previousVersion, // this can automatically be detected + "no-text", &hideTextChanges); if (args.length >= 2) { @@ -335,35 +337,42 @@ int main(string[] args) w.formattedWrite("$(VERSION %s, =================================================,\n\n", nextVersionDate); scope(exit) w.put(")\n"); - // search for raw change files - alias Repo = Tuple!(string, "path", string, "headline"); - auto repos = [Repo("dmd", "Language changes"), - Repo("druntime", "Runtime changes"), - Repo("phobos", "Library changes")]; + if (!hideTextChanges) + { + // search for raw change files + alias Repo = Tuple!(string, "path", string, "headline"); + auto repos = [Repo("dmd", "Language changes"), + Repo("druntime", "Runtime changes"), + Repo("phobos", "Library changes")]; - auto changedRepos = repos - .map!(repo => Repo(buildPath("..", repo.path, "changelog"), repo.headline)) - .filter!(x => x.path.exists) - .map!(x => tuple!("headline", "changes")(x.headline, x.path.readTextChanges.array)) - .filter!(x => !x.changes.empty); + auto changedRepos = repos + .map!(repo => Repo(buildPath("..", repo.path, "changelog"), repo.headline)) + .filter!(x => x.path.exists) + .map!(x => tuple!("headline", "changes")(x.headline, x.path.readTextChanges.array)) + .filter!(x => !x.changes.empty); - // print the overview headers - changedRepos.each!(x => x.changes.writeTextChangesHeader(w, x.headline)); + // print the overview headers + changedRepos.each!(x => x.changes.writeTextChangesHeader(w, x.headline)); - if (revRange.length) - w.put("$(BR)$(BIG $(RELATIVE_LINK2 bugfix-list, List of all bug fixes and enhancements in D $(VER).))\n\n"); + if (revRange.length) + w.put("$(BR)$(BIG $(RELATIVE_LINK2 bugfix-list, List of all bug fixes and enhancements in D $(VER).))\n\n"); - w.put("$(HR)\n\n"); + w.put("$(HR)\n\n"); - // print the detailed descriptions - changedRepos.each!(x => x.changes.writeTextChangesBody(w, x.headline)); + // print the detailed descriptions + changedRepos.each!(x => x.changes.writeTextChangesBody(w, x.headline)); + + if (revRange.length) + w.put("$(BR)$(BIG $(LNAME2 bugfix-list, List of all bug fixes and enhancements in D $(VER):))\n\n"); + } + else + { + w.put("$(BR)$(BIG List of all bug fixes and enhancements in D $(VER).)\n\n"); + } // print the entire changelog history if (revRange.length) - { - w.put("$(BR)$(BIG $(LNAME2 bugfix-list, List of all bug fixes and enhancements in D $(VER):))\n\n"); revRange.getBugzillaChanges.writeBugzillaChanges(w); - } } w.formattedWrite("$(CHANGELOG_NAV_LAST %s)\n", previousVersion); From a6eb5bd2ea0ecc44717771be1b95cfef93cee5be Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Tue, 22 Nov 2016 20:20:52 +0100 Subject: [PATCH 4/5] Address review comments --- changed.d | 78 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 10 deletions(-) diff --git a/changed.d b/changed.d index a525094..4b57546 100755 --- a/changed.d +++ b/changed.d @@ -26,11 +26,14 @@ rdmd changed.d "v2.071.2..upstream/stable" It is also possible to directly preview the generated changelog file: --- -rdmd changed.d "v2.071.2..upstream/stable" && dmd ../dlang.org/macros.ddoc ../dlang.org/html.ddoc ../dlang.org/dlang.org.ddoc ../dlang.org/doc.ddoc ../dlang.org/changelog/changelog.ddoc changelog.dd -Df../dlang.org/changelog/pending.html +rdmd changed.d "v2.071.2..upstream/stable" && dmd ../dlang.org/macros.ddoc ../dlang.org/html.ddoc ../dlang.org/dlang.org.ddoc ../dlang.org/doc.ddoc ../dlang.org/changelog/changelog.ddoc changelog.dd -Df../dlang.org/web/changelog/pending.html --- If no arguments are passed, only the manual changes will be accumulated and Bugzilla won't be queried (faster). + +A manual changelog entry consists of a title line, a blank separator line and +the description. */ // NOTE: this script requires libcurl to be linked in (usually done by default). @@ -157,7 +160,7 @@ auto getBugzillaChanges(string revRange) type = "bugs"; break; - case "enhancement": + case "enhancement": type = "enhancements"; break; @@ -183,7 +186,7 @@ Returns: The parsed `ChangelogEntry` ChangelogEntry readChangelog(string filename) { import std.algorithm.searching : countUntil; - import std.file: read; + import std.file : read; import std.path : baseName, stripExtension; import std.string : strip; @@ -201,7 +204,7 @@ ChangelogEntry readChangelog(string filename) ChangelogEntry entry = { title: (cast(string) fileRead[0..firstLineBreak]).strip, description: (cast(string) fileRead[firstLineBreak..$]).strip, - basename: filename.baseName.stripExtension // let's be safe here + basename: filename.baseName.stripExtension }; return entry; } @@ -296,11 +299,59 @@ void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w) } } +string toString(Month month){ + string s = void; + with(Month) + final switch (month) { + case jan: + s = "January"; + break; + case feb: + s = "February"; + break; + case mar: + s = "March"; + break; + case apr: + s = "April"; + break; + case may: + s = "May"; + break; + case jun: + s = "June"; + break; + case jul: + s = "July"; + break; + case aug: + s = "August"; + break; + case sep: + s = "September"; + break; + case oct: + s = "October"; + break; + case nov: + s = "November"; + break; + case dec: + s = "December"; + break; + } + return s; +} + int main(string[] args) { auto outputFile = "./changelog.dd"; auto nextVersionString = "LATEST"; - auto nextVersionDate = "September 19, 2016"; + + auto currDate = Clock.currTime(); + auto nextVersionDate = "%s %02d, %04d" + .format(currDate.month.toString, currDate.day, currDate.year); + string previousVersion = "Previous version"; bool hideTextChanges = false; string revRange; @@ -314,6 +365,13 @@ int main(string[] args) "prev-version", &previousVersion, // this can automatically be detected "no-text", &hideTextChanges); + if (helpInformation.helpWanted) + { +`Changelog generator +Please supply a bugzilla version +./changed.d "v2.071.2..upstream/stable"`.defaultGetoptPrinter(helpInformation.options); + } + if (args.length >= 2) { revRange = args[1]; @@ -347,14 +405,14 @@ int main(string[] args) auto changedRepos = repos .map!(repo => Repo(buildPath("..", repo.path, "changelog"), repo.headline)) - .filter!(x => x.path.exists) - .map!(x => tuple!("headline", "changes")(x.headline, x.path.readTextChanges.array)) - .filter!(x => !x.changes.empty); + .filter!(r => r.path.exists) + .map!(r => tuple!("headline", "changes")(r.headline, r.path.readTextChanges.array)) + .filter!(r => !r.changes.empty); // print the overview headers - changedRepos.each!(x => x.changes.writeTextChangesHeader(w, x.headline)); + changedRepos.each!(r => r.changes.writeTextChangesHeader(w, r.headline)); - if (revRange.length) + if (!revRange.empty) w.put("$(BR)$(BIG $(RELATIVE_LINK2 bugfix-list, List of all bug fixes and enhancements in D $(VER).))\n\n"); w.put("$(HR)\n\n"); From dd82e0cd0aa487fc724e995b41495dc9c6510265 Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Fri, 16 Dec 2016 02:32:56 +0100 Subject: [PATCH 5/5] Add dlang.org,installer and tools repos to the changelog builder --- changed.d | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/changed.d b/changed.d index 4b57546..fb3d10e 100755 --- a/changed.d +++ b/changed.d @@ -399,9 +399,12 @@ Please supply a bugzilla version { // search for raw change files alias Repo = Tuple!(string, "path", string, "headline"); - auto repos = [Repo("dmd", "Language changes"), + auto repos = [Repo("dmd", "Compiler changes"), Repo("druntime", "Runtime changes"), - Repo("phobos", "Library changes")]; + Repo("phobos", "Library changes"), + Repo("dlang.org", "Language changes"), + Repo("installer", "Installer changes"), + Repo("tools", "Tools changes")]; auto changedRepos = repos .map!(repo => Repo(buildPath("..", repo.path, "changelog"), repo.headline))