Merge pull request #186 from wilzbach/add_changelog_dir_builder

add changelog from directory builder

changed.d

@@ -1,27 +1,39 @@
 #!/usr/bin/env rdmd
+
 // Written in the D programming language
 
 /**
 Change log generator which fetches the list of bugfixes
 from the D Bugzilla between the given dates.
+Moreover manual changes are accumulated from raw text files in the
+Dlang repositories.
 It stores its result in DDoc form to a text file.
 
-Copyright: Dmitry Olshansky 2013.
+Copyright: D Language Foundation 2016.
 
 License:   $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0).
 
 Authors:   Dmitry Olshansky,
-           Andrej Mitrovic
+           Andrej Mitrovic,
+           Sebastian Wilzbach
 
 Example usage:
+
 ---
-rdmd changed.d --start=2013-01-01 --end=2013-04-01
+rdmd changed.d "v2.071.2..upstream/stable"
-rdmd changed.d --start=2013-01-01 (end date implicitly set to current date)
 ---
 
-$(B Note:) The script will cache the results of an invocation, to avoid
+It is also possible to directly preview the generated changelog file:
-re-querying bugzilla when invoked with the same arguments.
+
-Use the --nocache option to override this behavior.
+---
+rdmd changed.d "v2.071.2..upstream/stable" && dmd ../dlang.org/macros.ddoc ../dlang.org/html.ddoc ../dlang.org/dlang.org.ddoc ../dlang.org/doc.ddoc ../dlang.org/changelog/changelog.ddoc changelog.dd -Df../dlang.org/web/changelog/pending.html
+---
+
+If no arguments are passed, only the manual changes will be accumulated and Bugzilla
+won't be queried (faster).
+
+A manual changelog entry consists of a title line, a blank separator line and
+the description.
 */
 
 // NOTE: this script requires libcurl to be linked in (usually done by default).
@@ -32,6 +44,21 @@ import std.net.curl, std.conv, std.exception, std.algorithm, std.csv, std.typeco
     std.stdio, std.datetime, std.array, std.string, std.file, std.format, std.getopt,
     std.path;
 
+import std.range.primitives, std.traits;
+
+struct BugzillaEntry
+{
+    int id;
+    string summary;
+}
+
+struct ChangelogEntry
+{
+    string title; // the first line (can't contain links)
+    string description; // a detailed description (separated by a new line)
+    string basename; // basename without extension (used for the anchor link to the description)
+}
+
 auto templateRequest =
     `https://issues.dlang.org/buglist.cgi?bug_id={buglist}&bug_status=RESOLVED&resolution=FIXED&`~
         `ctype=csv&columnlist=component,bug_severity,short_desc`;
@@ -49,12 +76,6 @@ auto dateFromStr(string sdate)
     return Date(year, month, day);
 }
 
-struct Entry
-{
-    int id;
-    string summary;
-}
-
 string[dchar] parenToMacro;
 shared static this()
 {
@@ -105,17 +126,14 @@ auto getIssues(string revRange)
 }
 
 /** Generate and return the change log as a string. */
-string getChangeLog(string revRange)
+auto getBugzillaChanges(string revRange)
 {
     auto req = generateRequest(templateRequest, getIssues(revRange));
     debug stderr.writeln(req);  // write text
     auto data = req.get;
 
     // component (e.g. DMD) -> bug type (e.g. regression) -> list of bug entries
-    Entry[][string][string] entries;
+    BugzillaEntry[][string][string] entries;
-
-    immutable bugtypes = ["regressions", "bugs", "enhancements"];
-    immutable components = ["DMD Compiler", "Phobos", "Druntime", "dlang.org", "Optlink", "Tools", "Installer"];
 
     foreach (fields; csvReader!(Tuple!(int, string, string, string))(data, null))
     {
@@ -149,49 +167,282 @@ string getChangeLog(string revRange)
             default: assert(0, type);
         }
 
-        entries[comp][type] ~= Entry(fields[0], fields[3].idup);
+        entries[comp][type] ~= BugzillaEntry(fields[0], fields[3].idup);
     }
+    return entries;
+}
 
-    Appender!string result;
+/**
+Reads a single changelog file.
 
-    result ~= "$(BUGSTITLE Language Changes,\n";
+An entry consists of a title line, a blank separator line and
-    result ~= "-- Insert major language changes here --\n)\n\n";
+the description
 
-    result ~= "$(BUGSTITLE Library Changes,\n";
+Params:
-    result ~= "-- Insert major library changes here --\n)\n\n";
+    filename = changelog file to be parsed
 
-    result ~= "$(BR)$(BIG List of all bug fixes and enhancements:)\n\n";
+Returns: The parsed `ChangelogEntry`
+*/
+ChangelogEntry readChangelog(string filename)
+{
+    import std.algorithm.searching : countUntil;
+    import std.file : read;
+    import std.path : baseName, stripExtension;
+    import std.string : strip;
+
+    auto fileRead = cast(ubyte[]) filename.read;
+    auto firstLineBreak = fileRead.countUntil("\n");
+
+    // filter empty files
+    if (firstLineBreak < 0)
+        return ChangelogEntry.init;
+
+    // filter ddoc files
+    if (fileRead.length < 4 || fileRead[0..4] == "Ddoc")
+        return ChangelogEntry.init;
+
+    ChangelogEntry entry = {
+        title: (cast(string) fileRead[0..firstLineBreak]).strip,
+        description: (cast(string) fileRead[firstLineBreak..$]).strip,
+        basename: filename.baseName.stripExtension
+    };
+    return entry;
+}
+
+/**
+Looks for changelog files (ending with `.dd`) in a directory and parses them.
+
+Params:
+    changelogDir = directory to search for changelog files
+
+Returns: An InputRange of `ChangelogEntry`s
+*/
+auto readTextChanges(string changelogDir)
+{
+    import std.algorithm.iteration : filter, map;
+    import std.file : dirEntries, SpanMode;
+    import std.string : endsWith;
+
+    return dirEntries(changelogDir, SpanMode.shallow)
+            .filter!(a => a.name().endsWith(".dd"))
+            .map!readChangelog
+            .filter!(a => a.title.length > 0);
+}
+
+/**
+Writes the overview headline of the manually listed changes in the ddoc format as list.
+
+Params:
+    changes = parsed InputRange of changelog information
+    w = Output range to use
+*/
+void writeTextChangesHeader(Entries, Writer)(Entries changes, Writer w, string headline)
+    if (isInputRange!Entries && isOutputRange!(Writer, string))
+{
+    // write the overview titles
+    w.formattedWrite("$(BUGSTITLE %s,\n\n", headline);
+    scope(exit) w.put("\n)\n\n");
+    foreach(change; changes)
+    {
+        w.formattedWrite("$(LI $(RELATIVE_LINK2 %s,%s))\n", change.basename, change.title.escapeParens);
+    }
+}
+/**
+Writes the long description of the manually listed changes in the ddoc format as list.
+
+Params:
+    changes = parsed InputRange of changelog information
+    w = Output range to use
+*/
+void writeTextChangesBody(Entries, Writer)(Entries changes, Writer w, string headline)
+    if (isInputRange!Entries && isOutputRange!(Writer, string))
+{
+    w.formattedWrite("$(BUGSTITLE %s,\n\n", headline);
+    scope(exit) w.put("\n)\n\n");
+    foreach(change; changes)
+    {
+        w.formattedWrite("$(LI $(LNAME2 %s,%s)\n", change.basename, change.title.escapeParens);
+        scope(exit) w.put(")\n");
+        w.formattedWrite("    $(P %s  )\n", change.description.escapeParens);
+    }
+}
+
+/**
+Writes the fixed issued from Bugzilla in the ddoc format as a single list.
+
+Params:
+    changes = parsed InputRange of changelog information
+    w = Output range to use
+*/
+void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w)
+    if (isOutputRange!(Writer, string))
+{
+    immutable components = ["DMD Compiler", "Phobos", "Druntime", "dlang.org", "Optlink", "Tools", "Installer"];
+    immutable bugtypes = ["regressions", "bugs", "enhancements"];
 
     foreach (component; components)
+    {
         if (auto comp = component in entries)
         {
             foreach (bugtype; bugtypes)
             if (auto bugs = bugtype in *comp)
             {
-            result ~= format("$(BUGSTITLE %s %s,\n\n", component, bugtype);
+                w.formattedWrite("$(BUGSTITLE %s %s,\n\n", component, bugtype);
                 foreach (bug; sort!"a.id < b.id"(*bugs))
                 {
-                result ~= format("$(LI $(BUGZILLA %s): %s)\n",
+                    w.formattedWrite("$(LI $(BUGZILLA %s): %s)\n",
                                         bug.id, bug.summary.escapeParens());
                 }
-            result ~= ")\n";
+                w.put(")\n");
             }
         }
+    }
+}
 
-    return result.data;
+string toString(Month month){
+    string s = void;
+    with(Month)
+    final switch (month) {
+        case jan:
+            s = "January";
+            break;
+        case feb:
+            s = "February";
+            break;
+        case mar:
+            s = "March";
+            break;
+        case apr:
+            s = "April";
+            break;
+        case may:
+            s = "May";
+            break;
+        case jun:
+            s = "June";
+            break;
+        case jul:
+            s = "July";
+            break;
+        case aug:
+            s = "August";
+            break;
+        case sep:
+            s = "September";
+            break;
+        case oct:
+            s = "October";
+            break;
+        case nov:
+            s = "November";
+            break;
+        case dec:
+            s = "December";
+            break;
+    }
+    return s;
 }
 
 int main(string[] args)
 {
-    if (args.length != 2)
+    auto outputFile = "./changelog.dd";
+    auto nextVersionString = "LATEST";
+
+    auto currDate = Clock.currTime();
+    auto nextVersionDate = "%s %02d, %04d"
+        .format(currDate.month.toString, currDate.day, currDate.year);
+
+    string previousVersion = "Previous version";
+    bool hideTextChanges = false;
+    string revRange;
+
+    auto helpInformation = getopt(
+        args,
+        std.getopt.config.passThrough,
+        "output|o", &outputFile,
+        "date", &nextVersionDate,
+        "version", &nextVersionString,
+        "prev-version", &previousVersion, // this can automatically be detected
+        "no-text", &hideTextChanges);
+
+    if (helpInformation.helpWanted)
     {
-        stderr.writeln("Usage: ./changed <revision range>, e.g. ./changed v2.067.1..upstream/stable");
+`Changelog generator
-        return 1;
+Please supply a bugzilla version
+./changed.d "v2.071.2..upstream/stable"`.defaultGetoptPrinter(helpInformation.options);
     }
 
-    string logPath = "./changelog.txt".absolutePath.buildNormalizedPath;
+    if (args.length >= 2)
-    std.file.write(logPath, getChangeLog(args[1]));
+    {
-    writefln("Change log generated to: '%s'", logPath);
+        revRange = args[1];
 
+        // extract the previous version
+        auto parts = revRange.split("..");
+        if (parts.length > 1)
+            previousVersion = parts[0];
+    }
+    else
+    {
+        writeln("Skipped querying Bugzilla for changes. Please define a revision range e.g ./changed v2.072.2..upstream/stable");
+    }
+
+    auto f = File(outputFile, "w");
+    auto w = f.lockingTextWriter();
+    w.put("Ddoc\n\n");
+    w.formattedWrite("$(CHANGELOG_NAV_LAST %s)\n\n", previousVersion);
+
+    {
+        w.formattedWrite("$(VERSION %s, =================================================,\n\n", nextVersionDate);
+        scope(exit) w.put(")\n");
+
+        if (!hideTextChanges)
+        {
+            // search for raw change files
+            alias Repo = Tuple!(string, "path", string, "headline");
+            auto repos = [Repo("dmd", "Compiler changes"),
+                          Repo("druntime", "Runtime changes"),
+                          Repo("phobos", "Library changes"),
+                          Repo("dlang.org", "Language changes"),
+                          Repo("installer", "Installer changes"),
+                          Repo("tools", "Tools changes")];
+
+            auto changedRepos = repos
+                 .map!(repo => Repo(buildPath("..", repo.path, "changelog"), repo.headline))
+                 .filter!(r => r.path.exists)
+                 .map!(r => tuple!("headline", "changes")(r.headline, r.path.readTextChanges.array))
+                 .filter!(r => !r.changes.empty);
+
+            // print the overview headers
+            changedRepos.each!(r => r.changes.writeTextChangesHeader(w, r.headline));
+
+            if (!revRange.empty)
+                w.put("$(BR)$(BIG $(RELATIVE_LINK2 bugfix-list, List of all bug fixes and enhancements in D $(VER).))\n\n");
+
+            w.put("$(HR)\n\n");
+
+            // print the detailed descriptions
+            changedRepos.each!(x => x.changes.writeTextChangesBody(w, x.headline));
+
+            if (revRange.length)
+                w.put("$(BR)$(BIG $(LNAME2 bugfix-list, List of all bug fixes and enhancements in D $(VER):))\n\n");
+        }
+        else
+        {
+                w.put("$(BR)$(BIG List of all bug fixes and enhancements in D $(VER).)\n\n");
+        }
+
+        // print the entire changelog history
+        if (revRange.length)
+            revRange.getBugzillaChanges.writeBugzillaChanges(w);
+    }
+
+    w.formattedWrite("$(CHANGELOG_NAV_LAST %s)\n", previousVersion);
+
+    // write own macros
+    w.formattedWrite(`Macros:
+    VER=%s
+    TITLE=Change Log: $(VER)`, nextVersionString);
+
+    writefln("Change log generated to: '%s'", outputFile);
     return 0;
 }